
    RЦi                         S r SSKrSSKrSSKrSSKrSSKrSSKrSSKJrJ	r	  \R                  " \5      r " S S5      rg)zCheckpoint Saver

Track top-n training checkpoints and maintain recovery checkpoints on specified intervals.

Hacked together by / Copyright 2020 Ross Wightman
    N   )unwrap_modelget_state_dictc            
       l    \ rS rSrSSSSSSSSS\4
S jrS	 rS
 rSS jrSS jr	SS jr
SS jrS rSrg)CheckpointSaver   N
checkpointrecovery F
   c                 r   Xl         X l        X0l        X@l        XPl        / U l        S U l        S U l        SU l        SU l	        SU l
        Xl        Xl        X`l        Xpl        SU l        Xl        U
(       a  ["        R$                  O["        R&                  U l        Xl        Xl        U R*                  S:  d   eg )Nr   Tz.pth.tarr   )model	optimizerargs	model_ema
amp_scalercheckpoint_files
best_epochbest_metriccurr_recovery_fileprev_recovery_filecan_hardlinkcheckpoint_dirrecovery_dirsave_prefixrecovery_prefix	extension
decreasingoperatorltgtcmpmax_history	unwrap_fn)selfr   r   r   r   r   checkpoint_prefixr   r   r   r   r#   r$   s                Z/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/utils/checkpoint_saver.py__init__CheckpointSaver.__init__   s    " 
"	"$ !#"$"$  -(,.#$",8;;(++&"1$$$    c                    U R                   (       a;   [        R                  R                  U5      (       a  [        R                  " U5        [        R                  " X5        g ! [
        [        4 a  nSU l          S nAN3S nAff = fNF)r   ospathexistsunlinkOSErrorNotImplementedErrorreplacer%   srcdstes       r'   _replaceCheckpointSaver._replaceB   s_    *77>>#&&IIcN 	

3 01 *$)!!*s   :A$ $B4B  Bc                 >   U R                   (       aR   [        R                  R                  U5      (       a  [        R                  " U5        [        R
                  " X5        g [        R                  " X5        g ! [        [        4 a  nSU l          S nAN3S nAff = fr,   )
r   r-   r.   r/   r0   linkr1   r2   shutilcopy2r4   s       r'   
_duplicateCheckpointSaver._duplicateK   sn    *77>>#&&IIcN! 	S 01 *$)!!*s   AA; ;BBBc                 T   U[        U R                  5      R                  R                  5       [	        U R                  U R
                  5      U R                  R                  5       SS.nU R                  b(  U R                  R                  US'   U R                  US'   U R                  b0  U R                  R                  5       X@R                  R                  '   U R                  b#  [	        U R                  U R
                  5      US'   Ub  X4S'   [        R                  " XA5        g )N   )epocharch
state_dictr   versionrC   r   state_dict_emametric)typer   __name__lowerr   r$   r   rD   r   r   state_dict_keyr   torchsave)r%   	save_pathrB   rG   
save_states        r'   _saveCheckpointSaver._saveW   s    $--335(T^^D224

 99 !%Jv!%Jv??&9=9S9S9UJ556>>%+9$..$..+YJ'(#)x 

:)r*   c                    [        [        U R                  5      U5      nU R                  U-
  nUS:  d  [        U R                  5      U::  a  g U R                  US  nU HA  n [        R                  SR                  U5      5        [        R                  " US   5        MC     U R                  S U U l        g ! [         a/  n[        R                  SR                  U5      5         S nAM  S nAff = f)Nr   zCleaning checkpoint: {}z(Exception '{}' while deleting checkpoint)minlenr   r#   _loggerdebugformatr-   remove	Exceptionerror)r%   trimdelete_index	to_deletedr7   s         r'   _cleanup_checkpoints$CheckpointSaver._cleanup_checkpointsj   s    3t,,-t4''$.!s4#8#89\I)),-8	AT7>>qAB		!A$  !% 5 5m| D  THOOPQRSSTs   $=B::
C3$C..C3c                    US:  d   e[         R                  R                  U R                  SU R                  -   5      n[         R                  R                  U R                  SU R                  -   5      nU R                  X1U5        U R                  X45        U R                  (       a  U R                  S   OS n[        U R                  5      U R                  :  d  Ub  U R                  X%S   5      (       Ga  [        U R                  5      U R                  :  a  U R                  S5        SR                  U R                  [        U5      /5      U R                  -   n[         R                  R                  U R                  U5      nU R                  XG5        U R                  R                  Xr45        [!        U R                  S U R"                  (       + S9U l        S	nU R                   H  n	US
R%                  U	5      -  nM     [&        R)                  U5        Ub  U R*                  b   U R                  X R*                  5      (       aT  Xl        X l        [         R                  R                  U R                  SU R                  -   5      n
U R                  XJ5        U R*                  c  S$ U R*                  U R,                  4$ )Nr   tmplastr   -c                     U S   $ )Nr    )xs    r'   <lambda>1CheckpointSaver.save_checkpoint.<locals>.<lambda>   s    adr*   )keyreversezCurrent checkpoints:
z {}

model_best)NN)r-   r.   joinr   r   rP   r8   r   rT   r#   r"   r_   r   strr>   appendsortedr   rW   rU   infor   r   )r%   rB   rG   tmp_save_pathlast_save_path
worst_filefilenamerN   checkpoints_strcbest_save_paths              r'   save_checkpointCheckpointSaver.save_checkpointx   s&   zzT%8%8%$..:PQd&9&96DNN;RS

=0m4262G2GT**2.T
%%&)9)99~xx1..4(()T-=-==))!,xx!1!13u: >?$..PHT%8%8(CIOON6!!(())<=$*%%" OO+%D! 7O**7>>!#44 +LL)!t'7'7'?488FTdTdCeCe"'#) !#d.A.A<RVR`R`C`!a?#//7|`d>N>NPTP_P_=``r*   c                 F   US:  d   e[         R                  R                  U R                  SU R                  -   5      nU R                  X15        SR                  U R                  [        U5      [        U5      /5      U R                  -   n[         R                  R                  U R                  U5      nU R                  X55        [         R                  R                  U R                  5      (       aO   [        R                  SR                  U R                  5      5        [         R                  " U R                  5        U R"                  U l
        XPl        g ! [         a8  n[        R!                  SR                  X`R                  5      5         S nANUS nAff = f)Nr   recovery_tmpre   zCleaning recovery: {}z Exception '{}' while removing {})r-   r.   rn   r   r   rP   r   ro   r8   r/   r   rU   rV   rW   rX   rY   rZ   r   )r%   rB   	batch_idxrs   rv   rN   r7   s          r'   save_recoveryCheckpointSaver.save_recovery   s,   zzT%6%68WX

=(88T113u:s9~NORVR`R``GGLL!2!2H=	m/77>>$1122e5<<T=T=TUV		$112 #'"9"9"+  e@GGKbKbcddes   8AE 
F (.FF c                     [         R                  R                  U R                  U R                  5      n[
        R
                  " US-   U R                  -   5      n[        U5      n[        U5      (       a  US   $ S$ )N*r   r   )	r-   r.   rn   r   r   globr   rq   rT   )r%   recovery_pathfiless      r'   find_recoveryCheckpointSaver.find_recovery   s^    T%6%68L8LM		-#->?uu::uQx-2-r*   )r   r   r   r   r   r   r   r"   r   r   r   r#   r   r   r   r   r   r   r   r$   )N)r   )rI   
__module____qualname____firstlineno__r   r(   r8   r>   rP   r_   rz   r   r   __static_attributes__rg   r*   r'   r   r      sN    
 *&")%V
*&E%aN,$.r*   r   )__doc__r   loggingr   r-   r<   rL   r   r   r   	getLoggerrI   rU   r   rg   r*   r'   <module>r      s?       	   / 

H
%_. _.r*   