
    RЦi                         S r SSKrSSKrSSKrSSKrSSKJrJrJ	r	  SSK
Jr  \R                  " \5      r " S S\5      rg)zCosine Scheduler

Cosine LR schedule with warmup, cycle/restarts, noise, k-decay.

Hacked together by / Copyright 2021 Ross Wightman
    N)TupleListUnion   )	Schedulerc            #          ^  \ rS rSrSr              SS\R                  R                  S\S\	S\	S\	S	\S
\S\	S\
S\
S\\\   \\\4   \S4   S\	S\	S\S\	S\
SS4"U 4S jjjrS\S\\	   4S jrSS jrSrU =r$ )CosineLRScheduler   aD  
Cosine decay with restarts.
This is described in the paper https://arxiv.org/abs/1608.03983.

Inspiration from
https://github.com/allenai/allennlp/blob/master/allennlp/training/learning_rate_schedulers/cosine.py

k-decay option based on `k-decay: A New Method For Learning Rate Schedule` - https://arxiv.org/abs/2004.05909
N	optimizer	t_initiallr_min	cycle_mulcycle_decaycycle_limitwarmup_twarmup_lr_initwarmup_prefixt_in_epochsnoise_range_t	noise_pct	noise_std
noise_seedk_decay
initializereturnc                   > [         TU ]  USU
UUUUUS9  US:  d   eUS:  d   eUS:X  a!  US:X  a  US:X  a  [        R                  S5        X l        X0l        X@l        XPl        X`l        Xpl	        Xl
        Xl        Xl        U R                  (       aL  U R                   Vs/ s H  nUU-
  U R                  -  PM     snU l        [         TU ]=  U R                  5        g U R                   Vs/ s H  nSPM     snU l        g s  snf s  snf )Nlr)param_group_fieldr   r   r   r   r   r   r   r   zjCosine annealing scheduler will have no effect on the learning rate since t_initial = t_mul = eta_mul = 1.)super__init___loggerwarningr   r   r   r   r   r   r   r   r   base_valueswarmup_stepsupdate_groups)selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   v_	__class__s                      W/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/scheduler/cosine_lr.pyr    CosineLRScheduler.__init__   s   & 	"#'!! 	 		
 1}}{{>i1n1AOO>? #"&& ,*==OSO_O_ `O_!!n"4!EO_ `DG!$"5"56,0,<,< =,<q,< =D !a !>s   "D.Dtc                 <   XR                   :  a,  U R                   Vs/ s H  o R                  X-  -   PM     nnU$ U R                  (       a  XR                   -
  nU R                  S:w  a  [
        R                  " [
        R                  " SXR                  -  SU R                  -
  -  -
  U R                  5      5      nU R                  U-  U R                  -  nUSU R                  U-  -
  SU R                  -
  -  U R                  -  -
  nO+XR                  -  nU R                  nXR                  U-  -
  nU R                  U-  nU R                   Vs/ s H  oU-  PM	     n	nU R                  n
X@R                  :  ab  U	 Vs/ s HS  nU R                  SXR                  -
  -  S[
        R                  " [
        R                  Xj-  -  XZ-  -  5      -   -  -   PMU     nnU$ U R                   Vs/ s H  oR                  PM     nnU$ s  snf s  snf s  snf s  snf )Nr   g      ?)r   r$   r   r   r   mathfloorlogr   r   r#   r   r   r   cospi)r&   r,   slrsit_it_currgammar'   lr_max_valuesklr_maxr(   s                r*   _get_lrCosineLRScheduler._get_lrQ   s   }}8<8I8IJ8I1&&.8ICJ4 
1 !!%~~"JJtxxA,>!dnnBT,U(UW[WeWefgnn)DNN:a$..A"55!dnn:LMPTP^P^^^'nnnnq01$$)E040@0@A0@1Y0@MAA### #0"/ KK#++)=">!dhhtwwY_YdOdgjgoOoFpBp"qq"/   
 -1,<,<=,<q{{,<=
5 K B
 >s   H
HAH1Hc                 \   [        SU=(       d    U R                  5      nU R                  S:X  a  U R                  U-  nOM[	        [
        R                  " U R                  * U R                  U-  S-
  -  SU R                  -
  -  5      5      nU R                  (       a  X R                  -   $ U$ )Nr         ?)	maxr   r   r   intr.   r/   r   r   )r&   cyclesr,   s      r*   get_cycle_length"CosineLRScheduler.get_cycle_lengtho   s    Q2$"2"23>>S 'ADJJ$..F2JQ2NOSTW[WeWeSefghA$($6$6q== =A=    )
r   r   r   r   r   r   r   r   r$   r   )        r?   r?   r   r   rF   FTNgq=
ףp?r?   *   r?   T)r   )__name__
__module____qualname____firstlineno____doc__torchoptim	OptimizerrA   floatboolr   r   r   r    r<   rC   __static_attributes____classcell__)r)   s   @r*   r	   r	      s,    !!# $&"' $JN#"  ##1>{{,,1> 1> 	1>
 1> 1> 1> 1> "1>  1> 1> !cE#s(OS$!FG1> 1> 1> 1>  !1>" #1>$ 
%1> 1>f e <> >rE   r	   )rL   loggingr.   numpynprM   typingr   r   r   	schedulerr   	getLoggerrH   r!   r	    rE   r*   <module>r[      sA        % %   

H
%b>	 b>rE   