<Цinj*tSSKJrJrJrJrJr SSKrSSKJr SSKJ r J r J r J r J r JrJrJrJrJrJrJrJrJrJrJrJrJrJr SS/r"SS\5rS S \S \S \S \ S \ S \S 3 -\lS\\S\\S\\S\\S\\S\\S\\S\\S\S\S\\ \4S\\ \4S\\ \4S\ S\ S\S\S\4$S jr!S\\S\\S\\S\\S\\S\\S\\S\\S\S\S\\ \4S\\ \4S\\ \4S\ S\ S\S\S\4$S!jr"S\\S\\S\\S\\S\\S\\S\\S\\S\S\S\ S\ S\\ \4S\ S\ S\S\S\S"S4&S#jr#\ "\!S$9S(S\\S\\S\\S\\S\\S\\S%\\S\S\S&\\S\\S\\S\S\S\ S\ S\\ \4S\ S\ S\4(S'jj5r$g)))castListOptionalTupleUnionN)Tensor)_capturable_doc_default_to_fused_or_foreach_device_dtype_check_for_fused_differentiable_doc_disable_dynamo_if_unsupported _foreach_doc _fused_doc!_get_capturable_supported_devices_get_scalar_dtype _get_value _maximize_doc _params_doc_stack_if_compiling_use_grad_for_differentiable _view_as_real DeviceDictDeviceDtypeDict OptimizerParamsTAdamadamc^\rSrSrSSSSSSS.S\S\\\4S\\\\4\\\44S\S \S \ S \ \ S \ S \ S\ S\ \ 4U4Sjjjjr U4Sjr Sr \SSj5rSrU=r$)r"FN)foreachmaximize capturabledifferentiablefusedparamslrbetaseps weight_decayamsgradr!r"r#r$r%c n>[U[5(a8U(aU (d [S5eUR5S:wa [S5eSU::d[SU35eSU::d[SU35eSUSs=::aS:dO [S US35eSUSs=::aS:dO [S US35eSU::d[S U35e[US[5(a[US[5(d;[US[5(a[US[5(d [S 5e[US[5(a;U (dU(a [S 5eUSR5S:wa [S5e[US[5(a;U (dU(a [S5eUSR5S:wa [S5e[ UUUUUUUU U U S9 n [ T U]X5 U (a,U (a [S5eSUl U(a [S5egg)NElr as a Tensor is not supported for capturable=False and foreach=Truer zTensor lr must be 1-elementzInvalid learning rate: zInvalid epsilon value: r?z#Invalid beta parameter at index 0: z#Invalid beta parameter at index 1: zInvalid weight_decay value: z0betas must be either both floats or both TensorszKbetas[0] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[0] must be 1-elementzKbetas[1] as a Tensor is not supported for capturable=False and foreach=Truez!Tensor betas[1] must be 1-element) r'r(r)r*r+r"r!r#r$r%z)`fused` does not support `differentiable`Tz0`fused` and `foreach` cannot be `True` together.) isinstancer ValueErrornumelfloatdictsuper__init__ RuntimeError_step_supports_amp_scaling)selfr&r'r(r)r*r+r!r"r#r$r%defaults __class__s O/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/torch/optim/adam.pyr6 Adam.__init__#s) b& ! !z [xxzQ !>??by6rd;< <cz6se<= =eAh$$B58*MN NeAh$$B58*MN Nl";L>JK K a% ( (Za%-H-H58V,,E!Hf1M1MOP P eAh ' '' aQx~~1$ !DEE eAh ' '' aQx~~1$ !DEE%!)   * "#NOO.2D + "#UVV c >[TU]U5 URGH2nURSS5 URSS5 URSS5 URSS5 URSS5 URSS5nUSHnURR U/5n[ U5S :wdM0[R"US 5(aMP[US 5nUS(d US(a'[R"U[US 9URS 9O[R"U[5S 9US 'M GM5 g)Nr+Fr"r!r#r$r%r&rstepis_fuseddtypedevicerD) r5 __setstate__ param_groups setdefaultstategetlentorch is_tensorr3tensorrrE)r9rJgroupr%pp_statestep_valr;s r<rGAdam.__setstate__ps U#&&E   Y .   Z /   Y -   \5 1   -u 5$$Wd3E8_**..B/w<1$U__WV_-M-M$WV_5H!.%.  $"3U"C#$88 #\\(:K:MNFO %'r>cSnUSGHKn U RcMU[R"U 5-nURU 5 U RR(a [ S5eURU R5 UR U n [U 5S:XaUS(a [U 5 US(d US(a*[R"S[USS9U RS 9O[R"S [5S 9U S '[R"U [RS 9U S'[R"U [RS 9U S'US(a&[R"U [RS 9U S'URU S5 URU S5 US(aURU S5 US(aU S R(a [ S5eUS(a3[R "US5(aUS(d [ S5eURU S 5 GMN U$)NFr&zJAdam does not support sparse gradients, please consider SparseAdam insteadrr%r#rArCr.rFr@) memory_formatexp_avg exp_avg_sqr+max_exp_avg_sqr$zB`requires_grad` is not supported for `step` in differentiable moder!r'r-)gradrM is_complexappend is_sparser7rJrLr zerosrrErO zeros_likepreserve_format requires_gradrN) r9rPparams_with_gradgradsexp_avgs exp_avg_sqsmax_exp_avg_sqs state_steps has_complexrQrJs r< _init_groupAdam._init_groups xAvv!u//22  ''*66##&d QVV$ 1 u:?W~5a8!.%.  "3U7^"L#$88 #\\#5F5HI&M(-'7'7)>)>(E)$+0*:*:)>)>+E,'Y'272B2BU-B-B3./i 01""5#67##**51A+BC)*uV}/J/J&\ )$d 44!,/&_""5=1{!|r>cUR5 SnUb%[R"5 U"5nSSS5 URH|n/n/n/n/n/n/n USupUR UUUUUUU 5n [ UUUUUU 4USU U U USUSUSUSUSUS US US [ US S5[ US S5S.6 M~ U$!,(df  N=f)zPerform a single optimization step. Args: closure (Callable, optional): A closure that reevaluates the model and returns the loss. Nr(r+r'r*r)r"r!r#r$r% grad_scale found_inf)r+ribeta1beta2r'r*r)r"r!r#r$r%rmrn) _cuda_graph_capture_health_checkrM enable_gradrHrjrgetattr) r9closurelossrPrcrdrerfrgrhrorpris r<r@ Adam.steps0 --/  ""$y%&&E-/ "$E%'H(*K,.O(*K >LE** K    i(';">2%Lz*i( .$%56Gn"4t<!$ T:) ''T [%$s C  C)r8)gMbP?)g?g+?g:0yE>rFN)__name__ __module__ __qualname____firstlineno__rrr3rrboolrr6rGrjrr@__static_attributes__ __classcell__)r;s@r<rr"s $(COKW#' $ $KWKW %- KWU5&=)5+??@ KW  KW  KWKW$KWKWKWKW~KWKWZ.IV"8"8r>a Implements Adam algorithm. .. math:: \begin{aligned} &\rule{110mm}{0.4pt} \\ &\textbf{input} : \gamma \text{ (lr)}, \beta_1, \beta_2 \text{ (betas)},\theta_0 \text{ (params)},f(\theta) \text{ (objective)} \\ &\hspace{13mm} \lambda \text{ (weight decay)}, \: \textit{amsgrad}, \:\textit{maximize}, \: \epsilon \text{ (epsilon)} \\ &\textbf{initialize} : m_0 \leftarrow 0 \text{ ( first moment)}, v_0\leftarrow 0 \text{ (second moment)},\: \widehat{v_0}^{max}\leftarrow 0\\[-1.ex] &\rule{110mm}{0.4pt} \\ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\ &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\ &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\ &\hspace{5mm}\textbf{if} \: \lambda \neq 0 \\ &\hspace{10mm} g_t \leftarrow g_t + \lambda \theta_{t-1} \\ &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\ &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\ &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\ &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\ &\hspace{5mm}\textbf{if} \: amsgrad \\ &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_{t-1}}^{max}, \widehat{v_t}) \\ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big) \\ &\hspace{5mm}\textbf{else} \\ &\hspace{10mm}\theta_t \leftarrow \theta_{t-1} - \gamma \widehat{m_t}/ \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\ &\rule{110mm}{0.4pt} \\[-1.ex] &\bf{return} \: \theta_t \\[-1.ex] &\rule{110mm}{0.4pt} \\[-1.ex] \end{aligned} For further details regarding the algorithm we refer to `Adam: A Method for Stochastic Optimization`_. z Args: a lr (float, Tensor, optional): learning rate (default: 1e-3). A tensor LR is not yet supported for all our implementations. Please use a float LR if you are not also specifying fused=True or capturable=True. betas (Tuple[float, float], optional): coefficients used for computing running averages of gradient and its square (default: (0.9, 0.999)) eps (float, optional): term added to the denominator to improve numerical stability (default: 1e-8) weight_decay (float, optional): weight decay (L2 penalty) (default: 0) amsgrad (bool, optional): whether to use the AMSGrad variant of this algorithm from the paper `On the Convergence of Adam and Beyond`_ (default: False) z a= .. Note:: A prototype implementation of Adam and AdamW for MPS supports `torch.float32` and `torch.float16`. .. _Adam\: A Method for Stochastic Optimization: https://arxiv.org/abs/1412.6980 .. _On the Convergence of Adam and Beyond: https://openreview.net/forum?id=ryQu7f-RZ r&rdrerfrgrhrmrnr+rirorpr'r*r)r"r#r$c  UcUbe[RR5(aE[U [5(de[U [5(de[U [5(de[U [ 5(aU R U R4U 0nOSn[U5GHunnU(dUUOUU*nUUnUUnUUn[RR5(ddU(a][5nUR RUR R:XaUR RU;d SUS35eUS- nU S:waURUU S9n[R"U5(a{[R"U5n[R"U5n[R"U5nU(a[R"UU5UU'[R"U5nUR nUb0URnUU4nUU;aU R!UUSS9UU'UUnOU nUR#USU- 5 UR%U 5R'UUR)5SU - S9 U(dU(aUnSU U-- nSU U-- n U U- n!U!R+5n"U R-5n#U(auU(aUUR/5n$OUUn$UUR1[R2"U$U55 UUR-5U#U"-- R5UU"- 5n%O(UR-5U#U"-- R5UU"- 5n%UR7UU%5 O[9U5nSU U-- nSU U-- n U U- n!U S -n#U(aB[R2"UUUUUS 9 UUR-5U#- R5U5n%O"UR-5U#- R5U5n%UR7UU%U!*S9 U(dGM[R"UU5(dGM[R:"UU5UU'GM g) NIIf capturable=True, params and state_steps must be on supported devices: .r ralphaT)rErD non_blocking)value?)out)rMjit is_scriptingr0r3rrErD enumeratecompiler is_compilingrtypeaddr\ view_as_realtolerp_mul_addcmul_conjnegsqrtclonecopy_maximumadd_addcdiv_rview_as_complex)&r&rdrerfrgrhrmrnr+rirorpr'r*r)r"r#r$ beta1_dictiparamr[rXrYstep_tcapturable_supported_devicesrErDkey device_beta1r@bias_correction1bias_correction2 step_size step_size_negbias_correction2_sqrtrZdenoms& r<_single_tensor_adamrTsE*  )"33 3 yy"e$$$$%''''%'''' %  27,, 1Le0T  f%5'uQxeAhY1+ ^ Q~~**,,+L+N ( !!V]]%7%77LL%%)EE {[[wZxxyz {F !  1 88E86D   E " "%%d+D((1G++J7J%*%7%78J%K"&&u-E  !KKE5/C*$"'((&TX("Y 31;CL L  dA ,-''diikU'K D 5$;  5$; --I%MMOM$4$9$9$; !!%4Q%7%=%=%?N%4Q%7N"((~z)RS $A&++-1F1VW$s]*+ OO%)>)NO$s]*+ NN7E *f%D 5$;  5$; --I$4c$9 ! oa0*/RSBTU)+0025JJPPQTU#*-BBHHM NN7E)N < 7u''q 22!&!6!6q7I!JOA M&r>c | ^,[U5S:Xag[U [5(aU(d [S5e[U [5(a1U(d [ S5eU R 5S:wa [ S5e[U [5(a1U(d [ S5eU R 5S:wa [ S5e[ RR5(d>U(a7[SS 9m,[U,4S j[X555(d S T,S 35eUcUbeU(aS 5e[R"XX#XE/5n[U [5(a'[U R5S:waU RU 0OSnUR!5GHuunnnnnnn[#[$[U5n[#[$[U5n[#[$[U5n[#[$[U5n[#[$[U5nUSRn UbU U;aU R'U SS9UU 'U(aUU OU n!U (a<U(a'[#[$[U5n"[)UUUUU"5 O[)UUUU5 U(a[ R*"U5n[ RR5(d>USR,(a*[ R."U[ R0"SSS9SS9 O[ R."US5 U S:wa4U(a[ R."UUU S9 O[ R2"UUU S9n[ R4"UUSU!- 5 [ R6"UU 5 [U [ R5(a[ R8"USU - 5n#Sn$OUn#SU - n$[ R:"UU#UU$5 AA#U(Ga{[ R<"U U5n%[ R<"U U5n&[ R>"U%S5 [ R>"U&S5 [ R@"U&5 [ RB"U%U 5 [ RD"U%5 [ RF"U&5 U%n'U&n(U(aE[#[$[U5n"[ RH"U"U5 [ RJ"U"5n)O[ RJ"U5n)[ RB"U)U(5 [ R."U)U5 [ RB"U)U'5 [ RL"UUU)5 GMUV*s/sHn*SU [OU*5-- PM n%n*UV*s/sHn*SU [OU*5-- PM n&n*[QU%V+s/sH n+U U+- S-PM sn+5n'U&V+s/sHn+U+S-PM n(n+U(aE[#[$[U5n"[ RH"U"U5 [ RJ"U"5n)O[ RJ"U5n)[ RB"U)U(5 [ R."U)U5 [ RL"UUU)U'5 GM gs sn*fs sn*fs sn+fs sn+f)Nrr-zHbeta1 as a Tensor is not supported for capturable=False and foreach=Truer zTensor beta1 must be 1-elementzHbeta2 as a Tensor is not supported for capturable=False and foreach=TruezTensor beta2 must be 1-elementF) supports_xlac3># UHTupURRURR:H=(a URRT;v MV g7frw)rEr).0rQr@rs r< %_multi_tensor_adam..sN 4 HHMMT[[-- - > !== >3sAArrz#_foreach ops don't support autogradcpuTrErr/)rErr))rLr0rr7r1r2rMrrrallzipr"_group_tensors_by_device_and_dtypestrrEvaluesrrrr _foreach_negis_cpu _foreach_add_rO _foreach_add_foreach_lerp_ _foreach_mul_ _foreach_mul_foreach_addcmul_ _foreach_pow _foreach_sub_ _foreach_neg_ _foreach_div__foreach_reciprocal__foreach_sqrt__foreach_maximum_ _foreach_sqrt_foreach_addcdiv_rr)-r&rdrerfrgrhrmrnr+rirorpr'r*r)r"r#r$grouped_tensorsrdevice_params_ device_grads_device_exp_avgs_device_exp_avg_sqs_device_max_exp_avg_sqs_device_state_steps__ device_params device_gradsdevice_exp_avgsdevice_exp_avg_sqsdevice_state_stepsrErdevice_max_exp_avg_sqsscaled_device_gradsrrrrrexp_avg_sq_sqrtr@bcrs- @r<_multi_tensor_adamrs!* 6{a"fj S  %  Z  ;;=A => >%  Z  ;;=A => > >> & & ( (Z'H( $ v3    wWWsVttu v  w  )"33 3DDD BB LO eV $ $U\\):e)C u   " " $  T&\>: DL-8 tF|-=>!$v,0CD!$v,0CDq!((  !fJ&>!&d!KJv -7z&)U  )-d6l@RS#("5"56L"M"'"5"56H"I   1F G    5    ;  # #M?O T;M :L$EZ---:L  ;M :L$EZ---:L  ,FV,WFVb2g^FV,WXI7G$H7GRW7G !$H)-d6l@RS#("5"56L"M"'"5"56H"I   1F G    5  # # _ %p  -X$HsZ*'Z/ Z4 *Z9returnc 8U(dgU(a [S5eUbURU0O0nUbURU0O0n[U [5(a'[ U R5S:waU RU 0OSn[ R "XX#XE/5nUR5GHnuunnuunnnnnnn[[[U5n[[[U5n[[[U5n [[[U5n![[[U5n"URS:XaUcUbeSun#n$Ub URUURUSS95n#Ub URUURUSS95n$UbUU;aU RUSS9UU'UUn [R"U"S5 [R"UUU U!UU"UU U U U UUU#U$S 9 U$cGMJ[R "U"U$/[#U"5-5 GMq g) Nz9Adam with fused=True does not support differentiable=Truermps)NNT)rrr ) r+r'rorpr*r)r"rmrn)r7rEr0rrrritemsrrrrIrrMr _fused_adam_rrL)%r&rdrerfrgrhrmrnr+rirorpr'r*r)r"r#r$grad_scale_dictfound_inf_dictlr_dictrrErrrrrrrrrrrrdevice_grad_scaledevice_found_infs% r< _fused_adamrsQ* VWW,6+A  J'r*3)>  9%B&b&11c"))n6MBSW  BB LO            "  T&\>: DL-8 tF|-=>!$v,0CD!$v,0CD ;;% $); ;;.8++  ! / : : f4 @!   -88 V$ ?    6#8 ee6eEGFOB .2      " %(& "  '   "%5$6=O9P$P S !r>)single_tensor_fnr!r%c U c5Uc2[XSS9unnU(a[U[5(a U(dSnU cSn UcSn[RR 5(d"[ SU55(d [S5eU(a.[RR5(a [S5eU (a.[RR5(a [S5eU (a*[RR5(d[nO7U(a*[RR5(d[nO[nU"UUUUUUU U UUUUUUUUU U S9 g) zfFunctional API that performs Adam algorithm computation. See :class:`~torch.optim.Adam` for details. NF) use_fusedc3V# UHn[U[R5v M! g7frw)r0rMr)rts r<radam..Ys!5-8 1ell##[s')zPAPI has changed, `state_steps` argument must contain a list of singleton tensorsz6torch.jit.script not supported with foreach optimizersz4torch.jit.script not supported with fused optimizers) r+rirorpr'r*r)r"r#r$rmrn) r r0rrMrrrr7rrrrr)r&rdrerfrgrhr!r#r$r%rmrnrir+rorpr'r*r)r"rfuncs r<rr)sHD }1 e 7 z"f--jG } >> & & ( (5-8522 ^  599))++STT ''))QRR UYY++-- //11!"  ! %%r>)NFFNNNF)%typingrrrrrrMr optimizerr r r r rrrrrrrrrrrrrrr__all__r__doc__r|r3rrrrrVr>r<rs65 . 6 i9iZ&N         %OA LNK LNK <NK6lNKf NK &\ NK f NK NKNKNKNK  NK  NK eVmNKNK !NK"#NK$%NK&'NKb` L` <`6l`f ` &\ ` f ` ````  `  ` eVm`` !`"#`$%`&'`F^ L^ <^6l^f ^ &\ ^ f ^ ^^^^ ^ ^ eVm^^ !^"#^$%^&'^( )^B 1DE#  #'"&U LU <U6lUf U &\ U f Ud^UUU D>U UUU"#U$ %U& 'U( eVm)U*+U, -U./UFUr>