͑i%SSKJr SSKrSSKrSSKrSSKJrJrJrJ r J r J r J r J r SSKrSSKJr SSKJrJrJrJrJr SSKJr SSKJr SS KJrJr \(abSS KJr SS K J!r! SS K"J#r#J$r$ SS KJ%r% SSK&J'r' SSK(J)r) SSK*J+r+ SSK,J-r- SSK.J/r/J0r0 \ Sr1\ \2\3\4\3S4\5\34r6S\7S'"SS\ 5r8\ "SS\2S5r9\ "SS\2S5r:/SQr;SSSS.r<\Rzr>Sq?Sr@"SS 5rA\A"5rBSAS!jrCSBSCS"jjrDSDS#jrESDS$jrFSDS%jrGSDS&jrHSDS'jrISDS(jrJSDS)jrKSES*jrLSFS+jrMSGSHS,jjrNSIS-jrOSJS.jrPSKS/jrQSLS0jrRSMS1jrSSNS2jrT\SOSPS3jj5rU"S4S55rVSQS6jrW\ SRSSS7jj5rX\ SRSTS8jj5rX\SUSVS9jj5rXSWSXS:jjrYSOSYS;jjrZ\ SZSSS<jj5r[\ SRSTS=jj5r[SUSVS>jjr[S[S\S?jjr\S[S]S@jjr]g)^) annotationsN) TYPE_CHECKINGAnyCallableLiteralProtocolTypeVarUnionoverload)core)_current_expected_place_dygraph_tracer dygraph_onlyin_dynamic_or_pir_mode in_pir_mode)signature_safe_contextmanager)OptimizerWithMixedPrecision) black_list white_list) Generator)AbstractContextManager) TypeAlias TypeGuard)Tensor) PlaceLike) _DTypeLiteral)Layer) _StateDict)OperatorProgramO0ODO1O2.r _CustomListcP\rSrSrSSjrS SjrS SjrS SjrSrg) _OptimizerLike<cgN)selflossstartup_program parameters no_grad_sets T/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/amp/auto_cast.pyminimize_OptimizerLike.minimize=s BEcgr,r-r.s r3step_OptimizerLike.stepEsr6cgr,r-)r. state_dicts r3set_state_dict_OptimizerLike.set_state_dictGsr6cgr,r-)r. set_to_zeros r3 clear_grad_OptimizerLike.clear_gradIsr6r-N) r/rr0r!r1 list[Tensor]r2z set[Tensor]returnz2tuple[list[Operator], list[tuple[Tensor, Tensor]]]rDNone)r<zdict[str, Tensor]rDrF)r@boolrDrF) __name__ __module__ __qualname____firstlineno__r4r9r=rA__static_attributes__r-r6r3r)r)<sI E E% E%  E %  E @  E $L Update black and white list according to users' custom list. r#z+Custom white list overlap custom black list)setcopyrr ValueErrorremoveadd)custom_white_listcustom_black_listleveldtype _white_list _black_listop_names r3 _update_listrzs }e e ''))JL/67K))JL/67K.(G+ !NOO)(G%""7+ OOG $)(G%""7+ OOG $)  ##r6cz[5nU(a*UR[RR:Xaggg)z= Judge whether current code block is in `amp_guard` context. TF)r _amp_levelr AmpLevelr%tracers r3 _in_amp_guardrs0 F    0 0 0r6cv[5nU=(a' UR[RR:H$r,)rr|r r}r&r~s r3_in_pure_fp16_guardrs*  F  ;f''4==+;+;;;r6c[RRR5nUSS:=(d [R"5$)z0 Judge whether current gpu support float16 amp. r)paddledevicecudaget_device_capabilityis_compiled_with_rocm)props r3_is_gpu_float16_supportedrs8 ==   3 3 5D 7a< 967799r6c@[RRR5n[RR5nUb'US:wa![ UR S5S5S:nOSnUSS:=(a U=(d [R"5$)z1 Judge whether current gpu support bfloat16 amp. False.r F)rrrrversionintsplitr)r cuda_versioncuda_version_checks r3_is_gpu_bfloat16_supportedrs ==   3 3 5D>>&&(LLG$; !3!3C!8!;<B" 7a< .. P&2N2N2PPr6c[5n[R"UR55[RR :$)z_ Judge whether current xpu device support float16 amp. Only XPU2 and XPU3 support float16 amp. )r r get_xpu_device_version get_device_id XPUVersionXPU2places r3_is_xpu_float16_supportedrs: $ %E ##E$7$7$9: ??   r6c[5n[R"UR55[RR :$)z Judge whether current xpu device support bfloat16 amp. Only XPU3 support bfloat16 amp. Although XPU2 supports bfloat16 computing, but XPU2's bfloat16 operators haven't been widely covered. )r r rrrXPU3rs r3_is_xpu_bfloat16_supportedrs: $ %E ##E$7$7$9: ??   r6c[5nUR5S:H=(dG UR5S:H=(d- UR5S:H=(d UR5S:H$)z; Judge whether current custom device support bfloat16 amp. npu intel_hpu iluvatar_gpu metax_gpu)r get_device_typers r3$_is_custom_device_bfloat16_supportedrsf $ %E 5( 2  "k 1 2  "n 4 2  "k 1 r6crSnUR(dSnU$[U[RR[RR [RR [RR[RR45(aSnU$URS:Xd{US:Xaw[U[RR[RR[RR[RR45(aSnU$)NFTfloat16)_cast_to_low_precision isinstancernn BatchNorm BatchNorm1D BatchNorm2D BatchNorm3D SyncBatchNorm_dtype LayerNormInstanceNorm1DInstanceNorm2DInstanceNorm3D)layerrvneed_keep_fp32s r3rrsN  ' '8 5  II   II ! ! II ! ! II ! ! II # #      ,,) # )    ## (( (( ((     r6ct/n/nSnUc/nGO[U[RR5(aU/nO[U[5(a-[ U[RR5(aU/nO[U[ 5(aUHn[U[RR5(aURU5 M?[ U[RR5(aURU5 M{[U5e O [U5e[[U55H"nX&RSS9H nSUl M M$ [U5n[[U55H4nXRSS9Hn[Xs5(dMSUl M M6 g)Nz^excluded_layers must be either a nn.Layer instance/type or a list of nn.Layer instances/types.T include_selfF)rrrrtype issubclasslistappend TypeErrorrangelen sublayersrtuple)modelsexcluded_layersexcluded_layers_instancesexcluded_layers_types error_messageitemidxrs r3set_excluded_layersrsq!#tM OVYY__ 5 5%4$5! OT * *z00"1 1 OT * *#D$ 00)006D&))//22%,,T2 .. $ &&S234.3==> E,1E ( 5 ""78S[![***=E%77/4,>"r6c>U(a#UR5Hn[XAX#5 M URR5HupVUcM U"Xb5nM URR5HupXUcM U"X5URU'M! X lgr,)children _pir_apply _parametersitems_buffersr) r.funcrvinclude_sublayersrkeyparam param_appliedbufs r3rr:s ]]_E uE =%&&,,.    .M/MM'') ?!%c!1DMM# *Kr6c[RR5n[RR5n[RR U5 UR 5nUR HnUR5S:XdMUR5SUR:XdM<URS5R5n[R"Xa5nSUl [RRXpR5 URU5 O SSS5 UR!U5 [RR U5 [R"R%5 UR 5n[RR'UR5nUR(UlUR*UlURUl UR,UlUR.UlUR0UlUR2UlUR4UlUR6UlUR95nUR;U5 URU5 UR=U5 SSS5 g!,(df  GN|=f!,(df  g=f)Nzbuiltin.set_parameterparameter_namerT)rstaticdefault_main_programdefault_startup_program program_guard global_blockopsrdattrsoperandsourcecast persistable_pir_opsupdate_parameter remove_opset_parameters_frompirreset_insertion_point_to_start parameter trainable stop_gradient optimize_attr regularizerdo_model_average need_clipis_distributed is_parameterget_defining_opreplace_all_uses_with value_assign)trvmainstartupblockopr cast_params r3_pir_transformrOs == - - /Dmm335G  $ $W -$$&))B 44HHJ/0AFF: 1 ,,.#[[6 )- &00VVD# . W%  $ $T * 113!!#__..qvv6  {{ #$??  !" #$??  !" &'&8&8 # {{ $%$4$4 !"#..       +  z"! + * . - + *s&2J$J$5A:J$'D4J6$ J36 Kc^SU4Sjjn[R"5 [R"S[S9 [ XX5 SSS5 XlU$!,(df  N=f)Ncb>T(a[R"U5(dU$[X5$r,)ris_floating_pointr)rrv floating_onlys r3 transform_pir_to_impl..transformys& &":":1"="=Ha''r6ignore)category)rrrvrrDz Tensor | None)warningscatch_warningsfilterwarnings UserWarningrr)r.rvrrrs ` r3 _pir_to_implr ssJ (  ";?4E= #K K # "s &A A#c [X5 [[U55HnXRSS9Hn[ XA5(aMUS:Xah[ U[ RRR[ RRR45(aURUS9 M[5(a[UUSSS9 MURUSSS9 M M U$)NTrr)rvF)rvrr)rrrrrrrincubaterFusedFeedForwardFusedMultiHeadAttention _amp_decoraterr _to_impl)rrvrrrs r3amp_initializers 0S[![***=Ee++ !jOO&&77OO&&>>''##%#0}}&+"& 5+>"2 Mr6cUHnn[U[RR5(d[ S[ U5S35e[U[R 5(dMe[ S5e g)NzOCurrent train mode is pure fp16, models should be paddle.nn.Layer, but receive rzFor distributed AMP training, you should first use paddle.amp.decorate() to decorate origin model, and then call paddle.DataParallel get distributed model.)rrrr RuntimeErrorr DataParallel)rmodels r3 check_modelsrsk%11abfglbmannop  eV00 1 1n  r6c`SSKJnJn [U[R R UU45$)NrDygraphShardingOptimizerDygraphShardingOptimizerV2)Upaddle.distributed.fleet.meta_optimizers.dygraph_optimizer.dygraph_sharding_optimizerrrrr optimizer Optimizer)rrrs r3_is_valid_optimizerr s3     & & $ &  r6cfUH+n[U5(aM[S[U5S35e g)NzzCurrent train mode is pure fp16, optimizers should be paddle.optimizer.Optimizer or DygraphShardingOptimizer, but receive r)r rr) optimizersrs r3check_optimizersr#sS "9--MNRS\N]M^^_`  r6c #^# [5(dS5e[5n[nUqUR5nUS;a [ S5e[ U[ RRR5(a URnUR5nU(aUS;a [ S5eUnU[5l US:Xa[Rn ODUS:Xa[R n O-US:Xa[R"n OUS :Xa[R$n ['XX45up[)5(aU(d[R$n S n[R*"5n U R,n W U l[R."5up[R0"X5 U R2nXlU [R":XaU R4nX\lS v UqXl[R0"X5 UU lU [R":XaWU lg g [75nU(d [ S 5eU(aUR8R;5(ddUR8R=5(dEUR8R?5(d&[@RB"S UR8S35 SnU(GaUR8R=5(aUS:Xak[E5(d\[RF"[I5RK55n[@RB"[RL"U5S35 SnOpUS:Xaj[O5(d[[RF"[I5RK55n[@RB"[RL"U5S35 SnUR8R?5(a-[Q5(dUS:Xa[@RB"S5 SnUR8R;5(Ga8US:Xa[S5(dw[ RTRVRY5n[@RB"S[ RTRVR[5SUSSUSS35 SnOUS:Xa[]5(d[ RTRVRY5n[ R^RW5n[@RB"S[ RTRVR[5SUSSUSSUS3 5 SnU(d[R$n S n[5R`(a[5Rb(dSmSnU4Sjn[dRf"S5S ;a`[5RhH4n[kUS!5(aMS Ul6URoU"U55 M6 S"[dRpS'O[RrRuU5 S#[5l1U(apUR,n W UlUR/5upUR1X5 UR2nUUlU [R":XaUR4nUUlS v U(a?UqW UlUR1WW5 WUlW [R":XaWUlg g g !UqXl[R0"X5 UU lU [R":XaWU lff=f!U(a?UqW UlUR1WW5 WUlW [R":XaWUlfff=f7f)$a Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode. If enabled, the input data type (float32 or float16) of each operator is decided by autocast algorithm for better performance. Commonly, it is used together with `GradScaler` to achieve Auto-Mixed-Precision in imperative mode. It is used together with `decorator` to achieve Pure fp16 in imperative mode. Args: enable(bool, optional): Enable auto-mixed-precision or not. Default is True. custom_white_list(set|list|tuple|None, optional): The custom white_list. It's the set of ops that support fp16 calculation and are considered numerically-safe and performance-critical. These ops will be converted to fp16. custom_black_list(set|list|tuple|None, optional): The custom black_list. The set of ops that support fp16 calculation and are considered numerically-dangerous and whose effects may also be observed in downstream ops. These ops will not be converted to fp16. level(str, optional): Auto mixed precision level. Accepted values are "O1" and "O2": O1 represent mixed precision, the input data type of each operator will be casted by white_list and black_list; O2 represent Pure fp16, all operators parameters and input data will be casted to fp16, except operators in black_list, don't support fp16 kernel and batchnorm. Default is O1(amp). dtype(str|core.DataType, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'. use_promote(bool, optional): Whether op's dtype is 'float32', accord 'Promote to the Widest' principle, use 'float32' to calculate. Only active on 'AMP-02'. Default is True. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> import paddle >>> data = paddle.uniform([10, 3, 32, 32], paddle.float32, -1, 1) >>> conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> conv2d = paddle.amp.amp_decorate(models=conv2d, level='O2') >>> with paddle.amp.amp_guard(): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float16 >>> # doctest: -SKIP ... >>> with paddle.amp.amp_guard(enable=False): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float32 >>> # doctest: -SKIP z3We only support 'amp_guard' in dynamic or pir mode.r"!level should be O0, OD, O1 or O2.rbfloat16z7If enable amp, dtype should be 'float16' or 'bfloat16'.r$r%r&r#r_N;current_tracer is None, maybe it is not in imperative mode.zXamp_guard can only be enabled on CUDAPlace, XPUPlace, and CustomPlace, current place is z, so it makes no effect.Frz does not support float16 amp.r'z does not support bfloat16 amp.z%CustomPlace only support float16 amp.z`For float16, amp only support NVIDIA GPU with Compute Capability 7.0 or higher, current GPU is: z, with Compute Capability: rrrzFor bfloat16, amp only support NVIDIA GPU with Compute Capability 8.0 or higher and CUDA Version 11.0 or higher, current GPU is: z, current CUDA Version is: c[UR5n[U5HNupEUR5(dMX5R 5URU-X5R 5'MP [ R R5n[ R RU5n[ R"UUUUUS9$)N)dims process_mesh placementsr) rshape enumerateis_shardget_dimr frameworkr _get_paddle_placer) local_tensormeshr, global_dimsr placementrs r3_dtensor_from_local&_guard.._dtensor_from_locals"<#5#56 &/ &;NC ))++'(9(9(;.master_grad_hooks()HH!1!3!D!D ,1C1C1O % 2 2'7'9'E'E!F &+G!1 2 > >$)$6$6!"!1 2 > >$)$6$6!""(&-"EIM$&E')5566%5%7%C%C%I%I%K  33F;&LJJ//(*;;KP "Gr6cZ>^[RR5UU4Sj5nU$)Nc>UbUR5(aTRc[RR UR 5R [R5R5URSTR-S9nT"UURUR5TlO7TRR 5RUR 55 UR5 ggg)Nz main_grad@)valuerrd)_is_initialized main_gradr r:r _local_valuerrr_rBrrdr+r,add_ _clear_data)tmp_gradtmpr7rs r3 param_hook=amp_guard.._update_main_grad_hook..param_hooks+0H0H0J0J ??2"&**"3"3&.&;&;&=!%fnn!5!&&.nn%1EJJ%> #4#C/B # ( 5 5 ( 3 3/EO "OO88:?? ( 5 5 7!,,.%1K+r6)rautogradno_grad)rrJr7s` r3_update_main_grad_hook)amp_guard.._update_main_grad_hooks*((*/+/*"!r6FLAGS_enable_tensor_fusion)Truetrue1rD0T);rlocalsrSupperrprrbaser DataTyperdlowerrlr] AMP_LEVELr$r%r&r#rzr_get_amp_attrsr|_get_amp_op_list_set_amp_op_list _amp_dtype _use_promoter_expected_place is_gpu_place is_xpu_placeis_custom_placerwarnrrr rrrrrrrrget_device_namerrrYrZosgetenvrXhasattrrD_register_grad_hookenvironr:_add_backward_final_hook)enablersrtrurv use_promoterToriginal_stater] amp_levelrwrx amp_attrsoriginal_amp_leveloriginal_white_listoriginal_black_listoriginal_amp_dtypeoriginal_use_promoter xpu_versionrrr>rNrr7s @r3 amp_guardrwsun " # #= #I"NM KKME ,,<==%))2233  KKME / /I I#,  }LL $LL $LL $LL +e K}}! I!I'') &11( 373H3H3J0 k7&11(  $#,#9#9 %0 " > *M#5  ! !"5 K#5I ILL()= &) !"M    " " / / 1 1%%2244%%5577 MMjkqlBlBkCC[\ F %%2244Y&0I0K0K"&"="=/1??A#KMM??;788VW#Fz)3M3O3O"&"="=/1??A#KMM??;788WX#F&&6688<>>j( EF%%2244Y&0I0K0K!==--CCEDMMz|B|I|I|N|N|^|^|`{aa|}ABC}D|EEFGKLMGNFOOPQ#Fz)3M3O3O!==--CCED#)>>#6#6#8LMM\]c]j]j]o]o]]]A\BB]^bcd^e]ffghlmnhogppKLXKYYZ[#F! I!I   . .$&KK $ P< "2yy56; ./@@E"5+66*.112H2OPA7N7N7P 4   # #K ="(!2!2  )F ILL('-':':$&1# ?  . $6!''')<%7! ,*>F'-O+M#5  ! !"5 K#5I ILL()= &)F . $6!''')<%7! ,*>F'-sAG _$\>(P+_C_3^7A_>A^_A_  _c(\rSrSrSSjrSSjrSrg) StateDictHookicXlgr, _save_dtype)r. save_dtypes r3r`StateDictHook.__init__s%r6c[RRR[RRR 55 UHYnXn[R "U5(dM$[R"X0R5nURUl XAU'M[ SSS5 g!,(df  g=fr,) rrWr1_dygraph_guarddygraphTracerrrr|rd)r.r<rrrs r3__call__StateDictHook.__call__s [[ " " 1 1&++2E2E2L2L2N O!"++E22$*KK7G7G$HM).M&&3sO "P O Os#B979B99 Cr{N)r}rhrDrF)r<rrDrF)rHrIrJrKr`rrLr-r6r3ryrys &4r6rycSSKJnJn [XU45(a UROUn[ US5(aXlgg)Nrr_multi_precision)rrrr _inner_optrhr)rmulti_precisionrrs r3_set_multi_precisionr"sO  2LM    y,--%4".r6cgr,r-rr"rurv master_weightr} master_gradrs r3 amp_decorater5%(r6cgr,r-rs r3rrDr6cUS;a [S5eUS;a [S5eUS:XaUcU$X4$[5nU(d [S5eURR5(dFURR 5(d'URR 5(dUcU$X4$URR 5(a2US:Xa[ 5(aUS:Xa[5(dUcU$X4$URR 5(a[5(dUS:XaUcU$X4$URR5(a2US:Xa[5(aUS:Xa[5(dUcU$X4$S n [U[RR5(aS n U/n[U5 O.[U[ 5(a[U5 S n O [#S 5e[%XUS 9 UbiS n ['U5(aS n U/n[)U5 O.[U[ 5(a[)U5 S n O [#S 5eUS Ln UHn [+X5 M U(a]S [-5l[1[3U55H6n [-5R4R7X R955 M8 UbbUS;a[SUS35e[1[3U55H5n X R;S S9HnUR=[?U55 M M7 U (aUbW (aX4$XS4$U$UbW (aUSU4$USUS4$US$)aw Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure fp16), the decorate will cast all parameters of models to FP16, except BatchNorm, InstanceNorm and LayerNorm. Commonly, it is used together with `amp_guard` to achieve Pure fp16 in imperative mode. Args: models(Layer|list of Layer, optional): The defined models by user, models must be either a single model or a list of models. Default is None. optimizers(Optimizer|list of Optimizer|None, optional): The defined optimizers by user, optimizers must be either a single optimizer or a list of optimizers. Default is None. level(str, optional): Auto mixed precision level. Accepted values are "O1" and "O2": O1 represent mixed precision, the decorator will do nothing; O2 represent Pure fp16/bf16, the decorator will cast all parameters of models to FP16/BF16, except BatchNorm, InstanceNorm and LayerNorm. Default is O1(amp) dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'. master_weight(bool|None, optional): For level='O2', whether to use multi-precision during weight updating. If master_weight is None, in O2 level optimizer will use multi-precision. Default is None. save_dtype(str|None, optional): The save model parameter dtype when use `paddle.save` or `paddle.jit.save`,it should be float16, bfloat16, float32, float64 or None. The save_dtype will not change model parameters dtype, it just change the state_dict dtype. When save_dtype is None, the save dtype is same as model dtype. Default is None. master_grad(bool, optional): For level='O2', whether to use float32 weight gradients for calculations such as gradient clipping, weight decay, and weight updates. If master_grad is enabled, the weight gradients will be float32 dtype after the back propagation. Default is False, there is only float16 weight gradients. excluded_layers(Layer|list of Layer, optional): Specify the layers not to be decorated. The weights of these layers will always keep float32 when level is O2. `excluded_layers` can be specified as an Layer instance/type or a list of Layer instances/types. Default is None, the weights of the whole model will be casted to float16 or bfloat16. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> # Demo1: single model and optimizer: >>> import paddle >>> paddle.device.set_device('gpu') >>> model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer = paddle.optimizer.SGD(parameters=model.parameters()) >>> model, optimizer = paddle.amp.amp_decorate(models=model, optimizers=optimizer, level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.amp_guard(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = model(data) ... print(output.dtype) paddle.float16 >>> # Demo2: multi models and optimizers: >>> model2 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer2 = paddle.optimizer.Adam(parameters=model2.parameters()) >>> models, optimizers = paddle.amp.amp_decorate(models=[model, model2], optimizers=[optimizer, optimizer2], level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.amp_guard(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = models[0](data) ... output2 = models[1](data) ... print(output.dtype) ... print(output2.dtype) paddle.float16 paddle.float16 >>> # Demo3: optimizers is None: >>> model3 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer3 = paddle.optimizer.Adam(parameters=model2.parameters()) >>> model = paddle.amp.amp_decorate(models=model3, level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.amp_guard(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = model(data) ... print(output.dtype) paddle.float16 )r%r&zYlevel should be O1 or O2, O1 represent AMP train mode, O2 represent Pure fp16 train mode.r&z'dtype only support float16 or bfloat16.r%r(rr'FTz9models must be either a single model or a list of models.rrvrzEoptimizers must be either a single optimizer or a list of optimizers.)rr'r_float64zPsave_dtype can only be float16 float32 or float64, but your input save_dtype is rrr) rprr`rarbrcrrrrrrrrrrrrrr r#rrlrYrrrXextendr1rregister_state_dict_hookry)rr"rurvrr}rrrmodels_is_listoptimizers_is_listuse_multi_precisionoptrrs r3rrSsf L  g   ++BCC }  M% % F  I  ++--  ! ! . . 0 0  ! ! 1 1 3 3  M% % **,, Y '@'B'B Z (B(D(D! )) ..00466 j  M% %**,, Y '@'B'B Z (B(D(D! ))N&&))//**V FD ! !V G  &O" z * *!& $J Z (  D ) ) Z (!% W ,58C  :-1*V%C   / / 6 6v{7M7M7O P& J Jbcmbnnop V%C..D.A..}Z/HIB&  !!))!},,M  !!ay*,,ay*Q-//!9 r6c[X!S9$)a Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode. If enabled, the input data type (float32, float16 or bfloat16) of each operator is decided by autocast algorithm for better performance. Commonly, it is used together with `GradScaler` and `decorator` to achieve Auto-Mixed-Precision in imperative mode. Args: device_type(str, optional): Device type.But because the paddle does not distinguish between devices, this parameter does not work enable(bool, optional): Enable auto-mixed-precision or not. Default is True. dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'. cache_enabled(bool, optional): whether to enable cache or not. Default is True. But this parameter is not used Note: paddle.cuda.amp. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> import paddle >>> conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.auto_cast(): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float16 >>> # doctest: -SKIP >>> with paddle.amp.auto_cast(enable=False): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float32 >>> # doctest: -SKIP )rlrv) auto_cast) device_typervenabled cache_enableds r3autocastr&s` G 11r6c[XX#XE5$)a Create a context which enables auto-mixed-precision(AMP) of operators executed in dynamic graph mode. If enabled, the input data type (float32, float16 or bfloat16) of each operator is decided by autocast algorithm for better performance. Commonly, it is used together with `GradScaler` and `decorator` to achieve Auto-Mixed-Precision in imperative mode. Args: enable(bool, optional): Enable auto-mixed-precision or not. Default is True. custom_white_list(set|list|tuple|None, optional): A default white list is already set. Usually there is no need to set custom white list. The set of ops should be considered numerically-safe and performance-critical. These ops will be converted to float16/bfloat16. custom_black_list(set|list|tuple|None, optional): A default black list is already set. You can set a custom black list according to the model. The set of ops are considered numerically-dangerous and whose effects may also be observed in downstream ops. These ops will not be converted to float16/bfloat16. level(str, optional): Auto mixed precision level. Accepted values are "O1", "O2" and "OD": At the O1 level, operators in the white list will use float16/bfloat16 inputs for calculations, and operators in the black list will use float32 inputs for calculations. At the O2 level, model's parameters will be casted to float16/bfloat16 by using `decorator`, and operators that have all float16/bfloat16 inputs will be converted to float16/bfloat16, and that have any float32 input will be converted to float32. For the OD level, operators in default white list will compute in float16/bfloat16, and the others will compute in float32. Default is O1. dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'. use_promote(bool, optional): Whether to promotes to fp32 when op has any float32 inputs. It is only supported when amp level is O2. Default is True. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> import paddle >>> conv2d = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.auto_cast(): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float16 >>> # doctest: -SKIP >>> with paddle.amp.auto_cast(enable=False): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float32 >>> # doctest: -SKIP >>> with paddle.amp.auto_cast(custom_black_list={'conv2d'}): ... conv = conv2d(data) ... print(conv.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float32 >>> # doctest: -SKIP >>> a = paddle.rand([2, 3]) >>> b = paddle.rand([2, 3]) >>> with paddle.amp.auto_cast(custom_white_list={'elementwise_add'}): ... c = a + b ... print(c.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float16 >>> # doctest: -SKIP >>> with paddle.amp.auto_cast(custom_white_list={'elementwise_add'}, level='O2'): ... d = a + b ... print(d.dtype) >>> # doctest: +SKIP("This has diff in xdoctest env") paddle.float16 >>> # doctest: -SKIP )rw)rlrsrtrurvrms r3rrYs^ #4U r6cgr,r-rs r3decoraterrr6cgr,r-rs r3rrrr6c[RR5(a[U[[ 45(ae[U[[ 45(aeU[ 5lUS;aUcU$[USUUSSSSSSSUSS9 nX4$US:Xa7[U/X7S9 USLn[X5 UcU$[USUUSSSSSSSUSS9 nX4$[S5e[UUUUUUUU5$) a Decorate models and optimizers for auto-mixed-precision. When level is O1(amp), the decorate will do nothing. When level is O2(pure float16/bfloat16), the decorate will cast all parameters of models to float16/bfloat16, except BatchNorm, InstanceNorm and LayerNorm. Commonly, it is used together with `auto_cast` to achieve Pure float16/bfloat16 in imperative mode. Args: models(Layer|list of Layer): The defined models by user, models must be either a single model or a list of models. Default is None. optimizers(Optimizer|list of Optimizer|None, optional): The defined optimizers by user, optimizers must be either a single optimizer or a list of optimizers. Default is None. level(str, optional): Auto mixed precision level. Accepted values are 'O1' and 'O2': O1 represent mixed precision, the decorator will do nothing; O2 represent Pure float16/bfloat16, the decorator will cast all parameters of models to float16/bfloat16, except BatchNorm, InstanceNorm and LayerNorm. Default is O1(amp) dtype(str, optional): Whether to use 'float16' or 'bfloat16'. Default is 'float16'. master_weight(bool, optional): For level='O2', whether to use multi-precision during weight updating. If master_weight is None, in O2 level optimizer will use multi-precision. Default is None. save_dtype(str|None, optional): The save model parameter dtype when use `paddle.save` or `paddle.jit.save`,it should be float16, bfloat16, float32, float64 or None. The save_dtype will not change model parameters dtype, it just change the state_dict dtype. When save_dtype is None, the save dtype is same as model dtype. Default is None. master_grad(bool, optional): For level='O2', whether to use float32 weight gradients for calculations such as gradient clipping, weight decay, and weight updates. If master_grad is enabled, the weight gradients will be float32 dtype after the back propagation. Default is False, there is only float16 weight gradients. excluded_layers(Layer|list of Layer, optional): Specify the layers not to be decorated. The weights of these layers will always keep float32 when level is O2. `excluded_layers` can be specified as an Layer instance/type or a list of Layer instances/types. Default is None, the weights of the whole model will be casted to float16 or bfloat16. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> # Demo1: single model and optimizer: >>> import paddle >>> paddle.device.set_device('gpu') >>> model = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer = paddle.optimizer.SGD(parameters=model.parameters()) >>> model, optimizer = paddle.amp.decorate(models=model, optimizers=optimizer, level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.auto_cast(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = model(data) ... print(output.dtype) paddle.float16 >>> # Demo2: multi models and optimizers: >>> model2 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer2 = paddle.optimizer.Adam(parameters=model2.parameters()) >>> models, optimizers = paddle.amp.decorate(models=[model, model2], optimizers=[optimizer, optimizer2], level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.auto_cast(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = models[0](data) ... output2 = models[1](data) ... print(output.dtype) ... print(output2.dtype) paddle.float16 paddle.float16 >>> # Demo3: optimizers is None: >>> model3 = paddle.nn.Conv2D(3, 2, 3, bias_attr=False) >>> optimizer3 = paddle.optimizer.Adam(parameters=model3.parameters()) >>> model = paddle.amp.decorate(models=model3, level='O2') >>> data = paddle.rand([10, 3, 32, 32]) >>> with paddle.amp.auto_cast(enable=True, custom_white_list=None, custom_black_list=None, level='O2'): ... output = model(data) ... print(output.dtype) paddle.float16 )r#r$r%Ng?F) r amp_listsrurvinit_loss_scalingincr_every_n_stepsdecr_every_n_nan_or_inf incr_ratio decr_ratiouse_dynamic_loss_scaling use_amp_guardrYrmr&rr%) rr1rrrrrlrYrrrrpr) rr"rurvrr}rrrs r3rrsGh##%%ftUm4444j4-8888-8* & &! 8("&)'+,0##-2"&$/ $ )) d] xu #0u"<   A! 8("&)'+,0##-2"&$/ $ ))@A A          r6c[5(a2[R"5nUR[R :g$[ 5nU(a'UR[RR :g$g)a Check whether auto-mixed-precision is enabled in the current context. Args: device_type (PlaceLike, optional): The device type to check. This argument is ignored for all devices sharing the same AMP state in paddlepaddle. Returns: bool: True if auto-mixed-precision is enabled, False otherwise. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> # Demo1: Check if auto-mixed-precision is enabled by default >>> import paddle >>> paddle.device.set_device('gpu') >>> print(paddle.is_autocast_enabled()) False >>> # Demo2: Enable auto-mixed-precision and check again >>> with paddle.amp.auto_cast(): ... print(paddle.is_autocast_enabled()) True F)rr r[r|rZr#rr}rrprs r3is_autocast_enabledr_sW2}}'') ##y||33 " $$ (8(88 8r6c[5(dg[5(a![R"5nUR$[ 5nUR$)a Get the auto-mixed-precision dtype in the current context if autocast is enabled else default AMP dtype(float16). Args: device_type (PlaceLike, optional): The device type to check. This argument is ignored for all devices sharing the same AMP state in paddlepaddle. Returns: _DTypeLiteral: The current AMP dtype. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:GPU) >>> # Demo1: Get default auto-mixed-precision dtype >>> import paddle >>> paddle.device.set_device('gpu') >>> print(paddle.get_autocast_dtype()) float16 >>> # Demo2: Enable auto-mixed-precision and get the dtype >>> with paddle.amp.auto_cast(): ... print(paddle.get_autocast_dtype()) float16 r)rrr r[r^rrs r3get_autocast_dtypersE2  }}'') ### "   r6)rDrV)r%r) rsr'rtr'ru_AmpLevelLiteralrvrrDztuple[set[str], set[str]])rDrG)rrrvrhrDrG)r list[Layer]r/Layer | list[Layer | type[Layer]] | type[Layer]rDrF)T) r.rrz0Callable[[Tensor, _DTypeLiteral], Tensor | None]rvrrrGrDrF)rrrvrhrDrF) r.rrvrrrGrrGrDr)rrrvrrrrDr)rrrDrF)rrrDzTypeGuard[_OptimizerLike])r"z list[Any]rDrF)TNNr%rT)rlrGrs_CustomList | NonertrrurrvrrmrGrDzGenerator[None, None, None])rr)rrGrDrF).......)rrMr"rNrurrvrr bool | Noner}_DTypeLiteral | NonerrGr6Layer | list[Layer | type[Layer]] | type[Layer] | NonerDztuple[_ModelsT, _OptimizersT])rrMr"rFrurrvrrrr}rrrGrrrDrM)Nr%rNNFN)rrMr"z_OptimizersT | Nonerurrvrrrr}rrrGrrrDz(tuple[_ModelsT, _OptimizersT] | _ModelsT)rTT) rz str | NonervrrrGrrGrDr)rlrGrsrrtrrurrvrrmrGrDr)......r,)rPlaceLike | NonerDrG)rrrDr)^ __future__rrorfrtypingrrrrrr r r r paddle.baser paddle.base.frameworkr rrrrpaddle.base.wrapped_decoratorrpaddle.static.amp.decoratorrrrrcollections.abcr contextlibrtyping_extensionsrrrpaddle._typingrpaddle._typing.dtype_liker paddle.nnrpaddle.nn.layer.layersr paddle.staticr r!rrrhrrnr'rir)rMrNAMP_RELATED_FLAGSAMP_RELATED_FLAGS_SETTINGr}rZrSrTrVrkrlrzrrrrrrrrrrrr rrr r#rwryrrrrrrrr-r6r3rs#    HC-)16(71/56"49eCHos3x#GHKH = = :wW 6~'7>N9OP &''+01 MM   ""(#$#$ $"$"$ $  $  $@ < : Q   !H#5 #5D#5 #5T#   :    *!#H     &  E D   ,0,0"$ D? D?)D?*D?  D?  D?  D?!D?D?N 4 455045 5&  #!!$'*  (  ( (  (  (  ( % ( ( ? (# (  ( !!$'*            %   ?    '+"$!%'+ O O#O O  O  O % OO ?O.OOh% 0202 0202 02  02h,0,0"$ Q Q)Q*Q  Q  Q  QQh "!$'*  (  ( (  (  (  ( % ( ( ? (# (  ( !!$'*            %   ?     '+"$!%'+ Q Q #Q  Q   Q  Q % Q Q  ?Q .Q h F !r6