ΑizSSKJr SSKrSSKrSSKrSSKrSSKrSSKJrJ r J r SSK r SSK r SSK Jr SSKJr SSKJr SSKJr SSKJrJr S S KJr \(aSS KJrJr SS KJr SS KJ r "SS\ 5r!/r"Sr#Sr$Sr%"SS5r&\&"5r'\RPSSj5r)"SS\5r*SSjr+Sr,SSjr-g)) annotationsN) TYPE_CHECKINGAny TypedDict) framework)PyLayer)EagerParamBase)get_rng_state_tracker)corein_dynamic_mode)logger)CallableSequence) NotRequired) Sequentialc.\rSrSr%SrS\S'S\S'Srg) _Ctx,intsegmentszNotRequired[bool]preserve_rng_stateN)__name__ __module__ __qualname____firstlineno__r__annotations____static_attributes__rl/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/fleet/recompute/recompute.pyrr,s#--r!rc[R"UR5n[SURUR UR URS.UD6nURU5 U$)N)shapedtype trainablenamer) copydeepcopy__dict__r r$r%r&r'_share_buffer_to)paramstate new_params r" _varbase_helpr/4s_ MM%.. )Ekkkk// ZZ   I 9% r!c|/nUGHn[U[RR5(dQ[ U5[ Ld,[US[RR5(dUR U5 M~[U[5(aUR [U55 M[ U5[ La/nUHn[U[RR5(de[U[5(aUR [U55 M_UR5nURUl UR U5 M UR [ U55 GMwUR5nURUl UR U5 GM [ U5$)Nr) isinstancer eagerTensortypetupleappendr r/detach stop_gradient)inputsoutinp detach_inpitmp_ixs r"detach_variabler@AsA C#tzz0011 IU "*SVTZZ=N=N*O*O JJsO  c> * * JJ}S) *  9 J!!TZZ%6%67777a00%%mA&67HHJE*+//E'%%e, JJuZ( )  JJL++ 1 => :r!c/nUHn[U[R5(aURUR5 M?[ U5[ LdMSUH?n[U[R5(dM$URUR5 MA M [U5(a[R"S5 gg)Nz[Recompute]: None of the inputs to current recompute block need grad, therefore there is NO need to recompute this block in backward !) r1paddler3r6r8r4r5allrwarning)r9necessary_for_each_inputinput_r=s r"check_recompute_necessaryrGes! ffmm , , $ + +F,@,@ A &\U "a//,33AOOD  #$$ O %r!c*\rSrSrSrSrSrSrSrg)CustomStatesManagervc SUlSUlg)__init__Ncustom_get_state_funccustom_set_state_func)selfs r"rLCustomStatesManager.__init__ys%)"%)"r!c<SnURbU5eXlgNz=The custom_state_manager does not support duplicate settings.)rN)rPrN assert_msgs r"set_custom_get_state_func-CustomStatesManager.set_custom_get_state_func~) K ))1=:=1%:"r!c<SnURbU5eXlgrS)rO)rPrOrTs r"set_custom_set_state_func-CustomStatesManager.set_custom_set_state_funcrWr!rMN) rrrr__doc__rLrUrYr rr!r"rIrIvs* ;;r!rIc## [R"5n[5R5n[R"U5 [5R U5 [ RR5n [R"5n [ RRU5 [R"U5 UbUceUceU"5n U"U5 Sv [R"U5 [5R U5 [ RRU 5 [R"U 5 Ub U"W 5 gg![R"U5 [5R U5 [ RRU 5 [R"U 5 Ub U"W 5 ff=f7fN) rB get_rng_stater get_states_tracker set_rng_stateset_states_trackernprandom get_stategetstate set_statesetstate) rng_statetracker numpy_state random_state custom_staterNrOorig_rng_stateorig_rng_trackerorig_numpy_stateorig_random_stateorig_custom_states r"switch_rng_state_trackerrrsf))+N,.AAC #..w7yy**,)II $ OOL!$000$00013l+ 5 ^,223CD ,-)*  # !"3 4 $ ^,223CD ,-)*  # !"3 4 $s C!G$E(A1GA2G  Gc4\rSrSr\S5r\S5rSrg)RecomputeFunctionc\ XlX lX0lXplUR(a[R "5Ul[5R5Ul [RR5Ul [R"5UlU"5UlX@lXPl[&R("5nUR*[,R.R0:XaSOSUlUR*[,R.R4:XaSUlObUR*[,R.R8[,R.R04;aSUlO[;SUR*35eUR<S:XaSUlO0UR<S;aSUlO[;S UR<35eURA5uUl!Ul"[RF"5 U"U0UD6n SSS5 /Ul$/Ul%[M[OU55V s/sHn SPM sn Ul(/n [SU5GHup[RT"U 5(aXR;aK[,RV"5(aU RY5OU R[5nUR]U 5 U R_U 5 URJR_U 5 URHR_S5 M[aU 5[bLGa'XR;d S U 35eU Vs/sHn[RT"U5PM nn[eU5(aU Vs/sHoRfPM nn[eU5(dU5(a [;S 5eU R_U 5 URJR_U 5 SURPU 'URHR_S5 GM[iU5(a [;S 5eURHR_U 5 GMURHR_U 5 GM URj"U 6 W $!,(df  GN=fs sn fs snfs snf) NFTO2O1zunsupported amp level: float16bfloat16float32r{zunsupported amp dtype: z;offload_indices should not contain tensor tuple in positionzJRecompute receive a tuple containing tensor holds different stop gradient.zHRecompute receive a tuple containing tensor and non-tensor at same time.)6 run_functionroffload_indiceskwargsrBr^ fw_rng_stater r_fwd_rng_state_trackerrbrcrdfwd_numpy_staterefwd_random_statefwd_custom_staterNrOr_dygraph_tracer _amp_levelr AmpLevelO0is_fw_autocastrw amp_levelrx ValueError _amp_dtype amp_dtype_get_amp_op_listamp_white_listamp_black_listno_gradr9tensor_indicesrangelenduplicate_tensor enumerate is_tensoris_compiled_with_cuda pin_memorycpur+r6r4r5rCr8anysave_for_backward)ctxr}rr~rNrOargsrtraceroutputs_ tensor_inputsr=argcpu_arga is_tensorstensors_stop_gradients r"forwardRecomputeFunction.forwards(!3-   ! !%335C %'::<  %#%))"5"5"7C #)??#4C #8#:C (= %(= %**,&&$--*:*::E     0 0 0 CM   4==#3#3T]]5E5E"F F CM6v7H7H6IJK K    )%CM   "9 9&CM6v7H7H6IJK K171H1H1J.C. ^^ "D3F3G /4SY/?@/?!/?@ oFA$$+++ 5577( WWY ,,S1$$S)""))!, !!$'ce# 3 33QRSQTU3UR@S9 [C[EU55n URF"U 0URHD6n SSS5 SSS5 O[R4R7UR8UR:UR<UR>UR@S9 [C[EU55n URF"U 0URHD6n SSS5 [KW [LRNRP5(aU 4n [SU 5[SU5:Xde/n /n [U[SU 55Hkn[KX[LRNRP5(dM0XR(aMEU RWX5 U RWX5 Mm [SU 5S:Xa [YS5e[R4R7SS9 [RZR]X5 SSS5 /n [W 5Hup~[KU[LRNRP5(a!U RWUR_55 MO[aU5[DLdMcXG(dMn[cSU55(aU RWS5 MU RW[ESU555 M [e5(a [EU 5n O [ U 5n U sSSS5 $!,(df  GN=f!,(df  GNH=f!,(df  GNZ=f!,(df  GN@=f!,(df  g=f) NTenablecustom_white_listcustom_black_listlevelr%rzHnone of output has requires_grad=True, this recompute() is not necessaryF)rc38# UHoRv M g7fr])r8.0r=s r" -RecomputeFunction.backward..s8Cq??Csc3@# UHoR5v M g7fr]) _grad_ivarrs r"rrs*G3a<<>>3s)3rBbasedygraphguardlistr9rr saved_tensorrr~tor_current_expected_placer8r _has_gradrrrrrrrrrNrOamp auto_castrrrrrr@r5r}rr1r r2r3rrr6 RuntimeErrorautogradbackwardrr4rCr )rrr9rrtensorsr=idxrdetached_inputsrforward_outputs_with_gradbackward_inputs_with_gradgradsr;s r"rRecomputeFunction.backward#s [[ & & (#**%F //N"33 &&(G#N3 ///JMM --EEG!  +++07 0H0HFK-4..0F#F %%,((11++,,,,1111JJ(("11*-*<*<*-*<*<!mm!mm )'6eFm&DO!..N3::NG(ZZ))--&)&8&8&)&8&8---- *'6eFm&DO!..N3::NG'4::#4#455"*w<3t9, ,,)+ % )+ %3w<(wz4::+<+<==#J444-44WZ@-44TW= ),-2"^ %%U%3((-4 E%o6c4::#4#455LL!12#Y%',<,A,A8C888 T* U*G3*G%GH7  e U Q) (H(F43e) (sB)TBT3AS1R<8SAT1S  BT#T8A!T S29A4T1T.Intermediate_HolderirN)rrrrr rr!r"Intermediate_Holderrs r!rc`>T"5nTR[R"U55 U$r])r6weakrefref)r?resr holder_lists r"pack*_recompute_without_reentrant..packs'!#7;;s+, r!c >^Sm[T5S:XGaUUU4SjnSnT(a[T T TTTT T 5 [R"S5 [RR TTTTTS9 [R RX5 T "T0TD6 SSS5 SSS5 SSS5 SSS5 Oy[R"S5 [RR TTTTTS9 [R RX5 T "T0TD6 SSS5 SSS5 SSS5 UT;a [S5eTRU5$!,(df  N=f!,(df  N=f!,(df  N=f!,(df  Nc=f!,(df  N=f!,(df  N=f!,(df  N=f)Nrc@>TS- mTTS- "5cgUcSTTTS- "5'g[US5(d URbUTTTS- "5'gUR5(a [RR U5nOp[RR UR URURS-[RRRUR5nURU5 UTTTS- "5'g)Nr main_gradcpy)hasattrgradis_distr r2r3r%r$r'VarDescVarType DENSE_TENSOR persistable_unsafe_share_buffer_to)inner_x tmp_tensorrstorageunpack_counters r" inner_pack@_recompute_without_reentrant..unpack..inner_packs !#~124<?AEGK(:;=>7K00GLL4LAHGK(:;=>((%)ZZ%6%6w%? %)ZZ%6%6#MM#MM#LL50 LL00==#// & 33J?AKGK(:;=>r!c[S5e)Nz*An unexpected backward called on a tensor!) Exception)rs r" inner_unpackB_recompute_without_reentrant..unpack..inner_unpacks LMMr!TrzaNot supported to retrieve a tensor saved by autograd multiple times that is no need to recompute.) rrrrBset_grad_enabledrrrsaved_tensors_hooksrpop)r?rrrrrrrrrNrOfunctionfw_cuda_rng_statefwd_cuda_rng_state_trackerrrrrrrrrs @r"unpack,_recompute_without_reentrant..unpacks w<1  4 N",)2'((--++D1JJ((-*8*8'' )OO77"d-f- 20++D1JJ((-*8*8'' )OO77"d-f- 2 G s {{1~521@21sF "E81 E' E E'"E8*F "F<3 F+ FF+$F< E$ E'' E5 1E88 F F  F F( $F++ F9 5F<< G N)rB get_device startswithget_cuda_rng_stater^splitdeviceget_all_custom_device_typerr r_rbrcrdrerrrr rrrwrxrrrWeakKeyDictionaryrr)rrNrOrrr cur_devicerrrrrrrrrrrrrrrrrs`````` @@@@@@@@@@@@@r"_recompute_without_reentrantrs &&(   ( ( & 9 9 ;  j & 4 4 6  z ! & 4 4 6    S !! $}}779 :!' 4 4Z @ Mj\YZ[  " # 6 6 8 #))--/!??,02  & & (F$//4==3C3CCUN DMM,,,   t}}//1A1AB B  I%   5 5 %+%<%<%>"NN  '')GK NNN`  , ,T :D+F+ ; N ; : Ns  J J)cURSS5nURSS5n[Rc[RbeS SjnS SjnO [Rn[Rn[ 5(dSSKJn U"U5"U0UD6$[R"5R(a>[U5nUR[UR555 [U5 U(GaURS/5n /n [U[R R"5(a![$R&"UR(5n O[$R&"U5n U R*"U0UD6n U R-5 [/U R0R5U R2R55HupUR4UR6:XaU RU 5 M2UR4UR8UR:4;aU R=U 5 MkUR4UR>:Xa!U RU R55 MUR4UR@:Xa [CS 5e[CS 5e [DRF"UUU UU/U Q76$[IUUUU/UQ70UD6$) al recompute intermediate activations to save then memory. Parameters: function(paddle.nn.Layer): layer of sequence of layers that describes part of forward pass of the model whose intermediate activations will be released to save memory in forward stage and will be recomputed in backward stage for gradient calculation. *args(Tensor): inputs to the function. **kwargs(Dict): Kwargs should only contain two kinds of key-value params, the one is part of function's key-value params, and the other contains 'preserve_rng_state' and 'use_reentrant'. the key-value pair of preserve_rng_state, which is used to indicate whether to save the forward rng. If it is True, then the last forward rng value will be restored when the forward recalculation of backpropagation is performed, its default value is True. the key-value pair of use_reentrant is used to indicate which implementation of recompute you will be used. 'use_reentrant=True' means to use the PyLayer implementation of recompute, 'use_reentrant=False' means to use the Hook implementation of recompute, its default value is True. Returns: Output of function on args. Examples: .. code-block:: python >>> # doctest: +REQUIRES(env:DISTRIBUTED, env:GPU) >>> import paddle >>> from paddle.distributed.fleet.utils import recompute >>> import random >>> paddle.seed(2023) >>> def get_fc_block(block_idx, input_size, is_last=False): ... block_name = "block_" + str(block_idx) ... block = paddle.nn.Sequential( ... (block_name + "_fc_0", paddle.nn.Linear(input_size, input_size, bias_attr=False)), ... (block_name + "_dropout", paddle.nn.Dropout(p=0.5)), ... (block_name + "_relu_1", paddle.nn.ReLU()), ... (block_name + "_fc_1", paddle.nn.Linear(input_size, input_size, bias_attr=False)), ... (block_name + "_relu_2", paddle.nn.ReLU()), ... ) ... if is_last: ... block.add_sublayer( ... block_name + "_fc_2", ... paddle.nn.Linear( ... input_size, 1, bias_attr=False ... ) ... ) ... else: ... block.add_sublayer( ... block_name + "_fc_2", ... paddle.nn.Linear(input_size, input_size, bias_attr=False) ... ) ... return block >>> class Naive_fc_net(paddle.nn.Layer): ... def __init__(self, input_size=10, ... recompute_blocks=[1, 3], ... recompute_kwargs={}): ... super().__init__() ... self.recompute_blocks = recompute_blocks ... self.recompute_kwargs = recompute_kwargs ... self.runfunc0 = get_fc_block(0, input_size, is_last=False) ... self.runfunc1 = get_fc_block(1, input_size, is_last=False) ... self.runfunc2 = get_fc_block(2, input_size, is_last=False) ... self.runfunc3 = get_fc_block(3, input_size, is_last=False) ... self.runfunc4 = get_fc_block(4, input_size, is_last=True) ... self.total_func = [self.runfunc0, self.runfunc1, self.runfunc2, self.runfunc3, self.runfunc4] ... def forward(self, inputs): ... nums = len(self.total_func) ... for i in range(nums): ... if i in self.recompute_blocks: ... inputs = recompute(self.total_func[i], inputs, **{"preserve_rng_state": True}) ... else: ... inputs = self.total_func[i](inputs) ... return inputs >>> def run_model(cuda_state, recompute_block=[], recompute_kwargs={}): ... gen = paddle.seed(10) ... gen.manual_seed(10) ... random.seed(10) ... if cuda_state: ... paddle.set_cuda_rng_state(cuda_state) ... batch_size, input_size = 1, 10 ... model = Naive_fc_net( ... input_size, ... recompute_blocks=recompute_block, ... recompute_kwargs=recompute_kwargs) ... optimizer = paddle.optimizer.SGD(learning_rate=0.01, parameters=model.parameters()) ... loss_ = [] ... param_ = [] ... grad_ = [] ... for _ in range(5): ... x = paddle.rand(shape=[batch_size, input_size], dtype="float32") ... y_pred = model(x) ... loss = y_pred.mean() ... loss_.append(loss.item()) ... loss.backward() ... optimizer.step() ... param_.append(model.parameters()[9]) ... grad_.append(model.parameters()[3]._grad_ivar()) ... optimizer.clear_grad() ... return loss_, param_, grad_ >>> cuda_state = paddle.get_cuda_rng_state() >>> # without recompute >>> loss_ref, param_ref, grad_ref = run_model( ... cuda_state, recompute_block=[] ... ) >>> loss, param, grad = run_model(cuda_state, recompute_block=[1, 2]) >>> print("normal_loss: {}, recompute_loss: {}".format(loss_ref, loss)) >>> # The result of the recompute_loss should be the same as the normal_loss. normal_loss: [0.0018744759727269411, 0.0, 0.035971127450466156, 0.0, 0.0], recompute_loss: [0.0018744759727269411, 0.0, 0.035971127450466156, 0.0, 0.0] rT use_reentrantcgr]rr?s r"recompute..tr!cgr]rrs r"rrrr!r) recomputer~zCurrently, keyword-only arguments are not supported when you want to send kwargs(dict parameter) to function with use_reentrant=True.zUnknown parameter kind.r])%rcustom_state_managerrNrOr *paddle.distributed.auto_parallel.interfacerrrrrextendvaluesrGr1rBnnLayerinspect signaturerbindapply_defaultszip arguments parameterskindVAR_POSITIONALPOSITIONAL_ONLYPOSITIONAL_OR_KEYWORDr6 VAR_KEYWORD KEYWORD_ONLYrrtapplyr)rrrpreserverrNrOstatic_auto_recompute check_argsr~ input_args dyfunc_sig bound_argsrr,s r"rr#s}`zz.5HJJ5M119#99AAA 3 3 4 J J 4 J J    %X.???  ",,$Z $v}}/0!*- **%6; h  0 0 **8+;+;>> # doctest: +REQUIRES(env:DISTRIBUTED) >>> import paddle >>> from paddle.incubate.distributed.fleet import recompute_sequential >>> input = paddle.ones(shape=[8, 10]) >>> model = paddle.nn.Sequential(paddle.nn.Linear(10, 10), paddle.nn.Linear(10, 2)) >>> output = recompute_sequential({'segments' : 1}, model, input) rrrTc^^^UUU4SjnU$)NcJ>[TTS-5HnTU"U5nM U$)Nr)r)inputr=beginendfuncss r"do_run7recompute_sequential.._run_func..do_runs*5#'*a+Lr!r)r'r(r)r*s``` r" _run_func'recompute_sequential.._run_funcs   r!r) getr1rBr rrchildrenrrr) r functionsrrrrr, segment_sizer(r's r"recompute_sequentialr3sBwwz1%H!5t<)VYY1122++-. y>X-L Cq,Q,7F"Q& e) ,   2   G S1Wc)nq0) rCs#  00 #03#2-$.y.  !H ";;.+,  !5!5HXX~ Pfx v:D :D8:D :D :D  :Dr!