Αi˒6SSKJr SSKrSSKJrJrJr SSKrSSK rSSK r SSK J r SSK Jr SSKJr SSKJr \(aSSKJr "S S \S S 9r"S S\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS\S S 9r"SS \S S 9r"S!S"\S S 9r "S#S$\S S 9r!"S%S&\S S 9r""S'S(\S S 9r#"S)S*\S S 9r$/r%S+q&S,r'\"\'5r(S-r)S.r*S/r+S0r,S1r-"S2S35r.\/r0"S4S55r1g)6) annotationsN) TYPE_CHECKINGAny TypedDict) _global_flags)wrap_decorator)distributed_strategy_pb2)logger) BuildStrategyc\\rSrSr%S\S'S\S'S\S'S\S'S\S'S\S 'S\S 'S rg ) _SyncConf!intk_stepmax_merge_var_numsend_queue_sizeboolindependent_recv_threadthread_pool_sizesend_wait_timesruntime_split_send_recvN__name__ __module__ __qualname____firstlineno____annotations____static_attributes__rr/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/fleet/base/distributed_strategy.pyr r !s+ !%%!%%r r F)totalc>\rSrSr%S\S'S\S'S\S'S\S'Srg ) _TrainerDescConf*strdump_fields_path list[str] dump_field dump_paramstat_var_namesrNrrr r!r$r$*s!!r r$c>\rSrSr%S\S'S\S'S\S'S\S'Srg) _FsClientParam0r&uriuserpasswd hadoop_binrNrrr r!r-r-0s   r r-c\rSrSr%S\S'S\S'S\S'S\S'S\S 'S\S 'S \S 'S \S 'S \S'S\S'S\S'S\S'Srg)_AmpConf6floatinit_loss_scalingruse_dynamic_loss_scalingrincr_every_n_stepsdecr_every_n_nan_or_inf incr_ratio decr_ratior(custom_white_listcustom_black_listcustom_black_varnames use_pure_fp16 use_pure_bf16use_fp16_guardrNrrr r!r4r46sJ  "&&!$$$$$$((r r4cH\rSrSr%S\S'S\S'S\S'S\S'S \S 'S rg ) _QATConfigDrchannel_wise_abs_maxr weight_bitsactivation_bitsr(not_quant_patternr&algorNrrr r!rDrDDs""$$ r rDc4\rSrSr%S\S'S\S'S\S'Srg ) _RecomputeConfigKr( checkpointsrenable_offloadz list[int]checkpoint_shaperNrrr r!rLrLKs##r rLc\rSrSr%S\S'S\S'S\S'S\S 'S\S 'S \S 'S\S 'S\S'S\S'S \S'S \S'Srg)_ShardingConfigPr&sharding_segment_strategyr6segment_broadcast_MBr(segment_anchorsrsharding_degreegradient_merge_acc_steproptimize_offload dp_degree mp_degree pp_degreepp_allreduce_in_optimize optimize_castrNrrr r!rRrRPsA#&&##""!$$"&&r rRc \rSrSr%S\S'Srg)_PipelineConfig]rmicro_batch_sizerNrrr r!r`r`]sr r`c*\rSrSr%S\S'S\S'Srg)_TensorParallelConfig`rtensor_parallel_degreetensor_init_seedrNrrr r!rdrd`s ##r rdc\\rSrSr%S\S'S\S'S\S'S\S'S\S'S\S'S \S 'S rg ) _HybridConfigdrrZr[r\ sep_degree cp_degreerWr(orderrNrrr r!ririds'r ric*\rSrSr%S\S'S\S'Srg)_LocalSGDConfigmrk_steps begin_steprNrrr r!roroms  r roc*\rSrSr%S\S'S\S'Srg)_AdaptiveLocalSGDConfigqr init_k_stepsrrrNrrr r!rtrtqs r rtc4\rSrSr%S\S'S\S'S\S'Srg) _DGCConfigurrampup_begin_step rampup_stepz list[float]sparsityrNrrr r!rxrxusr rxc*\rSrSr%S\S'S\S'Srg)_GradientMergeConfigzrrqravgrNrrr r!r~r~zs   r r~c>\rSrSr%S\S'S\S'S\S'S\S'Srg ) _LarsConfig~r6 lars_coefflars_weight_decayepsilonr(exclude_from_weight_decayrNrrr r!rr~s  #,,r rc*\rSrSr%S\S'S\S'Srg) _LambConfigr6lamb_weight_decayr(rrNrrr r!rrs  #,,r rTc^U4SjnU$)Nc>SqT"U0UD6$)NF)non_auto_func_called)argskwargsfuncs r!__impl__*__non_auto_func_called__..__impl__s$T$V$$r r)rrs` r!__non_auto_func_called__rs% Or cT/nUHnURRn0nUHmn[X%R5nUR[ R RRR:Xa [U5nXdUR'Mo URU5 M U$N) DESCRIPTORfieldsgetattrnamelabelgoogleprotobuf descriptorFieldDescriptorLABEL_REPEATEDlistappend)msgres_listitemrres_dictfvs r!get_repeated_msg_dictrsH''Aff%A??--==LLMG QVV  ! Or c0nURRnUHn[XR5nUR[ R RRR:XaSUR[ R RRR:wa [U5nO [U5nXAUR'M U$r)rrrrrrrrrrtype TYPE_MESSAGErr)rrrrrs r! get_msg_dictrsH ^^ " "F  C  77foo00@@OO O??--==JJKG)!, Or c|UGH5nUR5nURRnUGHnX%R:XdMURS:XaXRb[ XR5nUR [RRRR:waURXR5 M[X1UR5 MMURS:XdURS:XdM[XRXR5 GM GM8 gN)addrrrrrrrrrrrextendassign_configs_valuesetattr)rconfigkeynew_itemrrs r!assign_repeated_msgrs779$$++Aff}77a<ff~1#*3#7FF%99IIVVW%OOF66N;0!&&>J2WW\QWW\C8!r c\URRnUGHnUGHnX4R:XdMURS:XaXRb[ XR5nUR [ RRRR:waURXR5 M[XQUR5 MMURS:XdURS:XdM[XRXR5 GM GM gr)rrrrrrrrrrrrrr)rrrrrrs r!rrs ^^ " "FAff}77a<ff~1#*3#7FF%99IIVVW%OOF66N;/.I2WW\QWW\C8#r cURRR5nUHnXC;aM SUSU35e g)Nzkey:z not in )rfields_by_namekeys)rr field_namekey_listrs r!check_configs_keyrs@~~,,113H@$se8J< @@r cT\rSrSrSrSrSrSrSrSr Sr S r S r S r S rS rg)DistributedJobInfoz~ DistributedJobInfo will serialize all distributed training information Just for inner use: 1) debug 2) replicate experiments c8[R"5Ulgr)r rjob_infoselfs r!__init__DistributedJobInfo.__init__s0CCE r c$XRlgr)r worker_num)rrs r!_set_worker_num"DistributedJobInfo._set_worker_num #-  r c$XRlgr)r server_num)rrs r!_set_server_num"DistributedJobInfo._set_server_numrr cNURRRU5 gr)r worker_ipsr)rrs r!_set_worker_ips"DistributedJobInfo._set_worker_ipss   '' 3r cNURRRU5 gr)rserver_endpointsr)rrs r!_set_server_endpoints(DistributedJobInfo._set_server_endpointss &&--.>?r c8[U5URlgr)r&rorigin_startup)rorigin_startup_progs r!_set_origin_startup&DistributedJobInfo._set_origin_startup s'*+>'? $r c8[U5URlgr)r&r origin_main)rorigin_main_progs r!_set_origin_main#DistributedJobInfo._set_origin_main s$'(8$9 !r c8[U5URlgr)r&rdistributed_main)rdistributed_main_progs r!_distributed_main$DistributedJobInfo._distributed_mains),-B)C &r c$XRlgr)roptimizer_name)rrs r!_optimizer_name"DistributedJobInfo._optimizer_names '5 $r c$XRlgr)rstrategy)r dist_strategys r!_set_distributed_strategy,DistributedJobInfo._set_distributed_strategys !. r )rN)rrrr__doc__rrrrrrrrrrrrr r!rrs= F..4@@:D6/r rc(\rSrSrSrSSjrSSjrSSjrSSjr\ SSj5r \ R\ SSj55r \ SS j5r \ R\ SS j55r \ SS j5r\R\ SS j55r\ SS j5r\R\ SSj55r\ SSj5r\ SSj5r\R\ SSj55r\R\ SSj55r\ SSj5r\R\ SSj55r\ SSj5r\R\ SSj55r\RSSj5r\ SSj5r\R\ SSj55r\ SSj5r\R\ SSj55r\ SSj5r\R\ SSj55r\ SSj5r\R\ SSj55r\ SS j5r\RSS!j5r\ SS"j5r\R\ SS#j55r\ SS$j5r\R\ SS%j55r\ SS&j5r\R\ SS'j55r\ SS(j5r\R\ SS)j55r\ SS*j5r\R\ SS+j55r\ SS,j5r\R\ SS-j55r\ SS.j5r \ R\ SS/j55r \ SS0j5r!\!R\ SS1j55r!\ SS2j5r"\"R\ SS3j55r"\ SS4j5r#\#R\ SS5j55r#\ SS6j5r$\$R\ SS7j55r$\ SS8j5r%\%R\ SS9j55r%\ SS:j5r&\&R\ SS;j55r&\ SS<j5r'\'R\ SS=j55r'\ SS>j5r(\(R\ SS?j55r(\ SS@j5r)\)R\ SSAj55r)\ SSBj5r*\*R\ SSCj55r*\ SSDj5r+\+R\ SSEj55r+\ SSFj5r,\ SSGj5r-\-R\ SSHj55r-\ SSIj5r.\.R\ SSJj55r.\,R\ SSKj55r,\ SSLj5r/\/R\ SSMj55r/\ SSNj5r0\0R\ SSOj55r0\ SSPj5r1\1R\ SSQj55r1\ SSRj5r2\2RSSSj5r2\ SSTj5r3\3R\ SSUj55r3\ SSVj5r4\4R\ SSWj55r4\ SSXj5r5\5R\ SSYj55r5\ SSZj5r6\6R\ SS[j55r6\ SS\j5r7\7R\ SS]j55r7\ SS^j5r8\8R\ SS_j55r8\ SS`j5r9\9R\ SSaj55r9\ SSbj5r:\:R\ SScj55r:\ SSdj5r;\;R\ SSej55r;\ SSfj5r<\\>R\ SSkj55r>\ SSlj5r?\?R\ SSmj55r?\ SSnj5r@\@R\ SSoj55r@\ SSpj5rA\ARSSqj5rA\ SSrj5rB\BRSSsj5rB\ SStj5rC\CRSSuj5rC\ SSvj5rD\DRSSwj5rD\ SSxj5r\RSSyj5r\ SSzj5r\RSS{j5r\ SS|j5rE\ERSS}j5rE\ SS~j5rF\FR\ SSj55rF\ SSj5rG\GR\ SSj55rG\ SSj5rH\HR\ SSj55rHSSjrISSjrJSSjrKSrLg)DistributedStrategyiFc[R"5UlSn[5R U5(a%[ [5U5URlSn[5R U5(a%[[5U5URlSn[5R U5(a%[ [5U5URl Sn[5R U5(a%[ [5U5URl /SQUl /SQUl SUl [R"S5 g ) a DistributedStrategy is the main configuration entry for distributed training of Paddle. All of the distributed training configurations can be configured in DistributedStrategy, such as automatic mixed precision (AMP), Layer-wise Adaptive Rate Scaling (LARS), asynchronous update parameter server(ASGD), etc. DistributedStrategy can be serialized into protobuf file or deserialized from protobuf file Users who run local training usually configure BuildStrategy, and DistributedStrategy supports configurations from BuildStrategy. (FLAGS_cudnn_batchnorm_spatial_persistentFLAGS_conv_workspace_size_limitFLAGS_cudnn_exhaustive_searchFLAGS_sync_nccl_allreduce)dpppshardingsepcpmp) embedding layer_normz.b_Tz distributed strategy initializedN)r rrr is_publicr"cudnn_batchnorm_spatial_persistentrconv_workspace_size_limitcudnn_exhaustive_searchsync_nccl_allreducehybrid_parallel_ordersync_param_name_DistributedStrategy__lock_attrr info)rrs r!rDistributedStrategy.__init__ s6IIK 9 ? $ $S ) )EI$FDMM <0 ? $ $S ) );>$<DMM 3. ? $ $S ) ):>$;DMM 1* ? $ $S ) )6:=?3;O6PDMM -1 "+M 67r cUR(a4[X5(d$[USURR35e[ R XU5 g)Nz is not a attribute of )r hasattr TypeError __class__robject __setattr__)rrvalues r!rDistributedStrategy.__setattr__QsN   GD$6$6%.t~~/F/F.GH  4e,r c[US5nUR[UR55 SSS5 g!,(df  g=f)a Serialize current DistributedStrategy to string and save to output file Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.dgc = True >>> strategy.recompute = True >>> strategy.recompute_configs = {"checkpoints": ["x"]} >>> strategy.save_to_prototxt("dist_strategy.prototxt") wN)openwriter&r)routputfouts r!save_to_prototxt$DistributedStrategy.save_to_prototxtXs1 &# $ JJs4==) *  s %; A c[US5n[RRR [ UR 55UR5UlSSS5 g!,(df  g=f)a Load from prototxt file for DistributedStrategy initialization Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.dgc = True >>> strategy.recompute = True >>> strategy.recompute_configs = {"checkpoints": ["x"]} >>> strategy.save_to_prototxt("dist_strategy.prototxt") >>> strategy.load_from_prototxt("dist_strategy.prototxt") rN)rrr text_formatMerger&readr)rpb_filers r!load_from_prototxt&DistributedStrategy.load_from_prototxtksK$'3 1"OO77==AFFH t}}DM  s AA'' A5c[RR5nURRR R nUH|n[URRUR5nURS:Xa)[RRRU5n[XRU5 M~ U$)a$ Configure BuildStrategy for DistributedStrategy Note that the properties of BuildStrategy are valid in DistributedStrategy only if the property is non-distributed strategy. Examples: .. code-block:: python >>> import paddle >>> build_strategy = paddle.static.BuildStrategy() >>> build_strategy.fuse_elewise_add_act_ops = True >>> build_strategy.fuse_bn_act_ops = True >>> build_strategy.enable_auto_fusion = True >>> build_strategy.fuse_relu_depthwise_conv = True >>> build_strategy.fuse_broadcast_ops = True >>> build_strategy.fuse_all_optimizer_ops = True >>> build_strategy.enable_inplace = True >>> strategy = paddle.distributed.fleet.DistributedStrategy() >>> strategy.build_strategy = build_strategy reduce_strategy) paddlestaticr rbuild_strategyrrrrReduceStrategyr)rr*rrrs r!r*"DistributedStrategy.build_strategys4 446--88??ADMM88!&&AEvv** 33BB5I NFFE 2  r c URRRRnUHnURS:XdURS:Xa][ XR 5nUR S:Xa [U5n[URRUR U5 MURS:XdM[ URRUR 5R[ XR 55 M g)Nrrr'r) rr*rrrrrReduceStrategyFleetrr)rrrrrs r!r*r,s--88??Aww!|qww!|&&166../6E 44affeDA 44aff=DDHff-r c@[URR5$)a: Set the strategy of gradient scale Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.gradient_scale_configs = {'scale_strategy': 'avg'} Note that, strategy must be in 'avg', 'sum' or 'customized' )rrgradient_scale_configsrs r!r0*DistributedStrategy.gradient_scale_configss DMM@@AAr c[URRUS5 [URRU5 g)Nr0)rrr0r)rrs r!r0r1s5  MM 0 0  $ T]]AA6Jr c.URR$)a Indicating whether we are using asynchronous stochastic gradient descent updates for training. This property is valid when we are using parameter server training, which is implied by setting appropriate RoleMaker Default value: True Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> role_maker = fleet.PaddleCloudRoleMaker() >>> fleet.init(role_maker) >>> strategy = fleet.DistributedStrategy() >>> strategy.a_sync = True # by default this is True >>> # code block for defining loss and local optimizer >>> # sgd = fleet.distributed_optimizer(optimizer, strategy) )ra_syncrs r!r4DistributedStrategy.a_syncs.}}###r c[U[5(aXRlSS0Ulg[ S[ U535e)NrqrCThe type of `flag` is invalid, expected type is bool, but received ) isinstancerrr4a_sync_configs ValueErrorrrflags r!r4r5sF dD ! !#'MM #,a.D UVZ[_V`Uab r c@[URR5$)a Set a_sync update configurations. In general, asynchronous parameter server training has several configurable settings that can be configured through a dict. **Notes**: k_step(int): number of local optimization updates before communication max_merge_var_num(int): maximum number of merged gradients before communication send_queue_size(int): a buffer size of worker communication independent_recv_thread(bool): if we are using independent recv thread for communication thread_pool_size(int): number of thread pool send_wait_times(int): waiting time for sending gradients runtime_split_send_recv(bool): if we are using Tensor split for send and recv during runtime Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> role_maker = fleet.PaddleCloudRoleMaker() >>> fleet.init(role_maker) >>> strategy = fleet.DistributedStrategy() >>> strategy.a_sync = True # by default this is True >>> configs = {"k_steps": 1024, "send_queue_size": 32} >>> strategy.a_sync_configs = configs >>> # code block for defining loss and local optimizer >>> # sgd = fleet.distributed_optimizer(optimizer, strategy) )rrr9rs r!r9"DistributedStrategy.a_sync_configssNDMM8899r c[URRUS5 [URRU5 g)Nr9)rrr9rrconfigss r!r9r>s4  MM ( ('3C  T]]997Cr c@[URR5$)a Set trainer desc configurations. **Notes**: dump_fields_path(str): the path of dump fields dump_fields(list(str)): the fields that you want to dump dump_param(list(str)): the param that you want to dump stat_var_names(list(str)): Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> role_maker = fleet.PaddleCloudRoleMaker() >>> fleet.init(role_maker) >>> strategy = fleet.DistributedStrategy() >>> configs = {"dump_fields_path": "./dump_data", "dump_fields": ["xxx", "yyy"]} >>> strategy.trainer_desc_configs = configs >>> # code block for defining loss and local optimizer >>> # sgd = fleet.distributed_optimizer(optimizer, strategy) )rrtrainer_desc_configsrs r!rC(DistributedStrategy.trainer_desc_configs%s<DMM>>??r c.URR$)a set adam_d2sum Default value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> role_maker = fleet.PaddleCloudRoleMaker() >>> fleet.init(role_maker) >>> strategy = fleet.DistributedStrategy() >>> strategy.adam_d2sum = True # by default this is False >>> # code block for defining loss and local optimizer >>> # sgd = fleet.distributed_optimizer(optimizer, strategy) )r adam_d2sumrs r!rFDistributedStrategy.adam_d2sumEs*}}'''r c|[U[5(aXRlg[ S[ U535e)Nr7)r8rrrFr:rr;s r!rFrG\s: dD ! !'+MM $UVZ[_V`Uab r c[URRUS5 [URRU5 g)NrC)rrrCrr@s r!rCrDfs4  MM . .9O  T]]??Ir c.URR$)a Set fs client configurations. Note: uri(str): the uri of fs client user(str): the user_name of fs client passwd(str): the passwd of fs client hadoop_bin(str): Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> role_maker = fleet.PaddleCloudRoleMaker() >>> fleet.init(role_maker) >>> strategy = fleet.DistributedStrategy() >>> configs = {"uri": "xxx", "user": "xxx", "passwd": "xxx"} >>> strategy.fs_client_param = configs >>> # code block for defining loss and local optimizer >>> # sgd = fleet.distributed_optimizer(optimizer, strategy) )rfs_client_paramrs r!rK#DistributedStrategy.fs_client_paramns8}},,,r c[URRUS5 [URRU5 g)NrK)rrrKrr@s r!rKrLs4  MM ) )74E  T]]::GDr c.URR$r)rdownpour_table_paramrs r!sparse_table_configs(DistributedStrategy.sparse_table_configss}}111r c^^SSKJm URRnSSUU4SjjjmU(d[R "S5 gUH(nUR 5nX4lT"USU-X5 M* g)Nr)rct>URRGHnUS-UR-nURT R:Xa[ R "SU35 URT R:XanUS-U;aMnX%S-n[ R "SUSU35 [U5H/n[XR5R5nT "XX'5 M1 MT "[XR5XR5 M[ R "SU5 XR;aGMURT R:Xa)[XR5RX%5 GMQ[X%5[:Xa[XRX%U5 GM[XRX%5 GM g)N.z message: z.numz message num:  znot message: %s)rrrrrr debugrrrangerrrrr) r config_namerAindexfieldrnumidatarset_table_configs r!r^BDistributedStrategy.sparse_table_configs..set_table_configsW.."S(5::5::!=!==LL9TF!34{{o&D&DD&=7$%Vm4 }TF!C5%AB!&sA#*3 #;#?#?#AD,TD",)#C4dLL!2D9* {{o&D&DDZZ077 F .$6#CW]55IJ#CW]C5/r ztable configs is emptyztable_parameters.)r) rr&rXr&rAdict[str, Any]rYrreturnNone)google.protobuf.descriptorrrrOr r r table_name)rrA table_paramrd table_datarr^s @@r!rPrQs ?mm88 OP D D#& D1? DHK D  D D> KK0 1% (__. (2% '*4'&r cT^^^^^ ^ /SQm SS/m /SQmURRnSmSmUUUUU U 4SjnT(d[R"S5 gTHKnUS :XdUS :XaM[ TU5[ :waM)UR 5nXElU"UTU5 MM g) N)1sparse_table_classsparse_compress_in_savesparse_shard_numsparse_accessor_classsparse_learning_ratesparse_initial_g2sumsparse_initial_rangesparse_weight_boundssparse_fea_dimsparse_embedx_dimsparse_embedx_thresholdsparse_nonclk_coeffsparse_click_coeffsparse_base_thresholdsparse_delta_thresholdsparse_delta_keep_dayssparse_delete_after_unseen_dayssparse_show_click_decay_ratesparse_delete_thresholdsparse_convertersparse_deconvertersparse_enable_cachesparse_cache_ratesparse_cache_file_numsparse_beta1_decay_ratesparse_beta2_decay_ratesparse_ada_epsilonsparse_optimizersparse_ssd_unseenday_thresholdembed_sparse_optimizerembed_sparse_learning_rateembed_sparse_weight_boundsembed_sparse_initial_rangeembed_sparse_initial_g2sumembed_sparse_beta1_decay_rateembed_sparse_beta2_decay_rateembedx_sparse_optimizerembedx_sparse_learning_rateembedx_sparse_weight_boundsembedx_sparse_initial_rangeembedx_sparse_initial_g2sumembedx_sparse_beta1_decay_rateembedx_sparse_beta2_decay_ratefeature_learning_rate nodeid_slotsparse_load_filter_slotssparse_save_filter_slotssparse_zero_init use_gpu_graphDownpourSparseTableDownpourSparseSSDTable)DownpourSparseValueAccessorDownpourCtrAccessorDownpourCtrDoubleAccessorDownpourUnitAccessorDownpourDoubleUnitAccessorDownpourCtrDymfAccessorc`URSS5UlURSS5Ulg)Nr皙?ri0#)getrr)graphrs r!add_graph_config@DistributedStrategy.fleet_desc_configs..add_graph_config s-*2,,'+E '!) ]D AE r c$ URUS-S5nX0lUS:XaSUlURUS-S5URlURUS-S5URlURUS -S S /5nURR R U5 gUS:XaS UlURUS-S5URlURUS-S5URlUS :XaSURlURUS-S5URlURUS -S S /5nURR R U5 gUS:XaSUlURUS-S5URlURUS-S5URlUS :XaSURlURUS-S5URlURUS -S S /5nURR R U5 gUS:XaSUlURUS-S5URlURUS-S5URlUS :XaSURlURUS-S5URlURUS -S S /5nURR R U5 gUS:XaSUlURUS-S5URlURUS-S5URlURUS-S5URl URUS-S5URl URUS-S5URl URUS -S S /5nURR R U5 gUS:XaSUlURUS-S5URlURUS-S5URlURUS-S5URl URUS-S5URl URUS-S5URl URUS -S S /5nURR R U5 gg) NradagradnaiveSparseNaiveSGDRulerlrrng-C6?roi SparseAdaGradSGDRuleembed_rrmr adagrad_v2SparseAdaGradV2SGDRule std_adagradStdAdaGradSGDRuleadamSparseAdamSGDRulegMbP?rg?rg+?rg:0yE> shared_adamSparseSharedAdamSGDRule) rrr learning_rate initial_range weight_boundsrr initial_g2sumrbeta1_decay_ratebeta2_decay_rate ada_epsilon)sgdrprefixrboundss r!sparse_optimizer_configGDistributedStrategy.fleet_desc_configs..sparse_optimizer_configs%\\++YN&H(/*2,,33T+ '+3,,33T+ '"33c2Y ''..v69,1,4LL33T- )-5LL33T- )X%01CKK-,4LL33Q- )"33c2Y ))008</3,4LL33T- )-5LL33T- )X%01CKK-,4LL33Q- )"33c2Y ))008=0.,4LL33T- )-5LL33T- )X%01CKK-,4LL33Q- )"33c2Y ))0086).)133U*&*233T*&-5LL66-)-5LL66-)(0||114($"33c2Y&&--f5=04)133U*&*233T*&-5LL66-)-5LL66-)(0||114($"33c2Y&&--f5)1r cb >UHnUT;dM [SUS35e URSS5nUT;a[SU35eUS:XaSUlOSUlURS S 5UlURS S 5UlURS S5UlURSS5UlURSS5UlURSS5nUT;a[SU35eURS5S:aSURl O8URS5S:aSURl OSURl T RSS 5(dSURl URSS5URl URRS -URl URS!S"5URl US#:XaSURRlOS URRlURS$S%5URRlURS&S'5URRlURS(S)5URRlURS*S+5URRlURS,S5URRlURS-S.5URRlURS/S05URRlURS1S25URRlURS3S'5URRlURS4/5nURRR2R5U5 URS5/5nURRR6R5U5 URS6S 5URRlURS LaSURRlURS7S85nURS9S85nURR:R=5n S'U lXyl Xl!URR:R=5n S:U lXzl Xl!US:XdUS;:Xa=T "URRDUS85 T "URRFUS85 ONzstrategy key 'z ' not supportrhrzXsupport sparse_table_class: ['DownpourSparseTable, DownpourSparseSSDTable'], but actual rSSDSparseTableMemorySparseTablerjir}Tr~g/nB?rrFrkrzsupport sparse_accessor_class: ['DownpourSparseValueAccessor', 'DownpourCtrAccessor', 'DownpourCtrDoubleAccessor', 'DownpourUnitAccessor', 'DownpourDoubleUnitAccessor', 'DownpourCtrDymfAccessor'], but actual DoublerCtrDoubleAccessorDymfCtrDymfAccessorCtrCommonAccessoruse_cvmSparseAccessorrqrrrrrrsg?rtrrug?rvg?rwryg\(\?rzg?rxrrrrr{r|rrrembedx_)%r:r table_class shard_numenable_sparse_table_cachesparse_table_cache_ratesparse_table_cache_file_numrfindaccessoraccessor_class embedx_dimfea_dimembedx_thresholdctr_accessor_param show_scale nonclk_coeff click_coeffbase_thresholddelta_thresholddelta_keep_daysshow_click_decay_ratedelete_thresholddelete_after_unseen_daysssd_unseenday_thresholdload_filter_slotsrsave_filter_slots zero_inittable_accessor_save_paramrparam converter deconverterembed_sgd_paramembedx_sgd_paramgraph_sgd_param)rfrrrrrrrr save_data1 save_data2rrArsupport_sparse_accessor_classsupport_sparse_key_listsupport_sparse_table_classs r!set_sparse_table_configGDistributedStrategy.fleet_desc_configs..set_sparse_table_configsU55$~cU-%HII!**$&;K"<< nozn{|66)9 &)< &#)::.@$#GJ 39::%t4J 028#W2J .6N%BB ghvgwx""8,15H ##2$$V,15F ##25H ##2;;y$//5E ##2-3ZZ8KQ-OJ   **4*=*=*H*H1*LJ   '39::)24J   0!77DI ##66ADH ##66ABH**%sCJ   2 2 ?BH$aBJ   2 2 >EKJJ'EJ   2 2 AFLZZ($FJ   2 2 BFLZZ("FJ   2 2 B 94@    2 2 H 4c:    2 2 C K#,,FFJJLJ J #, %0 "#,,FFJJLJ J #, %0 ""77!%@@'''77(''88&"(''77(''88&) Z00@@& Ir zfleet desc config is empty dense_tabledatanorm_table)rrOr r rdictrrd) rrArerrdrfrrrrrs ` @@@@@r!fleet_desc_configs&DistributedStrategy.fleet_desc_configss2# h " $& ") %mm88  B m 6^ J JB KK4 5% -/!%55 +,4(__. (2%' GJ4GH&r c.URR$)a% Indicating whether we are using automatic mixed precision training Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.amp = True # by default this is false )ramprs r!rDistributedStrategy.amp}}   r c|[U[5(aXRlg[R "S5 g)Nz"amp should have value of bool type)r8rrrr warningr;s r!rr#( dD ! ! $MM  NN? @r c@[URR5$)a Set automatic mixed precision training configurations. In general, amp has several configurable settings that can be configured through a dict. **Notes**: init_loss_scaling(float): The initial loss scaling factor. Default 32768. use_dynamic_loss_scaling(bool): Whether to use dynamic loss scaling. Default True. incr_every_n_steps(int): Increases loss scaling every n consecutive steps with finite gradients. Default 1000. decr_every_n_nan_or_inf(int): Decreases loss scaling every n accumulated steps with nan or inf gradients. Default 2. incr_ratio(float): The multiplier to use when increasing the loss scaling. Default 2.0. decr_ratio(float): The less-than-one-multiplier to use when decreasing the loss scaling. Default 0.5. custom_white_list(list[str]): Users' custom white list which always execution fp16. custom_black_list(list[str]): Users' custom black list which forbidden execution fp16. custom_black_varnames(list[str]): Users' custom black variables' names. use_pure_fp16(bool): Whether to use the pure fp16 training. Default False. use_pure_bf16(bool): Whether to use the pure bf16 training. Default False. use_fp16_guard(bool): Whether to use `fp16_guard` when constructing the program. Default True. Only takes effect when `use_pure_fp16` is turned on. Examples: .. code-block:: python :name: example_1 >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.amp = True >>> strategy.amp_configs = { ... "init_loss_scaling": 32768, ... "custom_white_list": ['conv2d'] ... } .. code-block:: python :name: example_2 >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.amp = True >>> # pure fp16 >>> strategy.amp_configs = { ... "init_loss_scaling": 32768, ... "use_pure_fp16": True ... } )rr amp_configsrs r!rDistributedStrategy.amp_configs+stDMM5566r c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rrg. $--33WmLT]]66@r c.URR$)a Indicating whether we are using automatic sparsity training Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.asp = True # by default this is false )rasprs r!r DistributedStrategy.aspmrr c|[U[5(aXRlg[R "S5 g)Nz"asp should have value of bool type)r8rrr r rr;s r!r r ~rr c.URR$)a Indicating whether we are using quantization aware training Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.qat = True # by default this is false rqatrs r!rDistributedStrategy.qatrr c\[U[5(dS5eXRlgNz"qat should have value of bool type)r8rrrr;s r!rrs'$%%K'KK% r c@[URR5$)a Set quantization training configurations. In general, qat has several configurable settings that can be configured through a dict. **Notes**: channel_wise_abs_max(bool): Whether to use `per_channel` quantization training. Default is True. weight_bits(int): quantization bit number for weight. Default is 8. activation_bits(int): quantization bit number for activation. Default is 8. not_quant_pattern(list[str]): When the skip pattern is detected in an op's name scope, the corresponding op will not be quantized. algo(str): Other quantization training algorithm. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.qat = True >>> strategy.qat_configs = { ... "channel_wise_abs_max": True, ... "weight_bits": 8, ... "activation_bits": 8, ... "not_quant_pattern": ['skip_quant'] ... } rr qat_configsrs r!rDistributedStrategy.qat_configss6DMM5566r c[URRUS5 [URRU5 gNrrrrrr@s r!rr,$--33WmLT]]66@r c.URR$)a Indicating whether we are using forward recomputation for memory optimization Default value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.recompute = True >>> # suppose x and y are names of checkpoint tensors for recomputation >>> strategy.recompute_configs = {"checkpoints": ["x", "y"]} )r recomputers r!rDistributedStrategy.recomputes }}&&&r c|[U[5(aXRlg[R "S5 g)Nz(recompute should have value of bool type)r8rrrr rr;s r!rrs( dD ! !&*MM # NNE Fr c.URR$)ae Indicating whether we are using synchronized all reduce in each communication thread We note that system overhead is usually lower when sync_nccl_allreduce = True Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.sync_nccl_allreduce = True )rrrs r!r'DistributedStrategy.sync_nccl_allreduce}}000r c|[U[5(aXRlg[R "S5 g)Nz2sync_nccl_allreduce should have value of bool type)r8rrrr rr;s r!rr ( dD ! !04MM - NNO Pr c.URR$)a Indicating whether we are using hierarchical allreduce in collective communication Hierarchical allreduce often does allreduce within a certain node group and then do allreduce among the leaders of each group Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.use_hierarchical_allreduce = True )ruse_hierarchical_allreducers r!r%.DistributedStrategy.use_hierarchical_allreduces }}777r c|[U[5(aXRlg[R "S5 g)Nz9use_hierarchical_allreduce should have value of bool type)r8rrr%r rr;s r!r%r&* dD ! !7;MM 4 NNK r c.URR$)aP Number of ranks for low level node groups in hierarchical allreduce Default value: number of GPU cards on each single GPU machine Example: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.hierarchical_allreduce_inter_nranks = 8 )r#hierarchical_allreduce_inter_nranksrs r!r*7DistributedStrategy.hierarchical_allreduce_inter_nrankss}}@@@r c|[U[5(aXRlg[R "S5 g)NzAhierarchical_allreduce_inter_nranks should have value of int type)r8rrr*r rrrs r!r*r+s* eS ! !@EMM = NNS r c.URR$)aD Indicating whether we are using sync_batch_norm to do synchronous batch normalization among all training nodes. Default value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.sync_batch_norm = True )rsync_batch_normrs r!r/#DistributedStrategy.sync_batch_norm)s"}},,,r c|[U[5(aXRlg[R "S5 g)Nz.sync_batch_norm should have value of bool type)r8rrr/r rr;s r!r/r0<( dD ! !,0MM ) NNK Lr c.URR$)aA Indicating whether we are using fuse_all_reduce_ops for gradient fusion during backward phase of training Default value: True Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.fuse_all_reduce_ops = False )rfuse_all_reduce_opsrs r!r4'DistributedStrategy.fuse_all_reduce_opsDr!r c|[U[5(aXRlg[R "S5 g)Nz2fuse_all_reduce_ops should have value of bool type)r8rrr4r rr;s r!r4r5Ur#r c.URR$)a  Specifying the size of gradient to fuse in Mega-Bytes Default value: 32 Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.fuse_grad_size_in_MB = 50 )rfuse_grad_size_in_MBrs r!r8(DistributedStrategy.fuse_grad_size_in_MB]s }}111r c|[U[5(aXRlg[R "S5 g)Nz2fuse_grad_size_in_MB should have value of int type)r8rrr8r rr-s r!r8r9os( eS ! !16MM . NNO Pr c.URR$)ay Specifying the size of gradient to fuse in Mega-Bytes when the last group of each batch communicates. Making the last group small is useful to improve performance. Default value: 1 Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.last_comm_group_size_MB = 2 )rlast_comm_group_size_MBrs r!r<+DistributedStrategy.last_comm_group_size_MBws$}}444r cFUS:aXRlg[S5e)Nrz0last_comm_group_size_MB should be greater than 0)rr<r:r-s r!r<r=s! 1949MM 1OP Pr c.URR$)a= Indicating whether we are using find_unused_parameters to find unused parameters in DataParallel. Default value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.find_unused_parameters = True )rfind_unused_parametersrs r!r@*DistributedStrategy.find_unused_parameterss$}}333r c|[U[5(aXRlg[R "S5 g)Nz5find_unused_parameters should have value of bool type)r8rrr@r rr;s r!r@rAs* dD ! !37MM 0 NNG r c.URR$r)rfuse_grad_size_in_TFLOPSrs r!_fuse_grad_size_in_TFLOPS-DistributedStrategy._fuse_grad_size_in_TFLOPSs}}555r c|[U[5(aXRlg[R "S5 g)Nz8fuse_grad_size_in_TFLOPS should have value of float type)r8r6rrDr rr-s r!rErFs* eU # #5:MM 2 NNJ r c.URR$)z Specifying the number of NCCL communicator Default value: 1 Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.nccl_comm_num = 2 )r nccl_comm_numrs r!rI!DistributedStrategy.nccl_comm_nums"}}***r c|[U[5(aXRlg[R "S5 g)Nz+nccl_comm_num should have value of int type)r8rrrIr rr-s r!rIrJs( eS ! !*/MM ' NNH Ir c@[URR5$)a? Set recompute configurations. **Note**: checkpoints(list[str]): list of string name of checkpoints. In general, the recompute strategy of current implementation should have some manually assign checkpoints. enable_offload(bool): enable recompute checkpoints offload feature. this feature will offload the checkpoint to host memory to allow even larger batch size. since the memcpy from host to device takes time, it is a trade off between larger batch size and training speed. checkpoint_shape(list[int]): list of int that specific the shape of checkpoint. so far recompute-offload requires that all checkpoint to be same shape, and every dimension specific here should be determined ("-1" is not allowed). Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.recompute = True >>> strategy.recompute_configs = { ... "checkpoints": ["x", "y"], ... "enable_offload": True, ... "checkpoint_shape": [100, 512, 1024] ... } )rrrecompute_configsrs r!rM%DistributedStrategy.recompute_configss@DMM;;<>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.sharding = True )rrrs r!rDistributedStrategy.shardings*}}%%%r c|[U[5(aXRlg[R "S5 g)Nz'sharding should have value of bool type)r8rrrr rr;s r!rrR( dD ! !%)MM " NND Er c@[URR5$)aQ Set sharding configurations. **Note**: sharding_segment_strategy(string, optional): strategy used to segment the program(forward & backward operations). two strategise are available: "segment_broadcast_MB" and "segment_anchors". segment is a concept used in sharding to overlap computation and communication. Default is segment_broadcast_MB. segment_broadcast_MB(float, optional): segment by the parameters broadcast volume. sharding will introduce parameter broadcast operations into program, and after every segment_broadcast_MB size parameter being broadcasted, the program will be cut into one segment. This configuration will affect the communication speed in sharding training, and should be an empirical value decided by your model size and network topology. Only enable when sharding_segment_strategy = segment_broadcast_MB. Default is 32.0 . segment_anchors(list): list of anchors used to segment the program, which allows a finer control of program segmentation. this strategy is experimental by now. Only enable when sharding_segment_strategy = segment_anchors. sharding_degree(int, optional): specific the number of gpus within each sharding parallelism group; and sharding will be turn off if sharding_degree=1. Default is 8. gradient_merge_acc_step(int, optional): specific the accumulation steps in gradient merge; and gradient merge will be turn off if gradient_merge_acc_step=1. Default is 1. optimize_offload(bool, optional): enable the optimizer offload which will offload the moment vars to Host memory in order to saving GPU memory for fitting larger model. the moment var will be prefetch from and offloaded to Host memory during update stage. it is a strategy that trades off between training speed and GPU memory, and is recommended to be turn on only when gradient_merge_acc_step large, where the number of time of update stage will be relatively small compared with forward&backward's. Default is False. dp_degree(int, optional): specific the number of data parallelism group; when dp_degree >= 2, it will introduce dp_degree ways data parallelism as the outer parallelism for the inner parallelism. User is responsible to ensure global_world_size = mp_degree * sharding_degree * pp_degree * dp_degree. Default is 1. mp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each megatron parallelism group; and megatron parallelism will turn be off if mp_degree=1. Default is 1. pp_degree(int, optional): [Hybrid parallelism ONLY] specific the number of gpus within each pipeline parallelism group; and pipeline parallelism will turn be off if pp_degree=1. Default is 1. pp_allreduce_in_optimize(bool, optional): [Hybrid parallelism ONLY] move the allreduce operations from backward stage to update(optimize) stage when pipeline parallelism is on. This configuration will affect the communication speed of Hybrid parallelism training depended on network topology. this strategy is experimental by now.. Default is False. optimize_cast(bool, optional): [Hybrid parallelism ONLY] Move the cast op of AMP which cast fp32 param to fp16 param to optimizer. optimize_cast will persist fp16 param, it will take more memory, but will be faster, trade space for time. Recommend to turn on only when using pipeline or gradient_merge_acc_step large. Examples: .. code-block:: python >>> # sharding-DP, 2 nodes with 8 gpus per node >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.sharding = True >>> strategy.sharding_configs = { ... "sharding_segment_strategy": "segment_broadcast_MB", ... "segment_broadcast_MB": 32, ... "sharding_degree": 8, ... "dp_degree": 2, ... "gradient_merge_acc_step": 4, ... } )rrsharding_configsrs r!rV$DistributedStrategy.sharding_configs#spDMM::;;r c[URRUS5 [URRU5 g)NrV)rrrVrr@s r!rVrW]4  MM * *G5G  T]];;WEr c.URR$)a Run program using Executor other than ParallelExecutor. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.without_graph_optimization = True )rwithout_graph_optimizationrs r!r[.DistributedStrategy.without_graph_optimizationes}}777r c|[U[5(aXRlg[R "S5 g)Nz9without_graph_optimization should have value of bool type)r8rrr[r rr;s r!r[r\ur(r c.URR$)al This based on raw_program_optimizer program Set whether use same stream for calc and comm when fuse allreduce The default value for the calc_comm_same_stream is False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy._calc_comm_same_stream = True )rcalc_comm_same_streamrs r!_calc_comm_same_stream*DistributedStrategy._calc_comm_same_streams }}222r c|[U[5(aXRlg[R "S5 g)Nz7calc_comm_same_stream should have value of boolean type)r8rrr_r r)rsames r!r`ras* dD ! !26MM / NNI r c.URR$)ag Set whether fuse the grad for gradient merge. Note: this flag will only effect the gradient merge under pipeline mode The default value for the fuse_grad_merge is False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.fuse_grad_merge = True )rfuse_grad_mergers r!re#DistributedStrategy.fuse_grad_merges }},,,r c|[U[5(aXRlg[R "S5 g)Nz1fuse_grad_merge should have value of boolean type)r8rrrer r)rres r!rerfs( ot , ,,;MM ) NNN Or c.URR$)a This based on raw_program_optimizer program and allreduce the num of the fused op Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.fuse_grad_size_in_num = 2 )rfuse_grad_size_in_numrs r!ri)DistributedStrategy.fuse_grad_size_in_nums}}222r c|[U[5(aXRlg[R "S5 g)Nz5fuse_grad_size_in_num should have value of int32 type)r8rrrir r)rr[s r!rirjs* c3  25MM / NNG r c.URR$)a Indicating whether we are using pipeline parallelism for distributed training. Current implementation mainly focus on single GPU machine pipeline parallelism and data parallelism across GPU machine. The pipeline information is indicated through device_guard information in user-defined program. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.pipeline = True )rpipeliners r!rmDistributedStrategy.pipelines"}}%%%r c.URR$r)r is_fl_ps_moders r!rp!DistributedStrategy.is_fl_ps_modes}}***r c|[U[5(aXRlg[R "S5 g)Nz,is_fl_ps_mode should have value of bool type)r8rrrpr rr;s r!rprqs( dD ! !*.MM ' NNI Jr c.URR$r)rwith_coordinatorrs r!is_with_coordinator'DistributedStrategy.is_with_coordinators}}---r c|[U[5(aXRlg[R "S5 g)Nz/with_coordinator should have value of bool type)r8rrrtr rr;s r!rurvs( dD ! !-1MM * NNL Mr c|[U[5(aXRlg[R "S5 g)Nz'pipeline should have value of bool type)r8rrrmr rr;s r!rmrnrTr c@[URR5$)a Set pipeline parallelism configurations. In pipeline parallelism, different parts of neural networks are running on different GPUS. There are Tensor queue buffer between each pair of neighborhood GPUS that are responsible for synchronizing hidden Tensor results between GPUs. Pipeline parallelism consists of several producer-consumer style hardware pairs, such as GPU-GPU, CPU-GPU, GPU-XPU. The best way to speedup pipeline parallelism is to make the size of Tensor in Tensor queue smaller, so that we will have a faster producer for downstream consumers. **Notes**: **Detailed arguments for pipeline_configs** **micro_batch_size**: the number of small batches in each user defined batch Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.pipeline = True >>> strategy.pipeline_configs = {"micro_batch_size": 12} )rrpipeline_configsrs r!rz$DistributedStrategy.pipeline_configss8DMM::;;r c[URRUS5 [URRU5 g)Nrz)rrrzrr@s r!rzr{!rYr c.URR$)a Indicating whether we are using tensor parallel for distributed training. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.tensor_parallel = True )rtensor_parallelrs r!r~#DistributedStrategy.tensor_parallel)s}},,,r c|[U[5(aXRlg[R "S5 g)Nz.tensor_parallel should have value of bool type)r8rrr~r rr;s r!r~r9r2r c@[URR5$)a7 Set tensor_parallel configurations. **Notes**: **Detailed arguments for tensor_parallel_configs** **tensor_parallel_degree**: degree of tensor parallel **tensor_init_seed**: parameter initialization random seed Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.tensor_parallel = True >>> strategy.tensor_parallel_configs = {"tensor_parallel_degree": 4, ... "tensor_init_seed": 123} )rrtensor_parallel_configsrs r!r+DistributedStrategy.tensor_parallel_configsAs0DMMAABBr c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rr[s5  MM 1 1  % T]]BBGLr c@[URR5$)a Dynamic graph hybrid parallel strategy configuration. Five-way hybrid parallelism needs to meet the following relationships total_number_GPUs = dp_degree * mp_degree * pp_degree * sharding_degree * sep_degree **Note**: **dp_degree(int)**: set number of GPUs in a data parallel group. Default -1. This value should be an integer greater than 0. If it is not set, or set to -1, its value will be inferred based on the total number of cards. **mp_degree(int)**: set number of GPUs in a model parallel group. Default 1 **pp_degree(int)**: set number of GPUs in a pipeline parallel group. Default 1 **sep_degree(int)**: set number of GPUs in a sep parallel group. Default 1 **cp_degree(int)**: set number of GPUs in a context parallel group. Default 1 **sharding_degree(int)**: set number of GPUs in a sharding parallel group. Default 1 **order(list(string))**: set hybrid parallel dimensions, the order is from outside to inside. Default ['dp','pp','sharding','sep', 'mp'] Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.hybrid_configs = { ... "dp_degree": 1, ... "mp_degree": 2, ... "pp_degree": 1, ... "order":['dp','pp','sharding', 'sep', 'mp'] ... } )rrhybrid_configsrs r!r"DistributedStrategy.hybrid_configsesHDMM8899r cX[R"U5nSU;aUSUlURS5 [ UR R US5 SU;ahSUS;a!USSUlUSRS5 [UR R RUS5 URS5 SU;a>[UR R RUS5 URS5 [UR R U5 g)Nrmr mp_configsr  pp_configs) copydeepcopyr poprrrr rrr)rrA hybrid_configs r!rrs g. m #)6w)?D &   g & MM ( (-9I  7 " GL$99'.|'<=N'O$ %))*;<  ,,779N  KK % 7 "  ,,779N  KK %T]]997Cr c.URR$)a Indicating whether we are using Local SGD training. Default Value: False For more details, please refer to `Don't Use Large Mini-Batches, Use Local SGD `_. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.localsgd = True # by default this is false )rlocalsgdrs r!rDistributedStrategy.localsgds }}%%%r c|[U[5(aXRlg[R "S5 g)Nz'localsgd should have value of bool type)r8rrrr rr;s r!rrrTr c@[URR5$)aK Set LocalSGD training configurations. LocalSGD has a configurable setting that can be configured through a dict. **Notes**: k_steps(int) The local steps for training before parameter synchronization. Default 1. begin_step(int) The step of beginning training by localsgd. Default 1. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.localsgd = True >>> strategy.localsgd_configs = {"k_steps": 4, ... "begin_step": 30} )rrlocalsgd_configsrs r!r$DistributedStrategy.localsgd_configss,DMM::;;r c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rrrYr c.URR$)a Indicating whether we are using Adaptive Local SGD training. Default Value: False For more details, please refer to `Adaptive Communication Strategies to Achieve the Best Error-Runtime Trade-off in Local-Update SGD `_. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.adaptive_localsgd = True # by default this is false )radaptive_localsgdrs r!r%DistributedStrategy.adaptive_localsgds }}...r c|[U[5(aXRlg[R "S5 g)Nz0adaptive_localsgd should have value of bool type)r8rrrr rr;s r!rrs( dD ! !.2MM + NNM Nr c@[URR5$)a Set AdaptiveLocalSGD training configurations. AdaptiveLocalSGD has a configurable setting that can be configured through a dict. **Notes**: init_k_steps(int) The initial steps for training before adaptive localsgd. Then, the adaptive localsgd method will modify init_k_steps automatically. Default 1. begin_step(int) The step of beginning training by adaptive localsgd. Default 1. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.adaptive_localsgd = True >>> strategy.adaptive_localsgd_configs = {"init_k_steps": 1, ... "begin_step": 30} )rradaptive_localsgd_configsrs r!r-DistributedStrategy.adaptive_localsgd_configss2DMMCCDDr c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rrs5  MM 3 3  ' T]]DDgNr c.URR$)a Indicating whether we are using Deep Gradient Compression training. For more details, please refer to [Deep Gradient Compression](https://arxiv.org/abs/1712.01887). Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.dgc = True # by default this is false )rdgcrs r!rDistributedStrategy.dgc"s"}}   r c|[U[5(aXRlg[R "S5 g)Nz"dgc should have value of bool type)r8rrrr rr;s r!rr5rr c@[URR5$)ae Set Deep Gradient Compression training configurations. In general, dgc has several configurable settings that can be configured through a dict. **Notes**: rampup_begin_step(int): The beginning step from which gradient compression is implemented. Default 0. rampup_step(int): Time steps used in sparsity warm-up periods. Default is 1. \ For example, if the sparsity is [0.75, 0.9375, 0.984375, 0.996, 0.999], and the rampup_step is 100, \ it will use 0.75 at 0~19 steps, and 0.9375 at 20~39 steps, and so on. And when reach sparsity array \ ends, it will use 0.999 then and after. sparsity(list[float]): Get top important element from gradient tensor, the ratio is (1 - sparsity). \ Default is [0.999]. For example, if the sparsity is [0.99, 0.999], the top [1%, 0.1%] important \ element will be transmitted. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.dgc = True >>> strategy.dgc_configs = {"rampup_begin_step": 1252} )rr dgc_configsrs r!rDistributedStrategy.dgc_configs=s8DMM5566r c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rr[rr c.URR$)a/ Indicating whether we are using fp16 gradient allreduce training Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.fp16_allreduce = True # by default this is false )rfp16_allreducers r!r"DistributedStrategy.fp16_allreduceas }}+++r cd[U[5(d [S5eXRlg)Nz)fp16_allreduce must be value of bool type)r8rrrrr;s r!rrss'$%%GH H'+ $r c.URR$)a Gradient Merge, also called as Gradient Accumulation, is a strategy for large batch training. With this strategy, model parameter will not be updated until user-defined steps. For each step, the forward network and the backward network will run to calculate the gradient of model parameters. For every k step, the optimization network will run, applying a specific optimization method (such as SGD, Adam) to model parameters. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.gradient_merge = True >>> strategy.gradient_merge_configs = {"k_steps": 4, "avg": True} )rgradient_mergers r!r"DistributedStrategy.gradient_mergezs,}}+++r c|[U[5(aXRlg[R "S5 g)Nz-gradient_merge should have value of bool type)r8rrrr rr;s r!rrs( dD ! !+/MM ( NNJ Kr c@[URR5$)a the key-value configs of distribute_strategy **Note**: k_steps(int): the update period of the parameters. avg(bool): whether to average the gradients of each mini-batch, the default value is `True` Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.gradient_merge = True >>> strategy.gradient_merge_configs = {"k_steps": 4, "avg": True} )rrgradient_merge_configsrs r!r*DistributedStrategy.gradient_merge_configss(DMM@@AAr c[URRUS5 [URRU5 g)Ngradient_configs)rrrrr@s r!rrs4  MM 0 0';M  T]]AA7Kr c.URR$)a Set lars configurations. lars is used to deal with the convergence problems when the global batch size is larger than 8k. For more details, please refer to [Large Batch Training of Convolutional Networks](https://arxiv.org/abs/1708.03888). Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.lars = True # by default this is false )rlarsrs r!rDistributedStrategy.larss$}}!!!r c|[U[5(aXRlg[R "S5 g)Nz#lars should have value of bool type)r8rrrr rr;s r!rr( dD ! !!%MM  NN@ Ar c@[URR5$)aS Set Lars training configurations. **Notes**: **lars_coeff (float)**: trust ratio in lars formula. **lars_weight_decay** (float): weight decay coefficient in lars formula. **epsilon (float)**: argument is used to avoid potential division-by-zero when compute the local lr; **exclude_from_weight_decay (list[str])**: is a list of name strings of layers which will be exclude from weight decay in lars formula. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.lars = True >>> strategy.lars_configs = { ... "lars_coeff": 0.01, ... "lars_weight_decay": 0.0005, ... "epsilon": 0, ... "exclude_from_weight_decay": ['batch_norm', '.b_0'] ... } )rr lars_configsrs r!r DistributedStrategy.lars_configss8DMM6677r c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rr. $--44g~NT]]77Ar c.URR$)a Set lamb configurations. lamb is used to deal with the convergence problems for large batch size training, specially for attention-related model like BERT. For more details, please refer to [Large Batch Optimization for Deep Learning: Training BERT in 76 minutes](https://arxiv.org/abs/1904.00962). Default Value: False Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.lamb = True # by default this is false )rlambrs r!rDistributedStrategy.lambs(}}!!!r c|[U[5(aXRlg[R "S5 g)Nz#lamb should have value of bool type)r8rrrr rr;s r!rr rr c@[URR5$)aJ Set Lars training configurations. **Notes**: **lamb_weight_decay** (float): weight decay coefficient in lamb formula. **exclude_from_weight_decay (list[str])**: is a list of name strings of layers which will be exclude from weight decay in lamb formula. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.lamb = True >>> strategy.lamb_configs = { ... 'lamb_weight_decay': 0.01, ... 'exclude_from_weight_decay': [], ... } )rr lamb_configsrs r!r DistributedStrategy.lamb_configs s.DMM6677r c[URRUS5 [URRU5 g)Nr)rrrrr@s r!rr/ rr c.URR$)z Indicating whether we want to do current distributed training on clusters with elastic resources. Currently, this is configuration is not valid. )relasticrs r!rDistributedStrategy.elastic5 s}}$$$r c|[U[5(aXRlg[R "S5 g)Nz&elastic should have value of bool type)r8rrrr rr;s r!rr? s( dD ! !$(MM ! NNC Dr c.URR$)a Indicating whether we are using auto-parallel configuration This feature is currently an experimental feature. Currently, auto-parallelism can be used only when a user does not set any other strategy configs except auto. For details, please reference the following code example Default Value: False Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.auto = True >>> # if set other strategy at the same time, auto will not apply >>> # strategy.amp = True >>> optimizer = paddle.optimizer.SGD(learning_rate=0.01) >>> optimizer = fleet.distributed_optimizer(optimizer, strategy) )rautors r!rDistributedStrategy.autoG s6}}!!!r c|[U[5(aXRlg[R "S5 g)Nz#auto should have value of bool type)r8rrrr rr;s r!rrd s& dD ! !!%MM  NN@ Ar c.URR$)a Indicating whether we are using semi-auto parallel function This feature is currently an experimental feature. Currently, auto-parallelism can be used only when a user does not set any other strategy configs except semi-auto. For details, please reference the following code example Default Value: False Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.semi_auto = True >>> # if set other strategy at the same time, auto will not apply >>> # strategy.amp = True >>> optimizer = paddle.optimizer.SGD(learning_rate=0.01) >>> optimizer = fleet.distributed_optimizer(optimizer, strategy) )r semi_autors r!rDistributedStrategy.semi_autok s6}}&&&r c|[U[5(aXRlg[R "S5 g)Nz(semi-auto should have value of bool type)r8rrrr rr;s r!rr s& dD ! !&*MM # NNE Fr c.URR$)a Indicating whether we are using auto-search parallel function For details, please reference the following code example Default Value: False Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.auto_search = True )r auto_searchrs r!rDistributedStrategy.auto_search s&}}(((r c|[U[5(aXRlg[R "S5 g)Nz*auto-search should have value of bool type)r8rrrr rr;s r!rr s& dD ! !(,MM % NNG Hr c.URR$)aM Indicating whether we split the data. If True, we split the data. Default Value: True Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.split_data = True )r split_datars r!rDistributedStrategy.split_data s$}}'''r c|[U[5(aXRlg[R "S5 g)Nz)split_data should have value of bool type)r8rrrr rr;s r!rr s& dD ! !'+MM $ NNF Gr c.URR$)zN Indicating whether we are using quantization training Default Value: False rrs r!rr s}}   r c|[U[5(aXRlg[R "S5 gr)r8rrrr rr;s r!rr s& dD ! ! $MM  NN? @r c@[URR5$)a Set quantization training configurations. In general, qat has several configurable settings that can be configured through a dict. **Notes**: channel_wise_abs_max(bool): Whether to use `per_channel` quantization training. Default is True. weight_bits(int): quantization bit number for weight. Default is 8. activation_bits(int): quantization bit number for activation. Default is 8. not_quant_pattern(list[str]): When the skip pattern is detected in an op's name scope, the corresponding op will not be quantized. algo(str): Other quantization training algorithm. Examples: .. code-block:: python >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.qat = True >>> strategy.qat_configs = { ... "channel_wise_abs_max": True, ... "weight_bits": 8, ... "activation_bits": 8, ... "not_quant_pattern": ['skip_quant'] ... } rrs r!rr sDDMM5566r c[URRUS5 [URRU5 grrr@s r!rr rr c.URR$)aa Indicating whether we are using heter_ccl_mode for model training. This feature is currently an experimental feature. Currently, heter_ccl_mode can be used only for dataparallel with dygraph mode. Default Value: False Examples: .. code-block:: python >>> import paddle >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.heter_ccl_mode = True >>> # for initialize parallel env, only need to call >>> paddle.distributed.init_parallel_env() >>> # then the heterogeneous context will be created. )rheter_ccl_moders r!r"DistributedStrategy.heter_ccl_mode s.}}+++r c|[U[5(aXRlg[R "S5 g)Nz-heter_ccl_mode should have value of bool type)r8rrrr rr;s r!rr s& dD ! !+/MM ( NNJ Kr c.URR$)a Indicating whether to use exhaustive search method to choose convolution algorithms. Exhaustive search attempts all cuDNN algorithms to choose the fastest algorithm. This method is time-consuming, the chosen algorithm will be cached for the given layer specifications. Once the layer specifications (like batch size, feature map size) are changed, it will search again. Default Value: True Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.cudnn_exhaustive_search = False >>> optimizer = paddle.optimizer.SGD(learning_rate=0.01) >>> optimizer = fleet.distributed_optimizer(optimizer, strategy) )rrrs r!r+DistributedStrategy.cudnn_exhaustive_search s0}}444r c|[U[5(aXRlg[R "S5 g)Nz6cudnn_exhaustive_search should have value of bool type)r8rrrr rr;s r!rr: s* dD ! !48MM 1 NNH r c.URR$)a The workspace limit size in MB unit for choosing cuDNN convolution algorithms. The inner function of cuDNN obtain the fastest suited algorithm that fits within this memory limit. Usually, large workspace size may lead to choose faster algorithms, but significant increasing memory workspace. Users need to trade-off between memory and speed. Default Value: 4000 Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.conv_workspace_size_limit = 1024 >>> optimizer = paddle.optimizer.SGD(learning_rate=0.01) >>> optimizer = fleet.distributed_optimizer(optimizer, strategy) )rrrs r!r-DistributedStrategy.conv_workspace_size_limitD s0}}666r c|[U[5(aXRlg[R "S5 g)Nz7conv_workspace_size_limit should have value of int type)r8rrrr rr-s r!rr^ s* eS ! !6;MM 3 NNI r c.URR$)a( Indicates whether to use the mode CUDNN_BATCHNORM_SPATIAL_PERSISTENT function in batchnorm. This is only useful in cudnn. Default Value: True Examples: .. code-block:: python >>> import paddle >>> paddle.enable_static() >>> import paddle.distributed.fleet as fleet >>> strategy = fleet.DistributedStrategy() >>> strategy.cudnn_batchnorm_spatial_persistent = True >>> optimizer = paddle.optimizer.SGD(learning_rate=0.01) >>> optimizer = fleet.distributed_optimizer(optimizer, strategy) )rrrs r!r6DistributedStrategy.cudnn_batchnorm_spatial_persistenth s,}}???r c|[U[5(aXRlg[R "S5 g)NzAcudnn_batchnorm_spatial_persistent should have value of bool type)r8rrrr rr;s r!rr s* dD ! !?CMM < NNS r cURn/SQn[UR5[UR5[UR 5[UR 5[UR5[UR5/n[U5H4upE[5RU5(dM%X4[5U'M6 g)N)rrrr FLAGS_fuse_parameter_memory_size FLAGS_fuse_parameter_groups_size) rrrrrrrr8rD enumeraterr)rrrvaluesr\rs r! _enable_envDistributedStrategy._enable_env s==  << = 22 3 11 2 -- . -- . 11 2   oFA((--'-y $&r cRURR(a [(agg)NTF)rrrrs r!_is_strict_auto#DistributedStrategy._is_strict_auto s ==  "6"6r c  SnSnSnX#-U-nSSUS3-nSSRUSU-U5-nSS RS /U-5-S -nSS RS /U-5-S -nUS -n XRS 5- n XRS5- n XRS 5- n URRRn S n US -n U GHn SU R ;aMSU R ;aM([ [URU R 5[5(GaJ[URU R S-5(Ga[URU R 5(GaXS -- n XRU R SU R S35- n XS -- n [URU R S-5nURRn[RRnUS:a%[RRRnOSSKJn URnUHn[ [UUR 5U5(ay[UUR 5n[#U5HRunnUS:Xa)XRUR [%U55- n M5XRS [%U55- n MT MXRUR [%[UUR 555- n M GM[GM^XRU R [%[URU R 555- n GMXRU R [%[URU R 555- n GM U U-S -URS5-nUU - nUS -nUURS5- nUUS -- nURR&RRn U HSn UURU R [%[URR&U R 555- nMU UUS -- nUU- nU$)Nr&z z|{:^zs}| z|{{:>{}s}}{}{{:^{}s}}| rUz +r=+- zDistributedStrategy Overviewr*_configsz =True <-> z4.21.0r)_messagez&Environment Flags, Communication FlagszBuild Strategy)formatjoinrrrrr8rrrrr __version___upbrRepeatedScalarContainergoogle.protobuf.pyextrr&r*)rspacingmax_kmax_vlength h1_format h2_formatborderlinedrawsrstr_res env_drawsr my_configs config_fieldsprotobuf_versionrrffrr\r result_resbuild_strategy_strs r!__repr__DistributedStrategy.__repr__ sO(uVHF33 7>> 3=%  277C56>22S8#0036  !!"%% !!"@AA !!"%%))004K A166)QVV#gdmmQVV((,  J!J --:EE 22  .!.`  EI!EIN !!  ZZAA 9797vAA!!  ZZAA !!  ZZ!!778AA''"GG 11 Q Q 88" &&'AA )//0--$MM 11 Q Q 22"  Q!Q 55&##Q$Q 44&""#66%%&++$JJ ==BGG &&,__FF 7<7r$sf # 00" /8C:+&IU&"9E" % 9E Ye$9E$ )5 )5  )5)5Ye y-iu- -iu-  89",9.9.A "/"/Jp'p'r