ΑiNSSKJr SSKrSSKJr SqSSjr\S:Xa\"5 gg)) annotationsN)ContextctX[5q[R5(aSSKJn UR 5 g[R 5(G+a SSKnSSKnSSKnSSK nSSK nSSK nSSK J n SSKJn SSKJn Jn Jn Jn Jn JnJn SSKJn UR5n[R4R6R9S5(d [;S 5e[=[R4R6S 5nUR?U5nSSS5 URAS 5nURCURD5 URFRIURFRK[R4R65URFRM[R4R65ROS 5SS35nURQUSS9nURCURD5 URSS5nURUU5 URWU5 [R4RXR9S5(atURZR]S5S:Xa-UR^SSSSSS[R4RX/nOUR^S[R4RX/nOp[R4RXR9S5(a'UR^[R4RX/nO[R4RX/nURa[R4Rb5 URe[R4Rb5n[R4Rf(dSnO2[i[R4RfROS55n[R4RjnU5(a[qUROS5S5nO [qU5nUWS'UUS 'UUS-US!'UR]S"S5(dS#S$0US"'UR]S%S5nURFRIURFRK[R4R65URFRM[R4R65ROS 5SS&35n/nSnUS':GaBSS(K9J:n [R4RvRyS)5(de[R4RvR{S)5ROS5un!n"U "U!U"S*9n#U#R}S+5 U#RS 5 SSK@n$U$R5n%U$RU$RU%55nUS,:wdeUS"R]S-S5(GaPURS.5 S/U3n&U#RU&URS055(d:URS'5 U#RU&URS055(dM:[U#RS/55n'[iU'5n(U(U:wa`URS'5 U#RU&URS055 [U#RS/55n'[iU'5n(U(U:waM`[U'V)s/sHn)U)SR5PM sn)5nURS1[iU5S2US 35 UR]S3S45n*U*US3'U*[lLS5U;aU*OUR]S55n+UR]S6S5n,UR]S7S5n-UR]S8S5n.S9n/U"U5n0Sn1Sn2S:[R4lMURe[5n3UR]S;05R]SU4S"'U"U45n5U5R5n6Sn7U1S'- n1S?[oU15-n8U8[R4lOU6(GaUReU35qS@RU1U6S<U6SAU6SBU6SCU6SDU6SEU6SFU6SGU6SH5 n9U9[R4lQU "UU6U45n:U:[R4l1[RRSIU8SJU9SKU635 URSIU8SJU9SKU635 UR[5n;U;R5 U"[R4RSLUSMS#[R4RSN3SO9unU>S'-(aK[RRSPU935 URSPU935 S:U6SQ'SU6USMS#'U=U6S8'U>SR-(aK[RRSSU935 URSSU935 S:U6SQ'SU6USMS#'STU6S8'U>SU-(aEU>SR-(d;[RRSVU935 URSVU935 SU6S8'U>(dUS'-(d U>SR-(aOU1U6SW'U6S<n7U0R"S0U6D6 U;RSXSY9 U0RSZ5 U5R5n?UReU?5n6U5RU65 URS[5 U6(aGMU7c[;S\U435eU7US;S<'U0RSZ5 U0R5 UR5n@[RRS]U@U- S^35 URS]U@U- S^35 U"U5nAURS_[iUARR5S`35 UR]SaU5nBUARUB5 UAR5nCUARUC5 SbnDUCcWD5eWC(GaUR5nEUReU35qU/(a U+[lLSXn/SU>(dHURWL5nM[RRStUM35 URStUM35 O0[RRSu5 URSu5 WCR]Sg5(aU "WCUU0R5 WCSv(aU2S'- n2WCSvnN[iWARR5nOUARRnP[RRSwRUPUOUPU1- U2UN[UOUP- U*-Sx- SR555 URSwRUPUOUPU1- U2UN[UOUP- U*-Sx- SR555 U0RU5 UAR5nQUReUQ5nCUARUC5 GMU "[WCU5q[m[R4Rj[p5(d5[q[R4RjROS5S5O[R4RjnRU(Ga*USWRnSUUS;GaW#R]SyU935SnCUR5nTUC(d_UR5nUUUWT- US3Sz-:a[;S{U9S|35eURS[5 U#R]SyU935SnCUC(dM_URS}U9S~35 [RRS}U9S~35 URWCR55nCWARRS:5 UARUC5 U0RcSqWC;a WCSqU0l`U0R"S0WCD6 U0RUSMS#USMSrU-U.Ss9unLn>U>(dHURWL5nM[RRStUM35 URStUM35 O0[RRSu5 URSu5 WCR]Sg5(aU "WCUU0R5 WCSnVUV(aU2S'- n2WCSvnN[iWARR5nOUARRnP[RRSwRUPUOUPU1- U2UN[UOUP- U*-Sx- SR555 URSwRUPUOUPU1- U2UN[UOUP- U*-Sx- SR555 U0RU5 UAR5nQUReUQ5nCUARUC5 GMjUS"S#S$:wa-SnWS-US";a WCSDS':XaSnW[RSWW05 UR[5n;U;R5 UR5nX[UXWE- SR5WCS'[RRSRU8U9UCS55 URSRU8U9UCS55 U"[R4RSLUSMS#[R4RSN3SO9unS9nYU>SR-nZWRS':GaUSyU1SU3n&U "[R4R5n[WZ(aW#RU&STRS055(d9URS'5 U#RU&STRS055(dM9[RRSU&35 URSU&35 GOW[(aW#RU&SRS055(d9URS'5 U#RU&SRS055(dM9[RRSU&35 URSU&35 GO[U;S5(aU;RS:XaW#RU&SRS055(d9URS'5 U#RU&SRS055(dM9[RRSU&35 URSU&35 GOV[U;S5(dU;RRS:XaW#RU&SRS055(d9URS'5 U#RU&SRS055(dM9[RRSU&35 URSU&35 OW#RU&SRS055(d9URS'5 U#RU&SRS055(dM9[RRSU&35 URSU&35 [U#RSyU1S355n\[iU\5n(U(WR:waBURS'5 [U#RSyU1S355n\[iU\5n(U(WR:waMBW\V)s/sHn)U)SR5PM n]n)[RRSU1SU]35 URSU1SU]35 STU];aSXnYS9nZOSW];aSXnYSXnVU>S'-(aX[RRSU9S35 URSU9S35 S:WCSQ'SUCUSMS#'WZ(dU=OSTWCS8'S9nVU>SR-(aM[RRU9S35 URU9S35 S:WCSQ'SUCUSMS#'STUCS8'S9nVU>SU-(aPU>SR-(dF[RRSU9S35 URSU9S35 WZ(dSOSTWCS8'WV(d%WY(aUU>(dHURWL5nM[RRStUM35 URStUM35 O0[RRS5 URS5 WCR]Sg5(aU "WCUU0R5 U0RU5 U;RSXSY9 WAR5nQUReUQ5nCUARUC5 [oUR55nr[RRS5(a URS5R5nsOY[R"5(a URS5R5nsOURS5R5nsWsH8nt[oUtR55nuUuWr:wdM$URSWu-5 M: URS[5 UR5n@SU1SU3n&U,(azW@U- [qU,5:ahUS':a`W#RU&SRS055(d9URS'5 U#RU&SRS055(dM9OgGOUS':a_W#RU&SRS055(d9URS'5 U#RU&SRS055(dM9US':a[W#RSU1355n\[iU\5n(U(U:waBURS'5 [U#RSU1S355n\[iU\5n(U(U:waMBW\V)s/sHn)U)SR5PM n]n)SU];aO WC(aGMU0RU5 SnvUReU35qUS':aURZR]SS5nwUwceUWw:XaoU0RUSMS#USMSrU-U.Ss9unvn>U>(a [;S5eURWv5nxW#RS+Wx5(dURS'5 M*O[S5H@n)W#R]S+5SR5nxURUx5nvWv(dM@ O Wv(deO3U0RUSMS#USMSrU-U.Ss9unvn>U>(a [;S5eWv(a WvSQS::wdeUR5n@[RRSU@U- S^35 URSU@U- S^35 UR]SS95(aUS"R]S-S5(aUR5 U "UWvUS9S9nJS9[l~UJ[R4l1S+[R4lOURUv5nM[RRSUM35 URSUM35 UR]SS5(aUS[R4lQO`URFRIURFRK[R4R65S+5[R4lQUR[5n;U;R5 U;RS9SY9 gSSKJn UR[5n;U;R5 U;R5 g!,(df  G*N=f! [;S 5e=f! S,nG%N=fs sn)fs sn)fs sn)fs sn)fs sn)f![aLny[RRUy5 URUy5 URSR5 SnyAyGNSnyAyff=f)a1 Paddle distribution training entry ``python -m paddle.distributed.launch``. Usage: .. code-block:: bash :name: code-block-bash1 python -m paddle.distributed.launch [-h] [--master MASTER] [--rank RANK] [--log_level LOG_LEVEL] [--nnodes NNODES] [--nproc_per_node NPROC_PER_NODE] [--log_dir LOG_DIR] [--run_mode RUN_MODE] [--job_id JOB_ID] [--devices DEVICES] [--host HOST] [--servers SERVERS] [--trainers TRAINERS] [--trainer_num TRAINER_NUM] [--server_num SERVER_NUM] [--gloo_port GLOO_PORT] [--with_gloo WITH_GLOO] [--max_restart MAX_RESTART] [--elastic_level ELASTIC_LEVEL] [--elastic_timeout ELASTIC_TIMEOUT] training_script ... Base Parameters: - ``--master``: The master/rendezvous server, support ``http://`` and ``etcd://``, default with ``http://``. e.g., ``--master=127.0.0.1:8080``. Default ``--master=None``. - ``--rank``: The rank of the node, can be auto assigned by master. Default ``--rank=-1``. - ``--log_level``: The log level to set for logging.setLevel which can be CRITICAL/ERROR/WARNING/INFO/DEBUG/NOTSET, case insensitive. Default ``--log_level=INFO``. - ``--nnodes``: The number of nodes for a distributed job, it can be a range in elastic mode, e.g., ``--nnodes=2:3``. Default ``--nnodes=1``. - ``--nproc_per_node``: The number of processes to launch on a node. In gpu training, it should be less or equal to the gpus number of you system. e.g., ``--nproc_per_node=8`` - ``--log_dir``: The path for each process's log. e.g., ``--log_dir=output_dir``. Default ``--log_dir=log``. - ``--run_mode``: The run mode of job, can be:collective/ps/ps-heter/rpc. e.g., ``--run_mode=ps``. Default ``--run_mode=collective``. - ``--job_id``: The job unique id, it affects the log files' name. e.g., ``--job_id=job1``. Default ``--job_id=default``. - ``--devices``: The selected accelerate devices on nodes, can be gpu/xpu etc.. e.g., ``--devices=0,1,2,3`` will launch four training processes each bound to one device. - ``training_script``: The full path to the single GPU training program/script to be launched in parallel, followed by all the arguments for the training script. e.g., ``training.py`` - ``training_script_args``: The args of training_script. e.g., ``--lr=0.1`` Collective Parameters: - ``--ips``: [DEPRECATED] Paddle cluster nodes ips, e.g., ``--ips=192.168.0.16,192.168.0.17``. Default ``--ips=127.0.0.1``. Parameter-Server Parameters: - ``--servers``: User defined servers ip:port, e.g., ``--servers="192.168.0.16:6170,192.168.0.17:6170"`` - ``--trainers``: User defined trainers ip:port, e.g., ``--trainers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172"`` - ``--workers``: [DEPRECATED] The same as trainers. - ``--trainer_num``: Number of trainers on each node, can be 0. - ``--worker_num``: [DEPRECATED] The same as trainer_num. - ``--server_num``: Number of servers on each node, can be 0. - ``--heter_workers``: User defined heter workers ip1:port1;ip2:port2, e.g., ``--heter_workers="192.168.0.16:6172;192.168.0.17:6172"`` - ``--heter_worker_num``: Number of heter_workers in each stage (It recommend to set when in the emulated distributed environment using single node) - ``--heter_devices``: Type of heter_device in each stage - ``--gloo_port``: Gloo http Port. Default ``--gloo_port=6767``. - ``--with_gloo``: Using gloo or not. Default ``--with_gloo=0``. Elastic Parameters: - ``--max_restart``: The maximum restart times for an elastic job. Default ``--max_restart=3``. - ``--elastic_level``: The elastic level: -1: disable, 0: failed exit, peers hold, 1: internal restart. Default ``--elastic_level=-1``. - ``--elastic_timeout``: Seconds to wait before elastic job begin to train. Default ``--elastic_timeout=30``. IPU Parameters: IPU distributed launch only requires and allows three arguments ``--devices``, ``training_script`` and ``training_script_args``. The ``--devices`` is the number of IPU devices. e.g., ``--devices=4`` will launch the training program with four IPU devices. The ``training_script`` is only allowed to set as ``ipu``. The ``training_script_args`` includes arguments required by IPU distributed launch and illustrated as below. ``Examples 10`` has provided a example of paddle.distributed.launch with IPUs. - ``--hosts``: The hosts for IPU distributed training. Each host is able to include multiple processes. - ``--nproc_per_host``: The number of processes launched per host. Each process is able to include multiple replicas. - ``--ipus_per_replica``: The number of IPUs requested per replica. Each replica is able to include multiple IPUs. - ``--ipu_partition``: The partition name of IPU devices. - ``--vipu_server``: The ip of the IPU device manager. - ``training_script``: The full path to the IPU distributed training program/script to be launched in parallel. e.g., ``training.py``. - ``training_script_args``: The args of the IPU distributed training program/script. e.g., ``--lr=0.1``. Returns: - ``None`` Examples 0 (master, ip/port auto detection): .. code-block:: bash :name: code-block-example-bash0 # For training on multi node, run the following command in one of the nodes python -m paddle.distributed.launch --nnodes 2 train.py # Then the following info will be print # Copy the following command to other nodes to run. # -------------------------------------------------------------------------------- # python -m paddle.distributed.launch --master 10.0.0.1:38714 --nnodes 2 train.py # -------------------------------------------------------------------------------- # Follow the instruction above and paste the command in other nodes can launch a multi nodes training job. # There are two ways to launch a job with the same command for multi nodes training # 1) using the following command in every nodes, make sure the ip is one of the training node and the port is available on that node # python -m paddle.distributed.launch --master 10.0.0.1:38714 --nnodes 2 train.py # 2) using the following command in every nodes with a independent etcd service # python -m paddle.distributed.launch --master etcd://10.0.0.1:2379 --nnodes 2 train.py # This functionality works will for both collective and ps mode and even with other arguments. Examples 1 (collective, single node): .. code-block:: bash :name: code-block-example-bash1 # For training on single node using 4 gpus. python -m paddle.distributed.launch --devices=0,1,2,3 train.py --lr=0.01 Examples 2 (collective, multi node): .. code-block:: bash :name: code-block-example-bash2 # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 # On 192.168.0.16: python -m paddle.distributed.launch --devices=0,1,2,3 --master=192.168.0.16:8090 train.py --lr=0.01 # On 192.168.0.17: python -m paddle.distributed.launch --devices=0,1,2,3 --master=192.168.0.16:8090 train.py --lr=0.01 Examples 3 (ps, cpu, single node): .. code-block:: bash :name: code-block-example-bash3 # To simulate distributed environment using single node, e.g., 2 servers and 4 workers. python -m paddle.distributed.launch --server_num=2 --worker_num=4 train.py --lr=0.01 Examples 4 (ps, cpu, multi node): .. code-block:: bash :name: code-block-example-bash4 # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server and 2 workers. # On 192.168.0.16: python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01 # On 192.168.0.17: python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01 # Or with master, the following command run 2 server and 2 trainer on each node. python -m paddle.distributed.launch --master 192.168.0.16:9090 --server_num=2 --trainer_num=2 --nnodes 2 train.py Examples 5 (ps, gpu, single node): .. code-block:: bash :name: code-block-example-bash5 # To simulate distributed environment using single node, e.g., 2 servers and 4 workers, each worker use single gpu. export CUDA_VISIBLE_DEVICES=0,1,2,3 python -m paddle.distributed.launch --server_num=2 --worker_num=4 train.py --lr=0.01 Examples 6 (ps, gpu, multi node): .. code-block:: bash :name: code-block-example-bash6 # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server and 2 workers. # On 192.168.0.16: export CUDA_VISIBLE_DEVICES=0,1 python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01 # On 192.168.0.17: export CUDA_VISIBLE_DEVICES=0,1 python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.16:6172,192.168.0.17:6171,192.168.0.17:6172" train.py --lr=0.01 Examples 7 (ps-heter, cpu + gpu, single node): .. code-block:: bash :name: code-block-example-bash7 # To simulate distributed environment using single node, e.g., 2 servers and 4 workers, two workers use gpu, two workers use cpu. export CUDA_VISIBLE_DEVICES=0,1 python -m paddle.distributed.launch --server_num=2 --worker_num=2 --heter_worker_num=2 train.py --lr=0.01 Examples 8 (ps-heter, cpu + gpu, multi node): .. code-block:: bash :name: code-block-example-bash8 # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 1 server, 1 gpu worker, 1 cpu worker. # On 192.168.0.16: export CUDA_VISIBLE_DEVICES=0 python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.17:6171" --heter_workers="192.168.0.16:6172,192.168.0.17:6172" train.py --lr=0.01 # On 192.168.0.17: export CUDA_VISIBLE_DEVICES=0 python -m paddle.distributed.launch --servers="192.168.0.16:6170,192.168.0.17:6170" --workers="192.168.0.16:6171,192.168.0.17:6171" --heter_workers="192.168.0.16:6172,192.168.0.17:6172" train.py --lr=0.01 Examples 9 (elastic): .. code-block:: bash :name: code-block-example-bash9 # With the following command, the job will begin to run immediately if 4 nodes are ready, # or it will run after elastic_timeout if only 2 or 3 nodes ready python -m paddle.distributed.launch --master etcd://10.0.0.1:2379 --nnodes 2:4 train.py # once the number of nodes changes between 2:4 during training, the strategy holds Examples 10 (ipu): .. code-block:: bash :name: code-block-example-bash10 # With the following command, the job will begin to run the distributhed program with IPUs # Require `devices` as the number of IPUs # Require `training_script` to be set as `ipu` # Require `training_script_args` as the arguments of IPU distributed training instead of the arguments of the training program/script # Please Check the `IPU Parameters` for details python -m paddle.distributed.launch --devices 4 ipu --hosts=localhost --nproc_per_host=2 --ipus_per_replica=1 --ipu_partition=pod16 --vipu_server=127.0.0.1 train.py Examples 11 (rpc, cpu, single node): .. code-block:: bash :name: code-block-example-bash11 # Training on single node with two local servers python -m paddle.distributed.launch --master 127.0.0.1:8765 --nnodes 1 --nproc_per_node 2 --rank 0 --run_mode rpc train.py Examples 12 (rpc, cpu, multi node): .. code-block:: bash :name: code-block-example-bash12 # For training on multiple nodes, e.g., 192.168.0.16, 192.168.0.17 where each node with 2 servers. # On 192.168.0.16 python -m paddle.distributed.launch --master 192.168.0.16:8765 --nnodes 2 --nproc_per_node 2 --rank 0 --run_mode rpc train.py # On 192.168.0.17 python -m paddle.distributed.launch --master 192.168.0.16:8765 --nnodes 2 --nproc_per_node 2 --rank 1 --run_mode rpc train.py r)launchN)HistoryRecorder) AutoTuner)add_overlap_performancefind_error_from_log gen_new_args gen_new_ctxread_completedread_logread_step_time_log) controllersz.jsonz+Please use '.json' as the file name suffix.rz0Please check your auto tuner json whether valid. auto_tuner.z_auto_tuner.logw)modez4%(asctime)s - %(name)s - %(levelname)s - %(message)sz.py WITH_COVERAGEONz-uz-mcoveragerunz--branchz-pz.pyxes,:nodes gpus_per_nodenum_gpus search_algonamegridrz _history.csv) ETCDClientzetcd://)hostportbest_cfgz 127.0.0.1estimated_num_gpuszauto_tuner/ip/zlatin-1zThe total count of nodes is z and sorted ips are max_time_per_taski warmup_timemax_search_timebuffer max_mem_usageT model_cfgglobal_batch_sizeautogbs gbs_tuner_zbJob{}_GBSSearch/GBS{}_DP{}_MP{}_PP{}_Sharding_degree_{}_stage_{}_MBS{}_Recompute_{}_granularity_{} dp_degree mp_degree pp_degreesharding_degreesharding_stagemicro_batch_size use_recomputerecompute_granularityz$Launch task from auto tuner: job_id z , log_dir z , config z workerlog.0 metric_cfgz.gpu.log)path metric_file target_metric memory_filez#Read metric failed for parameters: timezOut of memory for parameters: OOMz)Read memory usage failed for parameters: job_idF)exitz./tuner_gbs_history.csvzRNo valid global batch size found, check memory or valid search time. cur_tuner_cfgz!AutoTuner for GBS search ends in zs.zLaunch z tasks by auto tuner: resume_csv_file_patha No config can search. Please check if there are any situations where GBS is unable to divide dp degree or shading degree, or if there are related configurations of the model such as hidden_size cannot be evenly divided by mp degree, num_ Layers cannot divide pp degree. acc_steps auto_tuner_z_Job{}_GBS{}_DP{}_MP{}_PP{}_VPP{}_Sharding{}_Stage{}_MBS{}_Recompute_{}_Granularity_{}_AccStep{} vpp_degreesharding_overlap _Overlap_refined_recomputec3@# UHoR5v M g7fN capitalize.0is ^/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/launch/main.py launch..&N~!||~~~_custom_search_dimc3@# UHoR5v M g7frRrSrUs rXrYrZr[r\ log_dir_namezLaunch task: job_id additional_metric_keyOptimizationDirection)metric directionr-r.zCurrent best config: z7Get best config failed. Currently no config can be run. error_infozaAuto Tuner Schedule: [{}/{}], Pruned nums {}, Error nums {}, Error info {}, Remaining time {} min<z auto_tuner/zWait z failedzReceive that task z has ended by etcd. has_error01$FLAGS_shard_bypass_dygraph_optimizer exec_timez(Task: job_id {}, log_dir {} ended in {}s/z Put OOM to OKz Put OK to sigintErrorz Put Error to zStatus of auto_tuner/z/: zRead metric of z failed.z OOM.zRead memory usage of zauto_tuner/mem/FLAGS_log_memory_stats)read_allocated_memory_log workerlogmax_peak_memory conversion step_time)r>filer@comm_bwd model_size_bampmax_seq_lengthi need_unifyunified_bw_ unified_bw_z Out of memoryzauto_tuner/error/ignorezPut Error info: z to zutf-8zStatus of auto_tuner/error/z put auto_tuner/z successfully.z-Get best config failed, no config can be run.npuzjoindirnamebasenamesplit FileHandler Formatter setFormatter addHandlertraining_scriptenvironget executableextendtraining_script_argsdeepcopydeviceslennnodes isinstancestrint+paddle.distributed.launch.utils.etcd_clientr$master startswith removeprefixdelete delete_prefixsocket gethostname gethostbynamegetfqdnsleepputencodelist get_prefixsorteddecodeinfor* max_restart search_oncerFformatlog_dirloggerinitrwarningadd_cfgfinalize store_history clean_historyalgo all_tasksresume_form_historypopget_cfg_from_resume history_cfgsraget_bestdumpshistoryidxroundloadsset_envshasattrropod exit_codemaxfloatrswalkappendsetgetpidpaddledeviceis_compiled_with_custom_devicepopen readlinesis_compiled_with_xpustripsystemrange ExceptionrGr)zrrrrrrrBrrr r r r r rrr start_timef tuner_cfgrauto_tuner_log_pathhandler formatter entrypointraw_argsrrrhistory_file_path sorted_ipsipr$ master_ipr&clientrhostnamer>ipssizerWr*r+r,r-r. is_first_taskrecorderrFerror_task_numsraw_ctx gbs_tuner_cfg gbs_tuner gbs_cur_cfgbest_gbs task_job_idr gbs_new_argscrcmemerr gbs_new_cfgend_timerrIcur_cfg error_msgtask_start_timer1rJkeydir_namenew_argscur_resume_cfg cur_best_cfgs to_json_strre task_nums cur_task_idnew_cfg actual_nnodesactual_exec_ipswait_start_time wait_end_timerhbypass_optimizer_flag task_end_time timeout_flagOOM_flag completedresultstatus mem_allnodesrursrootdirsfilesrx peak_memorysingle_dp_performancestep_time_metricrwryr{r}r seq_lengthbw comm_timemulti_dp_performancesingle_error_infoself_pid processesprocesspidr'collective_master_ipdataesz rXrrs *\ )C 3    ! !JA    :YY[ xx''0099JK K Qchh..4 IIaL 5 ""<0 % ggll GGOOCHH44 5ww 8 89??DQGH X %%&9%D&%% B  Y''" 88 # # , ,U 3 3zz~~o.$6NNHH,,  "nndCHH4L4LM XX % % . .x 8 8..#((*B*BCJ((223J#((778==!>!>?xxM 0 0 6 6s ;>)+DEEJJqM!**TbT>>)+DEE6,,-=>?3xfnJJqMJJtt^^I%>?v001ABCCs8D fn $C$@CqQqT[[]C$@A  23z?2CCWXbWccde &MM*=tD): %& 1 I- }- $--(94@x.! ot<  "9-!--$ MM+r * . ./BF K !MM)4M+0M- (!-0I#//1KH aKF&V4K)CHHOmmG,~FF 34 , , , 12 01 230 78 $+  ,k=  1=- :;-zRYQZZcdocpq :;-zRYQZZcdocpq $$S) $,)) -"+L"9&"A#&88??"38 < $ S&>JJ&&=gYGNN=gYG+-K'CGK , 7 ?@36K0&>JJ&&8 BNN%CG9#MN*,K'CGK , 7 ?@38K0&>#.JJ&&CG9MNNCG9M48K0*0K'CIK , 7 ?@36K0&>SF^)/ H%&':;  /;/  &&&'@A(335 "mmK8 !!+. 1 C+F hivhwx;CIk "#6 7  " "#< =  " " $yy{H JJOO3Hz4I3J"M  KK3Hz4I3J"M  y)  c*//33455K L  )}} "$5  &&';<((*7# 5 "-I-""iikO--(C(3%!M''1+,{+,?@  ";'(,-.-./  $-GK +3w>9@3:6$$/w/)1):):(6v>"+L"93# &&3 *;*&M3&*jj&?  *? }(MN &;K=$IJ U U{{#566/#Y0@0@!( 4I '1,!(!6J #JOO$=$= >I",//"5"5KJJOO{CC'%'&0+&!!*[!8"3!4"$!%!"  KK{CC'%'&0+&!!*[!8"3!4"$!%!"  **+<=(446G"mmG4G&&w/'/69(+%(Im,DD 12a7,/) >@U   %A EEG IIKM#()H!#LGK JJOO:AA'+*>  KK:AA'+*>  (XX%%)' 5f="xx/x8   FC Lf~Hq $VHAbT2*388+;+;< $jju||I/FGG 1 %jju||I/FGGJJOOk$$89KK+dV 45$jjt{{9/EFF 1 %jjt{{9/EFFJJOOj$78KK*TF 34Q))ahh"n$jjt{{9/EFF 1 %jjt{{9/EFFJJOOj$78KK*TF 34 H--!%%//Q2F$jjt{{9/EFF 1 %jjt{{9/EFFJJOOj$78KK*TF 34$jjw~~i/HII 1 %jjw~~i/HIIJJOOmD6$:;KK-v 67f//+fXQ0GHI6{m+JJqM!&"3"3k&4K"LMFv;Dm+ 288A!A$++-8 "7xs6( KL 3F83vhGHF?#(L#H'#(LIf~ ""_WIX#FG BC"$;? ,/786>3E( f~ ""gYe#45'%01"$;? ,/78+0( f~cVn ""%:7)8#LM!6wixHI7?4U("(;A ,/786>3E(\"$;? ,/787?4U(&v.g=;? ,/78$VHAbT2Dz **#go67>>yIJJqM!**#go67>>yIf///&0JKL6{fnJJqM!))OF81*EFFv;D fn 8>>v!! v >'C{go&>&F e|360/2E#JU7?3K-L)M0GO, (zz~~6>>"&*,1A1A)B%D$ %#{;;$&?HH,,d' '27N.9O([__*_<2=!&*C.=)* &GH  - 889]#;L#II&g5(/ ,0G0O(P%#,]#;L#I$ /))&"2 $M2<@DDu #m,\:>>&   .|<@@Nwz23 k*../?F -!-0>BB$e/(:A> / (9\#:6#B"CDE"B$0E$:R$? $01$4r$9 1Y %(94634 ")-c"Qy'>v'F&GHI0%m4\BFF(%2X=qA 2%bT9\+B6+J*KL 55=)"Qy/Fv/N.OP!6%:: 78A"HJ#,GK  1$%%o6$q(!26(!B4@,?HH,,-)01A5&,jj $ 1 8 8H M''!% 1 '-jj $ 1 8 8H M''  JJOO"23D2ET$ P#KK"23D2ET$ P'-jjt{{97M&N&N $ 1 '-jjt{{97M&N&NJJOOj,?@"KK*TF(;N>N*OP$.GL !JOO556I$//--K JJOOszz&(#"[04EEJ   KKszz&(#"[04EEJ  &qM ?$jj%gY/ 7+229= 1 %jj%gY/ 7+229= KK2$&6wi~ NO    'w '!)!2!2 .v6#L12IJ+ "3" M3 "jj7  "7 } EF 3K=AB  OP KL{{-..'H$A:$jjw~~i/HII 1 %jjw~~i/HIIA:$jjt{{9/EFF 1 %jjt{{9/EFFzf//2B6(0KLM6{fnJJqM!)),>2H$#O '3 33)) ( 1 1$\26:' 56MN!"/ !2! # $]zz(+ **Z66JJqM7rA&%zz*5a8??A#'::d#3  x# x$-- .v6#L12IJ+ .MHc  YHV,22299; -h.C-DBGH )(Z*?)@CD}}Z..)M2J2N2N $3 3  HHJ(IM (0%$jj*  +K=9: ' }56 == . .(8CHH !ww|| 8 89: CHH    S !    :   S !  ] 54 QOP PV ! "%A~9~ ?n"|9B%& **1-q) 1 &s$Bn.?BnBn.0Bn>&BohBo s0BoH*BoI Bo_9Boc83Bo!n Bn+n&Bn.n+Bn.n. Bn;n>Boo! Bp7o+ABp2p2Bp7__main__)returnNone) __future__rr!paddle.distributed.launch.contextrrr__name__rrXrEs0# 5 ~B* z HrD