
    Αil,                         S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJr  S SK	J
r
  SrS r " S S5      r " S S	\5      r " S
 S\5      rg)    N)KVClient)KVServeretcd://c                     [         R                  " U 5      n U R                  SS5      R                  S5      S   n[	        [
        R                  " U5      5      $ )N	candidatez127.0.0.1:8080:r   )jsonloadsgetsplitint	ipaddressIPv4Address)xip_xs     l/var/www/html/banglarbhumi/venv/lib/python3.13/site-packages/paddle/distributed/launch/controllers/master.py
_cmp_by_ipr      sF    

1A55./55c:1=Dy$$T*++    c                   h    \ rS rSrSrSrSrSrS rS r	S r
S	 rS
 rSS\\44S jjr\S 5       rSrg)Master#   zC
Master is a distributed store design to exchange info among nodes
mainstandbyparticipantc                 :    Xl         S U l        SU l        S U l        g )NF)ctxserverinitializedendpoint)selfr   s     r   __init__Master.__init__,   s     r   c                     [         eNNotImplementedErrorr    s    r   stopMaster.stop2       !!r   c                     g r$    r    statuss     r   
set_statusMaster.set_status5       r   c                     g r$   r,   r'   s    r   
get_statusMaster.get_status8   s    r   c                     g r$   r,   r'   s    r   restart_peerMaster.restart_peer;   r1   r   returnc                     [         er$   r%   )r    prefixkeyvaluesizeranks         r   
sync_peersMaster.sync_peers>   r*   r   c                     UR                   R                  (       a9  UR                   R                  R                  [        5      (       a  [	        U5      $ [        U5      $ r$   )argsmaster
startswithETCD_PROTOCOL
ETCDMaster
HTTPMaster)clsr   s     r   factoryMaster.factoryA   s9    88??sxx99-HHc?"c?"r   )r   r   r   r   N)__name__
__module____qualname____firstlineno____doc__MAINSTANDBYPARTICIPANTr!   r(   r/   r3   r6   listr   r?   classmethodrI   __static_attributes__r,   r   r   r   r   #   sU     DGK""c{ " # #r   r   c                   B    \ rS rSrS rS rS rS rS
S\\	44S jjr
Srg	)rG   I   c                    U R                   (       a  g [        R                  U l        U R                  R
                  R                  (       a  U R                  R
                  R                  U l        U R                  R                  S5      u  pUSU R                  R                  R                  4;   a  [        R                  " S[        R                  " 5       -  5        U R                  R                  R                  U[        U5      5      (       d1   [!        [        U5      5      U l        [        R$                  U l        GOfGOdU R                  R                  R-                  5       nU R                  R                  R                   SU 3U l        [!        U5      U l        [        R$                  U l        [/        S5        [0        R2                  R                  S5      S   S	S
/nUR5                  SU R                  /5        UR5                  [0        R6                  SS  5        [/        S5        [/        SR9                  U5      5        [/        S5        [        U R                  R
                  R:                  5      S:  a%  U R                  R(                  R+                  S5        SU R                  ;   a?  U R                  R=                  SU R                  R                  R                  5      U l        [?        U R                  5      U l         SU l         U RC                  5         g ! [&         aJ  nU R                  R(                  R+                  SU 35        [        R                  " S5         S nAGM  S nAff = f)Nr   z	127.0.0.1   zstart master failed 皙?z1Copy the following command to other nodes to run./rL   z-mzpaddle.distributed.launchz--master   zP-------------------------------------------------------------------------------- r   z9--rank set in the command may not compatible in auto modeT)"r   r   rT   roler   rB   rC   r   r   nodeiptimesleeprandomis_server_readyr   r   r   rR   	Exceptionloggerwarningget_free_portprintsys
executableextendargvjoinr>   replacer   client_start_server)r    rb   portecmds        r   	lazy_initHTTPMaster.lazy_initJ   s^   &&	88== HHMM00DM}}**3/HBk488==#3#344

1v}}./((--77CIFF!&.s4y&9$*KK	 88==..0D#xx}}//0$8DM"4.DKDIEF$$S)"-+C
 JJ
DMM23JJsxx|$(O#((3- (O488==%%&!+''O $--' MM11+txx}}?O?OPDMt}}-C % !//2Fqc0JK

3 !s   .L- -
N7>M<<Nc                     U R                   (       ai  U R                   R                  (       dM  U R                   R                  5         U R                  R                  R                  SU R                   35        g g g )NzKV server start at )r   startedstartr   rh   debugr   r'   s    r   rs   HTTPMaster._start_server}   sN    ;;t{{22KKHHOO!!$7"GH  3;r   c                     U R                   (       a\  U R                   R                  (       d@  U R                   R                  5         U R                  R                  R                  S5        g g g )NzKV server stopped)r   stoppedr(   r   rh   r|   r'   s    r   _stop_serverHTTPMaster._stop_server   sC    ;;t{{22KKHHOO!!"56  3;r   c                 $    U R                  5         g r$   )r   r'   s    r   r(   HTTPMaster.stop   s    r   r8   c                 r   US:  a  U/S4$ U R                   R                  R                  S5        U R                  5         U R                   R                  R                  5       (       d  U R                  R                  SS9(       a  OfU R                   R                  R                  S5        [        R                  " S5        U R                   R                  R                  5       (       d  M  US:  a   U R                  [        R                  :X  a  SOUnU S	U S	U 3nU R                   R                  R                  5       (       Gd  U R                  R                  Xs5      (       d=  U R                   R                  R                  S
5        [        R                  " S5        M  U R                  R                  U5      nU R                   R                  R!                  SU 35        U(       a  [#        U5      U:X  a  U R                   R$                  R&                  (       a0  [)        UR+                  5       [,        S9n	U	R/                  U5      n
X4$ US:  aQ  [1        UR3                  5       5      nUR5                  5         U Vs/ s H  oxU   PM	     n	nU	R/                  U5      n
X4$ S /U-  n	UR7                  5        H$  u  p|X[9        UR;                  S	5      S   5      '   M&     X4$ [        R                  " S5        U R                   R                  R                  5       (       d  GM  / S4$ s  snf )Nr[   r   Waiting peer start...   )timeoutzmaster not readyr\   aaaaaar]   zput value failedsync peers r;   rL         ?)r   rh   inforw   r.   is_donerr   wait_server_readyri   rc   rd   r`   r   rR   put
get_prefixr|   lenrB   sort_ipsortedvaluesr   indexrU   keyssortitemsr   r   )r    r:   r;   r<   r=   r>   kykrjsonretidxr   vs                r   r?   HTTPMaster.sync_peers   sX   !87A:45((//))++{{,,Q,7''(:;

3 ((//))++ dii6;;&>XChat1TF#((//))++;;??1,,''(:;

3KK**62EHHOO!!Kw"78Ut+88==(( Z@C))E*C8OAX

-DIIK-12T8TC2))E*C8O&4-C %56CR 012 !.9$

33 ((//))++4 1u 3s   	L4)rr   r   r   r`   r   NrK   )rM   rN   rO   rP   rw   rs   r   r(   rU   r   r?   rW   r,   r   r   rG   rG   I   s-    1fI
7
-c{ - -r   rG   c                   l   ^  \ rS rSrU 4S jrSS\\44S jjrSS jrS r	S r
S rS	 rS
 rS rSrU =r$ )rF      c                   > [         TU ]  U5        U R                  R                  R                  (       a4  U R                  R                  R                  R                  S5      U l        SS KnSSKJ	n  U R                  R                  S5      u  pEUR                  5       (       a  U" XES9U l        g UR                  XES9U l        g )Nr   r   r[   )
ETCDClientr   )hostrt   )superr!   r   rB   rC   removeprefixr   etcd3utils.etcd_clientr   r   is_auto_tuner_moderr   )r    r   r   r   r   rt   	__class__s         r   r!   ETCDMaster.__init__   s    88== HHMM00==iHDM2]]((-
!!##$$:DK,,D,<DKr   r8   c                    US:  a  U/S4$ U R                   R                  R                  S5        U SU SU 3nU R                  R	                  U5        U R                   R                  R                  SU SU 35        U R                   R                  R                  5       (       Gd  U R                  R                  XcR                  S5      5        [        U R                  R                  U5      5      n[        R                  " U5      nU R                   R                  R                  SU 35        [        U5      U:X  Ga  U R                   R                  R                   (       aD  U Vs/ s H  oS   R#                  5       PM     n	n[%        U	[&        S	9n
U
R)                  U5      nX4$ US:  a  U Vs/ s H  oS
   R*                  R#                  5       PM!     nnU Vs/ s H  oS
   R*                  R#                  5       PM!     nnUR-                  5         U Vs/ s H  oS   R#                  5       PM     n	nU Vs/ s H  oUR)                  U5         PM     n
nU
R)                  U5      nX4$ S/U-  n
U H|  u  p[/        UR*                  R#                  5       R1                  S5      S   5      nUS:  a)  U R                   R                  R3                  SU S35        UR#                  5       U
U'   M~     X4$ [4        R6                  " S5        U R                   R                  R                  5       (       d  GM  ggs  snf s  snf s  snf s  snf s  snf )zw
sync_peers gather all value for key under scope prefix
result always be sorted either by rank or alphabet of pod.name
r[   r   r   r]   z
sync path z value latin-1r   r   r^   NrL   zrank z error in syncr   )r   rh   r   rr   delete_prefixr|   r.   r   r   encoderU   r   copydeepcopyr   rB   r   decoder   r   r   r;   r   r   r   errorrc   rd   )r    r:   r;   r<   r=   r>   pathresultir   r   r   r   sorted_keysr   r   iis                    r   r?   ETCDMaster.sync_peers   s    !87A:453%q'!!&)
4&w?@((//))++KKOOD,,y"9:$++0089F]]6*FHHOO!!Kx"896{d"88==((5;<VdkkmVF< Z8C))E*C8OAX7=>v!aDHHOO-vD>>D"EfQ488??#4fK"E$$&5;<VdkkmVF<:EF+Q$**Q-0+CF))E*C8O&4-C & !5!5c!:2!>?6 HHOO11E"^2LM"#((*B	 !'
 9$

3= ((//))++ =
 ?"E<Fs   0M:&M&&M"MMc                   ^ ^^^^^ [        T S5      (       a&  T R                  R                  R                  S5        g SU 3T l        T R                   S3T l        T R                  R                  T R                  5        T R                  R                  T5      mT R
                   ST 3mT R                  R                  TTR                  S5      TS9  U 4S jnT R                  R                  T R
                  U5      mUUUUU U4S	 jn[        R                  " S
USS9T l        T R                  R                  5         g )Nheartbeat_prefixzHeartbeat already donez/paddle/z
/heartbeatr]   r   leasec                 N   > TR                   R                  R                  5         g r$   )r   r.   restart)eventr    s    r   _beat_watch2ETCDMaster.register_heartbeat.<locals>._beat_watch  s    HHOO##%r   c                    > TR                   R                  R                  5       (       d   TR                  5         TTR	                  5       ;  aO  TR
                  R                  TTR                  S5      TS9  TR                   R                  R                  S5        [        R                  " TS-  5        TR                   R                  R                  5       (       d  M  TR                   R                  R                  S5        TR
                  R                  T5        g ! [         a2  n TR                   R                  R                  SU  35         S n A NS n A ff = f)Nr   r   zHeartbeat register againzHeartbeat error r[   zHeartbeat done)r   r.   r   refreshfetch_peer_aliverr   r   r   rh   r|   rg   r   rc   rd   cancel_watch)ru   	beat_path
beat_watchr   pod_idr    ttls    r   
_heartbeat1ETCDMaster.register_heartbeat.<locals>._heartbeat  s   hhoo--//BMMOT%:%:%<<%v}}Y'?u (  --.HI 

37# hhoo--// HHOO!!"23KK$$Z0	 ! BHHOO)),<QC*@AABs   A3D$ $
E .(EE 	heartbeatT)nametargetdaemon)hasattrr   rh   ri   
job_prefixr   rr   r   r   r   r   add_watch_prefix_callback	threadingThreadbeat_threadr{   )	r    job_idr   r   r   r   r   r   r   s	   ` ``  @@@r   register_heartbeatETCDMaster.register_heartbeat   s   4+,,HHOO##$<=$VH-#'??"3: >!!$//2!!#& ,,-Qvh7		6==#;5I	& [[::!!;

	1 	1 %++Z
 	 r   c                     U R                   R                  U R                  5       Vs/ s H  oS   R                  5       PM     nnU R                  R
                  R                  SU 35        U$ s  snf )Nr   zpeer alive )rr   r   r   r   r   rh   r|   )r    r   
peer_alives      r   r   ETCDMaster.fetch_peer_alive'  sf    #';;#9#9$:O:O#P
#PaaDKKM#P 	 
 	J<89	
s   A0c                    US:  a  UOSn[         R                   " 5       U-   n[        U R                  5       5      nU R                  R                  R                  5       (       d  [         R                   " 5       U:  a  [        U R                  5       5      nXb:X  a  SU4$ Xe:w  a1  [         R                   " 5       U-   nUn[         R                  " S5        O[         R                  " S5        U R                  R                  R                  5       (       d  [         R                   " 5       U:  a  M  [        U R                  5       5      nXa:  a	  Xb::  a  SU4$ SU4$ )Nr^      Tg?r   F)rc   r   r   r   r.   r   rd   )r    replicas_minreplicas_maxr   endnp_prenps          r   wait_peer_readyETCDMaster.wait_peer_ready.  s   $q['aiikG#T**,-((//))++		c0AT**,-B!l++iikG+

3

3 ((//))++		c0A &&()""4":2;r   c                 N    U R                   R                  U R                  5        g r$   )rr   r   r   r'   s    r   r6   ETCDMaster.restart_peerF  s    !!$"7"78r   c                     U R                   R                  U R                  UR                  S5      U R                   R	                  S5      S9(       d
   SU 35       eg )Nr   iX  r   zset status failed )rr   r   r   r   r   r-   s     r   r/   ETCDMaster.set_statusI  sW    {{OOMM)$++##C(  
 	)  x(		) 
r   c                 |    U R                   R                  U R                  5      S   nUb  UR                  5       $ S$ )Nr    )rr   r   r   r   )r    r<   s     r   r3   ETCDMaster.get_statusP  s3    03!&!2u||~::r   c                 p    [        U S5      (       a%  U R                  R                  R                  5         g g )Nr   )r   r   r.   doner'   s    r   r(   ETCDMaster.stopT  s(    4''HHOO  " (r   )r   rr   r   r   r   rK   )
   )rM   rN   rO   rP   r!   rU   r   r?   r   r   r   r6   r/   r3   r(   rW   __classcell__)r   s   @r   rF   rF      sB    ="/ c{ / b(!T09);# #r   rF   )r   r   r	   re   rl   r   rc   )paddle.distributed.launch.utils.kv_clientr   )paddle.distributed.launch.utils.kv_serverr   rE   r   r   rG   rF   r,   r   r   <module>r      sT        
   > >,## ##Ln nb\# \#r   