
    :i?                        d dl Z d dlZd dlmZ d dlZd dlmZ ddlm	Z	m
Z ddlmZmZ g dZ e j        ddd	g          Zd
 ZdZ ej        di           aet*          _        t*          j        ret*          _        d Zd Zej        ej        fdZd Zd Zej        ddfdZ d Z!d Z"	 ddZ#d Z$d Z%d Z&d Z'd Z( ede e(           dS )    N)cast   )api	constants)_group_membership_management_update_group_membership)backend_registeredregister_backendconstruct_rpc_backend_optionsinit_backendBackendValueBackendTyper   %construct_rpc_backend_options_handlerinit_backend_handlerc                     d| j         z   S )NzBackendType.)name)selfs    Y/var/www/icac/venv/lib/python3.11/site-packages/torch/distributed/rpc/backend_registry.py_backend_type_reprr      s    DI%%    z
    An enum class of available backends.

    PyTorch ships with a builtin ``BackendType.TENSORPIPE`` backend.
    Additional ones can be registered using the
    :func:`~torch.distributed.rpc.backend_registry.register_backend` function.
r   valuenamesc                     | t           j        v S )z
    Checks if backend_name is registered as an RPC backend.

    Args:
        backend_name (str): string to identify the RPC backend.
    Returns:
        True if the backend has been registered with ``register_backend``, else
        False.
    )r   __members__)backend_names    r   r	   r	   3   s     ;222r   c                 F   t          |           rt          d|  d          d t          D             }t          | t	          ||          ifi |}t          j        d|          at          t          _        t          j	        rt          t          _	        t          |          S )a  Registers a new RPC backend.

    Args:
        backend_name (str): backend string to identify the handler.
        construct_rpc_backend_options_handler (function):
            Handler that is invoked when
            rpc_backend.construct_rpc_backend_options(**dict) is called.
        init_backend_handler (function): Handler that is invoked when the
            `_init_rpc_backend()` function is called with a backend.
             This returns the agent.
    zRPC backend z: already registeredc                 (    i | ]}|j         |j        S  )r   r   ).0members     r   
<dictcomp>z$register_backend.<locals>.<dictcomp>R   s    NNN&+v|NNNr   )r   r   r   r   )r	   RuntimeErrorr   dictr   enumEnumr   __repr____doc___backend_type_doc)r   r   r   existing_enum_dictextended_enum_dicts        r   r
   r
   @   s     ,'' NL,LLLMMMNN+NNN,6[%9  	
    )-7IJJJK-K 0/|$$r   c                 *     | j         j        ||fi |S N)r   r   )backendrpc_timeoutinit_methodkwargss       r   r   r   f   s/     ?7=>[ $*  r   c                 &     | j         j        |i |S r-   )r   r   )r.   argsr1   s      r   r   r   q   s    -7=-t>v>>>r   c                 x   t           j        }t          j        | |||          }|
J d            |dk    r?||                                k    r't          d| d|                                           |dk    r?||                                k    r't          d| d|                                           |S )Nz*Failed to initialize default ProcessGroup.zrank argument z doesn't match pg rank zworld_size argument z doesn't match pg size )rpc_constantsDEFAULT_PROCESS_GROUP_TIMEOUTdistProcessGroupGloorankr#   size)storer:   
world_sizeprocess_group_timeoutgroups        r   _init_process_groupr@   u   s    )G !%z;PQQEJ

--WDWWWWXXXbzUZZ\\99T:TTejjllTT
 
 	
 Lr   c                 .    ddl m}  || ||||          S )Nr   )TensorPipeRpcBackendOptions)r/   r0   num_worker_threads_transports	_channels) rB   )r/   r0   rC   rD   rE   r1   rB   s          r   1_tensorpipe_construct_rpc_backend_options_handlerrG      s?     .-----&&-   r   c                 :    t          fd| D                       S )Nc              3   n   K   | ]/}|j         d k    p|j         dk    od|j        cxk    ok     nc V  0dS )cpucudar   N)typeindex)r    ddevice_counts     r   	<genexpr>z/_tensorpipe_validate_devices.<locals>.<genexpr>   sl         	
%MAFf,Lag1L1L1L1L1L1L1L1L     r   )all)devicesrO   s    `r   _tensorpipe_validate_devicesrS      s;             r   c                 \   d t          |                                          D             }t          j        || |||f|           d |D             }d |D             }d |D             }d |D             }	t	          ||||	           t          | ||          }
t          |||
          }|
|fS )Nc                     g | ]}d di g f	S )rF   r   r   )r    _s     r   
<listcomp>zB_tensorpipe_exchange_and_check_all_device_maps.<locals>.<listcomp>   s     666Q"aR666r   c                     g | ]	\  }}}}|
S r   r   )r    r   rV   s      r   rW   zB_tensorpipe_exchange_and_check_all_device_maps.<locals>.<listcomp>   s     444-$1a444r   c                     i | ]
\  }}}}||S r   r   )r    r   countrV   s       r   r"   zB_tensorpipe_exchange_and_check_all_device_maps.<locals>.<dictcomp>   s#    GGG):uauGGGr   c                     i | ]
\  }}}}||S r   r   )r    r   rV   map_s       r   r"   zB_tensorpipe_exchange_and_check_all_device_maps.<locals>.<dictcomp>   s#    CCC&6dAtQtTCCCr   c                     i | ]
\  }}}}||S r   r   )r    r   rV   rR   s       r   r"   zB_tensorpipe_exchange_and_check_all_device_maps.<locals>.<dictcomp>   s#    EEE%8T1a4EEEr   )ranger;   r8   all_gather_object_validate_device_maps_create_reverse_mapping_create_device_list)my_namemy_device_countmy_device_maps
my_devicesr?   gathered	all_namesall_device_countsall_device_mapsall_devicesreverse_device_mapss              r   ._tensorpipe_exchange_and_check_all_device_mapsrm      s    
 	76%

"5"5666  	7O^ZH%   548444IGGhGGGCC(CCCOEEHEEEK)%6UUU 2'9oVV$ZATUUJ
**r   Tc                    | D ]}||         }t          t          |                    t          |          k    rt          d| d|           t          |||                   st          d| d| d||                    | D ]U}|rjt          ||                                                                       |           s0t          d| d||                                          d|            ||                                         D ]\  }}	t          t          |	                                                    t          |	          k    rt          d| d| d|	           ||         r\t          |	                                                              ||                   s!t          d| d	| d|	 d
||                    nIt          |	                                ||                   s!t          d| d| d|	 d||                    |                    |g           r]t          |	                                                              ||                   s!t          d| d| d|	 d
||                    }||v rIt          |	                                ||                   s!t          d| d| d|	 d||                    ̐Wd S )NzNode z" has duplicated devices
devices = z, has devices with invalid indices
devices = z
device count = z@ has invalid target node names in its device maps
device maps = z
node names = z5 has duplicated target devices in its device map for z
device map = z5 has unexpected source devices in its device map for z
devices = z? has source devices with invalid indices in its device map for z5 has unexpected target devices in its device map for z? has target devices with invalid indices in its device map for )	lenset
ValueErrorrS   keysissubsetitemsvaluesget)
rh   ri   rj   rk   is_static_groupnoderR   source_nodetarget_noder\   s
             r   r`   r`      s?     	 	d#s7||G,,WTWWgWWXXX+G5Ft5LMM 	< < <$< <"3D"9< <  	 ! 2 2 	3{'C'H'H'J'J#K#K#T#T$
 $
 	 , , ,!0!=!B!B!D!D, , ), ,  
 "1!=!C!C!E!E (	 (	K3t{{}}%%&&#d))33 +K + +-8+ +$(+ +  
 ;' 499;;''00[1IJJ $@ @ @1<@ @(,@ @ &1%=@ @   2		.{;   !GK G G-8G G$(G G '8&DG G   {B// 4;;==))22;{3KLL $@ @ @1<@ @(,@ @ &1%=@ @    111:V0=; ;1 !GK G G-8G G$(G G '8&DG G  G(	2 2r   c                    | st                      }|                                D ])}|                    |                                           *|                                D ])}|                    |                                           *|                    t          j        d                     t          |          } t          | d           } | S )NrJ   c                     | j         S r-   )rM   )rN   s    r   <lambda>z%_create_device_list.<locals>.<lambda>  s    !' r   )key)	rp   ru   updaterr   discardtorchdevicelistsorted)rf   re   rl   devices_setr\   s        r   rb   rb      s     '),"))++ 	, 	,Dtyy{{++++'..00 	, 	,Dtyy{{++++EL//000+&&

(9(9:::Jr   c                     i }|D ]9}| ||         v r-d ||         |                                           D             ||<   :|S )Nc                     i | ]\  }}||	S r   r   )r    kvs      r   r"   z+_create_reverse_mapping.<locals>.<dictcomp>  s+     ) ) )A1) ) )r   )rt   )rc   rh   rj   rl   rx   s        r   ra   ra   
  sj    GI  od+++) )!0!6w!?!E!E!G!G) ) )% r   c                  X   ddl m}  t          | t          j                              }|                                }t          j                                        }t          j        	                                r%|j
        rt          j                                         ||j        |j
        fS )Nr   TensorPipeAgent)rF   r   r   r   _get_current_rpc_agent_get_backend_optionsr   rK   rO   is_availablerR   initdevice_maps)r   agentoptsrO   s       r   _get_device_infosr     s    !!!!!!#"<">">??E%%''D:**,,Lz   T\ 
)4<77r   c           	         ddl m} t          ||           } |                                 }|j        }|                                 }i i i g f\  }}}}|D ]}	|	j        }
|
|k    rt          j        |
t                    \  }}}n@| 	                                }t          j                                        |j        |j        }}}|||
<   |||
<   |||
<   |                    |
           t!          ||||d           t#          |||          }|D ]H}
t%          ||
         ||
         |          ||
<   t          j        |
t&          |||
         |df           Id S )Nr   r   F)rw   T)r3   )rF   r   r   get_worker_infor   get_worker_infosr   rpc_syncr   r   r   rK   rO   r   rR   appendr`   ra   rb   r   )r   r   my_worker_inforc   all_worker_infosri   rj   rk   rh   worker_infoworker_namerO   
device_maprR   r   rl   s                   r   #_set_devices_and_reverse_device_mapr     s   !!!!!!%((E **,,N!G--//ACRR>Y' & &!&'!!03.1 1-L*gg --//D
''))  '.*L
 *6+&'1$#*K %%%%    2'9oVV ! 	
 	
#6$ok&BDW$
 $
K  	$ +k":<OQUV	
 	
 	
 	
 	
	
 	
r   c           
         ddl m}m} t          | t          j                  st          d|            t          ||          st          d|           t          j        	                                }t          |          }|rt          | ||          }	t          |||j        |j        |	          \  }
}t          j                                        r |rt          j                                          || |||||
|          }t#          j        |           t#          j        d |j                   |	                                                                 |S t/          | |d          5   || ||||i g           }t#          j        |           	 t1          |           n"# t2          $ r t#          j                      w xY w|cd d d            S # 1 swxY w Y   d S )Nr   )r   rB   z`store` must be a c10d::Store. z?`rpc_backend_options` must be a `TensorPipeRpcBackendOptions`. )timeoutT)rF   r   rB   
isinstancer8   Store	TypeErrorr   rK   rO   boolr@   rm   r   rR   r   r   r   _init_rpc_states_all_gatherr/   barrierwaitr   r   	Exceptionshutdown)r<   r   r:   r=   rpc_backend_optionsr   rB   rO   rw   r?   rl   rR   r   s                r    _tensorpipe_init_backend_handlerr   S  s    ?>>>>>>>eTZ(( CA%AABBB)+FGG 
cNacc
 
 	
 :**,,L:&&O D $E4<<'U+'(
 (
$W :""$$ 	 	 JOO  
 
 	U### 	&9&EFFFF 	 *%t<< 	 	 $O# E  ''' 4E::::    +	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	s*   .&GF%$G%GGGG
TENSORPIPE)T))collectionsr%   typingr   r   torch.distributeddistributedr8   rF   r   r   r6   _utilsr   r   __all__
namedtupler   r   r)   r&   r   r'   r(   r	   r
   DEFAULT_RPC_TIMEOUT_SECDEFAULT_INIT_METHODr   r   r@   DEFAULT_NUM_WORKER_THREADSrG   rS   rm   r`   rb   ra   r   r   r   r   r   r   <module>r      s/                           - - - - - - - - J J J J J J J J   &{%<>TU 
& & &  dim2666)  ,+K
3 
3 
3#% #% #%P 51   ? ? ?  , %?   &  + + +. QU@ @ @ @F
 
 
  8 8 81
 1
 1
hU U Up  5$    r   