
    [iK                     V   d dl Z d dlmZ d dlmZmZ d dlZd dlmZ d dl	m
Z
 d dlmZ ddgZd	ej        ee         z  eed
f         z  eeef         z  dej        dz  fdZ	 	 ddee         dee         deeeef                  dz  deeej        z  dz           dz  dee         f
dZdS )    N)Sequence)Anycast)ExceptionWrapper_get_device_index)Module	get_a_varparallel_applyobj.returnc                    t          | t          j                  r| S t          | t          t          f          r6t          t          |           D ] }t          |t          j                  r|c S !t          | t                    rHt          t          |                                           D ] }t          |t          j                  r|c S !d S )N)	
isinstancetorchTensorlisttuplemapr
   dictitems)r   results     S/var/www/icac/venv/lib/python3.11/site-packages/torch/nn/parallel/parallel_apply.pyr
   r
      s     #u|$$ 
#e}%% )S)) 	 	F&%,// #t )SYY[[11 	 	F&%,// 4    modulesinputs
kwargs_tupdevicesc                   
 t          |           t          |          k    s*J dt          |            dt          |                       |#t          |           t          |          k    sJ n8t          t          t          t          f         i           ft          |           z  }|#t          |           t          |          k    sJ ndgt          |           z  }d |D             }d |D             }t
          j                                        s
J d            t
          j                                        j	        t          j                    i t          j                    t          j                    c	 	 ddt          dt          d	t          d
t          t          t          f         dt          t
          j        z  dz  dt
          j        dz  ddffd
t          |           dk    ra
fdt%          t'          | ||||d                    D             }|D ]}|                                 |D ]}|                                 n. 
d| d         |d         |d         |d         |d                    g }t-          t          |                    D ]H}|         }	t/          |	t0                    r|	                                 |                    |	           I|S )a  Apply each `module` in :attr:`modules` in parallel on each of :attr:`devices`.

    Args:
        modules (Module): modules to be parallelized
        inputs (tensor): inputs to the modules
        devices (list of int or torch.device): CUDA devices

    :attr:`modules`, :attr:`inputs`, :attr:`kwargs_tup` (if given), and
    :attr:`devices` (if given) should all have same length. Moreover, each
    element of :attr:`inputs` can either be a single object as the only argument
    to a module, or a collection of positional arguments.
    zThe number of modules z& is not equal to the number of inputs Nc                 .    g | ]}t          |d           S )Tr   .0xs     r   
<listcomp>z"parallel_apply.<locals>.<listcomp><   s#    ;;;a D));;;r   c                 L    g | ]!}t           j                            |          "S  )r   acceleratorcurrent_streamr    s     r   r#   z"parallel_apply.<locals>.<listcomp>=   s)    DDDqu //22DDDr   zNo available accelerator found.imoduleinputkwargsdevicestreamr   c                    t          j        
           |Xt          |          }|35  t          d|  d          | <   d d d            n# 1 swxY w Y   d S |                                }t          |t           j                  r|j        }|t           j        	                    |          }	 t           j        
                    |          5  |5  t           j                            	          5  t          |t          t          f          s|f} ||i |}d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   d d d            n# 1 swxY w Y   5  || <   d d d            d S # 1 swxY w Y   d S # t          $ r9 5  t          d|  d|           | <   d d d            Y d S # 1 swxY w Y   Y d S w xY w)Nzin replica zQ, no device was provided and no tensor input was found; device cannot be resolved)where)enabledz on device )r   set_grad_enabledr
   r   
get_devicer   r,   indexr&   r'   device_indexampautocastr   r   	Exception)r(   r)   r*   r+   r,   r-   toutputautocast_enableddevice_typegrad_enabledlockresultss           r   _workerzparallel_apply.<locals>._workerG   s    	|,,,>%  Ay  !14A 4 4 4" " "GAJ              
 \\^^Ffel++ 	"\F>&55f==F	!..v662 22 2 	"";8H"II2 2 "%$77 %"HE1&112 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2 2  $ $#
$ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ $ 	 	 	  ->>>f>>  
                   	s   AAA3F E"E7(D+E+D//E2D/3E6EE	E	E	
EF EF  E!F &E9,F 9E==F  E=F G	F:,G	:F>	>G	F>	G	G	   c                 Z    g | ]'\  }\  }}}}}t          j        ||||||f           (S ))targetargs)	threadingThread)r!   r(   r)   r*   r+   r,   r-   r?   s          r   r#   z"parallel_apply.<locals>.<listcomp>q   s^     
 
 
 ;:FE666 a%O  
 
 
r   T)strictr   NN)lenr   r   strr   r   r&   is_availablecurrent_acceleratortyperD   Lockis_grad_enabledis_autocast_enabledintr	   r,   Stream	enumeratezipstartjoinranger   r   reraiseappend)r   r   r   r   streamsthreadsthreadoutputsr(   r9   r?   r:   r;   r<   r=   r>   s             @@@@@@r   r   r      sD   $ w<<3v;;&&&bWbbUXY_U`U`bb '&& 7||s:.....4S>2..03w<<?
7||s7||+++++&3w<<';;7;;;GDDGDDDG))++NN-NNNN#7799>K>DG!## #L" -1&*' ''' ' S#X	'
 el"T)' t#' 
' ' ' ' ' ' ' ' ' 'R 7||a
 
 
 
 ?HGVZ'$OOO? ?	
 
 
  	 	FLLNNNN 	 	FKKMMMM	 	71:vay*Q-WQZPPPG3v;;  f.// 	NNvNr   rG   )rD   collections.abcr   typingr   r   r   torch._utilsr   torch.cuda._utilsr   torch.nn.modulesr	   __all__r   r   r   r   r
   rI   rP   r,   r   r%   r   r   <module>rc      sq       $ $ $ $ $ $          ) ) ) ) ) ) / / / / / / # # # # # # (
)	S		!E#s(O	3d38n	D
\D   ( 37:>	h hfhSMh c3h(4/h cEL(4/047	h
 
#Yh h h h h hr   