
    9i             !       s   U d dl Z d dlZd dlZd dlZd dlZd dlmZmZ d dlm	Z	 d dl
mZ d dl mZmZ d dlmZmZ d dlmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlmc mZ  d dlm!Z!m"Z"m#Z# d d	l$m%Z% d d
l&m'Z' d dlm(Z(m)Z)m*Z*m+Z+m,Z, d dl-m.Z.m/Z/m0Z0m1Z1 d dl2m3Z4 d dl5m6Z6 ej7        j8        Z8g Z9e:e;         e<d<   ej=        j>        j?        Z? G d de          Z@	 	 ddedejA        deBdeBfdZC eeCejA        jD        d          ZE eeCejA        jD                  ZF eeCejA        jD        d          ZG eeCejA        jH                  ZIde#deJde#fdZK e%e?jL                   e1d           eFd!e#d"e#fd#                                    ZL e%e?jM                   e1d           eFd!e#d"e#fd$                                    ZM e%e?jN                   e1d           eFd!e#de#d%eOd&eOfd'                                    ZN e%e?jP                   e1d           eFd(e#d)eOd*eOd+eOd,eBd-e#fd.                                    ZP e%e?jQ        jR        g          d/             ZS e%e?jQ        j#        g          d0e#fd1            ZT e%e?jU                   e1            eFd2e#de#fd3                                    ZU e%e?jV                   e1d           eFd(e#d2e#fd4                                    ZV e%e?jW                   e1d           d(e#d2e#d5eOd6eOfd7                        ZW e%e?jX                   e1            eFd2e#de#fd8                                    ZX e%e?jY                   e1            eFd(e#d2e#de#fd9                                    ZY e%e?jZ                   e1d           d(e#d2e#d&eOfd:                        ZZ e%e?j[                   e1d           eFd(e#d2e#d;eOd<eBfd=                                    Z[ e%e?j\                   e1d           eFdd?e#d2e#d@e;fdA                                    Z\ e%e?j]                  eFd(e#dBe#fdC                        Z] e%e?j^                   e1            eFd2e#de#fdD                                    Z^ e%e?j_                   e1d           eFd(e#d2e#de#fdE                                    Z_ e%e?j`                  d2e#dFe#de#fdG            Z` e%e?ja                  d(e#d2e#dFe#debe#e#f         fdH            Za e%e?jc                   e1            eFd(e#d2e#dIe#dJeOdKeOdLeBd<eBde#fdM                                    Zc e%e?jd                   e1d           eFd(e#d2e#dNe#de#fdO                                    ZddPe#dQeJfdRZedSejf        fdTZg e%e?jh                   e1            eFe@ji        jj        fd2e#dUe#dQeJde#fdV                                    Zh e%e?jk                   e1d           eFd(e#dBe#dUe#dQeJfdW                                    Zk e%e?jl                  ddX            Zm e%e?jn                   e1            eFe@ji        jj        dYfd2e#dUe#dQeJd%eOfdZ                                    Zn e%e?jo        jp                  eFd(e#d2e#dUe#dQeJd%eOf
d[                        Zo e%e?jo        jq                  eFd(e#d2e#dUe#dQeJd%eOd e#fd\                        Zr e%e?js        jp                  eFd(e#d2e#dUe#dQeJd]eOf
d^                        Zs e%e?js        jt                  eFd(e#d2e#dUe#dQeJd]eOd e#fd_                        Zud(e#d2e#dUe#dFee#         dQeJd`eJdae#de#fdbZv e%e?jw                   e1d           eFd(e#d2e#deJde#fdc                                    Zw e%e?jx                   e1d           d(e#d2e#dUe#dFee#         dQeJd`eJdae#de#fdd                        Zx e%e?jy                   e1d           d(e#d2e#dUe#dFee#         dQeJd`eJdae#de#fde                        Zy e%e?jz                   e1            eFde@ji        jj        fd2e#dUe#dFee#         dQeJde#f
df                                    Zz e%e?j{                   e1d           eFde@ji        jj        fd(e#d2e#dUe#dFee#         dQeJde#fdg                                    Z{ e%e?j|                   e1            eFe@ji        jj        fdBe#dUe#dQeJde#fdh                                    Z| e%e?j}                   e1d           eFe@ji        jj        fd(e#d2e#dUe#dQeJde#f
di                                    Z} e%e?j~                   e1            ddBe#dke#dleOfdm                        Z~ e%e?j                   e1            dne#doe#de#fdp                        Z e%e?j                   e1            d(e#dqe:eJ         deJdreJdseJdteJfdu                        Z e%e?j        j#                  	 	 	 	 dd2e#deJdreeJ         dseeJ         dteJf
dw            Zde#deJdreeJ         dseeJ         debeJeJf         f
dxZ e%e?j                   e1            	 	 	 	 ddBe#dye#deJdreeJ         dseeJ         dteJfdz                        Z e%e?j                   e1            d(e#dqe:eJ         deJd{eJfd|                        Z e%e?j                   e1            d(e#dqe:eJ         d}eJd~eJdeJf
d                        Zd(e#d e#dejf        fdZ e%e?j                   e1d           eEd(e#de#deJdejf        fd                                    Z e%e?j                   e1            eEd(e#de#deJdejf        fd                                    Zd Z e%e?j                   e1            dBe#de:eJ         de:eJ         de:eJ         de:eJ         de#fd                        Z e%e?j                   e1            eFdBe#de:eJ         de:eJ         de:eJ         de:eJ         de:eJ         de#fd                                    Z e%e?j                   e1            d(e#de#d*eOfd                        Z e%e?j                   e1            d?e#de:eJ         deJdeJdteJde#fd                        Z e%e?j        jp                  eF	 dd(e#d2e#deeO         de#fd                        Z e%e?j                  e?j        jp                            e8j                  e?j        jp                            e8j                  dBe#dleOdeeB         fd                                    Z e%e?j                   e1dd          dBe#dleOdeeB         fd                        Z e%e?j                   e1            de#deJdeBfd                        Z e%e?j                   e1d          de#deJdeBfd                        Z e%e?j                   e1            	 	 	 ddFe#de#deJdeBdeBde#fd                        Z e%e?j                   e1            d(e#de#deJdeJdeBf
d                        Zde:eJ         fdZde:e#         deJdeJde:e#         fdZde:e#         fdZde:e#         deJfdZde:e#         deJdeJfdZ e%e?j        jp        e?j        jt        g          	 dde:e#         deJdeJdee#         de#f
d            Z e%e?j        jp        e?j        jt        g          	 	 dd2e#de:eJ         deJdee:e#                  dee:e#                  f
d            Z e%e?j        j#                  ddBe#deJdeJdebe#df         fd            Z e%e?j        jp                  	 ddBe#de:eJ         deJdebe#df         fd            Z e%e?j        j#                  dd2e#deJdeJdebe#df         fd            Ze?j        j                            e8j                  	 dd2e#de#deJdebe#df         fd            Z e%e?j                   e1d          eFdd2e#de#de#d%eJd)eJf
d                                    Z e%e?j                   e1            eF	 	 	 dd2e#de#de#d%eJd)eJdeBfd                                    Z e%e?j                   e1d          eFdd2e#de#de#d%eJd)eJf
d                                    Z e%e?j        jp                  eFd(e#dBe#de#de#dee#         deJdeJdeJdeJde:eB         debee#         ee#         ee#         f         fdȄ                        Z e%e?j        jt                  d(e#dBe#de#de#dee#         deJdeJdeJdeJde:eB         dej#        dej#        dej#        debee#         ee#         ee#         f         fdʄ            Zdee#         dee#         fd˄Z e%e?j        jp                  de#dBe#de:eJ         de#de#dFee#         dee#         de:eB         debee#         ee#         ee#         f         fdτ            Z e%e?j        jt                  de#dBe#de:eJ         de#de#dFee#         dee#         de:eB         dej#        dej#        dej#        debee#         ee#         ee#         f         fdЄ            Z e%e?j        jp                  dBe#de:eJ         dFee#         deeO         debe#e#f         f
dф            Z e%e?j        jp                  de#dBe#de:eJ         de#dFee#         de:eB         debee#         ee#         f         fd҄            ZdBe#dFee#         dee#         dee#         dee#         dLeBdeOdeOdeBdebe#e#e#ee#         ee#         f         fdׄZ e%e?j                   e1ddd٦          dBe#dFee#         dee#         dee#         dee#         dLeBdeOdeOdebe#e#e#f         fdڄ                        Ze?j        jp                            e8j                  e?j        jp                            e8j                  dBe#dFee#         dee#         dee#         dee#         dLeBdeOdeOdebe#e#e#f         fdۄ                        Ze?j        jp                            e8j                  dde:e#         fd܄            Z e%e?j        jp                  dBe#dFee#         dee#         de#de#deOdeOdebe#e#e#f         fd݄            Z e%e?j        jp                  dBe#dFee#         dee#         de#de#dLeBdeOdeOdebe#e#e#f         fdބ            Z e%e?j        j                  dBe#dFee#         dee#         dLeBdeOdeOdebe#e#e#f         fd߄            Z e%e?j        jp                  dBe#dFee#         dee#         de#de#dLeBdeOdeOdebe#e#e#e#e#f         fd            ZdBe#dFee#         dee#         de#de#deOdLeBde#fdZ e%e?j        jp                  dBe#dFee#         dee#         de#de#deOdeOdebe#e#e#e#f         fd            Z e%e?j        jp                  dBe#dFee#         dee#         de#de#deOdeOdebe#e#e#e#e#e#f         fd            Z e%e?j        jp                  dBe#dFee#         dee#         de#de#deOdeOdebe#e#e#e#f         fd            Z e%e?j                   e1dd          eFdd                                    Z e%e?j                   e1            ddddddddee#e)f         dSeejf                 deej                 deBdeBdeej                 fd                        Z e%e?j        e?j        e?j        g           e1            d                         Ze?j        jp                            e8j                   e%e?j                   e1dddd          dBe#dFe#dee#         dee#         dee#         dLeBdeOdeOfd                                    Zd Z e%e?j        jp                  de#dBe#dFee#         dee#         dee#         dee#         dee#         deBdeOde:eB         de#debe#ee#         ee#         f         fd            Z e%e?j        jp                  de#dBe#dFee#         dee#         dee#         dee#         dee#         deBdeOde:eB         debe#ee#         ee#         f         fd            Z e%e?j        jt                  de#dBe#dFee#         dee#         dee#         dee#         dee#         deBdeOde:eB         dej#        dej#        dej#        debe#ee#         ee#         f         fd            Z e%e?j                   e1dddɦ          dBe#d(e#dFe#dee#         dee#         dee#         dee#         deOfd                        Z e%e?j                   e1dddɦ          dBe#d(e#dFe#dee#         dee#         dee#         dee#         deOde#fd                        Z e%e?j                   e1            eFdBe#debeJeJf         fd                                    Zd2e+de+de:eJ         deJfdZ e%e?j                   e1            d2e+de+de:eJ         fd                        Z e%e?j                   e1            dBe+de+de:eJ         de:eJ         de:eJ         f
d                        Z e%e?j                  dvdde+deJd{e+de+d)e)f
d             Z e%e?j                   e1            dvdde+deJd{e+de+d)e)f
d                        Zdvdde+deJd{e+de+deBd)e)fdZ e%e?j        jp                  e?j        jp                            e8j                  dd                        Z e%e?j                  de+deJd{e+de+fd            Z e%e?j                   e1            de+deJd{e+de+fd                        Zde+deJd{e+de+deBf
dZ e%e?j                   e1ddN          eFd2e#debe#e#f         fd	                                    Z e%e?j                   e1            	 	 	 dde#d
eeBeJeOf         deeBeJeOf         deej                 fd                        Z e%e?j                  dd            Zݐd Zސd Z e%e?j        j                   e%e?j        j                   e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  dBe#dee:eJ                  dee:eO                  de#fd                                                                                                            Z e%e?j        j                   e%e?j        j                   e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  dBe#dee:eJ                  dee:eO                  de#fd                                                                                                            ZddZ e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 ddBe#de:eJ         deeO         de#fd                                                Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 ddBe#de:eJ         deeO         de#fd                                                Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 	 ddBe#de:eJ         deeO         deeO         de#f
d                                                Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 	 ddBe#de:eJ         deeO         deeO         de#f
d                                                Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 	 	 ddBe#de:eJ         deeO         deeO         deeO         de#fd                                                Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                  e?j        jp                            e8j                   e1dd          	 	 	 ddBe#de:eJ         deeO         deeO         deeO         de#fd                                                ZeF	 ddBe#de:eJ         de:eeO                  d eBde#f
d!            Zd" Zd# Zd$ Zd% Z	 dd&Zd' Zd( Zdd)Zdd*Zd+ Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d,                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d-                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d.                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d/                                     Zd0 Zdd1Zdd2Z d3 Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d4                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d5                                     Zd6 Zd7 Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d8                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d9                                     Z	 e%e?j
        j                  e?j
        j                            e8j                  e?j
        j                            e8j                  d:                                     Z e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  d;                                     Z e%e?j        j                   e%e?j        j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  e?j        j                            e8j                  d<                                                                                                 Z e%e?j        jp        e?j        jt        g           e1            	 ddBe#de:eJ         d=eBdeeO         de#f
d>                        Z e%e?j        jp        e?j        jt        g          e?j        jp                            e8j                   e1            	 	 ddBe#de:eJ         d=eBdeeO         deeO         de#fd?                                    Z e%e?j        jp        e?j        jt        g           e1            	 	 	 ddBe#de:eJ         d=eBdeeO         deeO         deeO         de#fd@                        ZddAZdB Zdyee#         dCee#         dDe#de#fdEZdCe,de#fdFZeFdBe#de:eJ         d=eBde:eeO                  de#f
dG            Z e%e?j        jp                  dHe#dIe#deBfdJ            Z e%e?j        e?j        g           e1            dK                         Z e%e?j        g          dL             Z e%e?j        g          ddM            Z e%e?j        g          dN             Z e%e?j        g          dO             Zd2e#dUe#dFee#         dQeJd`eJdebe#e#f         fdPZ e%e?j                   e1dda          d2e#dUe#dFee#         dQeJd`eJdebe#e#f         fdQ                        Z e%e?j                    e1dda          d2e#dUe#dFee#         dQeJd`eJdebe#e#f         fdR                        Z de#dSeOde#fdTZ!de#dSeOde#fdUZ"dVe#de,fdWZ#dXe,dYe#de#fdZZ$dYee#         de#fd[Z%d\eJd=eBdSejf        dej        fd]Z&d^e#d_eJd`eJd=eBfdaZ'd^e#dbeJd_eJd`eJd=eBf
dcZ(d^e#de:eJ         d=eBfddZ)d^e#de:eJ         d=eBfdeZ* e%e?j+                   e1            eFd^e#de:eJ         d=eBfdf                                    Z+	 	 	 	 ddHe#dge#dheJdieJd=eBdjeBde#fdkZ, e%e?j-                   e1            eF	 	 	 ddHe#dge#dheJdieJd=eBde#fdl                                    Z- e%e?j.                   e1d          eFdm                                     Z. e%e?j/                   e1            dde@ji        jj        fdn                        Z/doej#        dpej#        dqeBdeBfdrZ0e?j1        jp                            e8j                  e?j1        jt                            e8j                   e1ds          ddtdu                                    Z1 e%e?j2        jp        e?j2        jt        g          e?j2        jp                            e8j                   e1            eF	 	 ddBe#debeJeJf         d=eBdveeO         dweeO         de#fdx                                                Z3 e%e?j2        j                  e?j2        j                            e8j                  e?j2        j                            e8j                   e1            eF	 ddHe#deebeJeJf                  d=eBdeebeOeOf                  de#f
dy                                                            Z4 e%e?j5                   e%e?j6                   e%e?j7                  eF e1            dHe#debeJdf         de#fdz                                                            Z8 e%e?j9                   e%e?j:                   e%e?j;                  eF e1            dHe#debeJdf         de#fd{                                                            Z<dHe#debeJdf         d|eeJeJeJge#f         de#fd}Z= e%e?j>                   e%e?j?                   e%e?j@                   e1d           d~                                                 ZA e%e?jB                   e1dd          dddd                        ZB e%e?jC                   e1            dddd                        ZC e%e?jD        jp        e?jD        jt        g           e1            dejE        ddddse)dSeejf                 dejF        deej                 deBf
d                        ZG e%e?jD        jH        g          dejE        ddddre)dse)dSeejf                 dejF        deej                 deBfd            ZI e%e'          d             ZJ e%e?jK                  e?jK        jp                            e8j                   e1            dvdvde@ji        jj        fdBe#dUe#dle)de)dFee#         dQeJde#fd                                    ZK e%e?jL                  e?jL        jp                            e8j                   e1dd          dBe#dUe#dQeJdebe#e#f         fd                                    ZL e%e?jM        jp                  	 	 ddddde#de#d0e#deOdeBdee#         d*eeO         debe#e#f         fd            ZNd ZO e%e?jP        g           e1d          eFdd                                    ZP e%e?jQ                   e1            d                         ZQ e%e?jR                  d             ZR e%e?jS        jp        e?jS        jt        g          dddd2e#dSeejf                 dee#         de#fd            ZT e%e?jU        jp        e?jU        jV        g          dd2e#deeJ         fd            ZW e%ej>        j?        jX                  dd            ZX e%e?jY                   e1            dddd                        ZY e%e?jZ        jp                  ddd2ej#        deej                 dej#        fd            ZZdddZ[ddddZ\ e%e?j]                   e1            d                         Z] e%e?j^                  dd            Z^ eOe?j_        e?j`                    eOe?ja        e?j                    eOe?jb        e?j                    eOe?jc        e?jP                    eOe?jd        e?jQ                    eOe?je        e?jf                    eOe?jg        e?jX                    eOe?jh        e?ji                    eOe?jj        e?jU                    eOe?jk        e?jl                    eOe?jm        e?jn                    eOe?jo        e?jp                    eOe?jq        e?jr                    eOe?js        e?jt                    eOe?ju        e?jv                    eOe?jw        e?jx                    eOe?jy        e?jz                    eOe?j{        e?j|                    eOe?j}        e?j~                    eOe?j        e?j                    eOe?j        e?j                    eOe?j        e?j                    eOe?j        e?j                    eOe?j        e?j                    eOe?j        e?j^                   dS (      N)CallableIterable)nullcontext)Enum)partialreduce)chainproduct)AnycastOptionalUnion)	sym_floatsym_intTensorregister_decomposition)	out_dtype)IntLike
NumberTypesuggest_memory_format
TensorLikeTensorSequenceType)_maybe_convert_to_dtype_maybe_resize_out_safe_copy_outout_wrapper)_pytree)tree_map__all__c                       e Zd ZdZdZdZdS )	Reductionr         N)__name__
__module____qualname__NONEMEANSUM     O/var/www/icac/venv/lib/python3.11/site-packages/torch/_decomp/decompositions.pyr"   r"   1   s        DD
CCCr,   r"   Fftype_promotioncompute_dtype_onlyinclude_non_tensor_argsc                 R     t          j                    fd            }|S )Nc                  4   rt           t          j        j        fnt           ffdt	          j        | i |D             }t          j        |di\  fd}fd} 
t          ||           i t          ||          }	r|S t          ||          S )Nc                 4    g | ]}t          |          |S r+   )
isinstance).0xallowed_typess     r-   
<listcomp>z-type_casts.<locals>.inner.<locals>.<listcomp>E   s9     
 
 
!]++

 
 
r,   type_promotion_kindc                 \    t          | t                    r|                               S | S Nr5   r   to)r7   computation_dtypes    r-   increase_precz0type_casts.<locals>.inner.<locals>.increase_precO   s-    !V$$ tt-...r,   c                 \    t          | t                    r|                               S | S r<   r=   )r7   result_dtypes    r-   decrease_precz0type_casts.<locals>.inner.<locals>.decrease_precU   s,    !V$$ ttL)))r,   )	r   torchtypes_Numberpytreearg_tree_leavesutilselementwise_dtypesr   )argskwargs	flat_argsr@   rC   rr8   r?   rB   r0   r.   r1   r/   s         @@@r-   innerztype_casts.<locals>.inner@   s     .ESVU[())6) 	
 
 
 
+T<V<<
 
 
	
 +0*B+
,:+
 +
'<
	 	 	 	 		 	 	 	 	 Axt,,P0O0OPP 	.HM1---r,   )	functoolswraps)r.   r/   r0   r1   rO   s   ```` r-   
type_castsrR   :   sM     _Q. . . . . . . .@ Lr,   T)r/   r0   )r/   )r/   r1   r7   dimreturnc                 ~    t          ||                                 z
            D ]}|                     d          } | S N)rangerS   	unsqueeze)r7   rS   _s      r-   _unsqueeze_to_dimr[   w   s;    3=!!  KKOOHr,   
grad_inputout_gradyc                 <    | d||z  z
                                   z  S Nr#   conj_physicalr]   r^   s     r-   tanh_backwardrd   }   s#     q1q5y//1111r,   c                 <    | |d|z
  z                                   z  S r`   ra   rc   s     r-   sigmoid_backwardrf      s#     qAE{113333r,   beta	thresholdc                 |    ||z                                   }t          j        ||z  |k    | | |z  |dz   z            S N      ?)exprD   where)r]   r7   rg   rh   zs        r-   softplus_backwardro      sA     
TA;DI-xAS9QRRRr,   grad_outputalphascaleinput_scale	is_resultself_or_resultc                     ||z  }|}|}|r&t          j        |dk    | |z  ||z   z  | |z            S t          j        |dk    | |z  |z  t          j        ||z            z  | |z            S Nr   )rD   rm   rl   )	rp   rq   rr   rs   rt   ru   negcoefposcoef
negiptcoefs	            r-   elu_backwardr{      s     emGGJ 
{a*$(@A'!
 
 	
 {a*$w.>J;V1W1WW'!
 
 	
r,   c                 ,    t          j        | |          S r<   )rD   	full_likeselfvalues     r-   fill_scalarr      s    ?4'''r,   r   c                     t          j                                        dk    fd           t                              |           S )Nr   c                  4    d                                   dS )Nz@fill only supports 0-dimension value tensor but got tensor with z dimensionsrS   )r   s   r-   <lambda>zfill_tensor.<locals>.<lambda>   s     kSXS\S\S^S^kkk r,   )rD   _checkrS   atencopyr~   s    `r-   fill_tensorr      sI    	L		qkkkk   99T5!!!r,   r   c                 b    t          j        t          j        | dz   d          d          dz  S N   r   min   maxrD   clampr   s    r-   hardsigmoidr      s1     ;u{4!8333;;;a??r,   c                 J    t          j        |dk    |dk     z  | dz  d          S )Ng      g      @gUUUUUU?        rD   rm   rp   r   s     r-   hardsigmoid_backwardr      s3     ;	$y!  r,   min_valmax_valc                 D    t          j        ||k    ||k    z  d|           S )Nr   r   )rp   r   r   r   s       r-   hardtanh_backwardr      s%    
 ;DGO<c;OOOr,   c                 h    | t          j        t          j        | dz   d          d          z  dz  S r   r   r   s    r-   	hardswishr      s6     %+ek$(:::BBBBQFFr,   c           
      x    t          j        |dk    dt          j        |dk     | |dz  dz   z  |                     S )Nr   r         ?r   r   s     r-   hardswish_backwardr      sE     ;
D1HkdQh#-=>LL  r,   c                 6    t          j        ||k    d|           S rw   r   )rp   r   rh   s      r-   threshold_backwardr      s     ;ty(![999r,   negative_slopeself_is_resultc                 <    t          j        |dk    | | |z            S rw   r   )rp   r   r   r   s       r-   leaky_relu_backwardr      s!     ;taxkN.JKKKr,   nonegradapproximatec                    d}d}d}|dk    rh||z  dz  }d}||z  }||z  }	||||	z  z   z  }
t          j        |
          }d|z  }d|z   }d|z  }d||z  z
  }|dd|z  |z  z   z  }||z  |z  }| ||z   z  S |}||z  dz  }ddt          j        ||z            z   z  }|t          j        ||z  d	z            z  }| |||z  z   z  S )
Ng;f?g;f?gmBP?tanhr   gHm?r#   r   g      )rD   r   erfrl   )r   r   r   M_SQRT2	M_SQRT1_2
M_2_SQRTPIkBetakKappax_sqx_cuberO   
tanh_innerleftrightleft_derivativetanh_derivativeinner_derivativeright_derivativekAlphacdfpdfs                        r-   gelu_backwardr      s-    %G&I'Jf*$s*d{/0Z&&
TzJ+j:55 AF
T(9$9:/14DD)99::Y&,Q4&=1112eitd 2333sTCZ'((r,   inputc                     t          j        t          j        |                    }t          j        |          }||z  d||z  z
  z  }| ||z   z  S r`   )rD   r   Fsoftplussigmoid)rp   r   input_tanh_softplusinput_sigmoidouts        r-   mish_backwardr     sY      *QZ%6%677M%((M
-
1':=P'P#P
QC-344r,   c                 0    | t          j        |           z  S r<   )rD   r   r   s    r-   silur   "  s     %-%%%%r,   c                 Z    ddt          j        |           z   z  }| |z  d|d|z
  z  z   z  S r`   )rD   rl   )rp   r   r   s      r-   silu_backwardr   )  s=     1uy$'''(G AG(<$<==r,   weightc                 <    t          j        | dk    | || z            S rw   r   )r   r   s     r-   _prelu_kernelr   1  s    ;taxv}555r,   c                 ~    t          j        |dk    | || z            }t          j        |dk    d|| z            }||fS )Nr   r   r   )rp   r   r   
input_gradweight_grads        r-   _prelu_kernel_backwardr   6  sG     TAX{F[4HIIJ+dQhTK-?@@K$$r,   noiseloweruppertrainingc                     |r||z
  dk    r|                      |          S ||z   dz  }t                              | |||          S )Ngư>r$   )mulr   r   )rp   r   r   r   r   r   r   r   s           r-   rrelu_with_noise_backwardr   A  s[      
EEMD((u%%%%-1,''~~
 
 	
r,   bufferc                     |dk     }t          j        |dd          }t          j        |dd          }t          j        t          j        |                     }| |||d|z   z  z  z
  z  S )Nr   r#   rW   )rD   rm   rl   abs)rp   r   r   in_negative	max_derivsignrn   s          r-   log_sigmoid_backwardr   V  sj     (KKA..I;{Ar**D	59T??"##A)da1q5k&::;;r,   loss	reductionc                     |t           j        j        k    rt          j        |           S |t           j        j        k    rt          j        |           S | S r<   )r"   r)   r   rD   meanr*   sum)r   r   s     r-   apply_loss_reductionr   c  sE    IN(((z$	im)	)	)yr,   dtypec                     | t           j        k    rt           j        S | t           j        k    rt           j        S | t           j        k    rt           j        S d S r<   )rD   	complex32float16	complex64float32
complex128float64r   s    r-   to_real_dtyper   l  sJ    }	%/	!	!}	%"	"	"} 
#	"r,   targetc                 2    | |z
  dz  }t          ||          S )Nr$   )r   )r   r   r   r   s       r-   mse_lossr   {  s"     6MaDi000r,   c                 t    |t           j        j        k    rd|                                z  nd}|||z
  z  | z  S )N       @)r"   r)   r   numel)rp   r   r   r   norms        r-   mse_loss_backwardr     s>     #,y~/C"C"C3D56>"[00r,   c                     t          j        | ||          }|                     t          d                    }t          j        ||d          }t          j        |          }t          j        |||          S )N)rS   r   z-infTrS   keepdim)rD   softmaxeqfloatall
zeros_likerm   )r   rS   r   r   maskedmasked_rowszeross          r-   safe_softmaxr    si    
-#U
3
3
3CWWU6]]##F)FT:::KS!!E;{E3///r,   rk   c                     | |z
                                   }t          j        ||k     d|dz  z  |z  |d|z  z
            }t          ||          S )Nr   r$   )r   rD   rm   r   )r   r   r   rg   r   s        r-   smooth_l1_lossr
    sV     6M  D;td{C$'MD$8$t:KLLDi000r,   c                    |t           j        j        k    rd|                                z  nd}||z
  }t	          j        |          }|| z  }t	          j        ||k     ||z  |z  |t	          j        |          z            S rj   )r"   r)   r   r   rD   r   rm   r   )	rp   r   r   r   rg   r   r7   abs_x	norm_grads	            r-   smooth_l1_loss_backwardr    s    
 "+in.B!B!B3DvAIaLLE{"I;AEJqMM!  r,   c                 v    t          | ||||          }t          ||j                   t          ||d          S NT	copy_fromcopy_toexact_dtype)r  r   shaper   )rp   r   r   r   rg   r\   results          r-   smooth_l1_loss_backward_outr    s@     %[$	4PPFj&,///FJDQQQQr,   deltac           
          |t           j        j        k    rd|                                z  nd}||z
  }t	          j        || k     | | z  |z  t	          j        ||k    || z  |z  ||z  | z                      S rj   )r"   r)   r   r   rD   rm   )rp   r   r   r   r  r   r7   s          r-   huber_loss_backwardr    s    
 "+in.B!B!B3DvA;	UF
	e#AItk1E94!8k;QRR  r,   c                 v    t          | ||||          }t          ||j                   t          ||d          S r  )r  r   r  r   )rp   r   r   r   r  r\   r  s          r-   huber_loss_backward_outr    s@     !dFIuMMFj&,///FJDQQQQr,   ignore_indextotal_weightc                    |                                 dk     rdnd}|t          j        j        k    r| |z  } |                    |          }t          j        ||k    |d          }t          j        |          }	t          j        |	||d          }	|	                                 |                                  cxk    rdk    rn n|                     |          } |Ud t          |                                           D             }
|j
        d         |
|<   |                    |
          }| |z  } t          j        ||k    | d          } |	| z  S )Nr$   r   r#   g      c                     g | ]}d S r#   r+   r6   rZ   s     r-   r9   z&_nll_loss_backward.<locals>.<listcomp>  s    2221Q222r,   )rS   r"   r)   r   rY   rD   rm   r  scatterrX   r  reshape)rp   r   r   r   r   r  r  channel_dimsafe_targetr\   	new_shapes              r-   _nll_loss_backwardr(    sG    xxzzA~~!!1KIN(((!L0k**F+f4fa@@K!$''Jz;TJJJ~~+//++////a/////!++K8822dhhjj 1 1222	!'a	+	**!F*+f4k1EEK##r,   c                    |                                 dk    s
J d            t          j        |                                 |          }|                    |          }|dz  dk    sJ d| d|             |dz  }|                    |d|          }|                    |||          }t          j        |          }d|z
  |z  |z  | z  }	|| z  }t          j        ||	g|          S )Nr   z*glu does not support 0-dimensional tensorsr$   z.Halving dimension must be even, but dimension z	 is size rk   r   )rS   rI   canonicalize_dimsizenarrowrD   r   cat)
rp   r   rS   wrap_dimnIn	inputSize	firstHalf
secondHalfgradInputFirstHalfgradInputSecondHalfs
             r-   glu_backwardr5    s    88::>>>G>>>%dhhjj#66H
))H

C7a<<<QQQCQQ << qIHa33IXy)<<Jz22	!	!%77)CkQ  ,k99(*=>HMMMMr,   c           	      6   d|                                 cxk    rdk    sn J d            |                                 dk    s
J d            |                                 dk    o|                                 dk    }|s7|j        d         |j        d         k    sJ d|j         d|j         d            |                                dk    s)J d	|j         d
|                                 df            |-|                                |j        d         k    s
J d            |t          j        j        k    r|                                 dk    rp|                                  dk    r| j        d         |j        d         k    s;J d|j        d          d|                                   d| j        d                      nB|                                  dk    r|                                 dk    sJ d| j                     t          | ||||||          S )Nr   r$   input tensor should be 1D or 2Dr#   ;0D or 1D target tensor expected, multi-target not supportedsize mismatch (got input: 
, target: ):expected total_weight to be a single element tensor, got: z (z
 elements)rW   z<weight tensor should be defined either for all or no classesz7Expected a tensor of dimension 1 and tensor.size[0] == z but got: dimension z and tensor.size[0] == z7Expected a single element grad_output tensor, but got: )rS   r  r   r"   r(   r   r(  )rp   r   r   r   r   r  r  no_batch_dims           r-   nll_loss_backwardr>    sc    

a!B::<<1E  88::?8vzz||q'8L DJqMV\!_<<<JTZJJ6<JJJ =<< 1$$$DAA!3!3!5!5AAA'$$$
 >V\\^^tz"~===F >== IN(((TXXZZ1__  A%%+*;A*>$*Q-*O*O*O_djQRm _ _)oo//_ _HSHYZ[H\_ _ +P*O*O*O
   A%%+*;*;*=*=*B*B*BYkFWYY +C*B*B T669lL  r,   c           	      d   |                                 dk    sJ d|                                              |                                 dk    sJ d|                                              |j        d         |j        d         k    r8|j        d         |j        d         k    r|j        d         |j        d         k    sJ d|j         d	|j                     |                                dk    s(J d
|j         d|                                 d            t          | ||||||          S )N   zSonly batches of spatial inputs supported (4D tensors), but got input of dimension: r   zUonly batches of spatial targets supported (3D tensors) but got targets of dimension: r   r$   r#   r9  r:  r<  z ( z, elements))rS   r  r   r(  )rp   r   r   r   r   r  r  s          r-   nll_loss2d_backwardrA  F  sb    88::???j^b^f^f^h^hjj ?? ::<<1n`f`j`j`l`lnn 
 	
1a((JqMV\!_,,JqMV\!_,,,HDJHH&,HH -,, 1$$$	I"	I 	I'3'9'9';';	I 	I 	I %$$
 T669lL  r,   c           	      2   |dz
  t          j        t          j        |            |                     dd                    z  |t          j        t          j        |           |                     dd                    z  z
  }|||z  }t          ||          S )Nr#   r+   i)rD   maximumlog1pnew_fulllogr   )r   r   r   r   r   s        r-   binary_cross_entropyrG  i  s     QJ%-TEDMM"d33  uyb$0G0GHHHID f}i000r,   c                     d}| ||z
  z  t          j        |d|z
  z  |          z  }|||z  }|t          j        j        k    r||                                z  }|S )Ng-q=r#   r   )rD   r   r"   r)   r   r   )rp   r   r   r   r   EPSILONr  s          r-   binary_cross_entropy_backwardrJ    sk     GD6M*U[T9JPW-X-X-XXF&IN((($**,,&Mr,   c                 v    t          j        t          j        |  |z                      }t          ||          S r<   )rD   rD  rl   r   )r   r   r   r   s       r-   soft_margin_lossrL    s3     ;uy%&1122Di000r,   c                     || z  t          j        ||z            dz
  z  }|t          j        j        k    r||                                z  }|S r`   )rD   r   r"   r)   r   r   )rp   r   r   r   r\   s        r-   soft_margin_loss_backwardrN    sN     +%v})E)E)IJJIN((($**,,.
r,   r$   otherpc                 @    t                               | |z
  |          S )N)rP  )r   r   )r   rO  rP  s      r-   distrR    s     99UU]a9(((r,   x1x2c                 .   |                      d                              dd          }t          j        |t          j                  }|                     d                              dd          }t          j        |t          j                  }t          j        |                     d          ||gd          }t          j        |||gd          }|                    |j                  }|	                    d          
                                S )Nr$   rW   Tmemory_formatr   )powr   rD   	ones_likecontiguous_formatr-  r   matmulmT	clamp_minsqrt)	rS  rT  x1_normx1_padx2_normx2_padx1_x2_r  s	            r-   _euclidean_distrf    s     ffQiimmB%%G_WE4KLLLFffQiimmB%%G_WE4KLLLF
)RVVBZZ&12
6
6C
)R)2
.
.CZZFA##%%%r,   input_sizesstartendstepc                 ^    |                      |          }t          j        || ||||          S r<   )	new_zerosrD   slice_scatter)rp   rg  rS   rh  ri  rj  r\   s          r-   slice_backwardrn    s2     &&{33Jz;UCNNNr,   r#   c                 d   ddl m} |                                 }|dk    rt          d          t	          j        |                                 |          }t          |                                           }t          |                                           }|dk    rt          d          ||nd}	||nt          j
        }
|	dk     r|	||         z  }	|
dk     r|
||         z  }
|	dk     rd}	n|	||         k    r||         }	 ||
t          j
        k              r	||         }
n|
|	k     r|	}
n|
||         k    r||         }
|                                 |	||         z  z   }|
|	z
  }||z   dz
  |z  ||<   ||xx         |z  cc<   | j        rt          d          |                     |||          S )Nr   statically_known_truez,slice() cannot be applied to a 0-dim tensor.zslice step must be positiver#   z<Slice decomposition for quantized tensors aren't implemented)%torch.fx.experimental.symbolic_shapesrq  rS   RuntimeErrorrI   r*  listr+  stridesysmaxsizestorage_offsetis_quantizedNotImplementedError
as_strided)r   rS   rh  ri  rj  rq  ndimsizesstrides	start_valend_valrx  lens                r-   slice_forwardr    s    LKKKKK88::DqyyIJJJ

 S
1
1CE4;;==!!Gqyy8999*I_cc#+G1}}U3Z	{{5:1}}			U3Z		#J	W344 *	9			5:		*((**Y-EEN
I
C*q.T)E#JCLLLDLLL ?!J
 
 	
 ug~>>>r,   c                 x    | j         |         dt          ffd} ||dd          } |||          }||fS )zn
    Normalize start and end such that both are in the range
    [0, x.get_size()[dim]] and start <= end.
    rT   c                 ^    | |S | dk     r| z   } t          t          | |          |          S rw   r   r   )valr   r   defaultdim_sizes       r-   
clamp_wrapz(_normalize_start_end.<locals>.clamp_wrap  s7    ;N77.C3sE??E***r,   r   )r  int)r7   rS   rh  ri  r  r  s        @r-   _normalize_start_endr    sk     ws|H+# + + + + + + Jua1--E
*S%8
4
4C#:r,   srcc           	         t          j        | j        |          }| j        |         }t	          | |||          \  }}t          | j                  }||z
  |dz
  z   |z  ||<   |                    |          }|dk    r ||k    r|dk    r|                                S d g|                                 z  }t          j
        || j                  }	|	|z
  |z  ||<   t          j        || j        t          j                  }
|dk    rt          j        |
|	|k              }
||k    rt          j        |
|	|k               }
|dk    rt          j        |
|	|z
  |z  dk              }
dg|                                 z  }d||<   |
                    |          }
t                               |
t                               ||
|d          |           S )Nr#   r   devicer  r   rW   )rI   r*  r|  r  r  rt  expandclonerS   rD   aranger  onesboollogical_andviewr   rm   _unsafe_masked_index)r   r  rS   rh  ri  rj  r  src_sizeindicesidxmask
mask_shapes               r-   rm  rm    s    
 S
1
1C{3H%eS%==JE3EK  H5[D1H-$6HSM
**X

CzzcXoo$!))yy{{'+fuyy{{&:G
,x
5
5
5C%KD(GCL:hu|5:FFFDzz se|44
h sSy11qyy et';q'@AAuyy{{"JJsO99Z  D::dD55c4!LLeTTTr,   indexc                 Z    |                      |          }t          j        || ||          S r<   )rl  rD   select_scatter)rp   rg  rS   r  r\   s        r-   select_backwardr  E  s.     &&{33J
KeDDDr,   offsetdim1dim2c                 \    |                      |          }t          j        || |||          S r<   )rl  rD   diagonal_scatter)rp   rg  r  r  r  r\   s         r-   diagonal_backwardr  L  s0    
 &&{33J!*k64NNNr,   input_dtypec                 F    | j         |k    r|                    |          }|S r<   )r   r>   )rp   r\   r  s      r-   _cast_grad_to_input_dtyper  U  s)     K'']];//
r,   outputc                     | |z  }||t          j        ||d          z  z
  }t          | ||                                          S NTr   )rD   r   r  
contiguous)rp   r  rS   r  new_grad_outputr\   s         r-   _softmax_backward_datar  ]  sY     "F*O 6EIS$- - - $ J %[*kJJUUWWWr,   c                     | t          j        |          t          j        | |d          z  z
  }t          | ||          S r  )rD   rl   r   r  )rp   r  rS   r  r\   s        r-   _log_softmax_backward_datar  o  sM     uy0059d4 4 4   J %[*kJJJr,   c                     | |dz  z   ||dz
  z  z
  }t          t          j        t          j        |          } |d||                              d          } |d||z  |                              d          }	||	z   S )z/Utility function to implement im2col and col2imr$   r#   r   r  r   rW   )r   rD   r  int64rY   )
input_dkernel_d
dilation_d	padding_dstride_dr  blocks_d	arange_kwblocks_d_indiceskernel_grids
             r-    _im2col_col2im_indices_along_dimr  {  s     Q&x!|)DDHEKGGGI !yHh77AA!DD )Ax*4jAAKKBOOK k))r,   kernel_sizedilationpaddingru  c           
         t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            dd} |d            |d	            |d
d            |d           | j        t                    }t          j        |dv o t	          d dd          D                       fd           t          d t          dd                    D                       t          j        t	          d D                       fd           |dk    }|s|                     d          } | j        \  }}	}
}\  }}\  }}\  }}\  }}t          |
||||| j	                  }t          |||||| j	                  }t          j        | ||||f          }|                    d                              d          }|d d d d ||f         }|                    dddddd          }|                    d          }|                    d          }|                    ||	|z  |z  ||z            }|s|                    d          }|S )Nr$   c                      dS )Nz"im2col(): only 2D kernel supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  s    0T r,   c                      dS )Nz$im2col(): only 2D dilation supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  s    -S r,   c                      dS )Nz#im2col(): only 2D padding supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  s    ,Q r,   c                      dS )Nz"im2col(): only 2D stride supportedr+   r+   r,   r-   r   zim2col.<locals>.<lambda>  s    +O r,   Tc                      |rt          d  D                       nt          d  D                       }t          j        | fd           d S )Nc              3   "   K   | ]
}|d k    V  dS r   Nr+   r6   rP  s     r-   	<genexpr>z1im2col.<locals>.check_positive.<locals>.<genexpr>  &      ((Q1q5((((((r,   c              3   "   K   | ]
}|d k    V  dS r  r+   r  s     r-   r  z1im2col.<locals>.check_positive.<locals>.<genexpr>  &      ;R;RqAF;R;R;R;R;R;Rr,   c                       d  S Nz& should be greater than zero, but got r+   param
param_names   r-   r   z0im2col.<locals>.check_positive.<locals>.<lambda>      ZVVuVV r,   r  rD   r   r  r  strictconds   ``  r-   check_positivezim2col.<locals>.check_positive  q    ,2Rs((%((((((;R;RE;R;R;R8R8RVVVVV	
 	
 	
 	
 	
r,   r  r  r  Fr  ru  r   r@  c              3   "   K   | ]
}|d k    V  dS r  r+   r6   ds     r-   r  zim2col.<locals>.<genexpr>  &      ::!qAv::::::r,   r   c                  (    dt                      S )NzmExpected 3D or 4D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: tupler  s   r-   r   zim2col.<locals>.<lambda>       <-25\\< < r,   c              3   V   K   | ]$\  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z   V  %dS )r#   r$   Nr+   r6   r   paddilkersts         r-   r  zim2col.<locals>.<genexpr>  sd        "Cc3 	
S1s7]SC!G_,q0R77     r,   rX  c              3   "   K   | ]
}|d k    V  dS r  r+   )r6   cs     r-   r  zim2col.<locals>.<genexpr>  s&      ''aAE''''''r,   c                  X    dt          dd                     d d  d d d dS )	Nz!Given an input with spatial size rX  , kernel_size=, dilation=
, padding=	, stride=z9, the calculated shape of the array of sliding blocks is z*, but its components must be at least one.r  )r  r  output_sizer  r  ru  s   r-   r   zim2col.<locals>.<lambda>  s|     FE%*4E4E F F"F F/7F FF F%+F F 	F F F r,   r@  r   rW   r#   r      T)rD   r   r  r  r  r  ziprY   r  r  r   r  permuter+  r$  squeeze)r   r  r  r  ru  r  r|  batched_input	batch_dimr%  input_hinput_wstride_hstride_w	padding_h	padding_w
dilation_h
dilation_wkernel_hkernel_wblocks_row_indicesblocks_col_indicespadded_inputr  num_blocks_rownum_blocks_colr  r  s    ````                     @@r-   im2colr    s    
L[!!Q&(T(TUUU	LX!#%S%STTT	LW"$Q$QRRR	LV!#O#OPPP
 
 
 
 N;...N8Z(((N8Yu5555N68$$$KEu::D	L:3::uRSSz:::::	< 	< 	< 	<  
   &)"##J;'
 '
    K 
L'';'''''	F 	F 	F 	F 	F 	F 	F 	F 	F   AIM #""/4{,I{GWHh"Iy%J
$Hh9:y(EL  ::y(EL  5Iy) LMML+55b99CCBGG!!!QQQ 24FFGF^^Aq!Q1--F',,Q//N',,Q//N^^;)H4n~6U F  #""Mr,   r  c           
         t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            t          j        t                    dk    d            d d} |d	            |d
            |dd            |d            |d           | j        t                    }t          j        |dv o t	          d dd          D                       fd           d         d         z  }t          j        d         |z  dk    fd           d t                    D             }	|	d         |	d         z  t          j        d         k    fd           t          j        dk    fd           |dk    }
|
s|                     d          } | j        \  }}\  }}\  }}\  }}\  }}|                     d         d         |z  gz   |	z             } |                     dddddd          } t          |||||| j
                  }t          |d          }t          |||||| j
                  }d t                    D             }|                     d         d         t                    z  g|z             }d d ||f}t                              ||| d          }t!          j        || | | | f          }|
s|                    d          }|S )!Nr$   c                      dS )Nzonly 2D output_size supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  s    0O r,   c                      dS )Nzonly 2D kernel supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  s    0J r,   c                      dS )Nzonly 2D dilation supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  s    -I r,   c                      dS )Nzonly 2D padding supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  s    ,G r,   c                      dS )Nzonly 2D stride supportedr+   r+   r,   r-   r   zcol2im.<locals>.<lambda>  s    +E r,   Tc                      |rt          d  D                       nt          d  D                       }t          j        | fd           d S )Nc              3   "   K   | ]
}|d k    V  dS r  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r,   c              3   "   K   | ]
}|d k    V  dS r  r+   r  s     r-   r  z1col2im.<locals>.check_positive.<locals>.<genexpr>  r  r,   c                       d  S r  r+   r  s   r-   r   z0col2im.<locals>.check_positive.<locals>.<lambda>  r  r,   r  r  s   ``  r-   r  zcol2im.<locals>.check_positive  r  r,   r  r  r  Fr  ru  r  )r$   r   c              3   "   K   | ]
}|d k    V  dS r  r+   r  s     r-   r  zcol2im.<locals>.<genexpr>   r  r,   rX  c                  (    dt                      S )NzmExpected 2D or 3D (batch mode) tensor for input with possible 0 batch size and non-zero dimensions, but got: r  r  s   r-   r   zcol2im.<locals>.<lambda>  r  r,   r   r#   c                       dd          d  S )Nz|Expected size of input's first non-batch dimension to be divisible by the product of kernel_size, but got input.shape[-2] = rX  z and kernel_size=r+   )r  r  s   r-   r   zcol2im.<locals>.<lambda>  s(     %=B2Y% %"% % r,   c                 N    g | ]"\  }}}}}d |d|z  z   ||d z
  z  z
  d z
  |z  z   #S r#   r$   r+   r  s         r-   r9   zcol2im.<locals>.<listcomp>  sV       "Cc3 	
S1s7]SC!G_,q0R77  r,   rW   c                  @    d d d d d d  dd          d	S 
NzGiven output_size=r  r  r  r  z , expected input.size(-1) to be 	 but got rW   .r+   Lr  r  r  r  r  ru  s   r-   r   zcol2im.<locals>.<lambda>  |     B[ B B B BB B(/B B:@B B)*B B5:2YB B B r,   c                  @    d d d d d d  dd          d	S r  r+   r  s   r-   r   zcol2im.<locals>.<lambda>  r!  r,   r   r@  r  c                 $    g | ]\  }}|d |z  z   S r$   r+   )r6   orP  s      r-   r9   zcol2im.<locals>.<listcomp>6  s$    JJJ1!a!e)JJJr,   
accumulater  )rD   r   r  r  r  r  rY   r$  r  r  r  r[   rl  prodr   _unsafe_index_putr   r  r  )r   r  r  r  r  ru  r  r|  prod_kernel_sizecolr  out_hout_wr  r  r   r  r  r  r  r  indices_rowindices_coloutput_padded_sizer  r  r   r  s    `````                    @@r-   col2imr1    s    
L[!!Q&(O(OPPP	L[!!Q&(J(JKKK	LX!#%I%IJJJ	LW"$G$GHHH	LV!#E#EFFF
 
 
 
 N;...N8Z(((N7Ie4444N68$$$N;...KEu::D	L:3::uRSSz:::::	< 	< 	< 	<  
 #1~A6	Lb	$$)	% 	% 	% 	% 	%   &)(K'
 '
  C 	AQA	Lb	Q	B 	B 	B 	B 	B 	B 	B 	B 	B 	B   
L	A	B 	B 	B 	B 	B 	B 	B 	B 	B 	B   AIM #""KELE5Hh"Iy%J
$Hh MM58U1X1A%AB[PSVVWWEMM!Q1a++E2xY%, K $K33K2xY%, K KJK0I0IJJJ__	q58tK00014FF F {K
0C##FC4#HHFU6YJ
YJ
KLLF #""Mr,   r  c                     | |                     |           |z  z                      t          j        |                     }|S NrV  )type_asr  rI   r   )rp   r  rr   rN   s       r-   native_dropout_backwardr5  C  sJ     
[11E9	:AA1+>> 	B 	 	A Hr,   
input_size	dimensionr+  c                 <   t          |          dk    rt          j        | d          S t          j        t          |          |          }t          j        ||         | j        t          j                  }|                    d||          	                                }| 
                    d|dz             	                    ||dz             } |                     |          }d|z  |fz   }t                              ||| d                                          S )Nr   r  rW   r#   r<   Tr&  )r  rD   squeeze_copyrI   r*  r  r  int32unfoldflattenmovedimrl  r   r)  r  )	r   r6  r7  r+  rj  rS   r  r\   r  s	            r-   unfold_backwardr>  R  s    
 :!!$***

 Z)
<
<C
,z#t{%+
N
N
NC
**Qd
#
#
+
+
-
-C<<C!G$$,,S#'::D 
++JcMSF"E!!*eTd!KKVVXXXr,   epsc           
      >   |A|}d|z
  }t          j        t          j        ||k    ||k              | |d|z
  z  z  d          S t          j        t          j        |dk    |dk              | |d|z
  z  z  |                    dt	          d                              S )Nrk   r   r+   nan)rD   rm   r  rE  r  )rp   r   r?  lohis        r-   logit_backwardrD  e  s    
 2X{dbj$"*5543:./
 
 	
 {dck43;7743:./MM"eEll++
 
 	
r,   trainc                 ~    |r(|dk    r"t                               | ||          d         S |                                 S rw   )r   native_dropoutr  )r   rP  rE  s      r-   dropoutrH  z  s?      a""5!U33A66{{}}r,   out0out1c                 n   |r|dk    r|dk    r4t          j        |           t          j        | t           j                  fS | j        j        st          d          t          j        |           |k    }|| z  t          dd|z
  z            z  }||fS | t          j        | t           j                  fS )Nr   r#   r   z?result type Float can't be cast to the desired output type Longrk   )	rD   r  r  r   is_floating_pointrs  	rand_liker  rZ  )r   rP  rE  	bool_maskress        r-   rG  rG    s      Aa66$U++U-=e5:-V-V-VWW{, 	Q   OE**Q.	%%sQw"8"88YuuEJ???@@r,   half_to_floatc                    ddl m} |                                 } |r| j        t          j        k    sJ t          j        | t          j        j	                  \  }}| 
                    |          }  ||                                 dk              rt	          j        |           }n.t	          j        | |d          }t	          j        | |z
            }|t	          j        ||d          z  }|s|
                    |          }|S Nr   guard_or_falser:   T)r   )rr  rT  r  r   rD   halfrI   rJ   ELEMENTWISE_TYPE_PROMOTION_KINDDEFAULTr>   r   rl   amaxr   )	r7   rS   rP  rT  r?   rB   unnormalizedx_maxr  s	            r-   _softmaxr\    s    EDDDDD 	
A %w%*$$$$&+&>	uDL' ' '#| 	
A~aggii1n%% ,y||
1c4000yU++EIlCFFFFF )<((Mr,   )r  c                    ddl m} |                                 } |r| j        t          j        k    sJ t          j        | t          j        j	                  \  }}| 
                    |          }  ||                                 dk              r| }nt	          j        | |d          }| |z
  }t	          j        t	          j        t	          j        |          |d                    }||z
  }	|s|	
                    |          }	|	S rR  )rr  rT  r  r   rD   rV  rI   rJ   rW  rX  r>   r   rY  rF  r   rl   )
r7   rS   rP  rT  r?   rB   shiftedr[  shifted_logsumexpr  s
             r-   _log_softmaxr`    s    EDDDDD 	
A %w%*$$$$&+&>	uDL' ' '#| 	
A~aggii1n%% 
1c4000e)	%)EIg,>,>T"R"R"RSS((F )<((Mr,   rW   r  padding_idxscale_grad_by_freqsparsec                     |                                  dk    s
J d            |j        dk    r8|                     d|          }|j        dk    r|                    d          }|S | |         S )Nr$   z'weight' must be 2-Dr#   r   )rS   r|  index_selectr  )r   r  ra  rb  rc  r   s         r-   	embeddingrf    ss     ::<<14|q!!!W--<1++a..C
gr,   num_weightsc                    t          j        | t           j        j                  \  }}|                     |          } t          |t          j                  }|ri|                    |f          }t          j	        |          }t                              ||g|d          }||         }	| |	                    d          z  } t          ||k    | j                  }
|                     |
d          }|                     |f| j        |j        d          z             }t                              ||g|d                              |          S )NrU  Tr&  rW   r   )rI   rJ   rW  rX  r>   r   rD   longrl  rZ  r   r)  rY   r[   r|  masked_fillr  )rp   r  rg  ra  rb  r?   rB   countsr  grad_weights_scaler  r   grad_weights                r-   embedding_dense_backwardrn    sN    ',&>)N)V' ' '#| ..!233K%guz::G E""K>22w''''	4D'QQ#G_!$6$@$@$D$DDW3[5EFFD""4++D''	*7<>>:: K !!+y$4!PPSS  r,   c                     d}| D ]}||z  }|S r`   r+   )r7   rN   is      r-   r(  r(    s&    	A  	QHr,   tensors
num_chunksc                    g }| D ]}|                                 }||         |z   dz
  |z  |z  }|||         k    r>dgdz  |j        |z
  dz
  z  d|||         z
  gz   }t                              ||d          }|d |         t	          j        |dg          z   }|                    |                    |                     |S )Nr#   r   r$   rW   )r+  r|  r   constant_pad_ndrD   Sizeappendr$  )	rq  rS   rr  padded_tensorstensortensor_sizepad_along_dimr  	view_sizes	            r-   
_pad_chunkr|    s    
 N 9 9kkmm$S)J6:zIJVK,,,#'V[3.23C 007 C ))&#q99F%
J3C(D(DD	fnnY778888r,   c                 F    | d         j         }| D ]}|j         |k    r dS dS )Nr   FTr|  )rq  r|  rx  s      r-   have_same_ndimsr    s:    1:?D  ;$55 4r,   c                     | d                                          d |         }| D ]6}t          j        |                                 d |         |k    d            7d S )Nr   c                      dS )NzG_chunk_cat expects same sizes of 0,...,dim-1 dimensions for all tensorsr+   r+   r,   r-   r   z+leading_dimension_matches.<locals>.<lambda>%  s    ] r,   )r+  rD   r   )rq  rS   leading_dim_sizesrx  s       r-   leading_dimension_matchesr     sq    
))$3$/ 
 
KKMM$3$#44]]	
 	
 	
 	

 
r,   c                    t          j        |dk    d            t          j        t          |           dk    d            | d         j        }| d         j        }| D ]l}t          j        |                                dk    d            t          j        |j        |k    d            t          j        |j        |k    d            mt          |           r.t          j        | d         	                                |          }n>t          j        |dk    d            | D ]!}t          j        ||j
        k     d	            "t          | |           |S )
Nr#   c                      dS )Nz&_chunk_cat expects positive num_chunksr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>.  s    *R r,   r   c                      dS )Nz0_chunk_cat expects a non-empty input tensor listr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>0  s    "T r,   c                      dS )Nz#_chunk_cat expects non-empty tensorr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>5  s    1V r,   c                      dS )Nz8_chunk_cat expects all input tensors with the same dtyper+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>8      N r,   c                      dS )Nz8_chunk_cat expects all inputs tensors on the same devicer+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda><  r  r,   c                      dS )NzK_chunk_cat expects non-negative dim when input tensors have different ndimsr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>C  s    a r,   c                      dS )Nz3_chunk_cat expects dim < ndim for all input tensorsr+   r+   r,   r-   r   z._preprocess_chunk_cat_inputs.<locals>.<lambda>H      M r,   )rD   r   r  r   r  r   r  rI   r*  rS   r|  r  )rq  rS   rr  expected_dtypeexpected_devicerx  s         r-   _preprocess_chunk_cat_inputsr  )  s   
 
Lq"R"RSSS	LGqTT   QZ%Naj'O 	
 	
V\\^^a')V)VWWWLN*NN	
 	
 	
 	M_,NN	
 	
 	
 	
 w $WQZ^^%5%5s;;1Haa	
 	
 	
  	 	FLfk!MM    gs+++Jr,   r   c                     t          | ||          }t          | ||          }|t          j        ||dz             S t          j        ||dz   |           |S )Nr#   )r   )r  r|  rD   r-  )rq  rS   rr  r   rw  s        r-   
_chunk_catr  N  sa     'wZ
@
@Cj99N
{yq111	.#'s3333
r,   split_sizesc                     t                               | ||          }|d |D             S t          ||          D ],\  }}t          ||j                   t          ||d           -d S )Nr   c                 N    g | ]"}|                     t          j                   #S )rV  )r  rD   r[  )r6   ss     r-   r9   z)split_with_sizes_copy.<locals>.<listcomp>j  s)    OOO1e&=>>OOOr,   Tr  )r   split_with_sizesr  r   r  r   )r   r  rS   r   splitsr  splits          r-   split_with_sizes_copyr  _  s     ""4#">>F
{OOOOOO f-- 	N 	NMFEfek222UFMMMMMtr,   
split_size.c                 D    t           j                            | ||          S r<   )r   r  r   )r   r  rS   s      r-   unsafe_splitr  r  s    :UJ444r,   c                 D    t           j                            | ||          S r<   )r   r  r  )r   r  rS   s      r-   unsafe_split_with_sizesr  w  s      ((SAAAr,   c                    | j         }||         }dk    r|dk    sJ |                                 fS |z   dz
  z  }ddlm}  ||          }fdt	          |          D             }|z  |z
  z
  |d<   t          j        | ||          S )Nr   r#   )	guard_intc                     g | ]}S r+   r+   )r6   rp  r  s     r-   r9   zsplit.<locals>.<listcomp>  s    555!:555r,   rW   )r  detachrr  r  rX   rD   r  )r   r  rS   rg  r  chunksr  r  s    `      r-   r  r  ~  s    *K3HQ1}}}}#a'J6F @?????YvF5555uV}}555K J$7($BCKO;t[#...r,   tensor_indices_or_sectionsc                 F   |j         j        dk    sJ |j        t          j        k    sJ |                                t          j        dk    pdk    fd           dk    rA|                                }t          |t                    sJ | 
                    ||          S t          }t          j                                        x}r|j        x}r|j        } |            5  d |D             }d d d            n# 1 swxY w Y   | 
                    ||          S )Ncpur#   r   c                      d  dS )Nz{tensor_split expected tensor_indices_or_sections to be a zero-dimensional or one-dimensional tensor, but got a tensor with z dimsr+   )	split_dims   r-   r   zAtensor_split_tensor_indices_or_sections_py_impl.<locals>.<lambda>  s      M<EM M M r,   c                 6    g | ]}|                                 S r+   )item)r6   rp  s     r-   r9   zCtensor_split_tensor_indices_or_sections_py_impl.<locals>.<listcomp>  s     DDDAqvvxxDDDr,   )r  typer   rD   r  rS   r   r  r5   r   tensor_splitr   _guardsdetect_fake_mode	shape_envignore_fresh_unbacked_symbols)	r   r  rS   sectionsctx	fake_moder  r  r  s	           @r-   /tensor_split_tensor_indices_or_sections_py_implr    s    &,1U::::%+u{::::*..00I	LQ()q.	M 	M 	M 	M  
 A~~-2244(G,,,,,  3///77999I 	:",,I	: 9C
 SUU 	E 	EDD)CDDDG	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E 	E   #...s   )DD	Dmat1mat2c                     |                                  s2|                                 st          |          }t          |          }|t          j        ||          z  }|dk    r|S ||| z  z   S rw   )rL  
is_complexr  rD   mm)r   r  r  rg   rq   r   s         r-   addmmr    st     !!## DOO,=,= 4yyE


%(4&&
&Cqyy
 r,   use_geluc                     t          | ||||          }|r=| j        rt                              |d          S t                              |          S t                              |          S )Nr   )r   )r  is_cudar   gelurelu)r   r  r  rg   rq   r  r   s          r-   _addmm_activationr    sa     dD$
.
.C "< 	"99Sf955599S>>!99S>>r,   vecc                    |                                  s2|                                 st          |          }t          |          }|t          j        ||          z  }|dk    r|S |                                dk    r|| z  S ||| z  z   S rw   )rL  r  r  rD   mvr   )r   r  r  rg   rq   r   s         r-   addmvr    s     !!## DOO,=,= 4yyE


%(4%%
%Cqyy

yy{{ad{r,   r   rstdgammaNCHxWgroupoutput_maskc
           	      	   t          j        | ||d           t          j        || d           t          j        |d           t          j        |                                z  z  k    fd           t          j        j        fk    fd           t          j        d u p                                k    fd           z  }
t          j        |
z  k    fd           t          j        | |                                        	                    dg          }|                               	                    dg          }d }d }d }|	d	         rdd
|
z  z  }t          j        |
                    d	                                        |
          	                    d          }t          j        |
                    d	                                        |
          	                    d          }t          j        |
                    d                              d|
                    }n|                    |
          	                    d          }|                    |
          	                    d          }t          j        |
                    d          t          j        d|
f|j                            }|z  |z
  |z  |z  |z  |z  }| z  ||z  |z  z
  }|
                    d          }t          |d          }t          |d          }t          j        |                     |
          |          t          j        |                    |
          |          z   |z   }|                    |j                                      |j                  }|	d         r|                    |
          |                    |
          
                    d          z  z
  |
                    d          z  	                    d	g                                        }|	d         r|	                    d	g          }|||fS )NF)allow_cpu_scalar_tensorsc                      d z  z   dS )NzExpect input to have z	 elementsr+   )r  r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>	  s    >A>>> r,   c                  $    d  d dj          S )NzExpect mean to have shape (, z
, but got r  )r  r  r   s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s"    PaPP5PPDJPP r,   c                  @    d  d                                 nd S )NzExpect gamma to have z elements but got rW   )r   )r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s*    iiieN_U[[]]]egii r,   c                      d  d S )NzExpect number of channels z, to be evenly-divisible by number of groups r+   )r  r  s   r-   r   z,native_group_norm_backward.<locals>.<lambda>  s    cQcc\acc r,   r$   r   r   rk   rW   r#   r  r@  )rI   check_same_devicecheck_same_shaperD   r   r   r  r   r  r   rY   r$  r  r  r[   r>   r   )rp   r   r   r  r  r  r  r  r  r  cpgdsdbd_inputd_gammad_biasr  ds_valdb_valc1c2c3s     ` `````             r-   native_group_norm_backwardr    s    
UD$    
5+NNNN	4FFFF	LQ$>>>>>>   
L
q%j PPPPPP   
L+!+iiiii  
 u*C	L	S5[ccccc   
;	&	&	+	+Aq#	6	6	:	:s	:	C	CB			!Q	$	$	(	(aS	(	1	1B $G $G#F1~ ?39Yr5??1#5#566>>q%MMQQRSTTFYr5??1#5#566>>q%MMQQRSTTFr""a,, BB
 ZZ5#..22155FZZ5#..22155Fr""
Auc?4;??? B tmf$,t3d:Q>S4Z&4-!++\\"r1%%r1%%Ik))!UC==rBBiaS992>>? 	
 //%+..11%+>>1~ 
 E3''"''!UC*@*@4>>RTCUCU*UU..$$% SaSS\\WQZZ 	 1~ !QCWf%%r,   out2c
                    t          | |||||||||	
  
        }|
||f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )r  	enumerater   r  r   )rp   r   r   r  r  r  r  r  r  r  rI  rJ  r  r  r\   rp  rN   s                    r-   native_group_norm_backward_outr  M  s    " (UD$q!S% F d#J&!! Q Q1=jmQW555Q
14PPPPr,   c                 4    | |                      |          S | S r<   r>   )r7   r   s     r-   _maybe_castr  j  s    }ttE{{Hr,   grad_outnormalized_shapebiasc                   " |j         }|                                }	t          j        |j                  ""fd| |||fD             \  }
}}}|
J |	t          |          z
  }||d          }|d |         }g }g }t          |	          D ]3}||k    r|                    |           |                    |           4t          |          }t          |          }ddl	m
}  ||dk              s ||dk              rl|d         r|                    |          nd |d         r|                    ||d                    nd |d         r|                    ||d                    nd fS t          ||                                          }t          ||                                          }|J ||z
  |z  }||
|z  }n|
}||z  }t          j        ||d          }t          j        ||          }t          j        ||d          }t          j        ||          }||z
  |z
  }d }d } d }!|d         r||z  |z  }|d         r4|2t          |          dk    rt          j        |
|z  |d          } n|
|z  } |d         r@|>t          |          dk    rt          j        |
|d          }!n|
                                }!t#          ||j                  t#          | ||j        nd           t#          |!||j        nd           fS )Nc              3   b   K   | ])}|!|                     t          j                  n|V  *d S r3  )r>   rD   r[  r6   r7   r?   s     r-   r  z-native_layer_norm_backward.<locals>.<genexpr>  sZ       9 9  = 	
e.EFFF9 9 9 9 9 9r,   r   rp  r#   r$   TF)r  rS   rI   get_computation_dtyper   r  rX   rv  r(  rr  rq  rl  r[   rD   r   r   r  r  )#r  r   r  r   r  r   r  r  input_shape
input_ndimgrad_out_cast
input_castweight_cast	bias_castaxis
inner_dims
outer_dimsinner_dim_indicesouter_dim_indicesrp  r  Mrq  x_hat
grad_x_hatabr  r  r  rO   r  d_weightr  r?   s#                                     @r-   native_layer_norm_backwardr  q  s    +KJ3EK@@9 9 9 9 E640	9 9 95M:{I $$$,---DTUU#JUdU#J#%#%: ( (99$$Q''''$$Q''''ZAZAKKKKKKQ!V$$ 
(=(=a1f(E(E 
,7NDEOOK(((3>q>KEOOK.///t3>q>KEOOK.///t
 	

 T:>>#3#344DT:>>#3#344D!!!$$&E"[0

"
QA	*/66A	:u	%	%B	2($	/	/B	5"		BEBJE $G!%H#F1~ %!8u$1~ -+1 !!A%%y!68I5QQHH$u,H1~ +)/ !!A%%Y}.?GGFF"((**F 	GU[))Hf.@flldKKF$*:DJJEE r,   c          
          t          | |||||||          }||	|
f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )r  r  r   r  r   )r  r   r  r   r  r   r  r  rI  rJ  r  r  r\   rp  rN   s                  r-   native_layer_norm_backward_outr    s     (%)4vt[ F d#J&!! Q Q1=jmQW555Q
14PPPPr,   c                    g }t          t          |                    D ]/}|                    |                                 |z
  dz
             0t	          j        | j                  }|                     |          }|b|t          j	        t          j
        fv r$t          j        t          j	                  j        }n&t          j        t          j                  j        }n|}t          j        t          j        j        j                            t          j        |d                              |d          |                    }	|                    |	          }
||
                    |          }
| j        p
|d uo|j        }t	          j        |           }|t          j        t          j        fv }|s*|s(|
                                }
|	                                }	|
                    |           }||	fS )Nr#   r$   Tr   )rX   r  rv  rS   rI   r  r   r>   rD   r   r   finfor?  r   rsqrtopsr   addScalarrY  r   r   	is_nestedr   channels_lastchannels_last_3dr  r4  )r   r  r   r?  dims_to_reducerp  r?   upcasted_inputeps_valrqrst_inputupcasted_resultr
  rW  is_channels_lastr  s                  r-   _fused_rms_normr    s    !#N3'(()) 3 3eiikkAo12222 3EK@@XX/00N { @@@k%-004GGk%-004GG+ 		!!Ina((--.$-OOQX	
 	
 K %((55O)--f55 LF$$6$K6;KI/66M$) 
  /- /)4466!,,.. $$U++F;r,   c                    |j         }|                                }t          j        |j                  }|                     |t          j                  }	|                    |t          j                  }
|!|                    |t          j                  nd }|	J |t          |          z
  }||d          }|d |         }g }g }t          |          D ]3}||k    r|
                    |           |
                    |           4t          |          }t          |          }ddlm}  ||dk              s ||dk              rF|d         r|                    |          nd |d         r|                    ||d                    nd fS t          ||
                                          }||	|z  }n|	}d }d }|
|z  }|d         r(t          j        ||z  |d          }|||z  |z  z
  |z  }|d         r4|2|	|z  }t          |          dk    rt          j        ||d          }n|}t#          ||j                  t#          ||j                  fS )NrV  r   rS  r#   Tr   F)r  rS   rI   r  r   r>   rD   r[  r  rX   rv  r(  rr  rT  rl  r[   r   r  )r  r   r  r  r   r  r  r  r?   r  r  r  r  r  r  r  r  rp  r  r  rT  r  r  r   r  sum_vald_weight_full_shapes                              r-   _fused_rms_norm_backwardr    s    +KJ3EK@@KK)@    M +5;RSSJ  			#53J	KKK 
 $$$,---DTUU#JUdU#J#%#%: ( (99$$Q''''$$Q''''ZAZADDDDDD~a1f 
Q!7!7 
,7NDEOOK(((3>q>KEOOK.///t
 	

 T:>>#3#344D"[0

"
 $G!%HE1~ >)EJ.4EtTTTg 55=1~ ++1+e3 !!A%%y#):E  HH +H 	GU[))Hek** r,   running_meanrunning_varmomentum
functionalc	                    dgt          t          d|                                                     z   }	t          j        | j                  }
|}|}|r$t          j        | j                  }
|                     |
          }t          j        ||	dd          \  }}t          j	        ||z             }| |z
  |z  }t          j
        ||	          }t          j
        ||	          }|%||z  d|z
  |z  z   }|s|                    |           |g|                                 | j        d         z  }t          j
        ||	          }|||dz
  z  z  }||z  d|z
  |z  z   }|s|                    |           n||J |                    |
d          }|}|                    |
d          }|}|}dt          j        ||z             z  }| j        j        dk    r|}|}n*|                     d	          }|                     d	          }t%          ||                                 dz
            }t%          ||                                 dz
            }| |z
  |z  }|>|                                }t%          ||                                 dz
            }||z  }|>|                                }t%          ||                                 dz
            }||z   }| j        j        dk    r6|                    | j                  }|                    | j                  }|                    | j                  ||||fS )
Nr   r$   r   T)rS   
correctionr   r#   )r   r   r  r   )rt  rX   rS   rI   r  r   r>   rD   var_meanr  r  copy_r   r  r_  r  r  rl  r[   r<  )r   r   r  r  r  r   r  r?  r  reduction_dimsr?   new_running_meannew_running_var	input_acc
biased_varr   r  r  	save_mean	save_rstdnsqueezed_varunbiased_varinvstds                           r-   native_batch_norm_helperr,  b  sP    S4a 5 5666N3EK@@#!O +)!7DDHH#4H55	 >>a
 
 

D {:+,,$,$&M$77	M$77	#')3q8||6SS 5""#3444"A.A !=^DDL'1A;7L&5X8TTO 3!!/222'K,C,C,C#->TJJ'!nn+<4nHH%ejs!2334<%%$III--I--I uyy{{Q77"6599;;?;;$,&(!!"6599;;?;;&||~~ uyy{{Q77$|E!!LLu{L33	LLu{L33				$$ r,   r&  save_invstdc                 F    t          | |||||||d	  	        \  }}	}
}}||	|
fS NFr,  r   r   r  r  r  r   r  r?  r  r&  r'  rZ   s               r-   native_batch_normr2    sB     *Bvt\;(CQV* *&FIy!Q 9i''r,   c           
         |!|t                               | |||||          S |t          d          |t          d          |r!t                               | |||||||          S t                               | ||||||          S )Nz`running_mean is None, but running_var is provided. They should both be None or both be provided.z`running_var is None, but running_mean is provided. They should both be None or both be provided.)r   _native_batch_norm_legitrs  $_native_batch_norm_legit_no_training)r   r   r  r  r  r   r  r?  s           r-   native_batch_norm_decompositionr6    s      3,,648S
 
 	
 <
 
 	
 <
 
 	
  
,,64{HhPS
 
 	
 8864{Hc
 
 	
r,   c                 F   |                      |          }||z   dz
  |z  dk    rP|dk    rJfd|D             }|z  |z
  z
  ||dz
  <   t          j        j        j                            | ||          S t          j        j        j                            | |          S )Nr#   r   c                     g | ]}S r+   r+   )r6   rZ   r  s     r-   r9   z(unsafe_chunk_py_impl.<locals>.<listcomp>  s    222az222r,   )r+  rD   r  r   r  r  r  r   )rx  r  rS   r  r  r  s        @r-   unsafe_chunk_py_implr9    s    {{3HV#a'F2JQ8q==22226222",
V0Ch0N"OFQJy~5==fkSVWWW9>&--fj#FFFr,   c           
      N    t           j                            | ||||d||          S r/  )r   r4  r  )r   r   r  r  r  r  r?  s          r-   r5  r5    s7     (00	 	 	r,   c                 F    t          | |||||||d	  	        \  }}	}
}}||	|
fS r/  r0  r1  s               r-   r4  r4    sB     *Bvt\;(CQV* *&FIy!Q 9i''r,   c                 F    t          | ||d d |||d	  	        \  }}}}	}	|||fS r/  r0  )
r   r   r  r   r  r?  r  r&  r'  rZ   s
             r-   !_native_batch_norm_legit_no_statsr=  (  sA     *BvtT48S%* *&FIy!Q 9i''r,   c                 z    t          | |||||||d	  	        \  }}	}
}}|
J d            |
J d            ||	|
||fS )NT#new_running_mean should not be None"new_running_var should not be Noner0  )r   r   r  r  r  r   r  r?  r  r&  r'  r"  r#  s                r-   #_native_batch_norm_legit_functionalrA  7  sw    " 	!vt\;(CQU	 	 '')N'''&&(L&&&9i)9?JJr,   c           	         t           j                            | ||||d|          }d}|t           j        j        j        k    r t           j                            | |          }t          j        |t           j        | j        | j	                  S )a  
    Return a reserve tensor for batch norm, used only by cudnn to pass forward state to the
    backward pass. This is needed for `_batch_norm_with_update` and `_batch_norm_no_update`,
    which support a variety of backends including cudnn. We create this tensor here to get
    the correct shape in the traced graph if we detect that will call the cudnn kernel,
    and rely on DCE to avoid materializing this tensor.
    Tr   )r   layoutr  )
rD   _C_select_batch_norm_backend_BatchNormBackendCudnn(_get_cudnn_batch_norm_reserve_space_sizeemptyuint8rC  r  )	r   r   r  r  r  r?  r   backendreserve_sizes	            r-   _get_batch_norm_reserve_tensorrM  P  s      h11vt\;c G L%(,222xHH8
 
 ;EKU\   r,   c                 t    t          | ||||d||d	  	        \  }}}	}
}
t          | |||||d          }|||	|fS )NTFr   r,  rM  r   r   r  r  r  r  r?  r  r&  r'  rZ   reserves               r-   _batch_norm_with_updaterS  m  sp     *B
* 
*&FIy!Q -vt\;d  G 9i00r,   c                     t          | ||||d||d	  	        \  }}}	}
}t          | |||||d          }|

J d            |
J d            |||	||
|fS )NTrO  r?  r@  rP  )r   r   r  r  r  r  r?  r  r&  r'  new_rmnew_rvrR  s                r-   "_batch_norm_with_update_functionalrW    s      	!vt\;hT	 	 -vt\;d  G DCIy'66BBr,   c                 t    t          | ||||d||d	  	        \  }}}	}
}
t          | |||||d          }|||	|fS )NFrO  rP  rQ  s               r-   _batch_norm_no_updaterY    sp     *B
* 
*&FIy!Q -vt\;e  G 9i00r,   c                     |J t          j        |           |k                         t           j                  }|                    |           | z  d|z  z  }||fS )Nr   rk   )rD   rM  r>   rJ  r4  )r   rP  	generatorr  rO  s        r-   _fused_dropout_decompositionr\    s`     OE""Q&***==D
,,u


%q
1C;r,   )r   rC  r  
pin_memorynon_blockingrW  r  r]  r^  rW  c                   |r|t           j        k    s
J d            |r
J d            t          | t           j        t          t
          t          t          f          sJ |4|2|0t          | t           j                  r|                                 S | S d}t          | t           j                  r| }nt          j	        |           }|[||j
        k    rP|-|j        dk    r"t           j                            ||          }d}t           j                            |||          }|$|s"t           j                            ||          }d}|t          j        ||          S |S )NTODOFr  TrV  )rD   stridedr5   r   r  r  r  complexr  scalar_tensorr  r  _primsconvert_element_type
device_put)	r7   r   rC  r  r]  r^  rW  dtype_convertedx_tensors	            r-   _to_copyri    s`    85=000&000!!6!!!a%,UD'BCCCCC~%-M,Aa&& 	7799HO!U\"" *&q))f77!5!5|885IIH"O<**8V\JJ<44XuEE {8=AAAAOr,   c                 6    t                               |           S r<   )r   alias)r7   s    r-   nop_decompositionrl    s     ::a==r,   out3exponential_average_factorepsilonc           
      0   t                               | |||||||          \  }}	}
|r%||	|
|                     dt          j                  fS ||                    d          |                    d          |                     dt          j                  fS )Nr  r   )r   r2  rl  rD   rJ  )r   r   r  r  r  r   rn  ro  r  r  r  s              r-   cudnn_batch_normrq    s     $$"	 	GAq!  C1aU[AABB	EK00	 r,   c                     t          |          D ]<\  }}|dk    r1|| j        k     r| j        |         |k    s|                     |          } =| S r`   )r  r|  r  rY   )r7   broadcast_maskr  r  s       r-   _broadcast_batch_norm_backwardrt   	  sW    // " "
d199dQVmm0E0ED!!AHr,   rR  c                 2    t          | |||||||||	
  
        S r<   )native_batch_norm_backward)r  r   r   r  r  r&  r-  rE  r?  r  rR  s              r-   batch_norm_backwardrw  '	  s4     &  r,   c
                 >  & |j         }
||j         }n|
}t          j        |j                   &&fd| ||||||fD             \  }}}}}}}|j        }|                                }|dk    s
J d            d}t          t          |                    ||         z  }|}|}|r||J n||J |}t          j        ||z             }dg|z  }||         ||<   g }t          |          D ]}||k    r|
                    |           t          ||          }d|z  }t          j        ||          }t          j        |||z
  z  |          }t          ||z  |          }t          t          j        ||z  ||z            |          } |t          ||          dz  }!nt          ||z  |          }!|r||z
  | z  }"||"z
  |z
  |!z  }#n||!z  }#|	d         r||z  }$nd }$|	d         r|}%nd }%|#                    |
          t          |$|          t          |%|          fS )Nc              3   J   K   | ]}||                               n|V  d S r<   r  r  s     r-   r  z-native_batch_norm_backward.<locals>.<genexpr>^	  sM       	 	 $%=a	 	 	 	 	 	r,   r$   z$rank of the input must be at least 2r#   rk   )r   rI   r  r  rS   r(  rt  rD   r  rX   rv  rt  r   r   r>   r  )'r  r   r   r  r  r&  r-  rE  r?  r  r  weight_dtyper  r  r  running_mean_castrunning_var_castsave_mean_castsave_invstd_castr  
input_rankr  num_featuresr   r+  rs  reduction_axesrp  r   grad_output_sumdot_p	grad_mean
proj_scale
grad_scaleprojr\   rm  	grad_biasr?   s'                                         @r-   rv  rv  C	  s    +K|"3EK@@	 	 	 	 
	 	 	 +KJ???B???D[))**[->>LDF 5F$6$6$6$6 !,1A1M1M1M -344!"j 0N&t,N4 "N: % %99!!!$$$)$??DDi~>>OImzD'89>JJE./E~VVI/	%$,00 J
 3FNKKcQ

3[ .
 

  0T!Z/$t+y8JF

"Z/
1~ fn1~ #			 	k""K..I|,, r,   c
                    t          | |||||||||	
  
        }|
||f}t          |          D ]:\  }}|3t          ||         |j                   t	          |||         d           ;|S r  )rv  r  r   r  r   )r  r   r   r  r  r&  r-  rE  r?  r  rI  rJ  r  r  r\   rp  rN   s                    r-   native_batch_norm_backward_outr  	  s    " ( F d#J&!! Q Q1=jmQW555Q
14PPPPr,   save_varc                 L    t                               || |||||d|g d
  
        S NT)TTTr   rv  )r   rp   r   r  r  r&  r  ro  s           r-   miopen_batch_norm_backwardr  	  s@     **  r,   reserveSpacec	                 L    t                               || |||||d|g d
  
        S r  r  )	r   rp   r   r  r  r&  r  ro  r  s	            r-   cudnn_batch_norm_backwardr  	  s@     **  r,   c                 ^   | j         | j        t                    t          j        dv fd           | j        dd          D ]}t          j        |dk    fd           d         |d         z  dk    rd         |d         z  dk    rt          d t          dd          |          D                       }t          d t          dd          ||          D                       }t          j        j        	                    | ||          S d	 d
 fd} |d         |d                   \  }}}}	 |d         |d                   \  }
}}}| dt          |d          |
f         }|	s|st          j        |d          S d } |||||	d          \  }} |||||d          \  }}d }t          t          |j        d                   t          |j        d                             D ]'\  }}||d|d d |f         }||d|d d |f         z   }(|||z  z  S )Nr  c                      d  S )Nz9adaptive_avg_pool2d(): Expected 3D or 4D tensor, but got r+   r~  s   r-   r   z%adaptive_avg_pool2d.<locals>.<lambda>
  s    RDRR r,   rX  r   c                  *    dt                      dS )Nzjadaptive_avg_pool2d(): Expected input to have non-zero size for non-batch dimensions, but input has shape r  r  r  s   r-   r   z%adaptive_avg_pool2d.<locals>.<lambda>
  s&     I9>uI I I r,   rW   c              3   &   K   | ]\  }}||z  V  d S r<   r+   )r6   rp  r%  s      r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>
  s*      GG$!QqAvGGGGGGr,   c              3   4   K   | ]\  }}}||d z
  |z  z
  V  dS r#   Nr+   )r6   rp  r%  r  s       r-   r  z&adaptive_avg_pool2d.<locals>.<genexpr>
  sE       
 
 '1aAQ!O
 
 
 
 
 
r,   c                 6    t          j        | |z  |d          S )Ntruncrounding_moderD   divr  r  r  s      r-   start_indexz(adaptive_avg_pool2d.<locals>.start_index"
  s    yQ9999r,   c                 H    t          j        | dz   |z  |z   dz
  |d          S )Nr#   r  r  r  r  s      r-   	end_indexz&adaptive_avg_pool2d.<locals>.end_index%
  s*    y!a%1q1,awGGGGr,   c                    t          j        |t           j                  } |||           }| |z  dz   }| |z  }|dk    p||z  dk     }|r|dz  }n|dk    r|dz  }t          j        |t           j                  }|                    d          |z   }|rLt          j        | dz
  |j        |j                  }	t          j        ||	          } |||           }
|
|z
  }n|}||||fS )Nr  r#   r   rW   r  )rD   r  r  rY   rc  r   r  minimum)in_sizeout_sizeorangei0	maxlengthin_size_modadaptive	range_maxr  maxvali1lengthr  r  r  s               r-   compute_idxz(adaptive_avg_pool2d.<locals>.compute_idx(
  s1   hvU[III[733 x'!+	(#q(GH{,Ba,GH 	NIIANIL6MMM	ll2* 	 (!39SZ  F -V,,C 68W55B"WFFFFIx//r,   .r@  )r   rW   r   c                     t          |t                    r| |fS |dk     sJ ||                    d          k    }|dk    rt          |d          }t	          j        | |d          } t          ||           }| |fS )Nr   rW   rX  r@  r   )r5   r   rY   r[   rD   rj  )valsr  r  r  rS   r  s         r-   
maybe_maskz'adaptive_avg_pool2d.<locals>.maybe_maskP
  s    fg&& 	 < 7777 0 0 4 44Dbyy(q11$T455D&vt44F<r,   )r  rS   r   )r  r  r  rD   r   r  r  nnr  
avg_pool2dr[   r   r
   rX   )r   r  r  ru  kernelr  idxhlength_hrange_max_h
adaptive_hidxwlength_wrange_max_w
adaptive_wr  r  retrp  jr  r  r|  r  r  s                      @@@@@r-   adaptive_avg_pool2dr  
  s   
 \FKEu::D	LRRRR   [ 
 
FI I I I	
 	
 	
 	
 Ry;r?"a''E"IB,G1,L,LGG#eBCCj+*F*FGGGGG 
 
+.uRSSz;+O+O
 
 
 
 
 x"--eVVDDD: : :H H H0 0 0 0 0 0 0@ /:k%)[QS_.U.U+D(K.9k%)[QS_.U.U+D(K'a00$67D .j .z$H----       Zhjb  ND(  Zhjb  ND(
 Cdjn--uTZ^/D/DEE + +1;sAqqq!|$CCS!QQQ\**CC(X%&&r,   c                    t          j        d| d           t          t          j        | j        d |                    }t          t          j        |          }dg| j        z  }| j        d |          |d | <   |t                              || j	                  
                    |          |z  z                       d          }|                     t          | j        d |                    t          |          z             }t                              |                    d          |g|                     d          d          
                    |j                  S )Nmax_unpoolingd_forward_outr#   r  rW   Fr&  )rI   alert_not_deterministicr   operatorr   r  r|  r   r  r  r  r$  rl  rt  r)  )	r   r  r  rS   nchwindices_nc_shapeindices_flatr  s	            r-   _max_unpoolndr  p
  s>    
!"D#"D"D"DEEE	dj3$/	0	0B	k	*	*BsTY"j3$/UsdU$++b+55::;KLLrQQgbkk  ^^DEcTE!233d;6G6GGHHF!!r\NDLL,<,< "  
d6<r,   c                     t          j        j        t           j        k    fd           t          j        t	                    dk    fd           t          j         j        dv  fd           t          j         j        j        k     fd           t          d j                  D ]2t          j                                       dk     fd	           3t           d          S )
Nc                      d j          S )Nz2elements in indices should be type int64 but got: r   )r  s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s    TW]TT r,   r$   c                  *    dt                      dS )NzMThere should be exactly two elements (height, width) in output_size, but got 
 elements.r  r  s   r-   r   zmax_unpool2d.<locals>.<lambda>
  '    4;''4 4 4 r,   r  c                      d j          dS )NzLInput to max_unpooling2d should be a 3d or 4d Tensor, but got a tensor with  dimensions.r~  r   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s!    =%)Y= = = r,   c                  (    dj          d j          S NzBExpected shape of indices to be same as that of the input tensor (z%) but got indices tensor with shape: r  )r  r   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s1    BQUQ[ B B29-B B r,   r#   r   c                       dj          d  dS )NzZmax_unpooling2d(): Expected input to have non-zero size for non-batch dimensions, but got  with dimension  being empty.r  )rp  r   s   r-   r   zmax_unpool2d.<locals>.<lambda>
  s4    H:H H78H H H r,   )
rD   r   r   r  r  r|  r  rX   r+  r  )r   r  r  rp  s   ```@r-   max_unpool2dr  
  sT    
L$TTTT   
LKA	
 	
 	
 	
   
L	V	
 	
 	
 	
   
L
gm#	
 	
 	
 	
 	
   1di   
 
IIaLL1    	
 	
 	
 	
 wQ777r,   c                 
    t          j        j        t           j        k    d            t          j         j        dv  fd           t          j        t                    dk    fd           t          j        t                    dk    fd           t          j        t                    dk    fd           t          j         j        j        k     fd           t          d	 j                  D ]2t          j                                       d
k     fd           3t          j        d
         d
k    od	         d
k    od         d
k    fd           t           d          S )Nc                      dS )Nz(elements in indices should be type int64r+   r+   r,   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    .X r,   r@  r  c                      d j          dS )NzLInput to max_unpooling3d should be a 4d or 5d Tensor, but got a tensor with r  r~  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    w_d_iwww r,   r   c                  *    dt                      dS )NzVThere should be exactly three elements (depth, height, width) in output_size, but got r  r  r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  r  r,   c                  *    dt                      dS )NzRThere should be exactly three elements (depth, height, width) in stride, but got: r  r  ru  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    |ehioepep||| r,   c                  *    dt                      dS )NzSThere should be exactly three elements (depth, height, width) in padding, but got: r  r  )r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    ~fijqfrfr~~~ r,   c                  (    dj          d j          S r  r  )r  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s1    BQVQ\ B B29-B B r,   r#   r   c                       dj          d  dS )NzZmax_unpooling3d(): Expected input to have non-zero size for non-batch dimensions, but got r  r  r  )rp  r   s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s4    I ;I I89I I I r,   r$   c                      d  S )Nz5strides should be greater than zero, but got stride: r+   r  s   r-   r   zmax_unpool3d.<locals>.<lambda>
  s    PPP r,   )
rD   r   r   r  r|  r  r  rX   r+  r  )r   r  r  ru  r  rp  s   `````@r-   max_unpool3dr  
  s    
L$&X&X   
L
fwwww   
LKA	
 	
 	
 	
   
LFq||||   
LG~~~~   
Lw}$	
 	
 	
 	
 	
   1ej!! 
 
JJqMMA    	
 	
 	
 	
 
Lq	A9&)a-9F1IMPPPP  
 a888r,   )rq   rx  c                ,    t          | |||d|          S )NTinplacerq   
_index_addr7   rS   r  rx  rq   s        r-   
index_add_r  
  s     aeVTGGGGr,   c                ,    t          | |||d|          S )NFr  r  r  s        r-   	index_addr  
  s     aeVU%HHHHr,   r  c                   t          j        | j                  t          j        j        dk    fd           j        dk    r                    d          nd|j        dk    r|                              ndt          j        k    fd           dk    r`t          j        | j                  t          j        t          u p!t          j	        t                              fd           |z  }| j        dk    }|r|                     d          n| }dz  fz   }|rt          j        nt          j        }	 |	|||d          }
|r| S |r|
                    d          n|
                                S )	Nr#   c                      d j          dS Nz(Index should have dimension 1 or 0 (got r;  r~  r  s   r-   r   z_index_add.<locals>.<lambda>      H5:HHH r,   r   c                      d d d S )NzNumber of indices (z') should be equal to tensor.size(dim) (z), for dim=r+   )rS   
index_sizery  s   r-   r   z_index_add.<locals>.<lambda>  s"    sjssYdssmpss r,   c                  0    dt                      d dS )Nzalpha argument of type z cannot be safely cast to type !)r  )rq   python_types   r-   r   z_index_add.<locals>.<lambda>"  s"    hd5kkhhZehhh r,   r<   Tr&  )rI   canonicalize_dimsr|  rD   r   r+  dtype_to_typer   r  is_weakly_lesser_typer  rY   r   
index_put_	index_putr  r  )r7   rS   r  rx  r  rq   zero_dimrS  r  r  r   r  r  ry  s    ``  `     @@@r-   r  r  	  s    
!!&#
.
.C	L
aHHHH   #(*//AqJ&,kAoo&++c"""1K	Lz!ssssss   zz)!'224 E*4;;DDhhhhh	
 	
 	

 %v{H#	*QB
C-5(
"C#*>I
)BV
5
5
5C @!)?s{{1~~~s~~/?/??r,   r   c           
         t          j        t          |           dk    d            t          |           }| d                                         }|dd          }t	          d | D                       }|r||f}n||f}||z   }| d                             ||          }dt          |          z  }	t          |          D ]}
| |
         }t                              ||	d||                    d          z
  fz   |          }|rt          	                    ||d|
          }bt          	                    ||d|
          }|S )Nr   c                      dS )Nz#received an empty list of sequencesr+   r+   r,   r-   r   zpad_sequence.<locals>.<lambda>4  s    -R r,   r#   c              3   @   K   | ]}|                     d           V  dS r  r+  )r6   r7   s     r-   r  zpad_sequence.<locals>.<genexpr>8  s,      //!&&))//////r,   )r   r   rS   r  )
rD   r   r  r+  r   rE  rX   r   rt  r  )	sequencesbatch_firstpadding_valuesequences_sizemax_sizetrailing_dimsmax_lenout_dimsr   dim_paddingsrp  currseqrows                r-   pad_sequencer  1  sd    
LY!#%R%RSSS^^N|  ""HQRRLM//Y/////G -"G,^,-'H
A,

-
8
8CC...L>"" @ @A,""\Q',,q//(A$BBM
 
  	@%%c3AQ%??CC%%c3AQ%??CCJr,   c                 *    t          | |||d          S )NTr  _index_copyr7   rS   r  rx  s       r-   index_copy_r  L  s    q#ufd;;;;r,   c                 *    t          | |||d          S )NFr  r  r  s       r-   
index_copyr  Q  s     q#ufe<<<<r,   c                   t          j        | j        |          }t          j        j        dk    fd           | j        dk    }|r|                     d          n| }j        dk    r                    d          nd|z  fz   }|rt          j        nt          j        } ||||          }	|r| S |r|		                    d          n|	
                                S )Nr#   c                      d j          dS r  r~  r  s   r-   r   z_index_copy.<locals>.<lambda>]  r  r,   r   r<   )rI   r  r|  rD   r   rY   r   r  r  r  r  )
r7   rS   r  rx  r  r   rS  r  r  r   s
     `       r-   r  r  W  s     
!!&#
.
.C	L
aHHHH  
 v{H#	*QB"'*//EOOAuE
C-5(
"C#*>I
)BV
$
$C @!)?s{{1~~~s~~/?/??r,   c                    t          j        |                     d          |           }t          j        t          j        |                      }| j        s| j        r|                     d          }n|}|t          j        |          z
  |fS )Nr+   r  )rD   r  rl  rl   r   r  is_xpurD  )r   r   rn   r   s       r-   log_sigmoid_forwardr  m  s~     -r**D
1
1C	59T??"##A| t{ %%Q''r,   lowhighr[  c                     t          j        | j        t          |          t          |          | j        | j        |          S )N)r  r   r   r  r[  )prims_uniform_helperr  r   r   r  )r7   r  r   r[  s       r-   uniformr$  z  sB      	cNNt__gx   r,   c                 L    |                      t          | |||                    S r<   )r   r$  )r   r  r   r[  s       r-   uniform_r&    s"    ::gdCy99:::r,   c                 p   t          |           dz
  }|At          j        |d u d            t          j        t          |          |k    d            |S |t          j        |d u d            t          j        t          |          |k    d            g }t          |          D ]u\  }}t	          |          |k    r/|                    | |dz            t	          |          z             G|                    t          | |dz            |z                       v|S t          j        dd            d S )Nr$   c                      dS Nz9Must specify exactly one of output_size and scale_factorsr+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>      O r,   c                      dS N r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  s    R r,   c                      dS r)  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  r*  r,   c                      dS r,  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  s    r r,   Fc                      dS r)  r+   r+   r,   r-   r   z.upsample_compute_output_size.<locals>.<lambda>  s    R r,   )r  rD   r   r  r  rv  r   )r6  r  scale_factorsspatial_dimensionsrp  r  s         r-   upsample_compute_output_sizer3    sg   Z1,T!OO	
 	
 	
 	S%%);;ZZHHH 4OO	
 	
 	
 	S''+==zzJJJm,, 	C 	CDAq1vv{{"":a!e#4s1vv#=>>>>""7:a!e+<q+@#A#ABBBB	LRR    r,   c                     | d S | |         S r<   r+   )scalesr  s     r-   get_scale_valuer6    s    ~t#;r,   r1  c                     t          |                                 ||          }|r|nd gt          |          z  }t          | ||          S r<   r3  r+  r  _upsample_nearestr   r  r1  osizer5  s        r-   _upsample_nearest_vecr<    sO     ){MRRE&?TFSZZ,?  UE6222r,   c                     t          |                                 ||          }|r|nd gt          |          z  }t          | ||d          S NTexactr8  r:  s        r-   _upsample_nearest_exact_vecrA    sT     ){MRRE&?TFSZZ,?  UE6>>>>r,   c                    g }t          |          }|rdnd}t          |          D ]}||         }| j        | |z            }	||         |	|	||         z  z  n|	|z  }
t          j        |t          j        | j                  }||z   |
z                      t          j                  }t          |dz
  |z
            D ]}|	                    d          }|
                    |           |S )Nr   r   r  r#   rW   )r  rX   r  rD   r  r   r  r>   r  rY   rv  )r   r  r5  r@  r  num_spatial_dimsr  r  r;  isizerr   output_indicesinput_indicesrZ   s                 r-   !_compute_upsample_nearest_indicesrG    s    G;''"SSsF#$$ & & A--12/5ay/D*++%RW-e5=VVV(61U:>>u{KK'!+a/00 	8 	8A)33B77MM}%%%%Nr,   )preserve_memory_formatr  r5  c                 &    t          | ||g          S r<   r9  r   r  r5  s      r-   upsample_nearest1drL    s     UK&:::r,   c                 *    t          | ||gd          S r>  rJ  rK  s      r-   upsample_nearest_exact1drN  
  s     UK&FFFFr,   scales_hscales_wc                 (    t          | |||g          S r<   rJ  r   r  rO  rP  s       r-   upsample_nearest2drS    s     UK(H1EFFFr,   c                 ,    t          | |||gd          S r>  rJ  rR  s       r-   _upsample_nearest_exact2drU  %  s      UK(H1ETRRRRr,   scales_dc                 *    t          | ||||g          S r<   rJ  r   r  rV  rO  rP  s        r-   upsample_nearest3drY  4  s     UK(Hh1OPPPr,   c                 .    t          | ||||gd          S r>  rJ  rX  s        r-   _upsample_nearest_exact3dr[  B  s+     {Xx:$   r,   r@  c                 8   t          | |||          }d d g|z   }t                              | |          }|j        dk    rYt	          j        |           }| j        d         }| j        j        dk    r|dk     rt          j
        }|                    |          }|S )Nr?  r@  r#   cudarV  )rG  r   _unsafe_indexr|  rI   r   r  r  r  rD   r[  r  )	r   r  r5  r@  spatial_indicesr  r  rW  
n_channelss	            r-   r9  r9  T  s     8{F%  O Tl_,Gw//F{a3E:: [^
<&&:>>!3M"""??Mr,   c                      |r|rdn|rdn|rdndt                     z  dk    sJ t                                  fdt          dt                               D             S )Nr  r@  r   r$   r   c                 D    g | ]}t          ||z                      S r+   r  )r6   rp  
group_sizeparamss     r-   r9   z!gather_params.<locals>.<listcomp>z  s<       ./fQZ'())  r,   )r  rX   )rd  
has_biaseshas_projectionsrc  s   `  @r-   gather_paramsrg  o  s     o 

	 

	 


v;;#q(((#f++(((    38CKK3T3T   r,   c                     |r3| d|z           |d|z           }}| d|z  dz            |d|z  dz            }}n| |         ||         }}d\  }}||||fS )Nr$   r#   NNr+   )rd  hiddensrp  bidirectional
cur_params
cur_hiddenbidir_paramsbidir_hiddens           r-   params_hiddensrp    sr     0!'AAJ
%+AEAI%6A	8Jl!'GAJJ
%/"lz<==r,   c                     ||k    sJ |                     |                     d|||z
                       |                     dd|          S rw   )rv  r,  )rm  last_batch_size
batch_sizerj  s       r-   update_hidden_for_packedrt    sT    Z''''NN:$$Q
Oj4PQQRRRQ:...r,   c           	      ~    ||k    r| S ||k     sJ t          j        | |                    d|||z
            f          S rw   )rD   concatr,  )rm  rr  rs  
inp_hiddens       r-    update_hidden_for_packed_reverserx    s\     *$$Z''''<a*2NOO	
  r,   c           	         |d         }|d         }|r|d         nd }	|r|d         nd }
g }g }|r|d         n|d         }|                     dd|          }t          j        | t          |                    }|r|d d d         }|D ]d} | j        d         }||k    rn'|rt          ||||          }nt          ||||          } || |||	||
          }|}|                    |           e|r|                                 n)|                    |           |                                 t          j	        |d          }|st          j	        |d          n|}||fS )Nr   r#   r$   r   rW   )
r,  rD   r  rt  r  rx  rt  rv  reverser-  )inphiddenrd  re  	hidden_fnbatch_sizesrz  	ih_weight	hh_weightih_biashh_biasstep_outputrj  rr  rm  	split_inprp  r   
hidden_outs                      r-   one_layer_rnn_datar    s    q	Iq	I%/fQii4G%/fQii4GK"$G)0Dk"ook!nOq!_55JCk!2!233I $dddO	 ' 'IaLa 	9OQ JJ 2OQ J YsJ	7IwWW
:&&&& z"""
)K
#
#C.5E7A&&&:J
?r,   c                       fd}|S )Nc                 H     t          j        |||          | z             S r<   r   linearrp  rm  r  r  r  r  nonlinearitys         r-   rO   zrnn_cell.<locals>.inner  s&    |AHZGDDqHIIIr,   r+   r  rO   s   ` r-   rnn_cellr    s(    J J J J J Lr,   c                       fd}|S )Nc                 t    t          j        | ||          }  t          j        |||          | z             S r<   r  r  s         r-   rO   zrnn_cell_data.<locals>.inner  s9    HQ	7++|AHZGDDqHIIIr,   r+   r  s   ` r-   rnn_cell_datar    s(    J J J J J Lr,   c           	         |d         }|d         }|r|d         nd }|r|d         nd }	t          j        | ||          }
|r|
                    d          n|
}
|                    d          }g }|
D ]'} |||||||	          }|                    |           (|r|                                 t          j        |d          }||                    d          fS )Nr   r#   r$   r   )	r   r  fliprY   rv  rz  rD   r-  r  )r{  r|  rd  re  r}  rz  r  r  r  r  precomputed_inputrm  r  rp  r   s                  r-   one_layer_rnnr    s   q	Iq	I%/fQii4G%/fQii4Gi995<S)..q111BS!!!$$JK ' 'Yq*i)WUU
:&&&& 
)K
#
#C
""1%%%%r,   c                    |d         }|d         }|r|d         }|d         }nLt          j        |                                          }t          j        |                                          }|d                             d          }	|d                             d          }
g }d}|	                    d          }d}d}d}d}|                                 } |	                                }	|
                                }
t           j        j        j                            | |||||	|
|||||||||          }|d         |d         |d         }}}||	                    d          |	                    d          ffS )Nr   r#   r$   r   F)
rD   r  r+  rY   r  r  r   mkldnn_rnn_layerr  r  )r{  r|  rd  re  rz  w0w1w2w3hxcxr~  modehidden_size
num_layersrk  r  rE  outputsr^   hycys                         r-   mkldnn_one_layer_lstmr    sq   	B	B $AYAY[##[##			Q		B			Q		BKD''!**KJ MKE ..

C	B	Bin-55





! G$ 
GAJ
2rArzz!}}bjjmm,,,r,   c
                 8   |r|                      dd          n| } g }
t          |          D ]}t          ||||          \  }}}}|r||dz
  k     r|nd} |	| |||          \  }}|
                    |           |r( |	| |||d          \  }}|
                    |           |r-t	          j        ||g|                                dz
            } n|} |dk    r"|r ||dz
  k     rt	          j        | |d          } |r|                      dd          n| } | |
fS )Nr   r#   r   T)rz  )rE  )	transposerX   rp  rv  rD   r-  rS   rH  )r   r|  rd  re  r  rH  rE  rk  r  layer_fnfinal_hiddensrp  rl  rm  rn  ro  fwd_inp
fwd_hiddenbwd_inp
bwd_hiddens                       r-   _rnn_helperr  #  sz    &1;EOOAq!!!eEM: > >=KFA}>
 >
:
Jl $D
QU(:(:''&huj*jQQZ((( 	-"*(|\:t# # #GZ   ,,, 	Iw0'++--!2CDDEEEa<<E<a*q.&8&8M%===E%0;EOOAq!!!eE-r,   c	                    |                     d          }	t          ||d          }t          | |	|||||||t          t          t          t          j                            
  
        \  }
}|
t          j        |d          fS Nr   Fr}  )	unbindrg  r  r   r  r  rD   r   stackr   r  rd  re  r  rH  rE  rk  r  r|  r   r  s               r-   rnn_tanh_inputr  L       YYq\\F6:u55F$%*)=)=>>> C M1----r,   c	                    |                     d          }	t          ||d          }t          | |	|||||||t          t          t          t          j                            
  
        \  }
}|
t          j        |d          fS r  )	r  rg  r  r   r  r  rD   r  r  r  s               r-   rnn_relu_inputr  k  r  r,   c	                 
   |                     d          }	t          ||d          }t          | |	||||||dt          t          |t          t          j                            
  
        \  }
}|
t          j        |d          fS Nr   Fr~  r}  )	r  rg  r  r   r  r  rD   r  r  datar~  r  rd  re  r  rH  rE  rk  r|  r   r  s               r-   rnn_relu_datar         YYq\\F6:u55F$##EJ//	
 	
 	
 C  M1----r,   c	                 
   |                     d          }	t          ||d          }t          | |	||||||dt          t          |t          t          j                            
  
        \  }
}|
t          j        |d          fS r  )	r  rg  r  r   r  r  rD   r   r  r  s               r-   rnn_tanh_datar    r  r,   c                    t          j        |||          | z   }|                    d|          }|d                                         }	|d                                         }
|d                                         }|d                                         }|
|z  |	|z  z   }||                                z  }||nt          j        ||d           }||fS )Nr@  r   r#   r$   r   r   r  chunkr   r   )r{  r  r  r  r  	hr_weight	chunk_dimgateschunked_gatesin_gateforget_gate	cell_gateout_gater  r  s                  r-   	lstm_cellr    s    HRG,,s2EKK9--MA&&((G"**,,Ka %%''IQ''))H	r	Wy0	1B	BGGII	B ahr9d&C&CBr6Mr,   c           
         |d         }|d         }|r|d         nd }|r|d         nd }t          |          dk    r|d         nt          |          dk    r|d         nd }	|d                             d          }
|d                             d          }t          j        | ||          }|r|                    d          n|}g }|D ]0} t          | |
||||	d          \  }
}|                    |
           1|r|                                 t          j	        |d          }||

                    d          |
                    d          ffS )Nr   r#   r$   r   r  r@  r  )r  rY   r   r  r  r  rv  rz  rD   r-  r  )r{  r|  rd  re  rz  r  r  r  r  r  r  r  r  r  r   s                  r-   one_layer_lstmr    so   q	Iq	I%/fQii4G%/fQii4G[[A%%q		Fq8H8H6!99d  
		Q		B			Q		Bi995<S)..q111BSK   3B	7IQRSSSB2 
)K
#
#CA

1...r,   c           
      d   |d         }|d         }|r|d         nd }|r|d         nd }	t          |          dk    r|d         nt          |          dk    r|d         nd }
g }g }|r|d         n|d         }t          j        | t          |                    }|r|d d d         }|d         }|d         }|                    dd|          |                    dd|          }}|D ]5} | j        d         }t          j        | ||          } ||k     ru|                    |                    d|||z
            |                    d|||z
            f           |                    dd|          |                    dd|          }}||k    r^t          j	        ||                    d|||z
            fd          }t          j	        ||                    d|||z
            fd          }t          | ||||	|
d          \  }}|}|                    |           7|r|                                 ||f}na|                    ||f           |                                 t          | \  }}t          j        |d          t          j        |d          f}t          j        |d          }||fS )	Nr   r#   r$   r   r  r@  rW   r  )r  rD   r  rt  r,  r  r   r  rv  rv  r  rz  r  r-  )r{  r|  rd  re  r~  rz  r  r  r  r  r  r  rj  rr  r  orig_hxorig_cxr  r  rp  r  hidden0hidden1r   s                           r-   one_layer_lstm_datar    s   q	Iq	I%/fQii4G%/fQii4G[[A%%q		Fq8H8H6!99d  KG)0Dk"ook!nOCk!2!233I $dddO	QiGQiGq!_--q!_-- 	B
   IaLhsIw// NNIIaOa$788IIaOa$788   YYq!Q''1a););B W^^AO8KLLMq B W^^AO8KLLMq B 3B	7IQRSSSB2 B"X

Bx   =Yw**EIgq,A,AA

)K
#
#C
?r,   c                 >    d } || ||          rt           S t          S )a*  Check whether we could use decompose lstm with mkldnn_rnn_layer.
    All the below conditions need to be met:
        * ``torch._C._get_mkldnn_enabled()`` returns ``True``.
        * All the input args are on CPU.
        * The dtypes of args are either torch.float or torch.bfloat16.
        * Inference.
        * ``has_projections`` returns ``False``.

    Args:
        * input: the input sequence to LSTM
        * hx: a tuple of the input hidden state and cell state ``(h_0, c_0)`` to LSTM
        * params: the weight and bias tensors of LSTM
    c                 4   t           j                                        sdS | gt          |          z   t          t	          j        |                    z   }d |D             }t          |          dk    rdS |                                }|t          j        d          k    rdS d |D             }|D ]}|t           j	        t           j
        fvr dS  | j        rdS |d                             d          |d                             d          k    }|rdS dS )	NFc                     h | ]	}|j         
S r+   r  r6   ts     r-   	<setcomp>zEselect_one_layer_lstm_function.<locals>.use_mkldnn.<locals>.<setcomp>M  s    ---18---r,   r#   r  c                     h | ]	}|j         
S r+   r   r  s     r-   r  zEselect_one_layer_lstm_function.<locals>.use_mkldnn.<locals>.<setcomp>U  s    +++a!'+++r,   r   r$   T)rD   rD  _get_mkldnn_enabledrt  r	   from_iterabler  popr  r  bfloat16requires_gradr+  )	r   r  rd  rq  devicesr  dtypesr   rf  s	            r-   
use_mkldnnz2select_one_layer_lstm_function.<locals>.use_mkldnnH  s#   x++-- 	5'DHH$tE,?,G,G'H'HH--W---w<<15U\%((((5++7+++ 	 	EU[%.999uu :  	5Q%**Q--2a5::a==8 	5tr,   )r  r  )r   r  rd  r  s       r-   select_one_layer_lstm_functionr  9  s5      : z%V$$ $$r,   c	                    t          |          dk    s
J d            t          |||d                             d          |d                             d          k              }t          t	          |d         |d                             }	t          | ||          }
t          | |	||||||||

  
        \  }}t          t	          |           }|t          j        |d         d          t          j        |d         d          fS )Nr$   lstm expects two hidden statesr   r#   )	r  rg  r+  rt  r  r  r  rD   r  )r   r  rd  re  r  rH  rE  rk  r  r|  r  r   r  s                r-   	lstm_implr  k  s     r77a<<<9<<<6:r!uzz!}}1

1/MNNF#beRU##$$F-eR@@H$ C m,--MM!,a00%+mA>NPQ2R2RRRr,   c	                    t          |          dk    s
J d            t          |||d                             d          |d                             d          k              }t          t	          |d         |d                             }	t          | |	||||||dt          t          |          
  
        \  }
}t          t	          |           }|
t          j	        |d         d          t          j	        |d         d          fS )Nr$   r  r   r#   F)r~  )
r  rg  r+  rt  r  r  r   r  rD   r  r  s               r-   lstm_data_implr    s     r77a<<<9<<<6:r!uzz!}}1

1/MNNF#beRU##$$F$#=== C m,--MM!,a00%+mA>NPQ2R2RRRr,   c                 p   |                      dd          }t          j        |||                               dd          }|d         |d         z                                   }|d         |d         z                                   }	|d         |d         |z  z                                   }
||
z
  |	z  |
z   S )Nr   r#   r$   r   )r  r   r  r   r   r{  rm  r  r  r  r  chunked_igateschunked_hgates
reset_gate
input_gatenew_gates              r-   gru_cellr    s    YYq!__NXj)W==CCAqIIN #nQ&77@@BBJ #nQ&77@@BBJq!^A%6%CDJJLLH!Z/(::r,   c                    t          j        | ||                              dd          }t          j        |||                              dd          }|d         |d         z                                   }|d         |d         z                                   }	|d         |d         |z  z                                   }
||
z
  |	z  |
z   S )Nr   r#   r   r$   r  r  s              r-   gru_cell_datar    s    Xc9g66<<QBBNXj)W==CCAqIIN #nQ&77@@BBJ #nQ&77@@BBJq!^A%6%CDJJLLH!Z/(::r,   c	                     t          ||d          }t          | |                    d          ||||||dt          t          |t
                    
  
        \  }	}
|	t          j        |
d          fS )NFr   r  )rg  r  r  r   r  r  rD   r  )r  r~  r  rd  re  r  rH  rE  rk  r   r  s              r-   gru_impl_datar    sz     6:u55F$
		!"}UUU C M1----r,   c	                     t          ||d          }t          | |                    d          |||||||t          t          t
                    
  
        \  }	}
|	t          j        |
d          fS )NFr   r  )rg  r  r  r   r  r  rD   r  )r   r  rd  re  r  rH  rE  rk  r  r   r  s              r-   gru_implr    sw     6:u55F$
		!222 C M1----r,   c                     t          |                                 ||          }t          |d          }t          |d          }t          j        j                            | ||||          S Nr   r#   )r3  r+  r6  rD   r  r   _upsample_bilinear2d_aar   r  align_cornersr1  r;  scale_hscale_ws          r-   upsample_bilinear2d_aa_vecr     sb     ){MRREmQ//GmQ//G9>11umWg  r,   c                     t          |                                 ||          }t          |d          }t          |d          }t          j        j                            | ||||          S r  )r3  r+  r6  rD   r  r   _upsample_bicubic2d_aar  s          r-   upsample_bicubic2d_aa_vecr    sb     ){MRREmQ//GmQ//G9>00umWg  r,   c                     t          |                                 ||          }|r|nd gt          |          z  }t          | |||          S r<   )r3  r+  r  _upsample_linear)r   r  r  r1  r;  r5  s         r-   _upsample_linear_vecr    sL     ){MRRE+D]]$#e**1DFE5-@@@r,   r  c                 (    t          | |||g          S r<   r  )r   r  r  rP  s       r-   upsample_linear1dr	  "  s     E;zJJJr,   c                 *    t          | ||||g          S r<   r  )r   r  r  rO  rP  s        r-   upsample_bilinear2dr  -  s     E;(?STTTr,   c                 ,    t          | |||||g          S r<   r  )r   r  r  rV  rO  rP  s         r-   upsample_trilinear3dr  <  s&     {MHh+I  r,   c                 P    |r|dk    r| dz
  |dz
  z  ndS ||dk    rd|z  n| |z  S )Nr#   rk   r   r+   )r  r  r  rr   s       r-   _compute_scaler  M  sK     V5=\\#(S.11qH#/EAIIsU{{7XCUUr,   c                 &    |r| |z  S | |dz   z  dz
  S Nr   r+   )rr   	dst_indexr  s      r-   _compute_source_indexr  T  s)     /y  	C(3..r,   weightsweights_precisionc                     t          d t          | |          D                       d|dz
  z  z   }||z	  }t          j        |dd                              t          j                  S )Nc              3      K   | ]F\  }}|                     t          j                  |                     t          j                  z  V  Gd S r<   )r>   rD   r:  )r6   r  r  s      r-   r  z%_sum_tensors_uint8.<locals>.<genexpr>^  sX        26!QU[ADD---     r,   r#   r      )_sum_tensorsr  rD   r   r>   rJ  )r  r  r  r  s       r-   _sum_tensors_uint8r  [  s}       :=c7:K:K    	
 1$	%'F ((F;vq#&&))%+666r,   c                     t          j        |                                           }d}t          j        ||j                  }d|d|dz   z  z  z   }|dk    }||                                z
  S )N   r  r   r#   i   )rD   r  r   r  r  r   )r  
max_weightmax_weight_precision
precisionsvaluesr  s         r-   _compute_weight_precisionr!  e  sp    W%%))++J2:;LMMMJ:zA~!677FgD$((**,,r,   c           	           j         d         } j         dd          }t          |          t          j         t          j        j                  \  } fdfdt          t          |||                    D             }t          t          |           \  }g }	t          ddggz   D ]dd d gfdt                    D             z   }
t                               |
          }t          |          }|	                    |           et          t                              D ]h}||         |         z
                      dd	                                        fd
t          |	d d d         |	dd d                   D             }	it          |	          dk    sJ |	d         }t          j                   } j        j        dk    r|dk     rt*          j        }t/          |t*          j                  sJ |                    |          }                                 s|                                }|S )Nr#   r$   rU  c                    t          | |	|          }t          j        |j                                      
          }t          ||	                              d          } |j        |j        d         gdg|z  R  }|                    t          j	                  }|dz                       | dz
            }|||fS )Nr  r   r   r   r   r#   r   )
r  rD   r  r  r>   r  r   r$  r  r  )inp_sizer  r5  nsqueezescale_factorrp  x_f32r7   xp1r  r   r   s            r-   
get_valuesz$_upsample_linear.<locals>.get_values  s    %h-PP L%,777:::GG%lA}EEKKPSKTTek!n@sh/?@@@HHU[!!1umm1m--a}r,   c           	      F    g | ]\  }\  }}} |||d z
  |z
            S r!  r+   )r6   rp  r$  r  r5  r)  n_dimss        r-   r9   z$_upsample_linear.<locals>.<listcomp>  sL       +A+(F 	
8XvvzA~>>  r,   r   c                 H    g | ]}|         d k    r|         n|         S r  r+   )r6   kr  xp1sxss     r-   r9   z$_upsample_linear.<locals>.<listcomp>  s1    UUU!qtqyybeed1gUUUr,   r   rk   c                 L    g | ] \  }}|t          j        ||z
            z   !S r+   )rD   r   )r6   v1v2xscales      r-   r9   z$_upsample_linear.<locals>.<listcomp>  sD     
 
 
 B 27F+++
 
 
r,   r]     rV  )r  r  rI   rJ   rW  INT_TO_FLOATr  r  rt  r
   rX   r   r^  r   rv  reversedr   r>   r   r  r  rD   r[  r5   r   r  rL  round)r   r  r  r5  r`  	inp_sizesrZ   r   xs_f32vsr  vrp  r  rW  r  r   r)  r+  r.  r/  r3  s   ` `            @@@@@@@r-   r  r  n  s    QJABBI^^F'!AN  HAu
          /8	;//0
 0
  F CL))FB	B1vh'(  TlUUUUUUuV}}UUUUuc**#Au--
		!eFmm$$ 
 
)be#**34477>>
 
 
 
 b1gr!$Q$x00
 
 
 r77a<<<<UF /66M |F""zB/fel+++++];;F""$$  Mr,   r  r  c                 "    | j         |j         k    S r<   r  )r  r  s     r-   is_same_sizer=    s    7agr,   c                 8    t                               | |          S r<   )r   r  )r7   r  rK   s      r-   _reshape_aliasr?    s     99Qr,   c                 8    t                               | |          S r<   )r   r  )r7   r  s     r-   r^  r^    s    ::a!!!r,   c                 <    t                               | |||          S r<   )r   r  )r7   r  r   r'  s       r-   r)  r)    s    >>!WeZ888r,   c                    |D ]7}|3t          j        |j        t           j        t           j        fv d            8t          j        |j        t           j        k    d            ddlm}  ||                                 dk              r;t           j	        
                    | |          }|                     |j        |          S t          t          |                    D ]<}||         }|0|                    d|                     |          dz
            ||<   =t"                              | |                              | |          S )Nc                      dS Nz3tensors used as indices must be long or int tensorsr+   r+   r,   r-   r   z&_unsafe_masked_index.<locals>.<lambda>  r  r,   c                      dS Nz*tensors used as masks must be bool tensorsr+   r+   r,   r-   r   z&_unsafe_masked_index.<locals>.<lambda>      < r,   r   rS  r#   r  )rD   r   r   ri  r  r  rr  rT  r   _meta_registrationsmeta_index_TensorrE  r  rX   r  r   r+  r   r^  rj  )r7   r  r  fillr  rT  meta_resultrp  s           r-   r  r    sK     L
EI66MM  
 
L
ej <<  
 EDDDDD~aggii1n%% 3/AA!WMMzz++T2223w<<   ? ?
q		A>>GAJa))55teTBBBr,   c                 d   |D ]7}|3t          j        |j        t           j        t           j        fv d            8t          j        |j        t           j        k    d            |                                 dk    r|                                 S t          t          |                    D ]P}||         }|D|
                    |                     |           |                     |          dz
            ||<   Q|                    | d          }t                              | ||d          S )Nc                      dS rD  r+   r+   r,   r-   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>  r  r,   c                      dS rF  r+   r+   r,   r-   r   z5_unsafe_masked_index_put_accumulate.<locals>.<lambda>  rG  r,   r   r#   r  Tr&  )rD   r   r   ri  r  r  r   r  rX   r  r   r+  rj  r   r)  )r7   r  r  r   r  rp  masked_values          r-   #_unsafe_masked_index_put_accumulaterP    s$     L
EI66MM  
 
L
ej <<  
 	wwyyA~~wwyy3w<<   H H
!&&))QGGGAJ%%teQ//L!!!Wlt!LLLr,   c                    |                                  }d}|dk     rd}|9|dk    r,dg|z  }|j        d         ||<   |                    |          }n|}| |z  } t          j        ||k    |d          }	|	                    |          }
t          j        | ||
                              |           }t          j        ||k    |d          }|t          j	        j
        k    r |dk    r|                     dd          }||fS |r|                    | j                  }t          j        |||
                              |          }t          j        ||k    |d          }|                                }n+||k                                                        |           }|t          j        j
        k    r|                                }n,|t          j        j
        k    r|                                |z  }||fS )Nr#   r$   r   r+   r   )rS   r  r  rD   rm   rY   gatherr  r"   r(   r   rE  r  r   r>   r*   r)   )r   r   r   r   r  r+  r%  r  wr&  safe_target_r  r  wsums                 r-   _nll_loss_forwardrV    s    XXZZFKzzA::E "(aE+E""AAAax+f4fa@@K((55L l4l;;CCKPPPF[</;;FIN(((VaZZ}}R--|##HHTZ  |A{L99AA+NN{6\14;;xxzz,.335588>>IM'''	in*	*	*,<r,   c                 Z   |                                  dk    r|                                  dk    s
J d            |                                 dk    s
J d            |                                  dk    o|                                 dk    }|s7| j        d         |j        d         k    sJ d| j         d|j         d            | j        d	         }|E|                                 dk    r|                                |k    sJ d
| d|j                     t          | ||||          S )Nr   r$   r7  r#   r8  r9  r:  r;  rW   z/weight tensor should be defined either for all z7 classes or no classes but got weight tensor of shape: )rS   r  r   rV  )r   r   r   r   r  r=  	n_classess          r-   nll_loss_forwardrY  9  s<    88::>>dhhjjAooo/Pooo::<<1E  88::?8vzz||q'8L DJqMV\!_<<<JTZJJ6<JJJ =<< 
2I>fjjlla//FLLNNi4O4O4O	:) 	: 	:+1<	: 	: 5P4O4O
 T669lKKKr,   c                 (    t          | ||||          S r<   )rV  )r   r   r   r   r  s        r-   nll_loss2d_forwardr[  V  s     T669lKKKr,   Ac                 0    |dz   | z  |dz   z
  | z  | z  dz   S )Nr$   r   r#   r+   r7   r\  s     r-   _upsample_cubic_convolution1r_  d  s(    UaK1q5!Q&*Q..r,   c                 <    || z  d|z  z
  | z  d|z  z   | z  d|z  z
  S )Nr     r@  r+   r^  s     r-   _upsample_cubic_convolution2rb  h  s0    UQU]a!a%'1,q1u44r,   r  c                    d}| j         t          j         d          k    rt          j        | d| z
  gd          }t          j        | dz   d| z
  gd          }t          ||          }t	          ||          }t          j        |d          \  }}t          j        |d          \  }}	|||	|fS t          | dz   |          t	          | |          t	          d| z
  |          t          d| z
  |          fS )Ng      r  rk   r   r   r   )r  rD   r  rb  r_  r  )
r  r\  tt1tt2w03w12r  r  r  r  s
             r-    _upsample_get_cubic_coefficientsrh  l  s   Ax5<&&&&k1cAg,A...k1s7C!G,!444*322*322cq)))Bcq)))B2r2~ )S!44(A..(q!44(q!44	
 	
r,   coeffstsc                 n    t          |          }t          d t          | |          D                       S )Nc              3   &   K   | ]\  }}||z  V  d S r<   r+   r6   r  r  s      r-   r  z+_upsample_cubic_interp1d.<locals>.<genexpr>  s*      EEHRREEEEEEr,   )rh  r  r  )ri  rj  coeffs2s      r-   _upsample_cubic_interp1dro    s6    .r22GEEFG0D0DEEEEEEr,   c                 6    t          t          j        |           S r<   )r   rD   r  )rj  s    r-   r  r    s    %)R   r,   	num_stepsc                     | dk    rt          j        d||          S |s| dz
  | z  nd}t          j        | || ||          S )Nr#   r   r  )stepsr  r   )rD   rx  linspace)rq  r  r   r  r  s        r-   _linspace_from_neg_oneru    sY     A~~|AfE::::-:A)a-9	$	$A>1"ayuMMMMr,   thetahrS  c                    | j         }| j        }t          ||||                              d|d          }t          ||||                              |dd          }t	          j        d||          }t          j        j                            |ddd          }t          j        j                            |ddd          }t          j        j                            |d	dd          }||z   |z   S )
Nr#   )r#   r#   r#   r  )r   r$   constantr   r  r  r   r#   r#   )r$   r   	r   r  ru  r  rD   r  r  r  r  )	rv  rw  rS  r  r   r  grid_xgrid_ygrid_ones	            r-   _make_base_grid_4dr    s    KE\F $A}eVDDII!QPQRRF#A}eVDDII!QPQRRFz)5@@@H X $$VjPQ$RRFX $$VjPQ$RRFx"&&xV*TU&VVHF?X%%r,   r  c                    | j         }| j        }t          ||||                              dd|d          }t          ||||                              d|dd          }t          ||||                              |ddd          }	t	          j        d||          }
t          j        j                            |ddd          }t          j        j                            |ddd          }t          j        j                            |	d	dd          }	t          j        j                            |
d
dd          }
||z   |	z   |
z   S )Nr#   )r#   r#   r#   r#   r  )r   r   ry  r   rz  r  r$   r#   )r   r   r|  )rv  r  rw  rS  r  r   r  r}  r~  grid_zr  s              r-   _make_base_grid_5dr    s?   KE\F#A}eVDDII!QPQSTUUF#A}eVDDII!QPQSTUUF#A}eVDDII!QPQSTUUFz,eFCCCH X $$VjPQ$RRFX $$VjPQ$RRFX $$VjPQ$RRFx"&&xV*TU&VVHF?V#h..r,   c                     |\  }}}}t          | |||          }|                    ddd          | j                            d          z                      d          }|                    |||d          S )Nr  rW   r   r#   rX  r$   )r  r  r]  rY   r   )	rv  r+  r  r(  rZ   rw  rS  	base_gridgrids	            r-   _affine_grid_generator_4dr    sw    JAq!Q"5!QmLLLI NN2q!$$ux'9'9!'<'<<AA"EED99Q1a   r,   c                     |\  }}}}}t          | ||||          }|                    ddd          | j                            d          z                      d          }	|	                    ||||d          S )Nr  rW   r@  r#   rX  r   )r  r  r]  rY   r   )
rv  r+  r  r(  rZ   r  rw  rS  r  r  s
             r-   _affine_grid_generator_5dr    s}    MAq!Q"5!QOOOI NN2q!$$ux'9'9!'<'<<AA"EED99Q1a###r,   c                     t          j        t          |          dv d            t          |          dk    rt          | ||          S t	          | ||          S )Nr  c                      dS )NzCaffine_grid_generator needs 4d (spatial) or 5d (volumetric) inputs.r+   r+   r,   r-   r   z'affine_grid_generator.<locals>.<lambda>  s    U r,   r@  r  )rD   r   r  r  r  )rv  r+  r  s      r-   affine_grid_generatorr    sg     
LD		VUU   4yyA~~(MRRRR(MRRRRr,   r  interpolation_modepadding_mode_expand_gridc           	          !"#$%&'()*+,- t          j        dv fd           t          j        dv fd           dt          dt          dt          ffd-dt          dt          d	t          dt          fd
+dt          dt          dt          f+fd dt          dt          dt          f -fd} j        \  $%|j        \  })*}|dk    sJ r0|                    d)*|                              )*d          }dt          dt          dt          f$%fd&t          j         j                                      ddd          t          j         j                                      ddd          dt          dt          dt          dt          f&)*fddt          dt          dt          f fd"|d         }	|d         }
dk    r ||	%          } ||
$          }|
                                |
                                c'('dz   (}}'(dz   }}||}}||z
  ||z
  z  }||z
  ||z
  z  }||z
  ||z
  z  }|'z
  |(z
  z  }t          "fd'(|f|||f|||f|||ffD                       S dk    rM ||	%          } ||
$          }|                                }|                                } "||d          S  -|	%          } -|
$          }|
                                '|
                                (|'z
  ,|(z
  }s*,                    d          ,|                    d          }dt          dt          dt          f "$%fd#dt          dt          f#'(,fd!t          !fdt          d           D                       }t!          ||          S )!N)r   r#   r$   c                      d  S )NzInvalid interpolation mode r+   )r  s   r-   r   z"_grid_sampler_2d.<locals>.<lambda>  s    B.@BB r,   c                      d  S )NzInvalid padding mode r+   )r  s   r-   r   z"_grid_sampler_2d.<locals>.<lambda>  s    +Q<+Q+Q r,   coordsr+  rT   c                 B    r|dz  dz
  n|dz  }|dz  dz
  }| |z  |z   S r  r+   )r  r+  r   ofsr  s       r-   unnormalizez%_grid_sampler_2d.<locals>.unnormalize  s>     %2CtczCs
Sj3|c!!r,   	twice_low
twice_highc                 ^   ||k    rt          j        |           S |dz  }||z
  dz  }| |z
                                  }t          j        ||          }||z                                                      t           j                  }t          j        |dz  dk    ||z   ||z   |z
            S )Nr$   r   r#   r   )rD   r  r   fmodfloorr>   int8rm   )r  r  r  
coords_mincoords_spancoords2extraflipss           r-   reflect_coordinatesz-_grid_sampler_2d.<locals>.reflect_coordinates  s    
""#F+++]
!I-2J&++--
7K00;&--//222DD{AINEJ.j0H50P
 
 	
r,   c                     dk    r| S dk    rt          j        | d|dz
            S r | dd|dz
  z            }n | dd|z  dz
            }t          j        |d|dz
            S )Nr   r#   r$   rW   r   )r  r+  coords_reflectedr  r  r  s      r-   compute_coordinatesz-_grid_sampler_2d.<locals>.compute_coordinates  s    1MQ;vq$(333 Q#6#6vq!tax.#Q#Q  #6#6vr1t8a<#P#P ;/D1H===r,   c                 4     | |          } ||          S r<   r+   )r  r+  	coords_unr  r  s      r-   compute_source_indexz._grid_sampler_2d.<locals>.compute_source_index  s'    K--	""9d333r,   r$   r#   r/  ysc                     t          j        d| k    t          j        | k     t          j        d|k    |k                                   S rw   rD   r  )r/  r  iHiWs     r-   in_bounds_condz(_grid_sampler_2d.<locals>.in_bounds_cond"  sH     GU&rBw0A!r'2PR70S0STT
 
 	
r,   r  wsc                      | |          rndt          	
fd|                     t          j                  |                    t          j                  |fD                       S )Nr#   c              3   p   K   | ]0}t          j        |d                                         V  1dS r  )rD   rm   r  )r6   r  r  r  r  oHoWs     r-   r  z1_grid_sampler_2d.<locals>.clip.<locals>.<genexpr>1  sY       
 
 Ka##((Ar266
 
 
 
 
 
r,   r   )r  r>   rD   r  )r/  r  r  r  r  r  r  r  r  r  r  s      @@r-   clipz_grid_sampler_2d.<locals>.clip*  s    ~b"%%
 $AA1 
 
 
 
 
 
 
 
ee%+e..EK0H0H"M
 
 
 
 
 	
r,   ixiyc                 D     	| ||          \  }}}||f         |z  S r<   r+   )
r  r  rS  idx_xidx_yw_C_idxN_idxr  r  s
         r-   get_summandz%_grid_sampler_2d.<locals>.get_summand6  s4    4B??ubue+,r11r,   ).r   ).r#   r   c              3   :   K   | ]\  }}} |||          V  d S r<   r+   )r6   r  r  rS  r  s       r-   r  z#_grid_sampler_2d.<locals>.<genexpr>L  sK       
 
R KB""
 
 
 
 
 
r,   c                 N     |           } |          } ||d          S r`   r+   )r  r  r7   r^   r  r  r  r  s       r-   get_value_boundedz+_grid_sampler_2d.<locals>.get_value_boundedk  s;    ##B++A##B++A;q!Q'''r,   r  c                     | dz
  z   } dz
  |           |           dz   |           dz   |          f}t          |          S )Nr#   r$   )ro  )r  iy_ofscsr  ix_nwiy_nwtxs      r-   	get_coeffz#_grid_sampler_2d.<locals>.get_coeffp  sv    cAg&F!!%!)V44!!%00!!%!)V44!!%!)V44	B ,B333r,   c              3   .   K   | ]} |          V  d S r<   r+   )r6   r  r  s     r-   r  z#_grid_sampler_2d.<locals>.<genexpr>z  s+      ::#yy~~::::::r,   r@  )rD   r   r   r  r  r  r  r  r  r   r  r  r7  rY   r  rX   ro  ).r  r  r  r  r  r  r  rZ   twor7   r^   r  r  ix_neiy_neix_swiy_swix_seiy_sew_nww_new_sww_se
ix_nearest
iy_nearesttyri  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s.   ` ````                     @@@@@@@@@@@@@@@@@@@r-   _grid_sampler_2dr    s    
Li'BBBB   
L	!#Q#Q#Q#Q  "F "# "& " " " " " "

F 

s 

 

PV 

 

 

 


>F 
># 
>& 
> 
> 
> 
> 
> 
> 
> 
>4V 43 46 4 4 4 4 4 4 4 7LAq"bZNAr2s!8888 D yyAr2s++221aRCC
6 
v 
& 
 
 
 
 
 
 

 L18,,,11!Q1==EL18,,,11!Q1==E

 

V 

 

4F 

 

 

 

 

 

 

 

 

 

 

2 2F 2& 2 2 2 2 2 2 2 2 2
 	VAVAQ!!!R((!!!R((xxzz288::uqy%ueaiueu
urz*U
urz*
rEz*U
rEz* 
 
 
 
 t$t$t$t$	 
 
 
 
 
 	
 
q	 	 !!!R((!!!R((XXZZ
XXZZ
{:z1555[B[B



%Z%Z 	!aBaB	(& 	(f 	( 	( 	( 	( 	( 	( 	( 	( 	( 	(
	43 	46 	4 	4 	4 	4 	4 	4 	4 	4 	4 ::::q:::::'333r,   c                 *    t          | ||||          S )N)r  r  r  r  )r  )r  r  r  r  r  s        r-   grid_sampler_2dr  ~  s*     	-!#   r,   c                 L    t          j                                         dk    o                                dk     fd           t          j                             d                              d          k     fd            z                      d          S )Nr$   r#   c                  \    d                                   d                                  S )Nzmatrix @ vector expected, got r  r   r   r  s   r-   r   zmv.<locals>.<lambda>  s(    JJJswwyyJJ r,   r   c                      d                      d           d                      d           d                     d           dS )Nzsize mismatch, got input (r   r7   r#   z), vec (r;  r  r  s   r-   r   zmv.<locals>.<lambda>  sG    `TYYq\\``DIIaLL``RURZRZ[\R]R]``` r,   r   )rD   r   rS   r+  r   r  s   ``r-   r  r    s     
L

a*CGGIINJJJJJ   
L		!#`````   3J"""r,   c                     |,|dz
  |z  dz   }d|z
  | z  |t          j        |           z  z
  }nd|z
  | z  t          j        |           z
  }|||z  }t          ||          S r`   )r   
logsigmoidr   )r   r   r   
pos_weightr   
log_weightr   s          r-    binary_cross_entropy_with_logitsr    s}    
  1n.2
F
d"j1<3E3E&EFF
d"Q\$%7%77f}i000r,   tensor1tensor2is_outc                    	 | j         |j         k    r| |fn|| f\  }}ddlm	 |j         dk    r|j         dk    sdS |j        r|sdS | j         dk    rdS  	|                                dk              rdS |j        }|                                }dg}t          |dd                    D ] }|                    ||d         z             !t          	fd	t          |t          t          |                    |          D                       S )
Nr   rS  r   r$   FTr#   rW   c              3   \   K   | ]&\  }}} |d k              p ||k              V  'dS r  r+   )r6   r   r   r+  rT  s       r-   r  zshould_fold.<locals>.<genexpr>  s^        D% 	tqy!!B^^DEM%B%B     r,   )r|  rr  rT  r  r   r  ru  r6  rv  r  r  rt  )
r  r  r  t1t2t1_shape	t1_strideexpected_strider+  rT  s
            @r-   should_foldr    sY    $+<7<#?#?gwgwEWFBDDDDDDGqLLRW\\u	  t|qu~bhhjjAo&& txH		I cO!""&& ; ;tob&99::::    !$tH_5566"
 "
     r,   )pass_is_out)r  c                
   ddl m}m} |                                 }|                                }|dk    r|dk    sJ |dk    r|dk    rt	          j        | |          S |dk    r|dk    rt	          j        | |          S |dk    rA|dk    r;t	          j        t	          j        t	          j	        | d          |          d          S |dk    r|dk    rt	          j        | |          S t          | ||          rN||k    }|r|j        n| }|s|n|dk    r|                                 n| }	|j        }
t          |
d d                   }t          t           j        |          }|	                                dk    }|r |                    |	j        d                    |                    ||
d                   }|rUt          j        j                            |                    |	          |          }|r|j                                        n|S t          j        j                            |                    |	          |          S |dk    r|dk    r|dk    r|                     d          nd}|                     d          }| j        d d         }|dk    r|                    d          n|                    d          }|dk    r|                    d          nd}g }t3          |dz
            D ]*}|                    |                    |                     +|dk    r|dk    r ||d         |d         k              r~ ||d         dk              r*| j        r#t7          |                     d          |          S  ||d         dk              r*|j        r#t7          | |                    d                    S t          t	          j        ||                    }|||gz   }t;          |          }|                     |                              |||          }|dk    }|rC||gz   }|                    |                              ||          	                    d          }n1|||gz   }|                    |                              |||          }|}|dk    r|                    |           |dk    r|                    |           |r;|                    |                              d                               |          S |                    |                               |          S t	          j!        dd	            d S )
Nr   )rT  guard_or_truer#   r$   rW   rX  r   Fc                      dS )Nz/both arguments to matmul need to be at least 1Dr+   r+   r,   r-   r   zmatmul.<locals>.<lambda>J  s    $U r,   )"rr  rT  r  rS   rD   dotr  r  r  rY   r  r]  r  r  rt  r   r  r   rv  r$  r  r   _unsafe_viewr  r+  rX   r  r\  broadcast_shapesr(  r  bmmr  r   )r  r  r  rT  r  dim_tensor1dim_tensor2r  r  r  sizes_1output_shapefolded_dim1t2_is_matrix	t1_foldedr  r(  m1batch_tensor1m2rP  batch_tensor2rp  expand_batch_portiontensor1_expand_sizeexpand_batch_producttensor1_expanded
vector_rhstensor2_expand_sizetensor2_expandeds                                 r-   r\  r\    s    TSSSSSSS++--K++--K!q 0 0 0 0aK1,,y'***			kQ..x)))			kQ..}UXeogq&A&A7KKQOOO			kQ..x)))	Wgv	.	. hW  +-	$1WZZ'$XGG+:J:J799;;;PW 	 (GCRCL))X\<88 vvxx1} 	-,,, JJ{GBK88	 	O Y^00b1A1A<PPF-6B69'')))FB9>..y||B/?/?NNN			kQ.. !,aGLLQ\\"crc*!,qW\\"gll26F6F +aGLLQ#%{Q'' 	2 	2A  a1111
 1q  mA.-2BBCC ! ~mA.!344 ;9N ;gooa00':::~mA.!344 ;9N ;gwq'9'9:::  $"=-@@ 
  
 3aW<#$899 #>>*=>>FF !R
 
 !A%
 	"6""=233-r221  #7"a"@&~~.ABBJJ$b!    ,??"""??""" 	M#''(899AA"EEJJ<XXX#''(899>>|LLLUUUVVVVVr,   r  r  c                      j         \  }}t          |d         ||          }t          |d         ||          }t          j         t          j        j                  \  }}t          j        |d          j                  	                    |          }	t          j        |d          j                  	                    |          }
t          ||
|          }t          ||	|          }|                    d          }|                                }|                                }||z
                      dd          }||z
                      dd          }|	                    t          j                  }|	                    t          j                  }|dz
  ||dz   |d	z   f}|dz
  ||dz   |d	z   ft          |          t          |          }d
\   j        t          j        k    r:t%                    t%          |          fdD             fd|D             } fd fdt'          fd|D                       } j        t          j        k    rJ t)          ||          }n't+          d t-          ||          D                       }t          j                   }|                    |          }|S )Nr   r#   rU  r  r   rW   r   rk   r$   ri  c                     g | ]?}|d z  z  t          j        |          dz  z                       t           j                  @S r#   r   rD   r   r>   int16)r6   rS  weights_precision_xs     r-   r9   z.upsample_bicubic2d_default.<locals>.<listcomp>  V     
 
 
 !**+ejmmc.AAEEekRR
 
 
r,   c                     g | ]?}|d z  z  t          j        |          dz  z                       t           j                  @S r  r  )r6   rS  weights_precision_ys     r-   r9   z.upsample_bicubic2d_default.<locals>.<listcomp>  r  r,   c                     t          j        | ddz
            }t          j        |ddz
            }t                              d d ||g          }|S r  )rD   r   r   r^  )r  r/  y_idxx_idxr;  in_hin_wr   s        r-   load_boundedz0upsample_bicubic2d_default.<locals>.load_bounded  sT    B4!8,,B4!8,,utT5%&@AAr,   c                      t           fdD                       }j        t          j        k    rJ t	          |          S t          d t          |          D                       S )Nc              3   0   K   | ]} |          V  d S r<   r+   )r6   x_ofsr  r^   s     r-   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s/      BBll1e,,BBBBBBr,   c              3   &   K   | ]\  }}||z  V  d S r<   r+   rm  s      r-   r  zCupsample_bicubic2d_default.<locals>.get_x_interp.<locals>.<genexpr>  s*      JJRBGJJJJJJr,   )r  r   rD   rJ  r  r  r  )r^   src_xr   ixs_ofsr  r  	weights_xs   ` r-   get_x_interpz0upsample_bicubic2d_default.<locals>.get_x_interp  s~    BBBBB'BBBBB;%+%%&222%eY8KLLLJJCy4I4IJJJJJJr,   c              3   .   K   | ]} |          V  d S r<   r+   )r6   y_ofsr   s     r-   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s-      ;;%,,u%%;;;;;;r,   c              3   &   K   | ]\  }}||z  V  d S r<   r+   rm  s      r-   r  z-upsample_bicubic2d_default.<locals>.<genexpr>  s*      LL(2rb2gLLLLLLr,   rV  )r  r  rI   rJ   rW  r5  rD   r  r  r>   r  rY   r  r   r  rh  r   rJ  r!  r  r  r  r  r   r  )r   r  r  r  r  rZ   h_scale_factorw_scale_factorr   rp  r  x_floaty_floatr7   r^   yscaler3  iys_ofs	weights_ysrc_yr  rW  r   r  r  r  r  r  r  r  s   `                     @@@@@@@@r-   upsample_bicubic2d_defaultr,  M  sH    {Aq$ $D+a.-QQN#D+a.-QQN'5#H#U  HAu 	[^EL999<<5<IIA[^EL999<<5<IIA#NA}EEG#NA}EEG##GAA k  c**Fk  c**F	U[A	U[A1uaQA&G1uaQA&G088I088I/9,,{ek!!7	BB7	BB
 
 
 

 
 
	
 
 
 

 
 
	
      K K K K K K K K K ;;;;7;;;;;E{ek!!"...#E96IJJLLc%6K6KLLLLL /66M];;FMr,   c                 f   t          j        t          |          t          |          z   dk    d            |]|J t          t          t
          t
          f         t	          d t          | j        dd          |          D                                 }|r|nd\  }}t          | ||||          S )Nr#   c                      dS )Nz:Must specify exactly one of output_size and scale_factors.r+   r+   r,   r-   r   z(upsample_bicubic2d_vec.<locals>.<lambda>  s    L r,   c              3   Z   K   | ]&\  }}t          t          |          |z            V  'd S r<   )r   r   )r6   rS  rr   s      r-   r  z)upsample_bicubic2d_vec.<locals>.<genexpr>  sM        Au 	!u,--     r,   r$   ri  )	rD   r   r  r   r  r  r  r  r,  )r  r  r  r1  r  r  s         r-   upsample_bicubic2d_vecr0    s     
L[D///14LL   (((#s(O   #AGABBK ? ?    
 
 )6G}}<GW%amWgVVVr,   c                 0      fd}t           ||          S )Nc                     t          j        |  ||z   j                  }|dz
  |dz
  |                                z
                                  z
  S )Nr  r#   )rD   r  r  r   r   middler   dim_idxr  s       r-   r  z_reflection_pad.<locals>.idx  sM    ,ufunQXFFFzVaZ'++--7<<>>>>r,   _reflection_or_replication_padr  r  r  s   `  r-   _reflection_padr9    s9    ? ? ? ? ? *	  r,   c                 0      fd}t           ||          S )Nc                 v    t          j        |  ||z   j                  }t          j        |d|dz
            S )Nr  r   r#   )rD   r  r  r   r3  s       r-   r  z_replication_pad.<locals>.idx  s9    ,ufunQXFFF{7Avz222r,   r6  r8  s   `  r-   _replication_padr<    s9    3 3 3 3 3 *	  r,   idx_fnc                    t                    dz  t          j        |                                 dz   dz   fv fd           | j         d          }|                                 z
  }fdt                    D             }fdt                    D             }| }t                    D ]Z}d g|                                z  }	 |||         ||         ||                   |	||z   <   t                              ||	          }[t          j	        |          }
|
                    |
          }|S )Nr$   r#   c                  (    d  d dz    d dz    dS )Nreflection_padzd requires r#   zD or r$   zD inputr+   r   s   r-   r   z0_reflection_or_replication_pad.<locals>.<lambda>  s+    OOOqOOsQwOOO r,   c                 2    g | ]}d dz
  |z
  z           S r  r+   r6   rp  rS   r  s     r-   r9   z2_reflection_or_replication_pad.<locals>.<listcomp>  +    CCC1GAq1-.CCCr,   c                 8    g | ]}d dz
  |z
  z  dz            S r  r+   rB  s     r-   r9   z2_reflection_or_replication_pad.<locals>.<listcomp>  0    HHHWQ#'A+.23HHHr,   rV  )r  rD   r   rS   r  rX   r   r^  rI   r   r  )r  r  r=  	inp_shapenc_dimpadding_leftpadding_rightr  rp  r  rW  rS   s    `         @r-   r7  r7    sU   
 g,,!
C	L	C!GS1W%%OOOO   IUUWWs]FCCCCCc

CCCLHHHHHU3ZZHHHMF3ZZ 1 1&**,,. &a)A,a@PQQAJ##FC00 /77M];;FMr,   c                     t                    dz  d |j         d          D             fdt                    D             fdt                    D             g }t          |j                  D ]^}dg|j        z  }d||<   |                    t          j        |j        |         |j                                      |                     _|d           | d          d fd	t                    D             fd
t                    D             }fdt                    D             }fdt                    D             t          j
        t          j        fdt                    D                       }t                               |z   d          }	 fd}
t          j        d t                    D              D ]}|t!          dgz            k    rg }g }t                    D ]}||         dk    r|         }|         }n]||         dk    r||         }|         d|         f}n7||         dk    r+||         }|         |         |         z
  |         dz
  f}|                    |           |                    |            |
|	||          }	|	S )Nr$   c                     g | ]}|d z
  S r!  r+   )r6   rw  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>  s    
)
)
)Q1q5
)
)
)r,   c                 2    g | ]}d dz
  |z
  z           S r  r+   rB  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>  rC  r,   c                 8    g | ]}d dz
  |z
  z  dz            S r  r+   rB  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>  rE  r,   r#   rW   r  c                 H    | \  }}}t          j        ||k    ||k              S r<   r  )index_rangerp  lbubs       r-   index_range_conditionz7_reflection_pad_backward.<locals>.index_range_condition  s(    	2r b!r'222r,   c                 2    g | ]}|         |         z   S r+   r+   r6   rp  rH  xyzs     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>!  s&    ;;;1c!f|A&;;;r,   c                 2    g | ]}|         |         z
  S r+   r+   rT  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>"  s&    AAALOc!f,AAAr,   c                 J    g | ]}d |         z  |         z   |         z
   S r$  r+   )r6   rp  dhwrH  rU  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>#  s3    OOOqQQZ,q/1CF:OOOr,   c                 V    g | ]%}|         d |         |         z   |         z   f&S r  r+   )r6   rp  centerrX  rH  rI  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>'  sH       HIAs1vQ/-2BBC  r,   c                 2    g | ]} |                   S r+   r+   )r6   rp  rR  range_cs     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>+  s)    QQQ00<<QQQr,   r   c                 :   t                    D ];}||         d         ||         d         k     }t          |t                    r|r| c S <t          j        t
          j        
fd|D                       }t
                              	||z   d          }| |z   S )Nr$   r#   c                 &    g | ]} |          S r+   r+   )r6   rO  rR  s     r-   r9   z@_reflection_pad_backward.<locals>.accumulate.<locals>.<listcomp>9  s%    PPPK"";//PPPr,   r   )rX   r5   r  rP   r   r   r  r  )r   r   index_rangesrp  upper_less_than_lowerr  gr  rS   rp   rR  s          r-   r'  z,_reflection_pad_backward.<locals>.accumulate/  s     s 	 	A$0OA$6a9K$K!/66 ;P PPPP<PPP
 
 %%k4S#FFaxr,   c                     g | ]}g d S ))rW   r   r#   r+   r"  s     r-   r9   z,_reflection_pad_backward.<locals>.<listcomp>>  s    #C#C#C1JJJ#C#C#Cr,   r   )r  r  rX   r|  rv  rD   r  r  r  rP   r   r   r  r  	itertoolsr
   r  )rp   r7   r  r  rp  
view_shapeleft_reflectright_reflectr  r   r'  areaoutsr_  r   rO  r  rZ  rX  rS   rR  rH  rI  r\  rU  s   ` `             @@@@@@@@@r-   _reflection_pad_backwardri    s}   
 g,,!
C
)
)!'3$%%.
)
)
)CCCCCCc

CCCLHHHHHU3ZZHHHMG16]] S SS16\

1u|AGAJqx@@@EEjQQRRRR#A
3$%%.C3 3 3 <;;;;c

;;;FAAAAAeCjjAAALOOOOOOE#JJOOOM      MRSVZZ  G QQQQQeCjjQQQ D $$[$F
CHHD        !#C#Cc

#C#C#CD 4 45!s####s 	- 	-AAw!||Qi%ajaB"1o"1vq,q/:aA#A&"1vs1va0@'@#a&1*MKK,,,,z$l33Kr,   r   r   r   c                f    t          j        | ||          }t          j        | ||          }||fS )Nr   )rD   aminrY  )r   rS   r   rk  rY  s        r-   aminmaxrl  Y  s:     :dW555D:dW555D:r,   r   c                    t                               t          j        t          j        |           d|           |||          S )Nr   r   )r   r   rD   rm   isnan)r   rS   r   r   s       r-   nansumro  c  s7     88EKD 1 11d;;S'QV8WWWr,   r   rC  r  r]  rC  c          	      N    t           j                            d| d||||          S )Nr   r#   rp  r   r  
start_step)ri  r   rC  r  r]  s        r-   arange_defaultrt  i  s2     ;!!	3vf "   r,   c          	      N    t           j                            | |d||||          S )Nr#   rp  rr  )rh  ri  r   rC  r  r]  s         r-   arange_startrv  x  s2     ;!!sAU6&Z "   r,   c                      ddl m}  || i |S )Nr   )out_dtype_dense)!torch._higher_order_ops.out_dtyperx  )rK   rL   rx  s      r-   out_dtype_decomprz    s)    AAAAAA?D+F+++r,   marginc                 N   	
 t          j                    t          j                   j        d         
 j        d         	t          j        |dk    p|dk    d            t          j         j        dk    o	dk     fd           t          j        j        dk    o                                
k    
fd           Nt          j                  t          j        j        dk    o                                	k    	fd                               d          t          j         d          }||z
   z   }|	                    d          }|dk    r|n||z  }|         z  }t          j
        	 j        	          }t          j        |k    |d          }|t          j        j        k    r|                                S |t          j        j        k    r"|                                |j        d         z  S |                    d
          S )Nr   r#   r$   c                      dS )Nz only p == 1 and p == 2 supportedr+   r+   r,   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    +M r,   c                      d j          S NzMExpected non-empty vector or matrix with optional 0-dim batch size, but got: r  r  s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    m`e`kmm r,   c                      d  dj          S )Nz#inconsistent target size, expected r  r  )nframer   s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    UfUUv|UU r,   c                      d  dj          S )Nz#inconsistent weight size, expected r  r  )rS   r   s   r-   r   z#multi_margin_loss.<locals>.<lambda>  s    V#VVVV r,   r  r  r   )rD   
atleast_2d
atleast_1dr  r   r|  r   rY   rR  r^  r  r  rm   r"   r)   r   r   r*   r   )r   r   rP  r{  r   r   urn   r  rS   r  s   ``  `    @@r-   multi_margin_lossr    s"    U##Ef%%F[^F
+a.C	La!16#M#MNNN	L
a$C1Hmmmm   
Lq5V\\^^v5UUUUU   !&))K163!6VVVVV	
 	
 	
 a  FU000A
UA	AA!VVQAv
,s5<
0
0
0CC6M1a((AIN(((vvxx	im)	)	)uuww##vv!v}}r,   	is_targetc                    | j         |j         t          j        |           } t          j        |          }| j         d         }t          j        t	                    dk    o|dk    fd           t          j        t	                    dk    ok    fd           t          j        ||j                  }|dk    }t          j        t          j        |||          dd	          }||k     }t          j        ||d          }t          j	        | d|
          }	t          j        ||d          }
t          j
        ||
                    d          k    d          }d|	j                            d          z
  | z   }|                    d          }||z  }t          j        |d|          }|t          j        j        k    r)|                    d                                          }n@|t          j        j        k    r|                                }n|                    d          }|                    | j                                                }||fS )Nr#   r$   r   c                      d  S r  r+   )orig_input_shapes   r-   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    r`prr r,   c                      d d  S )Nzinconsistent target size: z for input of size: r+   )r  orig_target_shapes   r-   r   z0multilabel_margin_loss_forward.<locals>.<lambda>  s    f->ffTdff r,   r  rW   Tr   r  r   rk   )r   rW   )r  rD   r  r   r  r  r  rk  rm   rR  anyrY   Tr^  r"   r)   r   r   r   r*   r>   r   r$  )r   r   r   rS   r  is_endend_idxtarget_masktidx0r  tidx1r  rn   r  r  s                @@r-   multilabel_margin_loss_forwardr    sG    {U##Ef%%F
+a.C	L"/saxrrrr   
L!#M(9=M(Mfffff  
 ,s6=
1
1
1Cr\FjVS#66BMMMG-KKVQ//EU%000AKVR00E	#R!8!88a@@@Iacmmm###e+A	AA	CAIq!$$AIN(((EEgE##%%	im)	)	)EEGGEEgEU[))112CDDIi<r,   )	attn_maskrr   querykey	dropout_p	is_causalr  c                8    t          j        t          j                    fd           t          j                                         dk    o/                                dk    o                                dk     fd           t          j        dk    fd           t          j         j        d         j        d         k    oj        d         j        d         k    d            t
          j                             ||d |                     d                              d          k    		  	        \  }}|	                    d
ddd          
                    t           j                  	                    dd
dd          }||fS )Nc                      d j          S )Nz-query must be FP32, FP64, BF16, FP16 but got r   )r  s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>	  s    MMM r,   r@  c                      d                                  d                                   d                                  S )Nz,q, k, v must be a 4 dimensional tensor, got r  r   )r  r  r   s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s=    huyy{{hhcggiihh[`[d[d[f[fhh r,   r   c                      d  S )Nz&dropout probability must be zero, got r+   )r  s   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    "V9"V"V r,   r   c                      dS )Nz&q, k, v should have the same head sizer+   r+   r,   r-   r   z<scaled_dot_product_flash_attention_for_cpu.<locals>.<lambda>  s    8 r,   r#   )r  r  r  dropout_maskrr   
enable_gqar$   r   rV  )rD   r   rL  rS   r  r   "_scaled_dot_product_attention_mathr  r+  r  r  r[  )	r  r  r   r  r  r  rr   r  attns	   ````     r-   *scaled_dot_product_flash_attention_for_cpur    s    
L&&MMMM   
L		q@SWWYY!^@		q0@hhhhhh   
LSVVVV   
LA%+a.(KSYq\U[^-K88  
 :BB::a==CHHQKK/ C 
 
LFDF 	q!Q""	%"9	:	:	Aq!		 
 4<r,   c                 <    t          |           fd            }|S )Nc                  J     | i |}| d                              |          S rw   )r   )rK   rL   r   outplace_ops      r-   
inplace_opz$register_inplace.<locals>.inplace_opB  s-    k4*6**Aw}}S!!!r,   r   )aten_opr  r  s    ` r-   register_inplacer  A  s7    G$$" " " " %$" r,   c                 f   |                                  s2|                                 st          |          }t          |          }t          j        ||          }t          |t          j                  r|dk    r||z  }|dk    r|S t          |t          j                  r|dk    r| |z  } | |z   S )Nr#   r   )rL  r  r  rD   r  r5   numbersNumber)r   batch1batch2rg   rq   r  s         r-   baddbmmr  J  s     !!## DOO,=,= 4yyE

Yvv&&FeW^,,  

%qyydGN++ tqyyd{&=r,   c                 0    t          j        | |d          S )Nr  r  r  )r   rO  s     r-   floor_divider  \  s     9T58888r,   c                 L    t          j        t          j        | j        d          S r`   )rP   r   r  r   r  )r  s    r-   	sym_numelr  b  s    HL!'1555r,   r   r   c                    |"t           j                            | g |          S t           j                            | g ||          S )Nr   r  )r   r   dim_IntListIntList_out)r   r   r   s      r-   sum_defaultr  g  sE     {x##D"E#:::x##D"Es#CCCr,   c           	         t          | t          j                  s| S |Lt          j                            | t          t          |                                                               S t          j                            | |g          S r<   )	r5   rD   r   r   r  dimsrt  rX   rS   )r   rS   s     r-   squeeze_defaultr  t  sl     dEL)) 
{|  tE$((**,=,='>'>???|  u---r,   c                 4   t          fdt          t          | j                            D                       }|j        t
          j        k    rt
          j        nd }|                     d|d|          }| ||	                    |j                  z  z  |fS )Nc              3   (   K   | ]}|k    |V  d S r<   r+   )r6   rp  rS   s     r-   r  z)_weight_norm_interface.<locals>.<genexpr>  s'      @@1qCxxQxxxx@@r,   r$   T)r   r   )
r  rX   r  r  r   rD   r  r  r   r>   )r;  ra  rS   keep_dim
norm_dtyper   s     `   r-   _weight_norm_interfacer    s     @@@@c!'ll 3 3@@@@@H !5> 9 9tJ66!Xt:6>>DDGGAG$$$%t++r,   assume_uniqueinvertc                   t          | t          j                  st          j        | |j                  } t          |t          j                  s,|rt          j        | |          S t          j        | |          S |                                dt          |                                 d          z  k     rt          | ||          S t          | |||          S )Nr  g      $@g(\?r  r  )r5   rD   r   rx  r  ner  r   rY  isin_defaultisin_sorting)elementstest_elementsr  r  s       r-   isinr    s     h-- G<1EFFFmU\22 5 	58Hm4448Hm444tc(..*:*:E&B&BBBBHmFCCCCm=
 
 
 	
r,   )r[  c                *   |9t          j        |                                 t           j        | j                  }n9t          j        |                                 |t           j        | j                  }|| k                         | j                  }|S )Nr  )r[  r   r  )rD   randr+  r   r  r>   r   )r   r[  raw_prP  s       r-   	bernoullir    s|     
499;;emDKPPP
IIKK-;	
 
 
 
$*%%AHr,   r  c                P   |                                  dk    r t          j        | t          j                  S | j        d|j        z  z   }|                     |          }t          t          d|j         dz
  d                    }||k    	                    |          }|r| n|S )Nr   r   r!  rW   r#   r   )
r   rD   
empty_liker  r  r|  r  r  rX   r  )r  r  r  expanded_elem_shaper7   rS   rO  s          r-   r  r    s    ~~1
;;;;".4-2D+DD)**A
b=--1266
7
7C
"
"s
"
+
+C"C44s"r,   c                   |                                  }|                                 }|rt          j        ||g          }t          j        |d          \  }}|dd          |d d         k    }	t          j        |	ddgd          }	|r|	                                }	t          j        |	          }
|
                    d||	          }
|
d|                                          S t          j        |          \  }}t          j	        ||          }t          j
        ||                                k     |d          }||         |k    }|r|                                n|}|                    | j                  S )NT)stabler#   rW   r   F)r<  rD   r-  sortrt  logical_notr  r  r   searchsortedrm   r$  r  )r  r  r  r  elements_flattest_elements_flatall_elementssorted_elementssorted_orderduplicate_maskr  sorted_test_elementsrZ   r  test_idxcmps                   r-   r  r    ss   $$&&M&..00 + y-1C!DEE(-
<(M(M(M%(,0DD.~1vuMM 	:+7799N//q,??A((())"'*-?"@"@a !5}EE;s%9%?%?%A%AA3JJ"8,=#)2coos{{8>***r,   c                 <    |                      d          }||         S rV   )r$  )r   r  	flatteneds      r-   taker    s      R  IUr,   c                     |t           j        }|t           j        k    rt          |          }t                              | |j        |          S r3  )rD   r[  preserve_formatr   r   resizer  )r   rO  rW  s      r-   	resize_asr    sE    /----e44;;tU[;FFFr,   )FF)r   r<   r$  )r   NNr#   )rW   FFr  r  r{  )r#   r#   F)Fr   )r   rk   N)r   r#   N)Fri  )NNN)r   r   FT)r   r   Fr/  )r   F(  rP   rc  r  r  rv  collections.abcr   r   
contextlibr   enumr   r   r   r	   r
   typingr   r   r   r   rD   torch._meta_registrationstorch._primsrd  r"  torch._prims_common_prims_commonrI   torch.nn.functionalr  r  r   r   r   r   torch._decompr   ry  r   r   r   r   r   r   torch._prims_common.wrappersr   r   r   r   torch.utilsr   rG   torch.utils._pytreer   rD  DispatchKeyr    rt  str__annotations___opsr  r   r"   rW  r  rR   rX  compute_only_pw_cast_for_opmathpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_argsr5  pw_cast_for_int_to_realr  r[   rd   rf   ro   r  r{   rJ  r	  r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r   r   r   r   r   r)   r   r   _safe_softmaxr  r
  r  r  r\   r  r  r   r  r(  r5  r>  rA  rG  rJ  rL  rN  rR  rf  rn  slicer  r  rm  r  r  r  r  r  r  r  r1  r5  r>  rD  rH  py_implCompositeImplicitAutogradAutogradrG  r\  r`  rf  rn  r(  r|  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r,  r2  r6  unsafe_chunkr9  r5  r4  no_statsr=  rA  rM  rS  rW  rY  _fused_dropoutr\  ri  r  rW  r  lift
lift_freshrl  rq  rt  rw  rv  r  r  r  _adaptive_avg_pool2dr  r  r  r  r  r  r  r  r  r  r  r  r$  	Generatorr&  r3  r6  rL  r  rS  rY  r<  _upsample_nearest_exact1drU  r[  rA  rG  rN  r9  rg  rp  rt  rx  r  r  r  r  r  r  rnn_tanhr   r  rnn_relur  r  r  r  r  r  r  r  lstmr  r  r  r  grur  r  r  r   r  r  r  r  r	  r  r  r  r  r!  r  r=  r?  r  r^  r)  r  rP  rV  rY  r[  r_  rb  rh  ro  r  ru  r  r  r  r  r  r  r  r  r  r  r\  upsample_bicubic2dr,  r0  reflection_pad1dreflection_pad2dreflection_pad3dr9  replication_pad1dreplication_pad2dreplication_pad3dr<  r7  reflection_pad1d_backwardreflection_pad2d_backwardreflection_pad3d_backwardri  rl  ro  r  ra  rC  rt  rh  rv  rz  r  r  +_scaled_dot_product_flash_attention_for_cpur  r  r  r  r  r   r  r  rS   r  r  r  r  r  r  r  r  addbmm_addbmmaddmm_addmv_baddbmm_fill_gelu_r  
hardswish_	hardtanh_hardtanhhardsigmoid___iand____and____ilshift__
__lshift__r  r  index_reduce_index_reduce__ior____or____irshift__
__rshift____ixor____xor__leaky_relu_
leaky_relulogit_logitrelu_r  renorm_renormround_r7  scatter_r#  scatter_add_scatter_addscatter_reduce_scatter_reducesilu_r+   r,   r-   <module>r>     s`              



 . . . . . . . . " " " " " "       % % % % % % % % $ $ $ $ $ $ $ $ - - - - - - - - - - - -                # # # # # #          , , , , , , , , , , 0 0 0 0 0 0 7 7 7 7 7 7                         * ) ) ) ) ) ( ( ( ( ( ( h" c   z~         %$)	' ''9' ' "	' ' ' 'T #*'8@# # # 
 WuDL    &-W8@ & & & "
 "'uDQ    c f     *++\2F 2v 2 2 2   ,+2 -..\4v 4& 4 4 4   /.4 .//\S S6 S S5 S S S   0/S
 )**\


 
 	

 
 
 
 
   +*
2 )*++( ( ,+( )*++"V " " " ,+" ())@f @ @ @ @   *)@ 122\f F      32 .//\PP%P05P@EP P P  0/P ''GF Gv G G G   ('G /00F & V      10 /00\:F :& :U : : :  10: 011\LL%L7<LNRL L L   21L *++\) ) )f )3 ) ) )   ,+)< *++5v 5f 5 5 5  ,+5 	""&v && & & &   #"& *++\>v >V > > > >   ,+>
 *++6 6 66 6 6 6 ,+6 344%%
% % 66>	% % % 54% 677



 
 	

 
 
 
 
 
 
   87
$ 122\<f <F <F <v < < <   32<v #         &&3<>3G1 1
1 1-011 1 1   '&1 .//\11 &1061CF1 1 1   0/1 *++0 0 0 ,+0 +,, ^)		1 	1
	1	1 	1 		1 	1 	1   -,	1 4<==%/5BEMR    >= 4?@@
R
R

R 
R 	
R
 
R 
R 
R 
R  A@
R 0899		%	/5	BE	NS	 	 	  :9	 0455
R
R

R 
R 	
R
 
R 
R 
R 
R  65
R$$
$ $ V	$
 $ $ $ $ $ $ $@ )**\Nf NF N N N N N   +*N$ .//\''
' ' V	'
 ' ' ' ' ' '  0/'T 011\
  V	
        21B 122  $^)	1 1
11 V1 	1
 1 1 1   321& :;;\
  $^) 
  V	
       <;  -.. ^)1 111 1 	1 1 1   /.1 677\
 ^)		 		
	 	 		
 	 	 	   87	 	"") ) )v )% ) ) )  #") ,--& &F &v & & &  .-& +,,	O	Oc	O 
	O 		O
 
	O 	O 	O 	O  -,	O 
)** 3? 3?
3? 
3? C=	3?
 
#3? 3? 3? 3? +*3?l (4<SM
38_   . *++ $U $U$U	$U 
$U C=	$U
 
#$U $U $U $U  ,+$UN ,--E Ed3i Ec ERU E E E  .-E
 .//OO&*3iO9<ODGOORO O O  0/O%+:?+    344\ XX!'X.1X@EX X X !   54X 788 KK!'K.1K@EK K K !   98K* * *& $$MMcM 3iM #Y	M
 IM M M M  %$M` $$]]c] c] 3i	]
 #Y] I] ] ] ]   %$]@ 455
 
v 
e 
 
 
  65
 ,--Y
Y"3iY47Y?BYJMYY Y Y  .-Y" +344>B
 

%
,4UO

 
 
  54
& %%kCDDk2336 e HTN    43 ED &% +,,VVA& AU A8D> A A A  -,A && S      '&. )**F  T     +*0 '' $   	
      ('& 566  	
     76:DI    &\	  
&\	   *T&\    
tF| 
# 
 
 
 
"&\"	" " " " "J 0$/2EFGG
 !	 &\	  
&		
    HG  	')C)GH  "&	 
c 
 
$v,		
 d6l     )0115 5 5C 5c 5%PSBT 5 5 5 215 4<==67B BB $S	B03B
63;B B B >=B 
)**/ / /C /c /%:L / / / +*/" -55)  &/ &/
&/ &&/ 
&/ 63;	&/ &/ &/ &/T 
##  f F # #      $#" .//
  

  	
       0/" 
##	 	 	f 	6 	 	 	 	 	   $#	 7?@@S&S&S& S& 	S&
 FS& S& S& 
S& S& dS& 8FXf-x/??@S& S& S&  A@S&n 7;<<  	
 F   
  d , , , 8FXf-x/??@   =<88F# x/?     7?@@OOO 3iO 	O
 O VO 6
O dO 8FXf-x/??@O O O A@Of 7;<< 3i 	
  V 6
 d , , , 8FXf-x/??@   =<4 ,455333i3 V3 
%	3
 66>3 3 3 653l 5=>>GGG 3iG 	G
 VG dG 8FXf--.G G G ?>GTOOVO 6
O 6"	O
 &!O O O 
O O 6668F#3Xf5EEFO O O Od .//UK//((V( 6
( 6"	(
 &!( ( ( 
( 666!"( ( ( 0/ 0/(4 ''(<==''(MNN 
 
V 
 6
 
 6"	 

 &! 
  
  
 
 
 666!" 
  
  
 ON >= 
F "";#HIIG G4< G G G JIG AIJJV 6
 	
   
 666!"   KJ* 5=>>((V( 6
( 	(
 ( ( ( 
( 666!"( ( ( ?>(  5>??((V( 6
( 	(
 ( 
( 666!"( ( ( @?( @HIIKKVK 6
K 	K
 K K K 
K 6666612K K K JIK0V 6
 	
  
     : 4<==11V1 6
1 	1
 1 1 
1 6666)*1 1 1 >=14 ?GHHCCVC 6
C 	C
 C C 
C 6666669:C C C IHC4 2:;;11V1 6
1 	1
 1 1 
1 6666)*1 1 1 <;14 +,,VV     -, && $(%)37& & &VZ & EK &
 U\"& & & E/0& & &  '&&X diABB   CB &&{';<<-..VVVV,, 6
 6"	
 &!  !&    -, /. =<>   0899 V 6"	
 &!  &!  
 d  68F#Xf%556   :96 7?@@ddd Vd 6"	d
 &!d d &!d d 
d dd 68F#Xf%556d d d A@dP 7;<<""" V" 6"	"
 &!" " &!" " 
" d" ," ," ," 68F#Xf%556" " " =<"J 788VVV$$  6"	
 &!  v    %$ 980 677VVV$$  6"	
 &!  v     %$ 872 122c'v c'E#s(O c' c' c'   32c'L
)8<S	HK   0 )***8
*8*8 c*8 *8 *8  +**8Z )**494949 c49 I	49
 #Y49 49 49  +*49n (( H H HH	H H 	H H H H )(H '' I I II	I I 	I I I I  ('I$ %@ %@ %@%@	%@ %@ 	%@ %@ %@ %@ %@ %@P )122"";#HII   JI 322 ())<: <C <
 <J < < < *)< ((=* =3 =z =: = = =  )(=@@@$.@8B@PT@ @ @ @, 011Xx  (f (vv~)> ( ( (  !  21( %% $'$'+/	 	tS%	  c5 
! (	    &%  &&; ; ; '&;
  8   /344/344/344$$[%JKK$$[%9::$$[%JKK$$[%9::$$[%JKK$$[%9::	3	3$s)$	3 DK(	3 		3 	3 	3 ;: LK ;: LK ;: LK 54 54 54	3 6:;;6:;;6:;;#++K,QRR#++K,@AA#++K,QRR#++K,@AA#++K,QRR#++K,@AA	?	?$s)$	? DK(	? 		? 	? 	? BA SR BA SR BA SR <; <; <;	?   @ 08$:Q:UVWW (()NOO (()=>>Dd;;; #; ;;c; UO; 	; ; ; <; ?> PO XW; 	#+T-K-OP  '//0UVV'//0DEEDd;;; #G GGcG UOG 	G G G <; FE WV G 08$:Q:UVWW (()NOO (()=>>Dd;;; !% $	G GGcG uoG uo	G
 G G G <; ?> PO XWG 	#+T-K-OP  '//0UVV'//0DEEDd;;; !% $	S SScS uoS uo	S
 S S S <; FE WV S 08$:Q:UVWW (()NOO (()=>>Dd;;; !% $ $Q QQcQ uoQ uo	Q
 uoQ Q Q Q <; ?> PO XWQ 	#+T-K-OP  '//0UVV'//0DEEDd;;; !% $ $	 		c	 uo	 uo		
 uo	 	 	 	 <; FE WV 	 
 	 c %! 	
    4   > > >/ / /   FK+ + + +\    & & & &,/- /- /- /-d&  &  & R +,,[BCC[122. . 32 DC -,.8 +,,[BCC[122. . 32 DC -,.8 *++KABBK011. . 21 CB ,+.@ *++KABBK011. . 21 CB ,+.@  / / / /6= = = =@/ / /d 	((>??-..S S /. @? )(S> 	''=>>,--S S .- ?> ('S<; ; ;; ; ; &&{<=={+,,. . -, >= '&.6 ''=>>,--. . .- ?> ('.6 4899!))+*OPP!))+*>??  @? QP :9 3788 (()NOO (()=>>  ?> PO 98 04551566##K$IJJ##K$899%%k&KLL%%k&:;;&&{'LMM&&{';<<A A =< NM <; ML :9 KJ 76 65A /79O9STUU
 !%	K KKcK K uo	K
 K K K  VUK 	%t'?'CD  !))+*>??
 !% $U UUcU U uo	U
 uoU U U U  @? 
U 	&(A(EF  
 !% $ $
 

c
 
 uo	

 uo
 uo
 
 
 
  
V V V V/ / /7	&	7$,V$47IO77 7 7 7-'9 -f - - - - EEcE E %!	E
 E E E ER )122F v $    32 ,d.?@AA   BA +,--" " .-" /0119 9 9 219 2344C C 54C6 ABCCM M DCM23 
3 3  V3  	3 
 3  66>3  3  3  3 l -..X~&&L
LL VL 	L
 L 66>L L L '& /.L6 /00X~&&L
LL VL 	L
 L 66>L L L '& 10L/F /u / / / / /5F 5u 5 5 5 5 5
 
3E 
 
 
 
(F%7 FV F F F F F!Xf% !& ! ! ! !NN#'N05NEJ\N N N N&f & & &T & & & &"/f / / / /T / / / /"!V !49 !T ! ! ! !$V $49 $T $ $ $ $ 233S StCy S S S S   43S  c4 c4c4
c4 c4 	c4
 c4 c4 c4 c4 c4 c4L ,--   
  	
       .-    	# 	#   ! 	# =>>$).:N1 1 1  ?>1  d t    @ [BCC>??', vW vW vW vW  @? DCvWr 08$:Q:UVWW (()=>>
  $#P PPsCxP P e_	P
 e_P P P P   ?> XWPf /344$$[%JKK$$[%9::
 48	W WW%S/*W W E%,/0	W
 W W W   ;: LK 54
W. -..-..-..	v 	c3h 	F 	 	 	   /. /. /.
	 .//.//.//	 	sCx 	V 	 	 	   0/ 0/ 0/
	38_ c3_f,- 	   8 677677677\T T  87 87 87Tn %%UEu      &% $$X4 X X X X  %$X ,dko>?? $( =%)
 
 
	
 EK 
 L	

 U\"
 
 
 
  @?
 *+,,
 $( =%)  	 EK 	
 L U\"    -, 	"", , #", .//''(<== #^)) ))) ) 	)
 V) ) ) ) )  >= 0/)X ;<<$,44[5IJJX{##--- - 66>	- - - $# KJ =<-x HPQQ
 A #'!A A AA	A A 	A
 A A E?A 66>A A A RQAH   ''     (' )**9 9  +*9 ''6 6 ('6 )48<899 $( 		D 	D 	D
	D EK 	D 
&			D
 	D 	D 	D :9	D -t|/?@AA. .& .x} . . . BA. 	=>>, , , ?>, 	""38 
 
 
 
  #"
$ .// ,0  
, ( \	   0/$ 5: # # # # # <A + + + + +: 	""   #"
 ''G G G ('G  t{ + + +  dj ) ) )  dj ) ) )   - - -  TY ' ' '  TY ' ' '  $. 1 1 1   / / /  "D$4 5 5 5   - - -  !4? 3 3 3  $. 1 1 1  #T%6 7 7 7  t{ + + +  !4? 3 3 3   - - -  !4? 3 3 3  dj ) ) )  TY ' ' '  t{ + + +  dj ) ) )   - - -  "D$4 5 5 5  %t': ; ; ;  TY ' ' ' ' 'r,   