
    IЦi;                    o   % S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
JrJrJrJrJrJrJrJrJr  S SKJr  S SKrS SKrS SKrS SKrS SKJs  Jr  S SKJr  S SK J!r!  S SK"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-  S SK.J/r/J0r0  S S	K1J2r2J3r3J4r4J5r5J6r6  S
SK7J8r8  SSK9J:r:J;r;J<r<J=r=  SSK>J?r?J@r@  SSK<JArAJBrBJCrCJDrDJErEJFrFJGrGJHrHJIrIJJrJJKrKJLrLJMrM  SSKJNrNJOrOJPrPJQrQJRrRJSrSJTrTJUrUJVrVJWrW  SSKXJYrYJZrZ  SS1r[\R                  " \]5      r^0 r_\\\S\4   \`4   \S\4   4   \aS'   0 rb\\R                  R                  \\S\4      4   \aS'   \e" 5       rf\\R                  R                     \aS'   \R                  R                  rg\R                  R                  rh\R                  R                  ri\e" 5       rj\\R                  R                     \aS'   \e" 5       rk\\R                  R                     \aS'   \e" 5       rl\\R                  R                     \aS'   0 rm\\R                  R                  \R                  R                  4   \aS'   \R                  R                  rnS\S\4   S\\S\4      4S jroS rpS  rqS! rrS" rs\r" \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gR                  \gGR                   /5        \GR                  \GR                  \GR                  \GR                  \GR
                  \GR                  \GR                  \GR                  \GR                  \GR                  \GR                  \GR                  \GR                  S#.rS$\4S% jrS& rS' rS(\(4S) jrS* rS+ rS,\\   S-\\`\4   S.\S(\\(   S/\S\\\   \\`\4   4   4S0 jrS1 rS(\\(   4S2 jrS3\(GR0                  S34S(\\(   4S4 jjrS5 rGSvS6 jr      GSwS7 jrGSxS8 jrGSxS9\KS$\GR<                  4S: jjr\" \iGR@                  SS;9S9\KS$\GR<                  4S< j5       rS3S=.S9\KS$\GR<                  4S> jjr\" \gGRF                  GR<                  SS;9S9\KS$\GR<                  4S? j5       rS3S3S@.S9\KSA\GRJ                  4SB jjr\" \iGRN                  SS;9GSxS9\KSA\GRJ                  4SC jj5       rSSD\(GR0                  S3SSS3S3S4	SE jrSF r\" 5          GSxSG jr\" \gGRX                  S3SSH9SI 5       r\" \gGRZ                  S3SSH9SJ 5       r\" \gGR\                  \gGR^                  \gGR`                  \gGRb                  \iGRd                  /5      SK 5       r\" \gSL5      (       a  \" \gGRj                  5      " \5        \" \gGRl                  SS;9GSySM j5       r\" \gGRn                  SS;9GSySN j5       r\" \gGRp                  /5      GSySO j5       r\" \gGRr                  5      SP 5       r\" \gGRt                  5      SQ 5       r\" \gGRv                  5      SR 5       r\" \gGRx                  5      SS 5       r\" \gGRz                  GR|                  5      ST 5       r\" \gGR~                  5      SU 5       r\" \gGR                  SS;9SV 5       r\" \iGR                  SS;9SW 5       r\" \gGR                  SS;9SX 5       r\" \gGR                  5      SY 5       r\" \gGR                  SS;9\" \gGRF                  SS;9\" \gGR                  SS;9SZ 5       5       5       r\" \gGR                  SS;9S[ 5       r\" \gGR                  SS;9GSzS\ j5       r\" \gR                  SS;9GSyS] j5       rt\" \gGR                  SS;9GSyS^ j5       r\" \gR                  SS;9GSyS_ j5       ruGS{S` jr\" \nGR                  SS;9Sa\KSb\KSc\KSd\Se\Sf\S$\GR<                  S\K4Sg j5       r\" \nGR                  SS;9Sa\KSb\KSc\KSd\Se\Sf\S$\GR<                  S\K4Sh j5       r\" \nGR                  GR|                  SS;9Sa\KSi\Sj\Se\Sf\S$\GR<                  S\K4Sk j5       r\" \nGR                  GR|                  SS;9Sa\KSi\Sj\Se\Sf\S$\GR<                  S\K4Sl j5       r\" \nGR                  GR                  SS;9Sa\KSi\KSj\KSe\Sf\S$\GR<                  S\K4Sm j5       r\" \nGR                  GR                  SS;9Sa\KSi\KSj\KSe\Sf\S$\GR<                  S\K4Sn j5       r\" \gGR                  5      GS{So j5       r\" \gGR                  SS;9GS|Sp\Sq\Sr\4Ss jj5       r\" \gGR                  SS;9GS|Sp\Sq\Sr\4St jj5       r\" \gGR                  SS;9GS|Sp\Sq\Sr\4Su jj5       r\" \gGR                  SS;9Sv 5       r\" \gGR                  SS;9GS{Sw j5       r\" \gGR                  SS;9GS{Sx j5       r\" \gGR                  SS;9GS{Sy j5       r\" \gGR                  SS;9Sz 5       r\" \gGR                  SS;9S{ 5       r\" \gGR                  SS;9S| 5       rGS{S} jr\" \gGR                  5      GS}S j5       rGS~S jr\GR                  " S5      S 5       rGSyS\GR                  4S jjrGSyS\GR                  4S jjrGS~S\GR                  GR                  4S jjrGSS jrS r\" \R                  GR                  GR                  SS;9S 5       r\" \gGR                  SS;9S 5       r\" \gGR                  SS;9S 5       r\" \gGR                  GR                  SS;9S 5       r\" \gGR                  5      S 5       r\GR                  " S5      S 5       rS r\" \gGR                  GR|                  5      r\" \gGR                  GR                  5      r\" \gGR                  GR|                  5      r\" \gGR                  GR                  5      r\" \gGR                   5        \" \gGR                  5      S 5       r\" \gGR                  5      S 5       r\" \;GR                  SS;9S 5       Gr\" \;GR                  SS;9SA\GRJ                  4S j5       Gr\" \;GR
                  SS;9S 5       Gr\" \;GR                  SS;9S 5       Gr\" \;GR                  SS;9S S.S\\   S\KS\`Sp\4S jj5       Gr
\" \;GR                   SS;9S S.S\S\S\\   S\KSp\4
S jj5       GrS\KS\\`\GR                  \GR                  \GR                  4   4S jGrS\KS\\`\GR                  4   4S jGr\" \gGR                  GR                  SS;9S3S3SSS.S\KS\KS\S\S\\`   S\\K   S\K4S jj5       Gr\" \gGR                   SS;9S3S3S.Sa\KS\KS\S\4S jj5       GrS GrS GrS GrS GrS Gr\" \gGR,                  5        \" \gGR.                  5        \" \gGR0                  5        \" \gGR2                  5        \" \gGR4                  S3S9  \" \gGR6                  GR|                  S3S9  \" \gGR8                  5        \" \gGR:                  S3S9  \" \gGR<                  5        \" \gGR>                  5        \" \gGR@                  5        \" \gGRB                  GR|                  5        \" \gGRD                  GR|                  5        \" \gGRF                  5        \" \gGRH                  GRJ                  5        \" \gGRL                  GR|                  5        \" \gGRN                  GR|                  5        \" \gGRP                  5        \" \gGRR                  S3S9  \" \gR                  G\5        \" \gGRT                  G\5        \" \gGRV                  G\5        \" \gGRX                  G\5        \" \gGRZ                  G\5        \" \gGR\                  5        \" \gGR^                  5        \" \gGR^                  5        \" \gGR`                  5        \" \gGRb                  5        \" \gGRd                  5        \" \gGRf                  G\5        \" \gGRh                  5        \" \gGRj                  5        \" \gGRl                  5        \" \gGRn                  5        \" \gGRp                  5        \" \gGRr                  5        \" \gGRt                  5        \" \gGRv                  5        \" \gGRx                  G\5        \" \gGRz                  5        \" \gGR|                  G\5        \" \gGR~                  5        \" \gGR                  5        \" \gGR                  GR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  GR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  GR|                  5        \" \gGR                  GR|                  S3S9  \" \gGR                  G\5        \" \GR                  GR                  GR                  5        \" \GR                  GR                  GR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  G\5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  5        \" \gGR                  GR                  5        \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\S3S9  \" \gGR                  GR|                  G\5        \" \gGR                  GR|                  G\5        \" \gGR                  GR|                  G\5        \" \gGR                   GR|                  G\5        \" \gGR                  5        \" \gGR                  SS;9GSxS j5       Gr\" \gGR                  5      SS.S j5       GrS Gr\" \gS5      (       a  \" \gGR
                  5      " G\5        \" \iGR                  5      S 5       Gr\" \gGR                  SS;9S\S\4S j5       Gr\" \gGR                  SS;9GSS j5       GrS Gr\" \GR                  \gGR                  /5      SSSS3S.S j5       r\" \GR                  5      GSvS j5       Gr\" \GR                  5      S 5       Gr\" \gGR                  5      S 5       Gr\" \gGR                  5      S 5       GrS Gr\" \gGR"                  SS;9S 5       GrS Gr\" \GR&                  \gGR&                  /5      SSSSSSS.S j5       GrS GrS Gr\" \gGR,                  5      " G\" G\5      5      GrG\" G\" S5      5      GrG\" G\" S 5      5      GrS Gr\" \gGR4                  5      SSSSS.S j5       Gr\" \gGR6                  5      SSSSS.S j5       Gr\" \gGR8                  5      SSSSS.S j5       Gr\" \iGR:                  GR|                  5      S 5       Gr\" \GR<                  \gGR<                  /5      S 5       Gr\" \gGR>                  SS;9GSxS j5       Gr\" \gGR@                  SS;9GSS j5       GrS Gr GS~S jGrS GrGS~S jGr\" \gGRJ                  SS;9S 5       Gr\" \gGRL                  SS;9S 5       Gr\" \gGRN                  5      GSxS j5       Gr\" \gGRP                  5      GSxS j5       GrS GrS Gr\" \gGRV                  SS;9GSxS j5       Gr\" \;GRX                  SS;9GSxS j5       GrS Gr\" \gGR\                  GR|                  S3S9Gr\" \gGR`                  GR|                  S3S9Gr\" \gGR\                  SS;9S 5       Gr\" \gGR`                  SS;9S 5       Gr\S 5       Gr\" \gGRf                  SS;9GSyS j5       Gr\" \gGRh                  SS;9S\4S j5       GrSSDS.S\R                  R                  S\S\\`   S\4S jjGr\" \gGRl                  SS;9SS.S\S\\`   4S jj5       Gr\" \gGRn                  SS;9S\4S j5       Gr\" \gGRp                  SS;9S\4S j5       Gr\" \gGRr                  SS;9S\4S j5       Gr\" \gGRt                  SS;9SDS.S\S\4S jj5       Gr  GSS\\\   S4   S\S\4S jjGr\" \gGRx                  GR|                  5      GSySb\\   4S jj5       Gr\" \gGRz                  GR|                  5      GSySb\\   4S jj5       Gr\" \gR                  GR|                  5       GSvS\\   S\\   4S jj5       r~\" \gR                  GR|                  5       GSvS\\   S\\   4S jj5       r\" \gGR|                  GR|                  5         GSS\\   S\\   S\\   4S jj5       Gr\" \gGR~                  GR|                  5         GSS\\   S\\   S\\   4S jj5       GrS Gr\" \iGR                  GR|                  5      GS  5       Gr\" \gGR                  SS;9GS{GS j5       GrGS\GR                  S\\GR                  \4   4GS jGrGS\GR                  S\GR                  4GS jGrGS\GR                  S\GR                  S\GR                  4GS jGr GSGS jGrGS GrGS GrSGS	.GS
 jGr\" \iGR                  SS;9 GSxGS j5       Gr\" \iGR                  SS;9GS 5       Gr\" \gR                  5        \" \gR                  GR|                  S3S9Gr\" \gR                  SS;9GS 5       r|GSGS jGrGS GrGS GrGS Gr\" \gGR                  GR|                  S3S9Gr\" \gGR                  5      GS 5       Gr\" \gGR                  GR|                  S3S9Gr\" \gGR                  5      GS 5       Gr\" \gGR                  GR|                  S3S9GrGS Gr\" \gGR                  5      GS 5       Gr\" \gGR                  GR|                  5       GSGS j5       Gr\" \gR                  GR|                  S3S9Gr\" \gGR                  GR|                  S3S9Gr\" \gR                  SS;9     GSGS j5       rv\" \gGR                  SS;9     GSGS j5       GrGS Gr\" \gR                  GR|                  S3S9Gr\" \gR                  SS;9 GSyGS j5       rw\" \gGR                  GR|                  S3S9Gr\" \gGR                  SS;9 GSyGS j5       GrGS GrGS GrGSyGS\`4GS jjGrGS  Gr\" \gGR                  5      GSSGS!.GS" jj5       GrGS# GrGS$ GrGS% GrGS& Gr\" \gGR                  \iGR                  /5      GSySS3GS'.GS( jj5       Gr\" \gGR                  5      GSySS3GS'.GS) jj5       GrGS* Gr\GS+ 5       Gr\" \gGR                  GR                  S3S9Gr\" \gGR                  GR                  S3S9Gr\" \gGR                  GR                  S3S9Gr\" \gGR                  SDGS,9GS- 5       GrGSxGS. jGr\" \gGR                  5      GS/ 5       Gr\" \gGR                  SS;9GSxGS0 j5       Gr\GS1 5       Gr\GS2 5       Gr\" \gGR                  SDGS,9GSyGS3 j5       Gr\" \gGR                  /SDGS,9GS4 5       GrS9\<R                  S\\<GR                     4GS5 jGr\" \iGR                  /SDGS,9GS6 5       Gr\" \gGR                   \gGR                  GR                  /SD\(GR                  SH9GS7 5       Gr\" \gGR                  \iGR                  /SDGS,9GS8 5       Gr\" \gGR                  \iGR                  /5      GSSGS!.GS9 jj5       Gr\" \gGR
                  GR|                  5      Gr\" \gGR                  GR|                  5      Gr\" \gGR                  GR|                  5      Gr
\" \gGR                  GR|                  5      Gr\" \gGR                  GR|                  5      Gr\" \gGR
                  5      GSvGS: j5       Gr\" \gGR                  5      GSvGS; j5       Gr\" \gGR                  5      GS< 5       Gr	\" \gGR                  SS;9GSyGS= j5       Gr\" \gGR                  SS;9GSyGS> j5       Gr\" \gGR                  5      GSSGS!.GS? jj5       Gr\" \gGR                   5      GSGS@ j5       Gr\" \gGR$                  SS;9GSGSA j5       Gr\" \gGR(                  SS;9GSGSB j5       Gr\" \iGR,                  5      " G\" GSC5      5        \" \gGR.                  5      " G\" GSD5      5      Gr\" \gGR2                  5      " G\" GSE5      5      Gr\" \gGR6                  5      " G\" GSF\GR
                  GSG95      Gr\" \gGR:                  5      " G\" GSH\GR
                  GSG95      Gr\" \gGR>                  SDGSIGSJ9Gr\" \gGR                  GR                  S3S9Gr \" \gGR                  GR                  SS;9SS~S3GSK.GSL j5       Gr!\" \gGR                  GR|                  SS;9GSGSM j5       Gr@GSvGSN jGr"GSO Gr#G\"" \gGRH                  5      Gr$G\#" \gGRJ                  5      Gr%G\"" \gGRL                  5      Gr&G\"" \gGRN                  5      Gr'\" \gGRP                  5      Gr(G\#" \gGRR                  5      Gr)G\#" \gGRT                  5      Gr*\" \gGRV                  5      Gr+\" \gGRX                  SDGSP9Gr,G\#" \gGRZ                  5        G\#" \gGR\                  5        \" \gGR^                  5      Gr/\" \gGR`                  5      Gr0\" \gGRb                  5      Gr1\" \gGRd                  GSQGSR9Gr2\" \gGRf                  5      Gr3\" \gGRh                  5      Gr4\" \gGRj                  5      Gr5G\"" \gGRl                  5        G\"" \gGRn                  5      Gr7\" \gGRp                  \(GR                  S;9" G\75        G\"" \gGRr                  5        G\"" \gGRt                  5        G\"" \gGRv                  5        G\#" \gR                  5        \" \gGRx                  SSD\GR                  GSS9Gr<\" \gGRz                  SSD\GR                  GSS9Gr=\" \gGR|                  SSD\GR                  GSS9Gr>\" \gGR~                  SSD\GR                  GSS9Gr?\" \gGR                  5      Gr@\" \gGR                  5      GrA\" \gGR                  5      " G\@5        \" \gGR                  5      " G\A5        \" \gGR                  5      GrD\" \gGR^                  5      Gr/G\"" \gGR                  5      GrE\" \gGR                  5        \" \gGR                  GSTGSR9GrG\" \gGRv                  5        \" \gGR                  \GR                  GSG9  \" \gGR                  5      " G\D5        \" \gGR                  \GR                  GSG9  \" \gGR                  \GR                  GSG9  \" \gGR                  \GR                  GSG9  \" \gGR                  \GR                  GSG9GrM\" \gGR                  \GR                  GSG9  \" \gGR                  \GR                  GSG9  G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        G\"" \gGR                  5        SGSUGK`GJaGraGJbGrb  GSV GrcG\b HR  GrdG\c" \gG\d5       H  u  GreGrfGrg\" G\eG\dG\fG\gGSW9  M     G\c" \iG\d5       H  u  GreGrfGrg\" G\eG\dG\fG\gGSW9  M     MT     \" \gGR                  R                  G\SDGSP9Gri\" \gGR                  GR                  G\SDGSP9Grj\" \gGR                  GR                  G\SDGSP9  \" \gGR                  R                  G\5      Grl\" \gGR                  GR                  G\5        \" \gGR                  GR                  G\5      Grm\" \gGR                  R                  G\,5        \" \gGR                  GR                  G\,5        \" \gGR                  GR|                  G\D5        \" \gGR                  GR|                  G\/5        \" \gGR                  GR                  G\5        \" \gGR                  R                  G\5        \" \gGR                  GR                  G\5        \" \gGR                  R                  G\5      Grt\" \gGR                  GR                  G\5        \" \gGR                  GR                  G\5      Gru\" \gGR                  G\*5        \" \gGR                  G\$5        \" \gGR                  R                  G\@5        \" \gGR                  GR                  G\@5        \" \gGR                  R                  G\A5        \" \gGR                  GR                  G\A5        \" \gGR                  R                  G\@5        \" \gGR                  GR                  G\@5        \" \gGR                  R                  G\A5        \" \gGR                  GR                  G\A5        \" \gGR                  G\E5        \" \gGR                  G\G5        \" \gGR                  G\5        GSX GrG\" \gGR                   R                  \gGR                  R                  G\i5        G\" \gGR                   GR                  \gGR                  GR                  G\j5        G\" \gGR                  R                  \gGR                  R                  G\l5        G\" \gGR                  GR                  \gGR                  GR                  G\m5        G\" \gGR                  R                  \gGR                  R                  G\t5        G\" \gGR                  GR                  \gGR                  GR                  G\u5        GSY GrG\" \gGR                  G\5        G\" \gGR
                  G\05        G\" \gGR                  G\15        G\" \gGR                  G\25        G\" \gGR                  G\35        G\" \gGR                  G\45        G\" \gGR                  G\55        G\" \gGR                  G\5        G\" \gGR                  GR                  G\5        G\" \gGR                  GR                  G\5        G\" \gGR                  G\<5        G\" \gGR                  G\=5        G\" \gGR                   G\>5        G\" \gGR"                  G\?5        G\" \gGR$                  G\,5        G\" \gGR&                  G\(5        G\" \gGR(                  G\)5        \" \gGR*                  5      " G\05        \" \gGR,                  5      " G\15        \" \gGR.                  5      " G\35        \" \gGR0                  5      " G\45        \" \gGR2                  5      " G\55        G\" \gGR4                  \gGR*                  5        G\" \gGR6                  \gGR,                  5        G\" \gGR8                  \gGR.                  5        G\" \gGR:                  \gGR0                  5        G\" \gGR<                  \gGR2                  5        \" \gGR>                  5      GSvGSZ j5       Gr\" \gGR@                  GR                  5      GS[ 5       Gr\" \gGRB                  GR                  5      GS\ 5       Gr\" \gGRD                  5      GS] 5       Gr\/GRF                  " 5        H  u  GrGr\" \0" G\5      5      " G\5        M      \" \GRL                  5      GS^ 5       Gr\" \gGR                  5      GS_ 5       Gr\" \R                  GRP                  GRR                  5      GS` 5       Gr\" \R                  GRV                  GRX                  5      GSa 5       Gr\" \R                  R                  GRZ                  GR\                  5      GSb 5       Gr\" \R                  GR`                  GSc5      (       a;  \" \R                  GR`                  GR                  GR|                  5      GSd 5       Gr\" \R                  R                  GRd                  5      SS.GSe j5       GrS GSfGKGJGr  \" G\5        \" \!5      GSg 5       Gr\" \R                  GRl                  GRn                  5      GSh 5       Gr\" \R                  GRl                  GRp                  5      GSi 5       Gr\" \R                  GRl                  GRr                  SS;9GSj\<GRt                  GSk\`4GSl j5       Gr\" \SS;9GSm\<GRt                  S\4GSn j5       Gr\" \R                  R                  GRx                  GR|                  5      GSo 5       Gr\" \R                  GRl                  GRz                  SS;9GSp 5       GrSGSqGKGJGr  G\" 5         SGSrK9GJGr  \8" G\5        SGSsK9GJGr  G\GR                  " 5         G\GR                  " 5         SGStK9GJGr  G\GR                  " 5         SGSuK9GJGr  G\GR                  " 5         g(      N)defaultdict)	AnyCallableDictListOptionalSequenceSetTupleUnion)patch)associative_scan_op)triton_kernel_wrapper_mutation)canonicalize_dimcanonicalize_dimscheckdtype_to_typeelementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDget_computation_dtypeis_boolean_dtypeis_float_dtypeis_integer_dtypeNumber)magic_methodsmethod_to_operator)CeilDivFloorDivIdentity
IntTrueDivModularIndexing   )import_submodule   )configinductor_primsirtest_operators)decompositionsget_decompositions)	DtypeView
ExpandViewIndexingConstantIRNode	is_tritonops_wrapperPermuteView	Pointwise	ReductionSqueezeView	TensorBoxvalidate_irView)
ceildivdecode_device
is_dynamicis_gpuis_pointwise_use,needs_fallback_due_to_atomic_add_limitationspad_listlike#register_op_dtype_propagation_rulessympy_productuse_scatter_fallback)opsVztorchvision::roi_alignzaten::index_add.	lowerings_maybe_layout_constraints	fallbacksneeds_realized_inputsforeach_opsinplace_foreach_opsinplaceable_foreach_opsfnreturnc                    ^  [        T [        R                  R                  5      (       d  gT [        ;   a	  [        T    $ T [
        ;   a
  S[        T '   gU 4S jn[        T 5      nU" U5      $ )zHGet layout constraints. Returns None if there are no layout constraints.Nc                    > U [         R                  R                  R                  L a  [        [
        T'   [
        T   $ U [         R                  R                  R                  L a
  S [
        T'   g [        SU  35      e)NzUnknown layout constraint tag: )torch_CTagneeds_fixed_stride_orderconstrain_to_fx_stridesrE   flexible_layoutAssertionError)tagrK   s    W/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/torch/_inductor/lowering.pyhandle_layout_constraint_tag>maybe_layout_constraints.<locals>.handle_layout_constraint_tags   sf    %((,,777,C%b),R00EHHLL000,0%b) #B3%!HII    )
isinstancerO   _ops
OpOverloadrE   rD   get_layout_constraint_tag)rK   rX   rV   s   `  rW   maybe_layout_constraintsr_   f   sc    b%**//00	&&(,,	Y(,!"%J $B
'C',,rZ   c                    [         R                  R                  R                  [         R                  R                  R                  /nU H  nX R
                  ;   d  M  Us  $    [         R                  R                  R                  U 5      (       a$  [         R                  R                  R                  $ [        [         R                  R                  [        R                  5      $ N)rO   rP   rQ   rR   rT   tags_libraryutils
is_builtingetattrr%   #custom_op_default_layout_constraint)rK   tags_by_priorityrV   s      rW   r^   r^      s    --$$  ''>J   ~~&&r**xx||+++588<<!K!KLLrZ   c                 .    U (       d  [        SU 35      eg )Nzinductor does not support NotImplementedError)condmsgs     rW   
assert_nyirn      s    !$>se"DEE rZ   c                 `  ^  [        T [        [        [        45      (       a  T  Vs/ s H  n[	        U5      PM     sn$ [
        R                  T 5        [        T [        R                  R                  5      (       a.  [
        R                  U 4S jT R                  5        5       5        g g s  snf )Nc              3   <   >#    U  H  n[        TU5      v   M     g 7fra   )rf   ).0overloadrK   s     rW   	<genexpr>,add_needs_realized_inputs.<locals>.<genexpr>   s      %
2@hGB!!.   )r[   listtuplesetadd_needs_realized_inputsrG   addrO   r\   OpOverloadPacketupdate	overloads)rK   xs   ` rW   ry   ry      s    "tUC())689b)!,b99b!"ejj1122$$ %
24,,.%
 	
 3 :s   B+c                     [        U [        R                  R                  5      (       a*  U R	                  5        H  nU[
        [        X5      '   M     g U[
        U '   g ra   )r[   rO   r\   r{   r}   rE   rf   )rK   
constraintrr   s      rW   add_layout_constraintr      sF    "ejj1122H?I%gb&;< ' )3!"%rZ   )r   r$   r"                     	   
         dtypec                 p    [        U [        5      (       d  U $ U [        ;   d   SU  S35       e[        U    n U $ )Nzid z missing from DTYPE_ID_LOOKUP)r[   intDTYPE_ID_LOOKUPr   s    rW   decode_dtyper      s@    eS!!O#Os5'1N%OO#E"ELrZ   c                    [        U [        5      (       a9  [        U R                  5       5      =(       d    [	        U R                  5       5      $ [        U [
        R                  5      (       a  U R                  SL $ [        U [        5      $ )NT)	r[   r5   r   	get_dtyper   sympyExpr
is_integerr   r~   s    rW   is_integer_typer      s^    !Y.Q2B1;;=2QQ	Auzz	"	"||t##!S!!rZ   c                 ~    [        U [        5      (       a  [        U R                  5       5      $ [        U [        5      $ ra   )r[   r5   r   r   boolr   s    rW   is_boolean_typer      s-    !Y..!T""rZ   type_promotion_kindc                 ^    S nU Vs/ s H
  o2" U5      PM     nn[        USU 06u  pVU$ s  snf )Nc                     [        U [        [        R                  45      (       a  U $ [	        U R                  5       5      n[        R                  " S/U-  U R                  5       S9$ )Nr$   r   )	r[   r   r   Basiclenget_sizerO   zerosr   )inpdims     rW   construct_input+get_promoted_dtype.<locals>.construct_input   sJ    cFEKK011Jclln%C;;sSy@@rZ   r   )r   )r   argsr   arginps_r   s          rW   get_promoted_dtyper      s?    A -11DSOC DD1!4Q=PQHAL 2s   *c                 R   [        U [        [        45      (       d  U /n O[        U 5      n [        U 5       Hm  n[        U[        R                  R
                  5      (       d  M.  UR                  5        H+  n[        X5      nU[        ;  d  M  U R                  U5        M-     Mo     U $ ra   )
r[   rv   rw   rO   r\   r{   r}   rf   rD   append)aten_fnrK   rr   other_fns       rW   get_overloadsr      s}    ge}--)w-7mb%**5566LLN"209,NN8, +  NrZ   c                     [        U [        R                  R                  5      (       a  XR                  ;   $ [        U [        R                  R
                  5      (       a  XR                  5       ;   $ gNF)r[   rO   r\   r{   _qualified_op_namer]   name)op	namespaces     rW   in_namespacer     sQ    "ejj11221111	B

--	.	.GGI%%rZ   r   kwargs	broadcastconvert_input_to_boolc           	        ^ ^^^ [        T 5       VVs/ s H  u  pV[        U[        5      (       d  M  UPM      nnnTR                  5        VV	s/ s H  u  p[        U	[        5      (       d  M  UPM      n
nn	U(       d  U
(       d  T T4$ U(       d  U(       Ga  U(       a  [        R
                  mO{T  Vs/ s H=  n[        U[        [        R                  45      (       d  [        US5      (       d  M;  UPM?     nnUR                  S TR                  5        5       5        [        USU06mU(       a  T US      OTU
S      R                  5       mUU4S jnT  Vs/ s H
  o" U5      PM     snm TR                  5        VV	s0 s H  u  pX" U	5      _M     sn	nmU(       GaF  [        [        [         R"                  " U 4S jU 5       U4S jU
 5       5      5      6 n[        US   R%                  5       5      n['        X~S [)        U5       5       H
  u  pVUT U'   M     ['        X[)        U5      S  5       H
  u  pUTU'   M     [+        [)        T 5      5       HD  n[        T U   [,        R.                  5      (       d  M'  [0        R2                  " T U   U5      T U'   MF     T HD  n[        TU   [,        R.                  5      (       d  M'  [0        R2                  " TU   U5      TU'   MF     T T4$ s  snnf s  sn	nf s  snf s  snf s  sn	nf )Nr   c              3   L   #    U  H  n[        US 5      (       d  M  Uv   M     g7f)r   N)hasattr)rq   as     rW   rs   !transform_args.<locals>.<genexpr>&  s     !T_7@S!!_s   $	$r   r   c                    > [        U [        5      (       a  [        U T5      $ [        U [        R                  5      (       a   [        R                  " U R
                  TTS9$ U $ )Nvaluer   device)r[   r5   to_dtyper'   Constantr   )r   r   r   s    rW   promotetransform_args.<locals>.promote0  sJ    #y))U++C--{{%OO
rZ   c              3   .   >#    U  H
  nTU   v   M     g 7fra    )rq   ir   s     rW   rs   r   ?  s     3lT!Wl   c              3   .   >#    U  H
  nTU   v   M     g 7fra   r   )rq   kr   s     rW   rs   r   @  s     71VAYr   )	enumerater[   r5   itemsrO   r   r   r   r   r   extendvaluesr   
get_devicebroadcast_tensorsrv   	itertoolschainr   zipr   ranger'   r   r,   create)r   r   r   r   r   r   r~   args_indicesr   vkwargs_indicesr   promoting_argsr   broadcastedsizer   r   s   ``              @@rW   transform_argsr     s    #,D/N/$!Z95MA/LN$*LLNONDAjI6NaNNOV|3 JJE
 Aa&%++!67771g;N    !!!TV]]_!TT&5HE
 &2Da!vnQ>O7P
*, 	
	 %))Dq
D),2LLN;NDA!WQZ-N;'3l377
 KN++-.2EC4E&FGDADG HC4E4G(HIDAF1I J s4y!A$q'2;;//$++DGT:Q " A&)R[[11&--fQi>q	  <{ OO0 *;s.   K+K+K1-K13:K71K7K<Lc                    ^ [         R                  " T5      U4S j5       n[        U 5      n[        R	                  U5        [
        R	                  [        R                  X25      5        U$ )a  
Add a foreach lowering to lowerings dict.

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                  P   > [        U 5      S::  d   eT" U 0 UD6n[        U5        U$ )Nr"   )r   r6   )r   r   out	decomp_fns      rW   wrapped+_register_foreach_lowering.<locals>.wrappeda  s/    4yA~~((C
rZ   )	functoolswrapsr   rH   r|   rD   dictfromkeys)r   r   r   aten_fnss    `  rW   _register_foreach_loweringr   U  sV     __Y   W%Hx T]]856NrZ   c                    ^ ^^^^ [         R                  " T5      U UUUU4S j5       n[        T 5      m [        R	                  [
        R                  T U5      5        U$ )a  
Add a lowering to lowerings dict

Arguments:
    aten_fn: torch.ops.aten.* fn we are lowering
    decomp_fn: alternate implementation on our IR
    broadcast: True to apply broadcasting to tensor inputs
    type_promotion_kind: kind of type promotion applied to tensor inputs, `None` means no type promotion
    convert_input_to_bool: some logical ops require inputs are converted to bool
c                    > [        U 5      n [        U5      nSn[        U 5      S:X  a.  [        U S   [         [        45      (       a  Sn[        U S   5      n [        S T 5       5      (       d,  [        S UR                  5        5       5      (       a   S5       e[        XTTT5      u  pU(       a  U /n T" U 0 UD6n[        U5        U$ )NFr$   r   Tc              3   X   #    U  H   o[         ;   =(       d    [        US 5      v   M"     g7f)_c10d_functionalN)rF   r   )rq   rK   s     rW   rs   6_register_lowering.<locals>.wrapped.<locals>.<genexpr>  s&      
PW"9_DR1C DDPWs   (*c              3   *   #    U  H	  oS :H  v   M     g7f)r   Nr   rq   r~   s     rW   rs   r     s      $1qU
M   zout= ops aren't yet supported)
rv   r   r   r[   rw   allanykeysr   r6   )	r   r   unpackedr   r   r   r   r   r   s	       rW   r   #_register_lowering.<locals>.wrapped  s    t*!%ft9>ja4-@@HQ=D 
PW
 
 
  $*KKM   /./  &)%8:O
 6D((C
rZ   )r   r   r   rD   r|   r   r   )r   r   r   r   r   r   s   ````` rW   _register_loweringr   n  sO    $ __Y   : G$GT]]7G45NrZ   Fc                 :    [         R                  " [        U UUUS9$ )z#
Shim to support decorator syntax.
r   r   r   )r   partialr   )r   r   r   r   s       rW   register_loweringr     s&     /3 rZ   c                    / n[         R                  " [        U 5      [        U5      [        R                  R
                  S9 GHf  u  p4[        R                  R                  R                  R                  [        R                  " US5      SS9(       a  UR                  U5        Me  [        R                  R                  R                  R                  [        R                  " US5      SS9(       a  UR                  U5        M  [        R                  R                  R                  X45        [        [        R                  " U5      R                   5      [        [        R                  " U5      R                   5      :  a  UR                  U5        GMU  UR                  U5        GMi     [#        [        U5      5      $ )z
Broadcasting logic based on symbolic shapes.

We give the shapes 0 and 1 concrete values, while all other shapes
are symbolic sympy formulas.
)	fillvaluer$   Tsize_oblivious)r   zip_longestreversedr   SOnerC   graphsizevars	shape_envevaluate_exprEqr   guard_equalsr   expandfree_symbolsrw   )r   boutputr~   ys        rW   broadcast_symbolic_shapesr    s%    F%%hqk8A;%''++V77%%33HHQN4 4 
 MM!WW''55HHQN4 6 
 MM!GG))!/5<<?//03u||A7S7S3TTa a  W &!""rZ   c                   ^ Ub
  Ub   S5       eUc  Uc  [         R                  n[        S U  5       5      (       d  U $ [        S U  5       5      (       a2  U=(       d    [	        U SU06mU4S jnU  Vs/ s H
  oC" U5      PM     sn$ [        S U  5       5      n/ nU  GH!  n[        U[        [        45      (       an  UR                  [        R                  " [        R                  " XER                  5       UR                  5       S9[!        UR#                  5       5      5      5        M  [        U[$        R&                  5      (       ad  UR                  [        R                  " [)        XER                  5       UR                  5       S9[!        UR#                  5       5      5      5        GM  UR                  U5        GM$     U$ s  snf )	NzEonly one of override_return_dtype or type_promotion_kind may be givenc              3   l   #    U  H*  n[        U[        R                  [        [        45      v   M,     g 7fra   )r[   r   r   r   floatr   s     rW   rs   $promote_constants.<locals>.<genexpr>  s%     HAz!ekk3677   24c              3   l   #    U  H*  n[        U[        [        [        R                  45      v   M,     g 7fra   )r[   r   r  r   r   r   s     rW   rs   r    s%     
DV:a#uekk233Vr  r   c                    > [        U [        R                  5      (       a  [        R                  " U T[        S 5      S9$ [        R                  " U T[        S 5      S9$ )Nindexr   r   r   )r[   r   r   r'   r-   r9   r   r~   r   s    rW   
const_func%promote_constants.<locals>.const_func  sM    !U[[))**5t1D  {{%d@STTrZ   c              3   ~   #    U  H3  n[        U[        [        [        R                  45      (       d  M/  Uv   M5     g 7fra   )r[   r5   r,   r'   r   r   s     rW   rs   r    s&     WA:a)Z1U#Vaas   .=	=r   r  )r   DEFAULTr   r   r   nextr[   r   r  r   r,   r   r'   r   r   get_device_or_errorrv   r   r   r   r-   )inputsoverride_return_dtyper   r  r~   exr   r   s          @rW   promote_constantsr%    s   %)<)DONOD $)<)D=EEHHHH

DV
DDD% 
);*
)<*
	U (..v!
1v..	WW	WB
Ca#u&&JJ!!KK||~b>T>T>V '	 5;;''JJ!!$||~b>T>T>V '	 JJqM) , J3 /s   3Gc           	      D   ^ ^^^^^^ S S.S[         4UU UUUUU4S jjjnU$ )Nalphar"  c           	      ~  >^^^	^
^ Tb%  [        S U 5       5      (       a  T(       a   eT" U6 $ [        UT5      nT(       a'  U b#  U S:w  a  [        U5      n[        US   U 5      US'   OU b   eU Vs/ s H  o"R	                  5       PM     snm
US   R                  5       mT=(       d    US   R                  5       m[        [        US   R                  5       5      R                  5      m	USS   Hh  n[        U[        R                  5      (       a  M$  [        T5      [        UR                  5       5      :X  a  ML   ST ST SUR                  5        35       e   [        R                   S L=(       a    [#        [        R                   SS 5      S L=(       a    [        R                   R$                  R&                  S L=(       a^    [        R                   R$                  R&                  R)                  SS	5      =(       a#    T[*        R,                  [*        R.                  4;   mUUUU	U
UUU4S
 jnT(       d_  S nU H=  n[        UR                  5       R                  5      (       d  M-  UR                  5       n  O   U(       d  US   R                  5       nT=(       d    Wn[0        R2                  " UTUTS9$ s  snf )Nc              3   f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fra   r[   r.   r/   )rq   r   s     rW   rs   0make_pointwise.<locals>.inner.<locals>.<genexpr>  s&      /
BH3JsF#6	#6&   /1r$   r   zndim mismatch  current_nodelow_precision_pointwise_barrierFc                 f  > [        U 5      [        T5      :X  d   SU  ST 35       eT[        R                  :X  a  Tb  T" T	 Vs/ s H
  o" U 5      PM     sn6 $ T
(       a6  T(       a/  T[        R                  :X  a  T
" T	 Vs/ s H
  o" U 5      PM     sn6 $ / nT	 HP  nU" U 5      nT(       a-  [        R
                  " UTSS9n[        R
                  " UT5      nUR                  U5        MR     T" U6 nT(       a-  [        R
                  " UTSS9n[        R
                  " UT5      $ U$ s  snf s  snf )Nzwrong ndim r/  F)use_compute_types)r   rO   r   float64rB   r   r   )r  loadinputs_loadedr   downcastr   emulate_precision_castsrK   is_gpu_deviceloadersoverride_fn_when_gpu_float64override_fn_when_input_boolrangess        rW   inner_fn/make_pointwise.<locals>.inner.<locals>.inner_fn1  s   u:V,LE7!F8.LL,

"'B'N2W4UWTT%[W4UVV,!U]]*3g5Vgdd5kg5VWW "#Du+C.#&<<Ue#T!ll8U;!((- $ -(*  #||C%PH<<%88
- 5V 6Ws   D)D.r   r   r>  r=  )r   r%  rv   mulmake_loaderr   r   r;   r9   r   typer[   r'   BaseConstantr   rC   r  rf   r0  metagetrO   bfloat16float16r2   r   )r(  r"  r~   otherr>  r   r   r   r8  r9  r:  r=  allow_alpharK   override_devicer;  r<  r#  triton_fallbacks          @@@@@rW   innermake_pointwise.<locals>.inner  sc   &3 /
BH/
 ,
 ,
 #"?"F++"6+@A UaZf U3r
= =,23Fq==?F3##%%>)<)<)>}VAY-A-A-CDIIJABZEeR__55V I : At1VHAenn.>-?@A    GG4 96dB9$$))59 $$))--.OQVW9 %..%--88 	 	 	6 F!,,.--..\\^F  --/ *F	
 	
w 4s   2J:)r5   )rK   r#  rK  r<  r;  rJ  rL  rM  s   ``````` rW   make_pointwiserO    s$     )- N
y N
 N
` LrZ   c                 L   ^ ^ SS.S[         [         [              4UU 4S jjjnU$ )Nr$   r'  r"  c                   > S n[        [        R                  R                  R                  5      S:H  =(       d+    [        R                  R                  R
                  [        ;   n[        R                  R                  R                   H>  nUR                   H+  nUR                  S:X  a  UR
                  [        ;   a  M)  SnM-     M@     S nU H"  n[        U[        [        45      (       d  M   Un  O   Uc   S5       e/ nU HO  n[        U[        [        45      (       d   UR                  U/[        U5      -  5        M>  UR                  U5        MQ     U" [        U6 5      n	S /[        U5      -  n
U	R                  5        H  u  u  pn/ nU H  u  nnT(       a	  T" USU 06nOT" U6 nUX'   [        R                  R                  U[         R"                  5      (       d  MT  U(       d  M]  U(       d  Mf  UR%                  5         UR                  UR'                  5       5        M     U(       d  M  [        R                  R)                  U5        M     [+        S U
 5       5      (       d   eU
$ )Nc                 L   [        [        5      n[        U 5       H  u  p#[        U6 (       + =(       d    [        R
                  nS nU H4  n[        U[        5      (       d  M  UR                  R                  5       n  O   Uc   S5       eXU4   R                  X#45        M     U$ )Nz.foreach op should have at least one tensor arg)r   rv   r   r:   r%   #combo_kernel_foreach_dynamic_shapesr[   r5   datar   r   )	arg_pairsr   r   r   use_foreachr   ts          rW   
group_args9make_foreach_pointwise.<locals>.inner.<locals>.group_argse  s    d#C$Y/"D))WV-W-W  A!!Y//!"!2!2!4 
 &DCD&[)*111)< 0 JrZ   r   call_functionTz1at least one input must be a list to a foreach opr(  c              3   (   #    U  H  oS Lv   M
     g 7fra   r   r   s     rW   rs   8make_foreach_pointwise.<locals>.inner.<locals>.<genexpr>  s     2'QD='s   )r   rC   r  r0  userstargetrI   r   rH   r[   rv   rw   r   r   r   has_featureBackendFeatureFOREACHrealizeget_operation_nameregister_operation_listr   )r(  r"  rX  realize_outputsnodeusera_list_inputinputbroadcast_inputsgroupsoutputsr   rV  groupoperation_list
output_indr   r  rJ  pw_fns                     rW   rM  %make_foreach_pointwise.<locals>.innerb  s   	$ $$**+q0 Bww##**.AA 	 GG((..D

?2{8R&*O # /
 E%$//$ 
 $	?>	?$ EedE]33 ''#l2C(CD ''.	  C!123&3|,,,2LLN(!V5(*N "D66F"D\F&,# GG''0F0FGG#'NN$"))&*C*C*EF   ~//?- -;0 2'22222rZ   )r   r5   )rp  rJ  rM  s   `` rW   make_foreach_pointwiserr  a  s+    45 JtDO, J JX LrZ   r~   c                    ^^ U R                  5       mTT:X  a  U(       a  [        U 5      $ U $ UU4S jn[        UTS9" U 5      $ )Nc                 0   > [         R                  " U TTS9$ )N)	src_dtype)rB   r   )r~   r   ru  s    rW   	_to_dtypeto_dtype.<locals>._to_dtype  s    ||Au	::rZ   r#  )r   clonerO  )r~   r   copyrv  ru  s    `  @rW   r   r     s@    IEuQx&Q&; )5A!DDrZ   r   c                 D   UR                   (       d  U R                  5       R                   (       ag  U R                  5       (       a*  [        XS9n[        R
                  R                  X 5        U$ [        [        R                  R                  SS9" X5      $ [        XSS9$ )Nr   Fadd_to_fallback_setTrz  )
is_complexr   r   
empty_liker'   InplaceCopyFallbackr   fallback_handlerprimsconvert_element_typedefaultr   )r~   r   dsts      rW   _convert_element_typer    s~    1;;=33::<< Q,C""))#1J#**22  A4((rZ   r  c                   U R                  5       nX1:X  a  U(       a  [        U 5      $ U $ S nU" U5      nU" U5      nXV:w  a)  [        [        R                  R
                  5      " X5      $ [        [        R                  " X5      5      $ )Nc                     U R                   (       a   [        R                  " U 5      R                  $ [        R                  " U 5      R                  $ ra   )is_floating_pointrO   finfobitsiinfor   s    rW   _get_primitive_bitwidth1to_dtype_bitcast.<locals>._get_primitive_bitwidth  s5    "";;u%***;;u%***rZ   )	r   ry  r  atenviewr   r5   r+   r   )r~   r   rz  x_dtyper  src_bitsdst_bitss          rW   to_dtype_bitcastr    ss    kkmGuQx&Q&+ 'w/H&u-H		0::))!344rZ   c                 8   UR                   (       d  U R                  5       R                   (       a`  [        R                  " [        R
                  R                  [        R                  R                  R                  R                  X5      5      $ [        X5      $ ra   )r  r   r5   r   r'   ComplexViewrO   rB   r  r  r   r  r  s     rW   _view_dtyper    sa    1;;=33NN!!%)).."5"5";";QF
 	
 A%%rZ   rz  non_blockingr   c                    [        U5      nU R                  5       U:X  a  U(       a  [        U 5      $ U $ [        R                  " [
        R                  R	                  XU5      5      $ ra   )r9   r   ry  r5   r   r'   
DeviceCopy)r~   r   rz  r  s       rW   	to_devicer    sM    6"F||~uQx&Q&BMM00LIJJrZ   c                     [        XSUS9$ )NTr  )r  )r~   r   r  s      rW   _device_putr    s    QTEErZ   Tc
           	      |   U=(       d    U R                   n[        U5      n
U(       a  [        SU-   5      n[        SU-   X55        [        XU5        Ub  [        U5      n[        U
UUU(       a  WOSUU	S9n
[	        U UUUS9" U
5      n
[        [        U5      (       a  [	        [        [        U5      SUS9" U
5        U
$ )z3A pointwise function that maps ops.{name} to inputs
libdevice_N)r#  r<  r;  rJ  rL  r   )r   r   )__name__r0   r?   rO  r   r   r  rf   )r   r   r   r   r   r#  r<  rJ  use_libdevice_for_f64rL  rK   fn_libdevices               rW   register_pointwiser    s     #7##D	T	B"<$#67+4!4	
 (#8 #.&12M&N#	
3$?5J\PT'
B 
/3	

 	

B udE4  $"7	
 		
 IrZ   c                  2  ^^ Sn [        S5      mU4S jnU4S jn[        U5      [        U[        R                  S9/mU4S jn[	        [
        R                  5      " U5      n[        [        U 5      (       a  [	        [        [        U 5      SS9" U5        U$ )z2A pointwise function that maps ops.frexp to inputsfrexpc                     > T" U 0 UD6S   $ Nr   r   r   r   r  s     rW   frexp0register_frexp.<locals>.frexp0.      d%f%a((rZ   c                     > T" U 0 UD6S   $ Nr$   r   r  s     rW   frexp1register_frexp.<locals>.frexp11  r  rZ   rx  c                  0   > TS   " U 0 UD6TS   " U 0 UD64$ Nr   r$   r   )r   r   pw_fnss     rW   rK   register_frexp.<locals>.fn9  s*    ay$)&)6!9d+Ef+EEErZ   Nr{  )
r0   rO  rO   int32r   r  r  r   r  rf   )r   r  r  rK   r  r  s       @@rW   register_frexpr  )  s    D E)) 	vvU[[AF
F 





B udE4  $	
 	 IrZ   c                 .    [        XS9n[        X5      nU$ )NrJ  )rr  r   )r   pointwise_lowering_fnrJ  rK   s       rW   register_foreach_pointwiser  K  s    
 
  5	OB	#G	0BIrZ   )r   r   c           
      *   S n[        U[        [        45      (       a  [        U5      " U5      n[        U[        [        45      (       a  [        U5      " U5      nXU/n[	        US   US   [
        R                  S9n[        U5       VVs/ s H  u  pg[        U[        5      (       d  M  UPM      nnn[        U[        U Vs/ s H  odU   PM	     sn6 5       H	  u  pgXtU'   M     [        [        U5      5       H]  n[        XF   [        R                  5      (       d  M&  [        R                   " XF   [#        XHS      R%                  5       5      5      XF'   M_     ['        X5S9" US   [)        US   U5      [)        US   U5      5      $ s  snnf s  snf )Nc                  (    [         R                  " U 6 $ ra   )rB   wherer   s    rW   rK   where.<locals>.fnW  s    yy$rZ   r$   r"   r{  r   rx  )r[   r  r   constant_liker   r   r  r   r5   r   r   r   r   r'   r   r,   r   rv   r   rO  r   )	rl   r   r  rK   r   r   r   r~   indicess	            rW   r  r  U  sX     !eS\""!Q!eS\""!QQ<DQa.M.U.UE 't_I_TQ
1i0Hq_GIG.'0J'Qa'0JKLQ M3t9dgr{{++ ''d1:6F6O6O6Q1RSDG  ":Q$q'5)8DGU+C  J0Js   F
,F
Fc                     [        U 5      S:X  a)  [        U S   [        [        45      (       a  [	        U S   6 $ [
        R                  " [        U  Vs/ s H  oR                  5       PM     sn/ 5      n/ nU  Hg  nUR                  5       n[        U5      [        U5      :w  d   [        S [        XB5       5       5      (       a  [        X5      nUR                  U5        Mi     U$ s  snf )Nr$   r   c              3     #    U  GHC  u  p[         R                  R                  R                  R	                  [
        R                  " US 5      SS9=(       aL    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9(       + =(       d    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9(       + =(       aG    [         R                  R                  R                  R	                  [
        R                  " US 5      SS9v   GMF     g7f)r$   Tr   N)rC   r  r  r  r	  r   r
  rq   r   r  s      rW   rs   $broadcast_tensors.<locals>.<genexpr>x  s	     ,
& +! GG$$..<<At =   GG,,66DDAt E  	 ((22@@At A    ((22@@At A " +s   EE)r   r[   rv   rw   r   r   reducer  r   r   r   r  r   )r"  r~   r^  rl  sizess        rW   r   r   n  s    
6{aJvay4-@@ &),,(//!&#A&QJJL&#A2 F G

u:V$ ,
& E*',
 )
 )
* q!Aq1 2 N9 $Bs   C$
c                     U $ ra   r   r   s    rW   nopr    s    HrZ   
lift_freshc           	         [        U [        5      (       d   eUc)  [        [        R                  " U R                  5      5      $ [        U[
        [        R                  45      (       a)  [        R                  R                  R                  U5      O[        S U 5       5      n[        [        U R                  5       5      U5      n[!        [        U[        5      (       d  U4OU5      n/ n[#        U R                  5       5       H_  u  pEXB;   aD  [        R                  R                  R%                  [        R&                  " USSS95      (       a  MN  UR)                  U5        Ma     X0R                  5       :w  a  [+        X5      $ U $ )Nc              3   t   #    U  H.  n[         R                  R                  R                  U5      v   M0     g 7fra   )rC   r  r  evaluate_static_shaperq   ds     rW   rs   squeeze.<locals>.<genexpr>  s(     Jc177##99!<<cs   68r$   Tr   )r[   r5   r4   r   rT  r   r   r   rC   r  r  r  rw   r   r   r   rx   r   r	  r
  r   r  )r~   r   dims	new_shaper  ss         rW   squeezer    s    a####
{++AFF344 cC,-- 	
..s3JcJJ 
 C

-s
3CZU33v=DI!**,'I  ..uxx1T/RSSQ ( "+jjl!:4AArZ   c                 *    [        [        X5      5      $ ra   )ry  r  )r~   r   s     rW   squeeze_copyr    s    !!rZ   c                     [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ ra   )r  r[   r5   rT  r~   r   vals      rW   squeeze_r    sA    
!/Ca####c9%%%%XXAFHrZ   c                     [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isinfrx  r   	full_likerO   r   r0   rO  r~   rK   s     rW   r  r    @    qE44	W	B"EJJ?BBrZ   c                     [        U 5      (       a  [        U S[        R                  S9$ [	        S5      n[        U[        R                  S9" U 5      $ )NFr   isnanrx  r  r  s     rW   r  r    r  rZ   c                 p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nceilr   ry  r0   rO  r  s     rW   r  r    s0    qQx	V	B"a  rZ   c                 p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nfloorr  r  s     rW   r  r    0    qQx	W	B"a  rZ   c                 p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Nroundr  r  s     rW   r  r    s0    qQx!b!!$$rZ   c                 p    [        U 5      (       a  [        U 5      $ [        S5      n[        U5      " U 5      $ )Ntruncr  r  s     rW   r  r    r  rZ   c                 *   SSK Jn  [        U /5      u  n [        U [        R
                  5      (       a   [        R                  " U [        U5      5      $ [        U [        5      (       d   e[        U[        [        45      (       d   e[        U R                  5       5      [        U5      :X  a  U $ U" U R                  5       5      (       d  [        R                  R                  R                  [!        U R                  5       5      5      nUS:  aQ  U" U5      (       dD  U R#                  [        R                  R                  R                  [!        U5      5      U-  5        [        [        R                  " U R$                  [        U5      5      5      $ )Nr   )free_unbacked_symbols)%torch.fx.experimental.symbolic_shapesr  r%  r[   r'   rD  r,   r   rw   r5   rv   r   rC   r  r  	size_hintr@   
mark_reuserT  )r~   r  r  x_size_products       rW   r  r    s   KaS!DQ!R__%%  E%L11a####edE]++++QZZ\eEl* ..))33M!**,4OP
 A&;E&B&BLL  **=+?@NR Z&&qvvuU|<==rZ   c                     [        U5      nU H  nSX4'   M	     U n[        U5       H  u  pgUS:w  d  M  [        XV5      nM     [        XQ5      $ Nr.  )rv   r   	unsqueezer  )r   shapebroadcast_dimensionsr  broadcast_dimensionr   idxr~   s           rW   broadcast_in_dimr    sS    UA3!#  4 	
AA,7!!A  !rZ   c                 4    [        XR                  5       5      $ ra   )r  r   )r~   r  s     rW   	expand_asr    s    !ZZ\""rZ   c                 N  ^^^ [        U R                  5       5      m[        T5      [        T5      :  aK  [        R                  R
                  /[        T5      [        T5      -
  -  T-   m[        U [        T5      5      n [        T5      [        U R                  5       5      :X  d   e[        U R                  5       5      nSn[        [        T5      5       H  nTU   S:X  a  SnX$   TU   -  X$'   M     U(       a&  [        X R                  5       U R                  5       S9$ [        S [        TT5       5       5      (       a  [        [        X5      5      $ UUU4S jn[        R                   R"                  R%                  ['        T5      5      nUS:  aD  U R)                  [        R                   R"                  R%                  ['        U5      5      U-  5        U R+                  5       m[,        R.                  " U R                  5       U R                  5       U[        U5      S9$ )NFr   Tr   r   c              3   H   #    U  H  u  pUS :H  =(       d    US :H  v   M     g7fr$   Nr   r  s      rW   rs   repeat.<locals>.<genexpr>3  s$     
A*@$!AFa1f*@s    "c                    > [        U 5      [        T5      :X  d   e[        U 5      n [        [        T5      5       HI  nTU   S:w  d  M  TU   S:X  a  [        R                  R
                  X'   M5  [        X   STU   5      X'   MK     T" U 5      $ r  )r   rv   r   r   r  Zeror!   )r  r   old_sizerepeatsx_loaders     rW   r>  repeat.<locals>.inner_fn8  s}    5zS\)))Us7|$AqzQA;!#$ww||EH.uxHQKHEH % rZ   r@  )rv   r   r   r   r  r  r  r   emptyr   r   r   r   ry  r  rC   r  r  r  r@   r  rB  r2   r   )	r~   r  new_sizezero_tensorr   r>  old_size_productr  r  s	    `     @@rW   repeatr  !  s   AJJL!H
7|c(m#GGKK=CL3x=$@AHLDN#w<3qzz|,,,,AJJL!HK3w< 1:?KkGAJ. !
 X[[]1<<>JJ

A#gx*@
AAAVA())	 ww''11-2IJ!	GG&&}X'>?CSS	
 }}H||~kkmH~	 rZ   c                     [        U [        5      (       d   e[        U[        [        45      (       d   e[        [        R
                  " U R                  U5      5      $ ra   )r[   r5   rv   rw   r7   r   rT  )r~   r  s     rW   r  r  S  sH     a####edE]++++T[[/00rZ   c                     [        U [        5      (       d   e[        U[        [        45      (       d   e[        [        R
                  " U R                  [        U5      5      5      $ ra   )r[   r5   rv   rw   r1   r   rT  )r~   r  s     rW   permuter  \  sL    a####dT5M****[''d<==rZ   c                     [        U [        5      (       d   e[        XS5      n[        [        R                  R                  U R                  XX4US95      $ )Nr   clamp)r[   r5   _validate_dimr'   	SliceViewr   rT  )r~   r   startendstepr  s         rW   slice_r  c  sH    a####

"CR\\((Se(TUUrZ   c                    [        U [        5      (       aC  [        U R                  [        R                  5      (       a  U R                  R                  5       n U R                  5         [        R                  " U 5      (       d  [        SU  S35      e[        R                  " U 5      u  pE[        R                  " UR                  UR                  U Vs/ s H  n[        R                  " U5      PM     snU Vs/ s H  n[        R                  " U5      PM     sn[        R                  " U=(       d    S5      5      n[        [        R                  " XGS95      $ s  snf s  snf )Nzunrealized as_strided(z, ...)r   rT  layout)r[   r5   rT  r'   BaseViewunwrap_viewrb  is_storage_and_layoutrk   as_storage_and_layoutFixedLayoutr   r   r   r  ReinterpretView)r~   r   stridestorage_offsetstorage
old_layoutr  
new_layouts           rW   
as_stridedr)  j  s    !YJqvvr{{$C$CFF IIK##A&&!$:1#V"DEE2215G"&'$Qa$'"()&Qa&)^(q)J R''WHII	 	()s    E; E"c                 j    [        U [        5      (       d   e[        XX#5      R                  U l        U $ ra   )r[   r5   r)  rT  )r~   r   r$  r%  s       rW   as_strided_r+  }  s.    a####8==AFHrZ   c                 0    [        XX#5      n[        U5      $ ra   )r)  ry  )r~   r   r$  r%  results        rW   as_strided_copyr.    s    8F=rZ   c                   ^ ^^^ / mSnT  H1  nTR                  X"UR                  5       T   -   45        TS   S   nM3     T  Vs/ s H  o3R                  5       PM     snmUU UU4S jn[        T S   R                  5       5      nTS   S   UT'   [        R
                  " T S   R                  5       T S   R                  5       UUS9$ s  snf )Nr   r.  c           	        >^
^ [         R                  " U T   [        R                  5      n/ n/ n[	        [        T5      5       GHH  m
T
S:X  a%  [         R                  " S[        R                  5      O*[         R                  " TT
   S   [        R                  5      n[         R                  " TT
   S   [        R                  5      n[         R                  " X5      n[         R                  " X5      nT
S:X  a  UnO+T
[        T5      S-
  :X  a  UnO[         R                  " Xg5      nUR                  U5        [        U 5      m[        TT   TT
   S   -
  5      TT'   UR                  [         R                  " UU
UU4S jS5      5        GMK     US   n	[	        [        T5      S-
  SS5       H!  m
[         R                  " UT
   UT
   U	5      n	M#     U	$ )Nr   r$   c                     > TT    " T5      $ ra   r   )r   idx_loadinputs_loaderss   rW   <lambda>1pointwise_cat.<locals>.inner_fn.<locals>.<lambda>  s    N1-h7rZ           r.  r"   )rB   
index_exprrO   int64r   r   constantgeltand_r   rv   r   maskedr  )r  idx_dimmasksmasked_loadsr  r  
start_condend_condmasknext_valr   r2  r   r"  r3  inputs_rangess             @@rW   r>  pointwise_cat.<locals>.inner_fn  s   ..S5;;7s6{#A 6 Q,^^M!$4Q$7E 
 ..q!1!!4ekkBC/Jvvg+HAvc&kAo%!xx
5LLCyH %Xc]]15Ea5H%HIHSM

75 $D  #Fq("b1AyyaQH 2 rZ   r@  )r   r   rB  rv   r2   r   r   r   )r"  r   prev_endr   r>  r  r3  rE  s   ``    @@rW   pointwise_catrH    s    9;MHh3<<>#3F(FGH $R(  4::6Coo'6:N. .` F1I&&()H!"%b)HSMay##%Qi!!#	 k ;s   C	ri  scaleszero_pointsaxis	quant_min	quant_maxc           	        ^^^^^^^^	^
 [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eU R                  5       mTR                  5       m	TR                  5       m
UUUUUUU	UU
4	S jn[        R                  " U R                  5       TUU R                  5       S9$ )Nr$   expect scales 1 dimexpect zero_points 1 dim<Expecting input to have dtype torch.float32, but got dtype: Expecting axis to be < c                 h  >	 U T
   4nT" U 5      nT" U5      nT" U5      n[        TT[        R                  S9u  pVTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      nTR                  [        R                  :w  a%  [        R
                  " U[        R                  5      n[        R                  " U5      n[        R                  " X'-  5      U-   n[        R                  " U[        R                  " Xh5      5      n	[        R
                  " U	T5      $ Nr   )_create_constantsrO   float32r   rB   r   r  
reciprocalr  maximumminimum)r  channel_idxri  scale
zero_pointqminqmax	inv_scaler  clampedrK  r   input_loaderrM  rL  rI  scales_loaderrJ  zero_points_loaders             rW   r>  ;quantized_decomposed_quantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
&y)5==Q
<<5==(LL6E+j%++>JNN5)	ii)*Z7++dCKK$:;||GU++rZ   r@  )r   r   r   rO   rG  r   rV  rB  r2   r   r   ri  rI  rJ  rK  rL  rM  r   r>  ra  rb  rc  s    `````` @@@rW   )quantized_decomposed_quantize_per_channelrf    sF    v !Q&=(==&{##%&!+G-GG+ENN*.U]]*Z	EeooFWEXYZ*#  9	 U^^%5!6 789  $$&L&&(M$002, ," !~~	 rZ   c                 p  ^^^^^	^
 [        TR                  5       5      S:X  d   S5       e[        TR                  5       5      S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eT[        U R                  5       5      :  d!   S[        U R                  5       5       35       eU R                  5       mTR                  5       m	TR                  5       m
UUUU	UU
4S jn[        R
                  " U R                  5       [        R                  UU R                  5       S9$ )	Nr$   rO  rP  Expecting input to have dtype , but got dtype: rR  c                   > U T   4nT" U 5      nT	" U5      nT" U5      nTR                   [        R                  :w  a%  [        R                  " U[        R                  5      nT
R                   [        R                  :w  a%  [        R                  " U[        R                  5      n[        R
                  " [        R                  " U[        R                  5      U5      U-  nU$ ra   )r   rO   rV  rB   r   sub)r  rZ  ri  r[  r\  r  rK  ra  rI  rb  rJ  rc  s         rW   r>  =quantized_decomposed_dequantize_per_channel.<locals>.inner_fn  s    4ylS!k*'4
<<5==(LL6E-j%--@Jggcll5%--8*EM
rZ   r@  	r   r   r   rB  r2   r   r   rO   rV  re  s    ```    @@@rW   +quantized_decomposed_dequantize_per_channelrn    s)    v !Q&=(==&{##%&!+G-GG+U"T	'w.?@Q?RST"#  9	 U^^%5!6 789  $$&L&&(M$002  !mm~~	 rZ   r[  r\  c                   ^^^^ U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       eU R                  5       mUUUU4S jn[        R                  " U R                  5       T[        R                  " U[        U5      [        U5      S9U R                  5       S9$ )NrQ  c                 @  > T
" U 5      n[        SU-  U[        R                  S9u  pB[        R                  " X4-  5      U-   n[        TT[        R                  S9u  pg[        R
                  " [        R                  " XV5      U5      n[        R                  " UT	5      $ )N      ?r   )rU  rO   rV  rB   r  rY  rX  r   )r  r[  r\  ri  r_  r  r]  r^  r`  r   ra  rM  rL  s            rW   r>  Bquantized_decomposed_quantize_per_tensor_default.<locals>.inner_fnD  s    S! 1%K5==!
	 ii)*Z7&y)5==Q
++ckk#4d;||GU++rZ   r[  r\  r@  )r   rO   rG  r   rV  rB  r2   r   r   r   r   r  r   r   ri  r[  r\  rL  rM  r   r>  ra  s      ``` @rW   0quantized_decomposed_quantize_per_tensor_defaultru  1  s     ENN*.U]]*Z	EeooFWEXYZ* $$&L, , !""E%LS_
 ~~ rZ   c                 \  ^ U R                  5       U:X  d   SU SU R                  5        35       eU R                  5       mU4S jn[        R                  " U R	                  5       [
        R                  [        R                  " U[        U5      [        U5      S9U R                  5       S9$ )Nrh  ri  c                    > T" U 5      n[        X[        R                  S9u  p[        R                  " [        R
                  " U[        R                  5      U5      U-  nU$ rT  )rU  rO   rV  rB   rk  r   )r  r[  r\  ri  r  ra  s        rW   r>  Dquantized_decomposed_dequantize_per_tensor_default.<locals>.inner_fni  sI    S!-eu}}Uggcll5%--8*EM
rZ   rs  r@  )r   rB  r2   r   r   rO   rV  r   r   r  r   r   rt  s          @rW   2quantized_decomposed_dequantize_per_tensor_defaultry  X  s     	U"T	'w.?@Q?RST" $$&L !mm""E%LS_
 ~~ rZ   c                 >  ^^^^^^^^	 U R                  5       [        R                  :X  a  [        U [        R                  5      n U R                  5       [        R                  :X  d   SU R                  5        35       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       mTR                  5       mTR                  5       m	UUUUUUUU	4S jn[        R                  " U R                  5       TUU R                  5       S9$ )NrQ  r   r$   expect scale as scalar tensor"expect zero_point as scalar tensorc                   > T	" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      n[
        R                  " U[
        R                  " U5      -  5      U-   n[        TT
[        R                  S9u  pV[
        R                  " [
        R                  " XE5      U5      n[
        R                  " UT5      $ )Nr$   r   r   r   )r   r   r   rO   rV  rB   r   r  rW  rU  rY  rX  )r  ri  _scale_zero_pointr  r]  r^  r`  r   ra  rM  rL  r[  scale_loaderr\  zero_point_loaders           rW   r>  Aquantized_decomposed_quantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKiiv 667+E&y)5==Q
++ckk#4d;||GU++rZ   r@  )r   rO   rG  r   rV  r   r   rB  r2   r   r   
ri  r[  r\  rL  rM  r   r>  ra  r  r  s
    ````` @@@rW   /quantized_decomposed_quantize_per_tensor_tensorr  y  sf    ENN*.U]]*Z	EeooFWEXYZ*u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  $$&L$$&L"..0, , !~~	 rZ   c                   ^^^^^	 [        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       e[        TR                  5       5      S:X  d;  [        TR                  5       5      S:X  a  TR                  5       S   S:X  d   S5       eU R                  5       U:X  d   SU SU R                  5        35       eU R                  5       mTR                  5       mTR                  5       m	UUUUU	4S jn[        R
                  " U R                  5       [        R                  UU R                  5       S9$ )	Nr   r$   r{  r|  rh  ri  c                 :  > T" U 5      nT" [        TR                  5       5      S:X  a  SOS5      nT	" [        TR                  5       5      S:X  a  SOS5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      nTR                  [        R                  :w  a%  [
        R                  " U[        R                  5      n[
        R                  " [
        R                  " U[        R                  5      U5      U-  nU$ )Nr$   r~  r   )r   r   r   rO   rV  rB   r   rk  )
r  ri  r  r  r  ra  r[  r  r\  r  s
        rW   r>  Cquantized_decomposed_dequantize_per_tensor_tensor.<locals>.inner_fn  s    S!c%..*:&;q&@dbI'ENN4D0E0JPRS;;%--'\\&%--8Fu}},,,{EMMBKggcll5%--8+FO
rZ   r@  rm  r  s
    ``    @@@rW   1quantized_decomposed_dequantize_per_tensor_tensorr    sL    u~~ A%ENN"u~~'7':a'?'&'  z""$%*J!"a'J,?,?,A!,D,I,+,  	U"T	'w.?@Q?RST" $$&L$$&L"..0	 	 !mm~~	 rZ   c                   ^^^^^^^ U S   R                  5       R                  S:H  nU(       a  [        S U  5       5      (       au  U  H  nUR                  5         M     [        S U  5       5      (       a  [	        [
        R                  /U Q76 u  p[        [
        R                  R                  5      " X5      $ [        U 5      S:X  a  [        U S   5      $ [        U S   US5      n[        U S[        R                  06nU  Vs/ s H  n[        Xe5      PM     n nS[         ["        [$        R&                  4   S[$        R(                  4S	 jmS
 mUUU4S jm[+        U4S jU  5       5      nS[,        4UU4S jjm[.        R0                  (       a  [3        X5      $ U(       a(  [#        [$        R4                  R7                  X5      5      $ UU4S jmSnSmS[8        R:                  R<                  4S jm[        U 5      U::  d8  [        U 5      [.        R>                  ::  a  [        UU4S jU  5       5      (       a  [        U4S j[@        RB                  RD                   5       5      n	[+        U4S jU  5       5      =(       a    U	n
[        U4S jU  5       5      =(       a    [+        U4S jU  5       5      (       + nU
(       d  U(       a  U(       d  [3        X5      $ [#        [$        R4                  R7                  X5      5      $ s  snf )Nr   cpuc              3      #    U  H5  oR                  5       [        R                  [        R                  4;   v   M7     g 7fra   )r   rO   int8uint8rq   ri  s     rW   rs   cat.<locals>.<genexpr>  s*      DJ5ejj%++66Fs   =?c              3   Z   #    U  H!  n[        UR                  5       5      S :H  v   M#     g7f)r   N)r   r   r  s     rW   rs   r    s!     >ves5>>#$)vs   )+r$   r   r~   rL   c                 $   [        U [        5      (       aO  [        U R                  [        R                  5      (       a  U R                  R                  5       $ U R                  $ [        U [        R                  5      (       a  U R                  $ U $ ra   )r[   r5   rT  r'   r  r  
StorageBoxr   s    rW   unwrap_tensorcat.<locals>.unwrap_tensor  s_    a##!&&"++..vv))++vva''66MrZ   c                     [        U [        R                  5      =(       a$    [        U R                  [        R                  5      $ ra   )r[   r'   ComputedBufferrT  r3   )rW  s    rW   is_reductioncat.<locals>.is_reduction  s)    !R../TJqvvr||4TTrZ   c                   > [        U [        [        R                  45      (       a  T" T" U 5      5      $ T" U 5      =(       dD    [        U [        R                  5      =(       a#    [        U4S jU R                  5        5       5      $ )Nc              3   n   >#    U  H*  nT" [         R                  R                  U5      5      v   M,     g 7fra   )rC   r  
get_buffer)rq   readcan_fuse_reductions     rW   rs   2cat.<locals>.can_fuse_reduction.<locals>.<genexpr>  s/      .D #177#5#5d#;<<.s   25)r[   r5   r'   r  r2   r   get_read_names)rW  r  r  r  s    rW   r  cat.<locals>.can_fuse_reduction  sk    a)R]]344%mA&677O !R\\*  ,,. 	
rZ   c              3   4   >#    U  H  nT" U5      v   M     g 7fra   r   rq   rW  r  s     rW   rs   r    s     B6a.q116   c                 X  > [         R                  " U 5      (       a;  [         R                  " U SS9u  p[         R                  R	                  U5      (       + $ [        U [        [         R                  45      (       a  T" T" U 5      5      $ [        U [         R                  5      (       a  gg)NF)freezeT)	r'   r   r!  ConcatKernelcan_realize_into_without_copyr[   r5   r  r2   )r~   r&  r   should_lower_cat_inputr  s      rW   r  #cat.<locals>.should_lower_cat_input
  s     ##A&&11!EBJGDDWMMMa)R]]344)-*:;;a&&rZ   c                 `  > [        U [        [        R                  45      (       a  T" T" U 5      5      $ [        U [        R                  5      (       d  gU R                  5       R                  nU R                  5        H+  nUT" [        R                  R                  U5      5      -  nM-     U$ r  )r[   r5   r'   r  r2   inner_fn_opcountnum_opsr  rC   r  r  )r~   countr  op_countr  s      rW   r  cat.<locals>.op_count!  s    a)R]]344M!,-- !R\\**""$,,$$&DXagg00677E ' rZ   r   r"   r   c                 p    U [         R                  R                  [         R                  R                  4;   $ ra   )r  catr  constant_pad_ndr   s    rW   additional_pointwise_ops%cat.<locals>.additional_pointwise_ops6  s(    dhh&&(<(<(D(DEEErZ   c              3   :   >#    U  H  nT" U5      T:*  v   M     g 7fra   r   )rq   rW  MAX_SIMPLE_OP_COUNTr  s     rW   rs   r  ;  s     CFq22F   c              3   <   >#    U  H  n[        UT5      v   M     g 7fra   )r<   )rq   user  s     rW   rs   r  =  s#      
+ S":;;+ru   c              3   4   >#    U  H  nT" U5      v   M     g 7fra   r   rq   r   r  s     rW   rs   r  D  s     >v&s++vr  c              3   4   >#    U  H  nT" U5      v   M     g 7fra   r   r  s     rW   rs   r  I  s      "
39C"3''6r  c              3   4   >#    U  H  nT" U5      v   M     g 7fra   r   r  s     rW   rs   r  K  s     <V(++Vr  )#r   rC  r   rb  require_channels_lastr  r  r  r  r   ry  r  r   r   r  r   r   r5   r'   r  r.   r   r   r%   force_pointwise_catrH  r  r   rO   r\   r]   max_pointwise_cat_inputsrC   r0  r]  )r"  r   
cpu_deviceri  r   r   r   fusable_reductionMAX_COMPLEX_POINTWISE_CATpointwise_usesfuse_pointwise_usehorizontal_fuse_catr  r  r  r  r  r  r  s               @@@@@@@rW   r  r    s}   %%',,5Jc DJ  
 EMMO >v>>>-dhh@@IF 0 01&>>
6{aVAY
q	3
*C	%D%L%LE /55fshs"fF5
y"--78 
RYY 
U

 B6BBT   !!V)) //<==$ !"FUZZ%:%: F 6{//	V77	7CFCCC 
~~++
 
 >v>>Q> 	 " "
39"
 
 =<V<<< 	 "5>O --R__++F899K 6s   Loffsetdim1dim2c                   ^^^
^^ U R                  5       m[        T5      m[        TTS9m[        TTS9m[        TT:g  UU4S j5        [        R
                  R                  R                  [        R                  " US5      5      nU(       a\  [        R
                  R                  R                  [        R
                  R                  R                  TT   U-   TT   5      S5      nO[[        R
                  R                  R                  [        R
                  R                  R                  TT   TT   U-
  5      S5      nSm
U(       a  U* S4m
OSU4m
[        T5       VVs/ s H  u  pgUTT4;  d  M  UPM     nnnUR                  U5        U
UUUU4S jn	[        [        R                   R#                  XU	5      5      $ s  snnf )N)r  rankc                     > ST  ST 3$ )Nz(diagonal dimensions cannot be identical z, r   )r  r  s   rW   r4  diagonal.<locals>.<lambda>Z  s     HbQUPVWrZ   r   )r   r   c                    > U S   nS/[        T	5      -  nSn[        T5       H2  nUT:X  a  UTS   -   X$'   M  UT:X  a  UTS   -   X$'   M'  X   X$'   US-  nM4     U[        T	5      S-
  :X  d   eU$ )Nr.  r   r$   r"   )r   r   )
r  diag_idxoriginal_idxcur_dimr  base_idxr  r  num_dimsoriginal_shapes
        rW   	reindexerdiagonal.<locals>.reindexerv  s    r7sS00xADy"*Xa["8d"*Xa["8"%,1 ! #n-1111rZ   )r   r   r   r   rC   r  r  r	  r   Ltevaluate_maxevaluate_minr   r   r5   r'   GenericViewr   )ri  r  r  r  offset_negative	diag_sizer   r  r  r  r  r  r  s     ``      @@@rW   diagonalr  R  s   ^^%N>"H84D84D	W gg&&44UXXfa5HIOGG$$11GG))t$v-~d/C 	
	 GG$$11GG))t$nT&:V&C 	
	 HGQ<v;$^4N441$8MQ4EN	LL   R^^**5CDD' Os   &G8Gc                 ,    [        [        XX#5      5      $ ra   )ry  r  )ri  r  r  r  s       rW   diagonal_copyr    s    %455rZ   c                 J    [        U 5      n[        XRX45      n[        Xa5        U$ ra   )ry  r  	mutate_to)ri  srcr  r  r  r  r^  s          rW   diagonal_scatterr    s$    5\Ffd1FfMrZ   c           	          [         R                  " X R                  5       U   5      n[        [	        XX"S-   5      U5      $ r  )r7   handle_negative_indexr   r  r  )r~   r   r  s      rW   selectr    s5    

$
$S**,s*;
<C6!#Qw/55rZ   c                 n   [        XS5      nUn[        U[        [        45      (       d^  U R	                  5       U   n[
        R                  R                  R                  [        XA-   S-
  U5      5      nU/U-  nXES-
  U-  -
  US'   / nSnU H#  nXx-   n	UR                  [        XXySS95        U	nM%     U$ )Nr   r$   r.  Fr  )r  r[   rv   rw   r   rC   r  r  r  r   r   r  )
r~   r  r   sizes_x_sizechunksr-  r  r   r  s
             rW   splitr    s    

"CF edE]++c"!!77V^a'/
 6!zU22r
 FEl 	fQUu=>  MrZ   c                     [        XU5      $ ra   )r  )r~   r  r   s      rW   split_with_sizesr    s    3rZ   c                     [        XS5      n[        R                  R                  R	                  U R                  5       U   5      n[        U5       Vs/ s H  n[        XU5      PM     nnU$ s  snf r  )r  rC   r  r  r  r   r   r  )r~   r   r  r   r-  s        rW   unbindr    s\    

"CWW33AJJL4EFF).v7AfQQF7M 8s   A.c                 4  ^^ U R                  5       n[        U5      n[        XQ5      mUS:X  a  [        [	        U S5      US9$ UT   n[
        R                  R                  nUR                  X&5        UR                  ST5        [        Xb-
  T5      S-   nUR                  U5      S:  a,  U R                  UR                  [        X-  U5      5      5        / US T QUPUTS-   S  QUPn	UU4S jn
[        [        R                   R#                  X	U
5      5      $ )Nr   )r  r$   c                 D   > U S   U T   T-  -   n/ U S T QUPU TS-   S Q7$ )Nr.  r$   r   )r  dim_idxr   r  s     rW   r  unfold.<locals>.reindexer  s?    b'CHtO+8Tc8G8c#'B&788rZ   )r   r   r   r  r  rC   r  r  	guard_leqguard_ltr   r  r  r   r5   r'   r  r   )r~   	dimensionr   r  r  ndimdim_sizer  new_dim_sizeout_sizer  r   s      `       @rW   unfoldr    s   JJLEu:D
4
+Cqyi1o400SzHwwHt&aHOT2Q6L(#a'	X''0CX(NOPDtDlDU379-=DtDH9 R^^**1	BCCrZ   c                     [        XS5      n[        U R                  5       5      nUR                  U[        R
                  R                  5        [        X5      $ r  )r  rv   r   insertr   r  r  r  )r~   r   r  s      rW   r  r    sA    

"CQZZ\"IS%''++&rZ   c                     [        X5      n[        U [        5      (       d   e[        U[        5      (       d   eUR                  U l        U $ ra   )r  r[   r5   rT  r  s      rW   
unsqueeze_r    sB    
A
Ca####c9%%%%XXAFHrZ   c                    [         R                  R                  R                  R	                  [
        R                  " U5      5      n[        U R                  5       5      nUS:  a  XU-   -  nSUs=::  a	  X2-   :  d   e   eU$ r  )	rC   r  r  r  r	  r   sympifyr   r   )r~   r   r  r  s       rW   r  r    sn    
''


$
$
2
25==3E
FCqzz|D
Qwf}#dm#####JrZ   r.  c                     [        XS5      n[        R                  R                  R	                  U R                  5       U   5      S-  n[        XSU5      n[        XX"S-  5      n[        U[        U5      5      $ )Nr   r"   )	r  rC   r  r  r  r   r  rA  sigmoid)r~   r   new_lenr   r  s        rW   glur    sg    

"Cgg44QZZ\#5FG1LGqq'"Aqw!,Aq'!*rZ   c                 L   ^  U(       a  [         R                  T 5        U 4S jnU$ )Nc                  |   > S n[         R                  " U[        R                  R                  " T/U Q70 UD65      $ )Nc                 p    [        U [        R                  5      (       a  [        R                  " U 5      $ U $ ra   )r[   r'   r.   r5   r   r   s    rW   wrap_tensors7fallback_handler.<locals>.handler.<locals>.wrap_tensors
  s(    *4Q		*B*B9##A&IIrZ   )pytreetree_mapr'   FallbackKernelr   )r   r   r  kernels      rW   handler!fallback_handler.<locals>.handler	  s<    	J "++226KDKFK
 	
rZ   )rF   rz   )r  r~  r  s   `  rW   r  r    s    f
 NrZ   c                  0    [         R                  " S5        g )NzjTorchinductor does not support code generation for complex operators. Performance may be worse than eager.)warningswarnr   rZ   rW   _warn_complex_not_supportedr    s    MMtrZ   rW  c                 (   U R                  5       (       a}  U(       ak  UR                  [        R                  R                  R
                  R                  [        R                  R                  R                  R                  4;   a  g[        5         gg)z0Do not support reading or writing to this tensorFT)r  r^  rO   rB   r  r  r   r  r  r  r  rW  parents     rW   unsupported_input_tensorr    s_    ||~~fmmIINN%%IIOO0088(
 
 #%rZ   c                 j    [        X5      (       a  gU R                  =(       a    [        R                  $ )z2Do not support writing tensor but can read from itT)r  is_cpur%   disable_cpp_codegenr  s     rW   unsupported_output_tensorr  +  s%    **882222rZ   rf  c                 4   U R                   [        R                  R                  L a  gU R                   [        R                  R                  L a  gS n[
        R                  " U R                  0 U R                  D6 H  nU" X0SS9(       d  M    g   U" X SS9$ )NFc                 ~   [        U [        R                  R                  5      (       d  gSU R                  ;  a  g[
        R                  " U R                  S   5       H[  n[        U[        R                  R                  5      (       d  M.  U(       a  [        X15      (       a    gMI  [        X15      (       d  M[    g   g)NFr  T)r[   rO   fxNoderE  r  tree_leaves_subclasses
FakeTensorr  r  )rf  r  	is_outputrE  s       rW   check_skip_conditionCfallback_node_due_to_unsupported_type.<locals>.check_skip_condition;  s    $..		!&&tyy'78DdE$5$5$@$@AA,T:: ; ,D99 9 rZ   )r&  T)	r^  r  view_as_complexr  lift_fresh_copyr  arg_tree_leavesr   r   )rf  allow_cpu_inputsr'  r   s       rW   %fallback_node_due_to_unsupported_typer-  2  s    {{d**222 {{d**222* %%tyy@DKK@U;; A  d;;rZ   c                   ^ U [         ;  d  U(       d
   SU  35       eU(       a  [        [        R                  " S5      5      (       a  [	        U /5      (       a  [
        R                  (       a(  U [        R                  R                  R                  ;   ds  U(       dl  [        R                  R
                  R                  (       a4  S[        R                  R
                  l        [        R                  S5        [        SU  S35      eU4S jn[!        U [        R"                  R$                  5      (       a+  U R'                  5        H  n[)        X5      nU" U5        M     g [!        U [        R"                  R*                  [        R"                  R,                  45      (       a	  U" U 5        g [/        SU  S	[1        U 5       35      e)
Nz*both a fallback and a decomp for same op: CIFznA make_fallback error occurred in suppress_errors config, and suppress_errors is being disabled to surface it.zmake_fallback(a.  ): a decomposition exists, we should switch to it. To fix this error, either add a decomposition to core_aten_decompositions (preferred) or inductor_decompositions, and delete the corresponding `make_fallback` line. Get help from the inductor team if unsure, don't pick arbitrarily to unblock yourself.c                 j   > [        U 5        Tb  [        U T5        [        U S S9" [        U 5      5      $ Nr{  )ry   r   r   r  )op_overloadlayout_constraints    rW   register_fallback(make_fallback.<locals>.register_fallbackw  s8    !+.(!+/@A $G[)
 	
rZ   zUnsupported fallback z with type )r)   r   osgetenvr*   r%   fallback_randomrO   _decompdecompositions_for_rngextra_random_decomps_dynamosuppress_errorslogwarningrU   r[   r\   r{   r}   rf   r]   HigherOrderOperatorRuntimeErrorrC  )r   r3  r  override_decompr4  olr2  s    `     rW   make_fallbackrD  W  sV   
. O9	3B4893 	4!!t$$ ""emm::OOO ==//38EMM  0KKH RD !f f
 	

 "ejj1122,,.B!"/Kk* ! 
B..

0N0NO	P	P"22$k$r(LMMrZ   c                 P    SnU  H  nX-  nM	     [        U[        R                  S9$ )z
TorchInductor offset calculation differs from PyTorch eager offset
calculation for random ops (tl.rand vs torch.rand). In future, we should
strive for same impl for tl.rand and torch.rand.
r$   r   tensorrO   r8  )r  numelr  s      rW   philox_rand_offsetrI    s+     E	 %u{{++rZ   c                 @  ^^	^
^ [         R                  " UTU [         R                  R                  U 5      5      R	                  5       m
UR                  5       mUR                  5       m	UU	U
U4S jn[        R                  " UTU[        U 5      S9n[        U 5      nXx4$ )Nc                   > [         R                  " T" / 5      [        R                  5      n[         R                  " T" / 5      [        R                  5      n[         R                  " [         R
                  " T" U 5      [        R                  5      U5      n[         R                  " UU5      n[         R                  " UT5      $ ra   )rB   r   rO   r  rz   r7  rand)	r  seed_index_exproffset_index_exprrand_index_exprr-  r   offset_loader
random_posseed_loaders	        rW   r>  philox_rand.<locals>.inner_fn  s     ,,{2DLLr):EKKH''NN:e,ekk:<M
 
 ||FE**rZ   r@  )
r'   r"  FlexibleLayoutcontiguous_stridesmake_indexerrB  r2   r   rv   rI  )r   seedr  r$  r   r   r>  random_values_nodeoffset_noderP  rQ  rR  s        `   @@@rW   philox_randrZ    s     
,,T2	
 ln  ""$K&&(M+ + #))Dz	 %T*K**rZ   c           	          [         R                  (       a\  [        R                  " [        R
                  [        R                  R                  [        R                  R                  XU5      5      $ [        S5      e)Nz&should be handled in replace_random.py)r%   r8  r  r  r5   r   r'   r  r  native_dropoutr  rU   )r~   ptrains      rW   r\  r\    sW    $$T%8%8%@%@!N
 	

 EFFrZ   c                    [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       eU R                  5         [        U5      S:X  d  [        US   [        5      (       a  [        R                  R                  O[        R                  R                  n[        R                  " X /UQ76   U $ )Nr  Tthis should be handled in decomps unless config.fallback_random or the device is CPUr   )r%   r8  r   rO   r   rb  r   r[   r  r  
bernoulli_Tensorr'   InplaceBernoulliFallback)r~   r   r2  s      rW   ra  ra    s    !!Q\\^u||8 & ^]^  IIK t9>ZQ77 	__## 
 66HrZ   c                     [         R                  (       d/  U R                  5       [        R                  " S5      :X  d   S5       e[        [        U 5      /UQ76 $ )Nr  r`  )r%   r8  r   rO   r   ra  ry  )r~   r   s     rW   bernoulli_pre    sP    !!Q\\^u||8 & ^]^  eAh&&&rZ   c                     [         era   rU   r   s    rW   _foobarri    s    
rZ   c                 .    [         R                  S5        g )Nz1using triton random, expect difference from eager)r>  info)salts    rW   _warn_triton_randomrm    s    HH@ArZ   c                  J    [        [        R                  R                  5        g ra   )rm  rC   r  creation_timer   rZ   rW   warn_triton_randomrp    s    --.rZ   c                      UR                  SS 5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      eN	generatorz-should have been handled in replace_random.py)rF  fallback_rand_generatorr%   r8  popfallback_rand_defaultrU   r   r   s     rW   rL  rL    sV    zz+t$0&777			

;%$d5f55
H
IIrZ   c                      UR                  SS 5      b  [        U 0 UD6$ [        R                  (       a  UR	                  SS 5        [        U 0 UD6$ [        S5      err  )rF  fallback_randn_generatorr%   r8  ru  fallback_randn_defaultrU   rw  s     rW   randnr{    sV    zz+t$0'888			

;%%t6v66
H
IIrZ   c                 l    [         R                  " U5      n[         R                  R                  X5      $ ra   )r'   get_stride_orderExternKernelrequire_stride_order)input_tensorr$  stride_orders      rW   inductor_force_stride_orderr  	  s'    &&v.L??//KKrZ   c                     [        S5      e)Nz.should be handled in fuse_seed_creation_pass()rg  )r   s    rW   inductor_seedr    s    
I
JJrZ   c                 ~    [        5         [        R                  " [        R                  " U [        U5      5      5      $ ra   )rp  r5   r   r'   RandomSeedsr9   )r  r   s     rW   inductor_seedsr    s)    BNN5-2GHIIrZ   c                 z   ^ ^ UU 4S jn[         R                  " T R                  5       T R                  5       U/ S9$ )Nc                 N   > [         R                  " TR                  5       T5      $ ra   )rB   	load_seedget_name)r   r  seedss    rW   r>  &inductor_lookup_seed.<locals>.inner_fn  s    }}U^^-u55rZ   r@  )r2   r   r   r   )r  r  r>  s   `` rW   inductor_lookup_seedr    s9    6 !oo	 rZ   r  r   rW  modec                  ^^^	 [         R                  (       a   eTS;   d   e/ U Qn [        R                  nUR	                  5       n[
        R                  " XTU [
        R                  R                  U 5      US9R                  5       mUR                  5       m	UUU	4S jn[        R                  " UUU/ U QS9nUR                  5         U$ )N)rL  r{  r  c                    > [        [        T5      " T" / 5      [        R                  " T" U 5      [        R                  5      5      $ ra   )rf   rB   r7  rO   r  )r  r  rQ  rR  s    rW   r>  !inductor_random.<locals>.inner_fn3  s5    sD!ONN:e,ekk:
 	
rZ   r@  )r%   r8  rO   rV  r!  r'   r"  rT  rU  rV  rB  r2   r   rb  )
r   rW  r  r  r   r   r>  r-  rQ  rR  s
     `     @@rW   inductor_randomr  '  s    %%%%$$$$T7DMME%%'FtR..AA$GPVln  ""$K
 w	F NNMrZ   lowhighc                h  ^ ^^^	 [         R                  (       a   e/ UQn[        R                  nUR	                  5       n[
        R                  " XeU[
        R                  R                  U5      US9R                  5       mUR                  5       m	UU UU	4S jn[        R                  " UUU/ UQS9$ )Nr  c           	         > [         R                  " T" / 5      [         R                  " T" U 5      [        R                  5      [         R                  " T[        R
                  5      [         R                  " T[        R
                  5      5      $ ra   )rB   	randint64r7  rO   r  r8  )r  r  r  rQ  rR  s    rW   r>  "inductor_randint.<locals>.inner_fnP  sV    }}ONN:e,ekk:NN3,NN4-	
 	
rZ   r@  )r%   r8  rO   r8  r!  r'   r"  rT  rU  rV  rB  r2   r   )
r  r  r   rW  r  r   r   r>  rQ  rR  s
   ``      @@rW   inductor_randintr  C  s     %%%%T7DKKE%%'FtR..AA$GPVln  ""$K
 
 w	 rZ   tbc                     U R                  5       U R                  5       S   U R                  5       S   U R                  5       S   -  U R                  5       S   4$ Nr.  r   )r  r   
get_strider  s    rW   _boundaries_helperr  `  sO    

b
a2==?1--
	 rZ   c                 H    U R                  5       U R                  5       S   4$ r  )r  r  r  s    rW   _sorter_helperr  i  s    ;;="--/"---rZ   	out_int32rightsidesortersorted_sequenceselfr  r  r  r  c          	      L  ^ ^^^	^
 S nU" T 5      (       a  U" U5      (       a  Tb8  U" T5      (       d+  [        [        R                  R                  SS9" T UUTUTS9$ Ub  US:X  a  SmU(       a  [        R
                  O[        R                  m	UR                  5       m
T R                  5         Tb  TR                  5         [        T R                  5       5      S:X  a  U	UU UU
4S jnO
U	UU UU
4S	 jnUR                  5       n[        R                  " UT	UUR                  S
9$ )Nc                 ^    [         R                  R                  U [        R                  5      $ ra   )rC   r  r_  r`  	BUCKETIZEr  s    rW   r4  searchsorted.<locals>.<lambda>w  s    AGG$7$7
N$$%rZ   Fr}  r  r  Tr$   c           
         > T" U 5      n[         R                  " U[        T5      STTTc  S O
[        T5      Tc  S S9$ SS9$ )Nr   r  sorter_indices)rB   	bucketizer  r  )r  r  index_dtyper  r  r  values_loaders     rW   r>  searchsorted.<locals>.inner_fn  sV    $C=="?3%~t>&3I'-~t  <= rZ   c                    >^  T" T 5      nS[         4U U4S jjn[        R                  " U[        T5      U" T5      TTTc  S O
[	        T5      Tc  S S9$ U" T5      S9$ )Nr  c                    > U R                  5       n[        R                  " [        R                  " [
        R                  S [        US S TS S 5       5       5      T5      $ )Nc              3   .   #    U  H  u  pX-  v   M     g 7fra   r   )rq   r  r   s      rW   rs   Nsearchsorted.<locals>.inner_fn.<locals>.get_flattened_index.<locals>.<genexpr>  s     &U9Tqu9T   r.  )r  rB   r7  r   r  operatorrz   r   )r  stridesr  r  s     rW   get_flattened_index;searchsorted.<locals>.inner_fn.<locals>.get_flattened_index  sU    --/~~$$ &UWSb\3sPR89T&U  	 rZ   r  )r5   rB   r  r  r  )r  r  r  r  r  r  r  r  s   `  rW   r>  r    st    $C	   =="?3#O4%~t>&3I'-~t  <Ov;V rZ   r@  )r  r  searchsortedrb  rO   r  r8  rB  rb  r   r   r   r2   r   r  )r  r  r  r  r  r  validate_bucketizer>  r   r  r  s   `  ` `   @@rW   r  r  m  s    //!$'''9&'A'A 1 1 8 8eT
 	
 DGO!*%++K$$&M 
?##%&!+
	 
		 	0 __Fzz	 rZ   r  r  
boundariesc                p  ^^^^ [        TR                  5       5      S:X  d   e[        R                  R	                  U [
        R                  5      (       a3  [        R                  R	                  T[
        R                  5      (       d)  [        [        R                  R                  SS9" U TUTS9$ TR                  5         U R                  5       nU R                  5       mU(       a  [        R                  O[        R                   mUUUU4S jn["        R$                  " UTUU R                  5       S9$ )Nr$   Fr}  r  c                 ^   > T" U 5      n[         R                  " U[        T5      STT5      nU$ r  )rB   r  r  )r  r  r  r  r  ra  r  s      rW   r>  bucketize.<locals>.inner_fn  s6    5!--z*
 rZ   r@  )r   r   rC   r  r_  r`  r  r  r  r  rb  rb  r   rB  rO   r  r8  r2   r   )ri  r  r  r  r   r>  r  ra  s    ` `  @@rW   r  r    s     z""$%*** 	
E>#;#;<<GG
N,D,DEE 5 55Q:%
 	
 F$$&L!*%++K
 
 ~~	 rZ   c                     [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ ra   )r  tree_map_onlyr'   r.   r~  require_stride1r   r   r   s      rW   require_denser    s5    ''
		2??22TNLD <rZ   c                     [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ ra   )r  r  r'   r.   r~  require_contiguousr  s      rW   r  r  	  s5    ''
		2??55~LD <rZ   c                     [         R                  " [        R                  [        R                  R
                  X45      u  pX4$ ra   )r  r  r'   r.   r~  r  r  s      rW   r  r  		  s5    ''
		2??884.LD <rZ   c           
         ^ U4S jm[        U4S j[        XR                  5       5       5      nUR                  5        VVs0 s H  u  p4UT" X@R                  U   5      _M     nnnX4$ s  snnf )Nc           	        > [        U [        R                  5      (       as  [        R                  " UR                  S   R                  5       [        R                  R                  R                  5      n[        R                  R                  X5      $ [        U [        5      (       a,  U R                  5        Vs0 s H  o3T" X   X   5      _M     sn$ U $ s  snf Nr  )r[   r'   r.   r}  rE  r$  rC   r  r  r  r~  r  r   r   )r   fx_argr  keyapply_constraints       rW   r  1constrain_to_fx_strides.<locals>.apply_constraint	  s    c299%%..E"))+QWW-=-=-G-GL ??77JJc4  LOHHJWJS)#(FK@@JWW
 Xs   ;Cc              3   8   >#    U  H  u  pT" X5      v   M     g 7fra   r   )rq   r   r  r  s      rW   rs   *constrain_to_fx_strides.<locals>.<genexpr>	  s       9P+#%%9Ps   )rw   r   r   r   r   fx_noder   r   r   r   r  s        @rW   rS   rS   	  sl      9<T<<9P D EKLLNSNDAa!!^^A%677NFS< Ts   "A+c                    ^ ^ U 4S jm[        U4S j[        [        UT R                  5      5       5       5      nUR	                  5        VVs0 s H  u  p4UT" SUT R
                  U   5      _M     nnnX4$ s  snnf )Nc           
        >^ [        U[        R                  5      (       d  U$ UR                  S   nUR	                  5       n[        R
                  " U5      nU(       a=  US   S:w  a4  [        [        [        [        UR                  5       5      5      5      5      nT	R                  [        R                  R                  :X  a  U S;   a  [        U5      S:X  d   eSnUR                  (       d  [        R                   R#                  X5      $ Sm[        U[$        5      (       d   e[        UR                  5       5      S;  a  U$ U4S	 jn[        U[        5      (       a[  UR'                  5       bJ  U" U5      (       a=  [(        R*                  R-                  [        R                   R/                  U5      U5      $ U4S
 jn[        UR0                  [        R2                  5      (       ae  U" U5      (       dX  U" UR5                  5       5      (       a=  [(        R*                  R-                  [        R                   R/                  U5      U5      $ [        R                   R#                  X5      $ )Nr  r.  r   )r   r   r   )r   r$   r"   r   r   r   r   c           	        >^  [        UU 4S j[        [        T R                  5       5      S-
  5       5       5      n[        R
                  R                  R                  T R                  5       S   5      S:H  =(       d=    [        R
                  R                  R                  T R                  5       S   5      S:*  nU=(       a    U$ )Nc              3      >#    U  HE  n[         R                  R                  R                  TR	                  5       U   5      T-  S :H  v   MG     g7fr   N)rC   r  r  r  r  )rq   r   	ALIGNMENTr~   s     rW   rs   `sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensor.<locals>.<genexpr>J	  sB      "7A !!++ALLN1,=>JqP7s   AAr$   r.  )	r   r   r   r  rC   r  r  r  r   )r~   aligned_stridesaligned_last_dimr  s   `  rW   is_aligned_realized_tensorMsdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned_realized_tensorI	  s    ! "s1<<>2Q67" O   **1<<>"+=>!C E77##--ajjl2.>?1D  $77rZ   c                    > [         R                  R                  R                  U R	                  5       S   5      T-  S:H  $ r  )rC   r  r  r  r   )r~   r  s    rW   
is_aligned=sdpa_constraint.<locals>.apply_constraint.<locals>.is_aligned^	  s4    GG$$..qzz|B/?@9LQRRRrZ   )r[   r'   r.   rE  r$  r}  rv   r  r   r   r   r^  r  0_scaled_dot_product_efficient_attention_backwardr  is_cudar~  r  r5   maybe_get_striderC   r  try_match_insignificant_stridesrealize_inputrT  r  r  )
r  r   r  meta_valmeta_strider  r  r  r  r  s
           @rW   r  )sdpa_constraint.<locals>.apply_constraint%	  s   #ryy))J;;u%oo'**;7L,1s3<<>/B)C DEL NNDDLLMv|$)))
 (L??77JJ 	#y))))s||~f,J
	8 sF##$$&2*3//77::--c2K 	S chh,,c??coo/0077BB55c:K  33CFFrZ   c              3   @   >#    U  H  u  nu  p#T" XU5      v   M     g 7fra   r   )rq   r  r   r  r  s       rW   rs   "sdpa_constraint.<locals>.<genexpr>j	  s)      "DC# 	6**"Ds   r.  )rw   r   r   r   r   r   r  s   `    @rW   sdpa_constraintr  "	  sz    CGJ  "+Cgll,C"D D IOWa!"a):;;FW< Xs   $A8)r  c                    UnU R                  5       UR                  5       :w  a  [        X0R                  5       5      nU R                  5       UR                  5       :w  a  [        X0R                  5       5      nU R	                  5       UR	                  5       :w  a$  [        X0R	                  5       5      n[        U5      $ [        U5      $ ra   )r   r  r   r   r   r  ry  )r  r  r  r~   r   s        rW   rz  rz  4
  s    ACNN,,a*+~~3==?*Q()}}#,,.(Q(Sz8OrZ   )memory_formatc          	          [         R                  " U R                  5       U R                  5       U R	                  5       [        U R                  5       5      S9$ Nr@  )r2   r   r   r   rB  rv   r   )r~   r  s     rW   ry  ry  B
  s@     ||~kkmAJJL!	 rZ   c                 "   / n[        U [        5      (       a  [        U R                  [        R                  5      (       a  U R                  n [        U [        R                  5      (       aL  UR                  U R                  5       5        U R                  n [        U [        R                  5      (       a  ML  [        U 5      n [        U 5      n U(       a:  U R                  n US S S2    H  n[        R                  " XS9n M     [        U 5      n U $ )Nr.  r  )r[   r5   rT  r'   r#  r   
get_layoutry  )r~   reinterpret_view_layoutsr  s      rW   clone_preserve_reinterpret_viewr  M
  s    !!YJqvvr7I7I$J$JFFB..//$++ALLN;A B..// aLaAFF.tt4F""9A 5aLHrZ   r*  c                Z   ^^^ UUU4S jn[         R                  " [        U5      TUU /S9$ )Nc                 @   > [         R                  " TU S   -  T-   TS9$ )Nr   r   rB   r7  )r  r   r  r  s    rW   rK   iota.<locals>.fno
  s!    ~~dU1Xo5UCCrZ   r@  )r2   r   r9   )lengthr  r  r   r   requires_gradrK   s    ```   rW   iotar  e
  s1    D V$x	 rZ   r   r  c           	        ^^^^ U R                  5       UR                  5       :X  d   eU R                  5       m[        U TS5      m[        R                  R
                  R                  [        R                  " TS5      5      (       a  TU R                  5       T   -   m[        R                  R
                  R                  ST5        [        R                  R
                  R                  TU R                  5       T   5        [        [        UT5      U R                  5       5      nUR                  5       mUUUU4S jn[        R                  " U R!                  5       U R                  5       U[#        U R                  5       5      S9$ )Nr   c           	        > [         R                  " [         R                  " [         R                  " U T   [        R
                  5      [         R                  " T[        R
                  5      5      T" U 5      T" U 5      5      $ ra   )rB   r  eqr7  rO   r  )r  r   r  
src_loaderr  s    rW   r>   select_scatter.<locals>.inner_fn
  sX    yyFFs3x5uekk2 sOSM
 	
rZ   r@  )r   rB  r  rC   r  r  r	  r   r  r   r  r  r  r  r2   r   r   rv   )r~   r  r   r  r>  r  r  s     `` @@rW   select_scatterr  z
  s   ;;=CMMO+++}}H
3
"Cww%%ehhua&899

S))GGq%(GGeQZZ\#%67
3$ajjl
3C"J
 
 ||~kkmAJJL!	 rZ   c           	      J  ^ ^^^^^^	^
 T R                  5       UR                  5       :X  d   eT R                  5       m
[        T TS5      mT R                  5       T   m[        R
                  R                  T TTT5      u  mm[        T R                  5       5      n[        TT-
  TS-
  -   T5      UT'   [        X5      nUR                  5       m	UUUU	UUU U
4S jn[        R                  " T R                  5       T R                  5       U[        T R                  5       5      S9$ )Nr   r$   c                 V  >^ T	S:X  a  TT:X  a  T
S:X  a  T" U 5      $ [         R                  " U T   [        R                  5      n[	        U 5      m[        U T   T	-
  T
5      TT'   / nT	S:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T	5      [        R                  5      5      5        TT:w  a]  UR                  [         R                  " U[         R                  " [        R                  " T5      [        R                  5      5      5        T
S:w  a}  UR                  [         R                  " [         R                  " [        U T   T	-
  ST
5      [        R                  5      [         R                  " S[        R                  5      5      5        U(       d   e[        R                  " [         R                   U5      n[         R"                  " UUU4S j[%        T5      (       a  SOS5      n[         R&                  " UUT" U 5      5      $ )Nr   r$   c                     > T" T 5      $ ra   r   )src_idxr  s   rW   r4  1slice_scatter.<locals>.inner_fn.<locals>.<lambda>
  s
    Jw'rZ   r6  )rB   r7  rO   r8  rv   r   r   r:  r   r  r;  r  r!   r9  r   r  r<  r=  r   r  )r  r>  rC  src_valr  r   r  r  r  r  r  r~   r  s       @rW   r>  slice_scatter.<locals>.inner_fn
  s   A:#/daic?"..S5;;7s)C5 0$7A:KKNN5<<#6D (?KKNN5<<#4ekkB 19KKNN'C5(8!TBEKK LLEKK0	 t$/**' ##A

 yySM
 	
rZ   r@  )r   rB  r  r   r'   r  normalize_start_endrv   r   r  r2   r   r   )r~   r  r   r  r  r  src_sizer>  r  r  r  s   ` ````  @@@rW   slice_scatterr  
  s    ;;=CMMO+++}}H
3
"Czz|C H11!S%EJE3AJJL!HS5[D1H5t<HSM

C"J,
 ,
\ ||~kkmAJJL!	 rZ   c                 v    [        U [        [        45      (       a  [        U 5      S:  a  [	        U S   5      $ U $ r  )r[   rv   rw   r   _unwrapr   s    rW   r  r  
  s0    !dE]##A
qt}HrZ   r   r   r  
pin_memoryc                :  ^ ^ [        US [        R                  4;   SU 35        [        U(       + S5        [        [	        T 5      [
        5      (       a  T=(       d    [        R                  mOT=(       d    [        R                  " 5       m/ n[        T [        R                  5      (       a  U U4S jnO[        T [        [
        45      (       a  U U4S jnO[        T 5      S:X  d-  [        T S   [        [
        45      (       aE  [        T 5      S::  a6  UR                  [        R                  " [        T 5      5      5        U U4S jnO3[        R                  R!                  [        R"                  " T TUS95      $ [$        R&                  " [)        U5      TUUS	9$ )
Nlayout=r  c                 2   > [         R                  " TT5      $ ra   r  r  rT  r   s    rW   r>  tensor.<locals>.inner_fn
  s    >>$..rZ   c                 2   > [         R                  " TT5      $ ra   rB   r9  r  s    rW   r>  r  
  s    <<e,,rZ   r   r   c                    >^ ^ UUUU 4S jm[        T5      S:X  a  [        R                  " ST5      $ T" S[        T5      5      $ )Nc           	      n  > X:  d   eX-
  S:X  a  [         R                  " TU    T5      $ X-
  S-  U -   n[         R                  " [         R                  " [         R                  " TS   [
        R                  5      [         R                  " U[
        R                  5      5      T" X5      T" X!5      5      $ )Nr$   r"   r   )rB   r9  r  r;  r7  rO   r8  )r  r  midbinary_searchrT  r   r  s      rW   r   /tensor.<locals>.inner_fn.<locals>.binary_search
  s    {"{;!#<<UU;;{q(50yyFFuQx=S%++6 "%-!#+ rZ   r   )r   rB   r9  )r  r   rT  r   s   `@rW   r>  r  
  s<      4yA~||Au-- CI..rZ   r   r@  )rn   rO   stridedr[   r  r   r8  get_default_dtyper   r   r  r   r   IntegerrC   r  add_tensor_constantrG  r2   r   r9   )rT  r   r   r  r  r=  r>  s   ``     rW   rG  rG  
  s+   v$..'&0BC:~|,'$-%%$2002!F$$$	/ 
D5#,	'	'	- 
Ta:d1gs|<<TaemmCI./	/( ww**LLU6:
 	
 V$	 rZ   c                 |    [        U [        5      (       a  Ub  [        X5      n Ub  [        X5      n U $ [	        XUS9$ )Nr   )r[   r5   r   r  rG  )rT  r   r   s      rW   	as_tensorr'    sA    $	""D(DT*D$F33rZ   c                 2    [        U [        R                  S9$ rT  rF  )rT  s    rW   long_tensorr)  &  s    $ekk**rZ   c                    SSK Jn  U" [        R                  R                  R
                  [        R                  R                  R                  S   5      nUc   e[        U5      S:X  d   U5       e[        [        UR                  5       5      5      u  p4[        R                  " X4U 5      n[        R                  R                  U5      Ul        [        R                  R!                  U5        [        R                  R                  R                  S   n[#        U[$        R&                  [$        R(                  [$        R*                  45      (       a  UR,                  R.                  $ [0        R2                  " U5      $ )Nr   )resolve_unbacked_bindingsunbacked_bindingsr$   r  )r  r+  rC   r  r  r  r0  rE  r   r   iterr   r'   DynamicScalarregister_bufferr   register_operationr[   rO   SymIntSymFloatSymBoolrf  exprr   r  )rT  r+  r,  binding_symkeypathbufferr  s          rW   _local_scalar_denser8  +  s   O 2	""AGG$8$8$=$=>Q$R ((( !Q&9(99&   %6%<%<%> ?@KkD9F''))&1FKGGv& ''


#
#E
*C#ennemmDEExx}}}}S!!rZ   c                     g ra   r   )rT  rm   s     rW   _assert_scalarr:  Z  s     rZ   c                   ^^^ U m[        U [        [        45      (       d  [        TS5      (       a  TR                  m[        T[        [        45      (       a  UU4S jnO\[        T[
        R                  5      (       a  UU4S jnO5[        TR                  5       5      S:X  d   eTR                  5       mU4S jn[        R                  " UTU[        U5      S9$ )Nr   c                 2   > [         R                  " TT5      $ ra   r  r  r   r   s    rW   r>  _full.<locals>.inner_fnj  s    <<u--rZ   c                 2   > [         R                  " TT5      $ ra   r  r=  s    rW   r>  r>  o  s    >>%//rZ   r   c                    > T" / 5      $ ra   r   )r  value_loaders    rW   r>  r>  v  s    ##rZ   r@  )r[   r   r  r   r   r   r   r   r   rB  r2   r   rv   )
fill_valuer   r   r   r>  r   rA  s     `  @@rW   _fullrC  c  s    Ej3,//GE74K4K%#u&&	. 
E5;;	'	'	0 5>>#$)))((*	$ Dz	 rZ   c                 8    [        [        U5      5      " U 40 UD6$ ra   create_tensor_liketensor_constructor)r~   rB  r   s      rW   r  r    s    0<=aJ6JJrZ   c                 &   ^  S S S S SS S.U 4S jjnU$ )NF)namesr   r   r  r  r  c                 0  > [        U S L S5        [        US [        R                  4;   SU 35        [        U(       + S5        [        U5      nU=(       d    [        R                  " 5       n[        U5      S:X  a;  [        US   [        [        [        R                  45      (       a  [        US   5      nU H$  n[        U[        R                  5      (       d  M$   e   U Vs/ s H  n[        R                  " U5      PM     nn[        TX!U5      $ s  snf )Nnamed tensorsr  r  r$   r   )rn   rO   r"  r9   r#  r   r[   rv   rw   Sizer1  r   r  rC  )	rI  r   r   r  r  r  r   r  rB  s	           rW   rM  !tensor_constructor.<locals>.inner  s     	5D=/26dEMM22gfX4FGz><0v&2002t9>ja4

2KLLa>D A!!U\\2222 )-.AQ.Z55 /s   $ Dr   )rB  rM  s   ` rW   rG  rG    s(     6 6. LrZ   )rI  r   r  r   r  r  c           	          [        U S L S5        [        U5      n[        U5      S:X  a;  [        US   [        [
        [        R                  45      (       a  [        US   5      n[        US XX4S9$ )NrK  r$   r   r   r  r   r  )	rn   r9   r   r[   rv   rw   rO   rL  empty_strided)rI  r   r  r   r  r  r   s          rW   r
  r
    se     u}o.6"F
4yA~*T!WtUEJJ.GHHT!W~d%v rZ   c                 $   ^  SSSSSS.U 4S jjnU$ )zR
Shim to convert X_like(...) into X(...).  For example zeros_like() into zeros().
NF)r   r   r  r  r  c                  > [        U(       + S5        [        US [        R                  4;   SU 35        Uc  U R                  5       nO[	        U5      nU=(       d    U R                  5       n[        U R                  5       5      nT" XaX#US9$ )Nr  r  r  )rn   rO   r"  r   r   r   rv   r   )r~   r   r   r  r  r  r   creation_fns          rW   _constant_like*create_tensor_like.<locals>._constant_like  s~     	z><06dEMM22gfX4FG=KKME 'E)1<<>AJJL!f

 	
rZ   r   )rS  rT  s   ` rW   rF  rF    s#     duTX
 
 rZ   c                 *    [        [        U 5      5      $ ra   rE  )rB  s    rW   r  r    s    0<==rZ   c                 "   ^  S S S S S.U 4S jjnU$ )NrO  c                  > [        U[        [        45      (       d   e[        U(       + S5        [        US [        R
                  4;   SU 35        [        U5      =(       d    U R                  5       nU=(       d    U R                  5       nU Vs/ s H  n[        R                  " U5      PM     nn[        T[        U5      X!5      $ s  snf )Nr  r  )r[   rv   rw   rn   rO   r"  r   r   r   r   r$  rC  r9   )r~   r   r   r  r   r  r  rB  s          rW   _new_constant#new_constant.<locals>._new_constant  s     $u....z><06dEMM22gfX4FGU#4q{{})1<<>*./$Qa $/Zv!6DD 0s    C	r   )rB  rY  s   ` rW   new_constantr[    s     tDT	E 	E rZ   rO  c          	      |    Uc  U R                  5       nUc  U R                  5       n[        US UU[        U5      US9$ NrO  r   r   rP  r9   )r~   r   r   r  r   r  s         rW   	new_emptyr_    sF    }~V$ rZ   c                   [        U [        [        45      (       d   e[        U[        [        [        S 5      45      (       d   e[	        U(       + S5        [	        US [
        R                  4;   SU 35        [        U5      =(       d    [
        R                  " 5       nU=(       d     [
        R                  " S5      R                  n[        U5      n[        SXBU S9nUR                  5         UR                  R                  n[        R                   " UR                  S/[#        U 5      -  S9Ul        [        U[$        R&                  5      (       d   eU  Vs/ s H  n[(        R*                  " U5      PM     n nU(       a'  U Vs/ s H  n[(        R*                  " U5      PM     snO[$        R,                  R/                  U 5      n[$        R0                  " UUU US9Ul        U$ s  snf s  snf )Nr  r  r6  r   )rB  r   r   r   )r=  )r   r   r   r$  )r[   rv   rw   rC  rn   rO   r"  r   r#  rG  r   r9   rC  rb  rT  dataclassesreplacer   r'   r  r   r  rT  rU  r"  r  )	r   r$  r   r  r   r  	pointwiser7  r  s	            rW   rP  rP    s    dT5M****ftUDJ78888:~|,v$..'&0BC<5#:#:#<E/u||C(//F6"F6TJI^^  F%%fkk1#D	/JFKfb//0000%)*TELLOTD*  #))&Qa&)11$7 
 NN	FM  +)s    G' G,c          	      |    Uc  U R                  5       nUc  U R                  5       n[        UUUU[        U5      US9$ r]  r^  )r~   r   r$  r   r  r   r  s          rW   new_empty_stridedre    sH     }~V$ rZ   c                    U Vs/ s H,  n[         R                  R                  R                  U5      PM.     nn[	        [        [        U5      5      UR                  S9n[        R                  R                  X5      $ s  snf )N)r  )rC   r  r  r  sortedr   r   __getitem__r'   r~  r  )r~   r$  r  r  s       rW   copy_stridedri  &  s`    5;<Vagg((+VF<%F,&2D2DEL??//@@ =s   3A?c                 X    UR                  S5      c   S5       e[        U5      " U 40 UD6$ )Nr   z(dtype should be handled by decomposition)rF  rG  )r   rB  r   s      rW   fullrk  -  s2    ::g*V,VV*j)$9&99rZ   c                 2  ^^^^ [        U [        5      (       d   eUR                  5       S:X  a  [        XR	                  5       5      $ UR                  5       [        R                  :X  d   eU R	                  5       m[        T5      S:H  n[        U TU5      mU(       a  [        U S/5      n S/mU R                  5       mUR                  5       mUUUU4S jn[        R                  " U R                  5       U R                  5       UUR	                  5       S9$ )Nr   r$   c                    > [        U 5      n [        R                  " T" U 5      TT   5      n[        U 5      S:X  a  U/n OXT'   T" U 5      $ r  )rv   rB   indirect_indexingr   )r  
gather_idxr   index_loaderr   r  s     rW   rK   gather.<locals>.fnH  sJ    3i**<+<d3iH
s8q=,C!H}rZ   r@  )r[   r5   	get_numelr_  r   r   rO   r8  r   r  r  rB  r2   r   r   )	r~   r   r  sparse_gradr  rK   rp  r   r  s	    `    @@@rW   gatherrt  3  s     a####ANN,--??+++::<DY!^F
3
'C1qcNs}}H$$&L  ||~kkm~~	 rZ   c                   ^^^^	^
 U(       a   e[        U [        5      (       d   e[        U[        5      (       d   eS[        UR                  5       5      ;   d   eU R	                  5       m	UR	                  5       m[        UR                  5       5      mU R                  5       m
/ UR                  5       QT
SS  QmUUUU	U
4S jn[        R                  " U R                  5       U R                  5       UTS9$ )Nr   r$   c                    > [        U 5      [        T5      :X  d   U  ST 35       eT" U S T 5      n[        R                  " UTS   5      // U TS  Q-   nT" U5      $ )Nz != r   )r   rB   rn  )r  	var_index
weight_idxindices_loaderindices_ndimr  weight_loaderweight_sizes      rW   rK   embedding.<locals>.fnf  s{    3x3x=(@SEhZ*@@("3}#56	++I{1~FG K
K
 

 Z((rZ   r@  )
r[   r5   strr   rB  r   r   r2   r   r   )weightr  padding_idxscale_grad_by_freqsparserK   ry  rz  r  r{  r|  s         @@@@@rW   	embeddingr  Y  s    :fi((((gy))))C))+,,,,&&(M((*Nw'')*L//#K6!!#6k!"o6H) )   " 	 rZ   c           
      L   [        S U  5       5      (       d-   SU  Vs/ s H  o"c  M  UR                  5       PM     sn 35       e[        S U  5       5      (       a  [        S5      e[	        U 5       VVs/ s H  u  p#[        U[        5      (       d  M  UPM      nnn[        U5      S:  d   S5       eS /[        U 5      -  n[        U[        U Vs/ s H  o U   PM	     sn6 5       H(  u  p#UR                  5       U:w  a  [        S5      eX5U'   M*     XT4$ s  snf s  snnf s  snf )Nc              3      #    U  HY  nUc  M  UR                  5       [        R                  [        R                  [        R                  [        R
                  4;   v   M[     g 7fra   )r   rO   r8  r  r   r  rq   r   s     rW   rs   .check_and_broadcast_indices.<locals>.<genexpr>w  sA      A 	M%++u{{EJJLLs   A#AA#z)indices must be int64, byte or bool. Got c              3      #    U  H:  oc  M  UR                  5       [        R                  [        R                  4;   v   M<     g 7fra   )r   rO   r   r  r  s     rW   rs   r  |  s.      <Cq2%**ekk22Gs
   A8AzFallback for bool indicesr   z"requires at least 1 non-None indexz.Fallback when indices is on a different device)r   r   r   rk   r   r[   r5   r   r   r   r   )r  r   r   r~   
valid_idxsnew_indicess         rW   check_and_broadcast_indicesr  v  s4       g 
373d7aMAKKM73d2ef	g 
  <C   ""=>> )' 2O 2jI6N! 2JOz?QD DD&3w<'KJ 1
3S
1AJ
3S TU <<>V#%&VWWA V ""! 4e P 4Ts   DD5DDD!c	           
      ^  ^^^^^^^^^^ Sm[        TTSS  5       H  u  pX-
  S:w  d  M  SmM     [        T5       VVs/ s H  u  pUb  M
  X   PM     snnm/ TQU [        T5      [        T5      -   S  QmTS   nT(       a  TT-   mOTS U T-   TUS  -   mUUUUUUUUUU4
S jnTU4$ s  snnf )NFr$   Tr   c                   >
 [        U 5      [        T5      :X  d   e[        T5      [        T
5      :X  d   e[        T5      n/ nTS   nT(       a  SOUnSn[        TS   S-   5       H|  nXd:X  a  XQ-  nTU   c+  U[        U 5      :  d   eUR                  X   5        US-  nM=  TU   nUc   eT
U   nUR                  [        R                  " U" XXA-    5      UT	TS95        M~     / UQXS  QnTc  U$ T" U5      $ )Nr   r.  r$   r   wrap_neg)r   r   r   rB   rn  )r  r  	new_indexfirst_tensor_indexstart_offsetnext_idxr   loaderr   r   indexed_sizer  indices_loadersnon_consecutive_tensorsoutput_sizetensor_indicestensor_sizer  r  s            rW   rK   *index_output_size_and_inner_fn.<locals>.fn  s=   3x3{++++?#s<'8888;	+A.3q9K~b)A-.A  qz!#c(***  /A(+)))#A  ))s,2EFG#!)	 /&

^
	 %,yE(92EErZ   )r   r   r   )r  r  r  r  r  r  r  r   r  previouscurrentr   r  r  rK   r  r  s    ````````      @@rW   index_output_size_and_inner_fnr    s    $ $ 1CD"&*# E ,5W+=M+=969+=MKSKS&[)9C<O)O)Q"RSK'*!K/ ++,,-./ 	 F  FD ?_ Ns   	B)B)c                     [        XU5      u  p4n[        R                  " U R                  5       U R	                  5       UUS9$ r  )index_impl_helperr2   r   r   r   )r~   r  r   r  r>  r   s         rW   
index_implr    s>    0UCK1||~kkm	 rZ   c                   ^^ [        U[        [        45      (       d   eU R                  5       m[	        XR                  5       5      u  p[        U5      S:  d   S5       eU Vs/ s H  oUb  UR                  5       OS PM     nn[        XS      R                  5       5      nU R                  5       n[        [        U5      5       Vs/ s H  oQU   c  M
  X   PM     n	nU(       a  SU	;   a  SU;  a  [        S5      e[        [        U5      5       Vs/ s H  oXU   PM	     n	n[        UUUUUU	S UUS9	u  n
mUU4S jnXT4$ s  snf s  snf s  snf )Nr   z Must have at least one valid idxz0index is out of bounds for dimension with size 0r  c                     > T" T" U 5      5      $ ra   r   )r  index_inner_fnr  s    rW   r>  #index_impl_helper.<locals>.inner_fn  s    s+,,rZ   )r[   rv   rw   rB  r  r   r   r   r   
IndexErrorr  )r~   r  r   r  r  r   r  r  r  r  r  r>  r  r  s               @@rW   r  r    sL   ge}----}}H9'<<>RG~"F$FF"KRS7a-q}}TA7OS wa01::<=KZZ\F',S\':U':!ajIFI':LUl"q';KLL',S\':;':!1I':L;"@
#K- .009 T V <s   %E		EEEc                      [        XSS9$ ! [         a;    U R                  5         [        [        R
                  R                  SS9" X5      s $ f = f)NTr   Fr}  )r  rk   rb  r  r  r  rb  r~   r  s     rW   r  r    sM    
!D11 
			

 1 1uM
 	

s   	 AAAc                     [        XSS9$ NFr  )r  r  s     rW   _unsafe_indexr    s    a..rZ   c                 .    [        [        U 5      XU5      $ ra   )
index_put_ry  r~   r  r   
accumulates       rW   	index_putr     s    eAh<<rZ   c                 ,    [        [        U 5      XUSS9$ r  )index_put_impl_ry  r  s       rW   _unsafe_index_putr  %  s    58WjNNrZ   c                     UR                  5       U R                  5       :w  a  [        X R                  5       5      nU(       a  [        X5      n[        U [	        US   X 5      5      $ r  )r   r  rz   r  r  )r  r  r   r  s       rW   index_put_as_masked_fillr  *  sP    T__..%!23D T5U9::rZ   c                    [         R                  " 5       n[        U5      (       ap  U(       d  U(       ab  U(       d  SOSn[        R                  R
                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  " [        R                  R
                  R                  XX#5        U $ )Nzindex put with accumulate.zdeterministic index put.stack_trace Found from : 
 )rO   $are_deterministic_algorithms_enabledr/   rC   r  r0  rE  rF  disable_cudagraphs_reasonr'   IndexPutFallbackr^  )r  r  r   r  deterministicrm   r  s          rW   index_put_fallbackr  2  s    >>@MjM ! )+ 	
 ''..3377tLL;LE*;-8C,/),,33TFWKrZ   c                     [        XX#SS9$ )NTr  r  r  r  r   r  s       rW   r  r  B  s    4&DIIrZ   c                     [        XX#SS9$ r  r  r  s       rW   _unsafe_index_put_r  G  s    4&EJJrZ   c                    UR                  5       S:X  a  [        U5      S:X  a  US   R                  5       [        R                  [        R
                  1;   a_  US   n[        [        UR                  5       5      [        U R                  5       5      5       H  n[        US5      nM     [        X/X#5      $ [        R                  " 5       (       a  [        XX#5      $ U HH  nUc  M  UR                  5       [        R                  [        R
                  1;   d  M<  [        XX#5      s  $    U R                  5       n[        U5      n	U(       aQ  [        U R                  5       5      (       a3  U	S:X  a  [        U S/5      n [        XX#5      n U	S:X  a  [        U / 5      n U $ [        X R                  5       5      n [        XR!                  5       5      u  pU Vs/ s H  ob  UR%                  5       OS PM     nn['        U [(        5      (       d   eU R+                  5         U	S:X  a  [        U S/5      n [-        XS      R                  5       5      n[        [        U5      5       Vs/ s H  oU   PM	     nn[/        UUU
UUUS US9u  nn[1        X/5      n[2        R4                  " U R!                  5       U R                  5       UR%                  5       UUU(       a  SOS S9n[2        R6                  " S [2        R8                  " U 5      US9n[:        R<                  R?                  U5      Ul         [:        R<                  RC                  U5        U	S:X  a  [        U / 5      n U $ ! ["         a    [        XX#5      s $ f = fs  snf s  snf )Nr$   r   r.  r  
atomic_addr   r   r>  r=  output_indexerscatter_moder   r  rT  )"rr  r   r   rO   r   r  r   r   r  r  r  r  r=   r  r   r  r   rk   rB  r[   r5   rb  rv   r  r  r'   Scatterr  MutationLayoutSHOULDREMOVErC   r  r/  r   r0  )r  r  r   r  r   rC  r   r  r  x_ndimr  r   r  r  r  expected_vals_sizer>  scatterr7  s                      rW   r  r  L  s    	aLAAJ  "uzz5;;&??qzs4==?+S-ABAT2&D C'ffII 1133!$DD !2uzz5;;6O!O%dVHH  ]]_F[FB4>>CSTTQ;qc?D!$DQ;b>Dfnn./FE"=__&#
 LSS7a-q}}TA7OSdI&&&&LLN {D1# wa01::<=K',S\':;':!1I':L;#A	$  F/F jj nn##%!%/\TG ,,T2F
 ''))&1FKGGv&{D"~Ka  E!$DDE T <s   9M! M=&N!M:9M:r}  c                    ^^^^^	 [        XSSS9u  pEmTR                  5       mU R                  5       m	UUUUU	4S jn[        R                  " U R	                  5       U R                  5       UUS9$ )NFr  c                    >^  TR                   [        R                  :w  a,  [        R                  " T" T 5      [        R                  5      nOT" T 5      n[        R
                  " UUU U4S jT5      $ )Nc                      > T" T " T5      5      $ ra   r   )_unsafe_index_fnr  self_loaders   rW   r4  8_unsafe_masked_index.<locals>.inner_fn.<locals>.<lambda>  s    K8H8M,NrZ   )r   rO   r   rB   r   r=  )r  mask_valr  fillrC  mask_loaderr  s   ` rW   r>  &_unsafe_masked_index.<locals>.inner_fn  sK    ::#||K$4ejjAH"3'Hzz($NPTUUrZ   r@  )r  rB  r2   r   r   r   )
r  rC  r  r  r=  r   r>  r  r  r  s
    ` `   @@@rW   _unsafe_masked_indexr    ss    "3UU#F ""$K""$KV V  nn	 rZ   c           	          [        XS5      nU R                  5       n[        [        U5      5       Vs/ s H%  nX&   (       a  [	        X&   XV   * XV   S-
  5      OS PM'     nn[        XUSS9$ s  snf )Nr   r$   T)r  )r  r   r   r   r  r  )r~   rC  r  r   masked_valuer  r   clamped_indicess           rW   #_unsafe_masked_index_put_accumulater    sx    q)LJJLE s7|$$A 7>jgj58)UX\2dJ$   Q$OOs   ,A,c                 X    [         R                  " U[         R                  " X 5      5      $ ra   )rB   rX  rY  r   minmaxs      rW   r  r    s    ;;sCKK/00rZ   c                 J    [        U 5      n[        XRX45      n[        Xa5        U$ ra   )ry  r)  copy_)r  r  r   r$  r%  r  output_views          rW   as_strided_scatterr    s$    4[FV6BK	+MrZ   c                 0    [        [        U 5      XU40 UD6$ ra   )scatter_ry  )r~   r   r  r  r   s        rW   r  r    s    E!Hc#888rZ   r  include_selfr2  r  r  c          
      $   [        U[        5      n[        U UUR                  5       U(       a  UR                  5       O
[	        U5      U(       a  UR                  5       R                  OSU5      (       a  [        R                  " U UUUUUUS9  U$ g )Nznot implr  )r[   r5   rA   r   rC  r   r'   ScatterFallback)r2  r  r   r  r  r  r  src_is_tensors           rW   scatter_fallbackr    s     sI.M(d3i!.J  	%	
 rZ   r  c          	          US;   d   eUcV  [        [        R                  [        R                  R
                  R                  R                  5      n[        XPXX4S9nUb  U$ US:X  a  SnOUS:X  a  Sn[        XX#U5      $ )N>   Nrz   multiplyr  rz   sumr  prod)
rf   r  r  rC   r  r0  r^  _overloadnamer  scatter_reduce_)r  r   r  r  r  r2  fallback_results          rW   r  r    s    ....~dmmQWW-A-A-H-H-V-VW*s3
 &""	:	4e&99rZ   c                 .    [        [        U 5      XU5      $ ra   )scatter_add_ry  r~   r   r  r  s       rW   scatter_addr    s    a#c22rZ   c                     [        XX#S5      $ )Nr  )r  r  s       rW   r  r    s    15u55rZ   c                 0    [        [        U 5      XX440 UD6$ ra   )r  ry  )r~   r   r  r  reduction_typer   s         rW   scatter_reducer  #  s    58SOOOrZ   )r  c          
      (  ^ ^^^^ US;   d   e[        [        R                  R                  5       5      S:X  a"  S[        R                  R                  5       ;   d   S5       e[	        T[
        5      (       a  [        T T5      m[        [        R                  R                  T TUTUUS9nU(       a  U$ [	        T [        5      (       d   eS[        UR                  5       5      ;   d   e[        T R                  5       5      nUS:X  a  [        T S/5      m [	        T[        5      (       a*  [        TR                  5       5      S:X  a  [        TS/5      m[	        U[        5      (       a*  [        UR                  5       5      S:X  a  [        US/5      nUR                  5       S:X  a  T $ [        T T5      mT R!                  5         UR#                  5       m[	        T[        5      (       a  TR#                  5       OS mUUU 4S jnU UU4S	 jn	S
 n
U(       d  [$        R&                  " T R)                  5       T R                  5       U 4S jUR                  5       US S9n[$        R*                  " S [$        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        [$        R&                  " T R)                  5       T R                  5       U	UR                  5       UU
" U5      S9n[$        R*                  " S [$        R,                  " T 5      US9n[.        R0                  R3                  U5      Ul        [.        R0                  R7                  U5        US:X  a  [        T / 5      m T $ )N>   Nr  amaxaminmeanr  r$   twozKaten.scatter_reduce_.two is not the unique overload of aten.scatter_reduce_r  r   r   c                    > TR                  5       n[        U5      n[        U 5      n[        R                  " T" U 5      US:X  a  SOUT   SS9UT'   U$ )Nr   r$   F)r  )r   r   rv   rB   rn  )r  r  r  indirect_idxr   rp  r  s       rW   r  'scatter_reduce_.<locals>.output_indexerV  sU    5zCy11DAIq5:
S rZ   c                 l   > T(       a  T" U 5      $ [         R                  " TTR                  5       5      $ ra   rB   r9  r   )r  r  r  r  s    rW   rK   scatter_reduce_.<locals>.fn`  s*    c?" <<T^^%566rZ   c                     U S:X  a  gU b   eg )Nr  r  r   r  s    rW   backend_reduce_str+scatter_reduce_.<locals>.backend_reduce_strg  s    U? >!>rZ   c                 N   > [         R                  " STR                  5       5      $ r  r  )r  r  s    rW   r4  !scatter_reduce_.<locals>.<lambda>t  s    3<<4>>3C#DrZ   r  r  )r   r  r  r}   r[   r   r  r  r  r5   r~  r   r   r  rr  r  rb  rB  r'   r  r   r  r  rC   r  r/  r   r0  )r  r   r  r  r  r  r  r  r  rK   r
  zero_outr7  r  rp  r  s   `` `          @@rW   r  r  (  s   BBBBD  **,-2T))3355U UU	6 #vc"&  !O dI&&&&C)****t}}DqyD1##y!!c#,,.&9Q&>3n%##ENN,<(=(BUQC A
c
"CLLN$$&L&0i&@&@"dJ7 ::??$.."D>>#)
 ""006

 gg--f5	""6*
 jj nn~~%'/G ,,T2F
 ''))&1FKGGv&qyD"~KrZ   scales_xnexactc                 d  ^^^^^^ U R                  5         U R                  5       mU R                  5       T* S  mU R                  5       S T*  nT Vs/ s H,  n[        R                  R
                  R                  U5      PM.     snm[        U5      T:X  d   eUn[        TU5       VVs/ s H	  u  phXh-  PM     snnm[        U5       H  u  piU	c  M
  SU	-  TU'   M     U4S jmUUUUU4S jn
[        R                  " U R                  5       U R                  5       U
/ UQUQS9$ s  snf s  snnf )Nrq  c                   > [         R                  " U [        R                  5      n T(       a:  [         R                  " U [         R
                  " S[        R                  5      5      n [         R                  " U [         R
                  " U[        R                  5      5      n [         R                  " U [        R                  5      n [         R                  " XSS9$ )N      ?Fr  )
rB   r7  rO   rV  rz   r9  rA  r   r  rn  )r~   r[  r   r  s      rW   scale_fn$upsample_nearestnd.<locals>.scale_fn  s     NN1emm,3<<U]];<AGGAs||E5==9:LLEKK($$QE::rZ   c                    > U T* S  nU S T*  nT
" / UQ[        UTT5       VVVs/ s H  u  p4nT	" X4U5      PM     snnnQ5      $ s  snnnf ra   )r   )r  r~   r  r   r  r   i_sizes
inv_scalesr  r  r  s         rW   rK   upsample_nearestnd.<locals>.fn  s_    H!HWaW#aW:UV:UJA$8A$':UVW
 	
Vs   Ar@  )realize_hintrB  r   rC   r  r  r  r   r   r   r2   r   r   r   )r~   r  r  r  r  batchr   o_sizesor[  rK   r  r  r  r  s      ``      @@@@rW   upsample_nearestndr    s    NN}}HjjlA23GJJL1"EBIJ'Qqww55a8'JGx=AG$'$9:$9DA!%$9:Jh'%KJqM (	;
 
 ||~kkm!!!	 9 K
 ;s   3D'+D,c                     [        XU4SS9$ )Nr$   r  r  r~   r  rI  s      rW   upsample_nearest1dr$    s    avi1==rZ   c                     [        XU4SSS9$ )Nr$   Tr  r  r"  r#  s      rW   _upsample_nearest_exact1dr'    s    avi1DIIrZ   scales_hscales_wc                     [        XX#4SS9$ )Nr"   r!  r"  r~   r  r(  r)  s       rW   upsample_nearest2dr,    s     ax.BaHHrZ   c                     [        XX#4SSS9$ )Nr"   Tr&  r"  r+  s       rW   _upsample_nearest_exact2dr.    s     ax.BatTTrZ   scales_dc                     [        XX#U4SS9$ )Nr   r!  r"  r~   r  r/  r(  r)  s        rW   upsample_nearest3dr2    s     ax8.LPQRRrZ   c                     [        XX#U4SSS9$ )Nr   Tr&  r"  r1  s        rW   _upsample_nearest_exact3dr4    s     	X6!4 rZ   c                 .   ^  [        U 4S jU 5       5      $ )Nc              3   R   >#    U  H  n[         R                  " UT5      v   M     g 7fra   r  )rq   r   r   s     rW   rs   $_create_constants.<locals>.<genexpr>  s     6Aa''s   $')rw   )r   r   s   ` rW   rU  rU    s    6666rZ   c                    ^^^ U R                  5       mU R                  5       mUUU4S jn[        R                  " U R	                  5       U R                  5       UTS9$ )Nc                    > [        U 5      n [        U 5      [        T5      :X  d   eT H  nTU   S-
  X   -
  X'   M     T" U 5      $ r  )rv   r   )r  r   r  r  r  s     rW   r  rev.<locals>.loader  sN    3i3x3u:%%%Cc
Q#(2CH  }rZ   r@  )rB  r   r2   r   r   r   )r~   r  r  r  r  s    ` @@rW   revr;    sM     }}HJJLE ||~kkm	 rZ   c                   ^^^^^^^ [        U5      S-  S:X  d   e[        S U 5       5      (       a  [        U 5      $ U R                  5       n[	        [        [	        [        US S S2   USS S2   5      5      5      5      m[        U5      [        T5      -
  m/ mT H?  u  pETR                  [        R                  R                  R                  U5      U45        MA     [	        US T 5      n/ m[        TUTS  5       HC  u  u  pxn	TR                  U	5        UR                  [        R                  " X-   U-   5      5        ME     [        U5      [        U5      :X  d   e[        U R                  5       5      " T5      mUUUUU4S jmUUU4S jn
U R!                  5       m["        R$                  " U R'                  5       U R                  5       U
US9$ )Nr"   r   c              3   *   #    U  H	  oS :H  v   M     g7fr  r   rq   r]  s     rW   rs   "constant_pad_nd.<locals>.<genexpr>  s     
#7a67r   r$   c                 N  >^  / n[        T T	S  TT5       HL  u  nu  p4nUS:w  a  UR                  [        US5      5        US:w  d  M2  UR                  [        X%5      5        MN     [        R
                  " [        R                  U5      n[        R                  " UU U
4S jT5      $ )Nr   c                     > T" T 5      $ ra   r   )r  r  s   rW   r4  /constant_pad_nd.<locals>.mask.<locals>.<lambda>2  s	    rZ   )	r   r   range_mask_lowrange_mask_highr   r  rB   r<  r=  )r  rC  r  r  r  r  boundsrB  
mask_sizesr  r  s   `     rW   rC  constant_pad_nd.<locals>.mask*  s    (+E!"Ivz(J$C#faxN323qyOC89	 )K
 $/zz$ 7DDrZ   c                    > [        U S T 5      n[        U TS  T5       H  u  nu  p4UR                  X#-
  5        M     [        U5      [        U 5      :X  d   eT" U5      $ ra   )rv   r   r   r   )r  r  r  r  r  bounds_precomprC  r  s        rW   	offset_fn"constant_pad_nd.<locals>.offset_fn4  s`    rO	 #E!"I~ >C#SY' !?9~U+++IrZ   r@  )r   r   ry  r   rv   r  r   r   rC   r  r  lookup_precomputed_sizer   r  r   r   rB  r2   r   r   )r~   paddingrB  r  lhr  r  r  r   rJ  rE  rI  rC  rF  r  r  s     `        @@@@@@rW   r  r    s   L1"""

#7
###QxJJLE(4GCaCL'!$Q$- @ABCFE
S[ A 68Nqww//GGJANO  uRay/KJ qr3T$5<<
T(9:; 4 {s5z)))q{{}-j9JE E }}H||~kkm	 rZ   r   c                     [         R                  " [         R                  " U [        R                  5      [         R                  " [
        R                  " U5      [        R                  5      5      $ ra   )rB   r:  r7  rO   r8  r   r$  )r   r  s     rW   rC  rC  D  s@    66q%++&u}}S)5;;7 rZ   c                     [         R                  " [         R                  " U [        R                  5      [         R                  " U[        R                  5      5      $ ra   )rB   r;  r7  rO   r8  )r   r  s     rW   rD  rD  K  s7    66q%++&tU[[) rZ   c                 T    [         R                  " [        X5      [        X5      5      $ ra   )rB   r<  rC  rD  )r   r  r  s      rW   
range_maskrS  R  s#    88q  rZ   c                    ^ ^^^^^^^ T R                  5       T* S  mT R                  5       mT=(       d    S/T-  mUUUUUUU U4S jnU$ )Nr   c                 h  >^^ U S T*  mU T* S  m[         R                  " [        R                  [	        T5       Vs/ s H   n[        TU   TU   T
U   -   T
U   * 5      PM"     sn5      nT	(       a   [        R                  " UUUUUU4S jT5      $ [        R                  " UUUU4S jT5      $ s  snf )Nc                  .   > [        TTT S9" / TQTQ5      $ )Nr   )constant_boundary_condition)r   ihpad_fill_valueprefixr~   s   rW   r4  ;constant_boundary_condition.<locals>.load.<locals>.<lambda>k  s    3A~3O"fNrNrZ   c                     > T" / TQT Q5      $ ra   r   )rY  r[  r  s   rW   r4  r\  q  s    (>V>b>*BrZ   )r   r  rB   r<  r   rS  r=  )r  r   rC  rY  r[  r   rB  rO  rZ  rM  	padding_hr~   r  s      @@rW   r5  )constant_boundary_condition.<locals>.load`  s    uC45\HHLQRUJWJqZ1qtil2Yq\MBJW
  JJ  
	
 D"BJO
	
 Xs   'B/
r   rB  )	r~   rB  rM  rZ  r   r5  rO  r^  r  s	   ````` @@@rW   rX  rX  Y  sE     	


cTUA}}H$A39I
 
( KrZ   c                 f   [        U SXA   -  -   X!   S-
  -
  X1   S-
  -   X1   5      nU(       Ga  [        U SXA   -  -   X!   S-
  -
  SX1   S-
  -  -   X1   5      n[        R                  R                  R	                  US-
  X1   -  U -
  XA   -
  5      S:  a<  US-  n[        R                  R                  R                  SXsU   -  U -
  XA   -
  5        [        R                  R                  R	                  Xg-
  5      S:X  a.  [        R                  R                  R                  Xg5        SnXe4$ UnXe4$ )Nr"   r$   r   F)r   rC   r  r  r  r  r  )r~   r   kernel_sizer$  rM  	ceil_modex_outx_alts           rW   pooling_sizerf  w  s6   	A
Nknq01VY]CVYE GJ+.1"45VY]8KKVY
 77%%uqyFI&=&AGJ&NOSTTQJEGG&&q%)*;a*?'**LM77%%em49GG))%7I  ErZ   c                 l    [        U S5      n U S   U S   -  nUS:  =(       d    [        S U 5       5      $ )Nr"   r   r$      c              3   *   #    U  H	  oS :  v   M     g7fr  r   r  s     rW   rs   :should_fallback_max_pool2d_with_indices.<locals>.<genexpr>  s     $=HqUHr   )r>   r   )rb  dilationwindow_sizes      rW   'should_fallback_max_pool2d_with_indicesrm    s;    {A.Ka.;q>1K"=$=H$=!==rZ   assert_fallbackc                   US:X  a  SS/nUS:X  a  SS/nU(       d  Un[        US5      n[        US5      n[        US5      n[        US5      n[        U [        5      (       d   e[        U5      S:X  d   e[        U5      S:X  d   e[        U5      S:X  d   e[        U5      S:X  d   e[        U R	                  5       5      S;   d   e[        X5      nUb  Xe:X  d   eXX4U4$ )Nr   r$   r"   r  )r>   r[   r5   r   r   rm  )r~   rb  r$  rM  rk  ro  use_fallbacks          rW   max_pool2d_checksrr    s     !|a&1}q6{A.K&!$F7A&GHa(Ha####{q   v;!w<1x=Aqzz|&&&:;QL"...<??rZ   c           	      B  ^^^^ [        U TTTUSS9u  mmmpFU R                  5         U R                  5       Gt pxn	[        USTTTU5      u  p[        U	STTTU5      u  pU R                  nU[
        R                  L a  SO;UR                  (       a  [        S5      O[
        R                  " U5      R                  n[        U5      X/-   nTS   (       d  TS   (       d  U(       d  U(       a  [        XSS9mOU R                  5       mUUUU4S jn[        R                  " U R!                  5       U R#                  5       [$        R&                  " USS	9US
9n[        R                  " U R!                  5       [
        R(                  [$        R&                  " USS	9US
9nUU4$ )NFrn  r   r$   -infr"   rW  c                   > U Gt p#nS nS n[         R                  " [        TS   5      [        TS   5      5       H  u  pxUTS   -  U-   TS   -
  n	UTS   -  U-   TS   -
  n
T" / UQU	PU
P5      nU(       a_  [        R                  " UTS   -  U-   [
        R                  5      nUc  UnO+[        R                  " [        R                  " X5      X5      nUc  UnM  [        R                  " X5      nM     U(       a  U$ U$ r  )
r   productr   rB   r7  rO   r  r  gtrX  )r  return_indexr[  bhbwmaxvalmaxindexh_incw_incrY  iwr  r  rb  rM  r$  r  s                rW   rK   /_low_memory_max_pool2d_with_offsets.<locals>.fn  s   R%--+a.!5Q#8
LE fQi%''!*4BfQi%''!*4B,V,R,,-Cu{1~'='EuzzR#$H"yy)<eNH~S1
  OMrZ   rx  r@  T)rr  r  r   rf  r   rO   r   r  r  r  r  rv   rX  rB  r2   r   r   r   r   r   r  )r~   rb  r$  rM  rk  rc  r   r  rO  wh_out
ceil_mode1w_out
ceil_mode2r   	min_valuer  rK   r   offsetsr  s    ```                @rW   #_low_memory_max_pool2d_with_offsetsr    sl    1B	;51-K( NN::<LUq$Q;SE$Q;SEGGE EJJ 	$66eFmEKK<N<R<R  E{e^+HqzWQZ:.qC==? 2 

||~kkm""2E:	C ||~jj""2D9	G <rZ   c                    ^^^^^^ U R                  5       mUUU4S jmUUU4S jn[        R                  " U R                  5       [        R
                  UU R                  5       S9nU$ )Nc                 0  > [         R                  " T	[        R                  5      n[         R                  " UTS   -  T
S   -
  [        R                  5      n[         R                  " UTS   -  T
S   -
  [        R                  5      nXP-   nXa-   nXt-  U-   $ r  )rB   r7  rO   r8  )r}  r~  ry  rz  w_inhbasewbaserY  r  input_widthrM  r$  s            rW   increments_to_indexF_low_memory_max_pool2d_offsets_to_indices.<locals>.increments_to_index  s{    ~~k5;;7rF1I~
:EKKHrF1I~
:EKKH]]y2~rZ   c                    > U Gt pnT
" / UQUPUP5      n[         R                  " T	[        R                  5      nXE-  nXFU-  -
  nT" XgX#5      $ ra   rB   r9  rO   r  )r  r[  ry  rz  r  kw_constr}  r~  r  kernel_widthoffsets_loaders           rW   offsets_to_indicesE_low_memory_max_pool2d_offsets_to_indices.<locals>.offsets_to_indices  s\    R 1& 1" 1b 12<<ekk:"(*+"588rZ   r@  )rB  r2   r   r   rO   r8  r   )	r  r  r  r$  rM  r  r  r  r  s	    ````  @@rW   )_low_memory_max_pool2d_offsets_to_indicesr    sX     ((*N9 !!#kk#!	G NrZ   c                 `  ^^^^^^^^^^^ TS:X  a  SS/mUS:X  a  SS/nT(       d  Tm[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        U5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         U R                  5       n[        U[        5      (       a  [        UR                  R                  [        5      (       a  UR                  R                  n	[        R                  " S [        R                  " U	R                  5       U	R                  5       U	R                  5       S9U	S9n
U
R                  5         U
R                  5       nOUR                  5       nUS L=(       a    US   S:H  =(       d    US L=(       a    US   S:H  n[        S U 5       5      (       a  [!        XTTTXVU5      $ UR                  5       Gt pmU R                  5       Gt nmmUR#                  5       mU R#                  5       m[%        UR                  5       5      n['        UU4S j[)        TS   S-  5       5       5      m['        UU4S	 j[)        TS   S-  5       5       5      mTT-  nUS
:  a  [!        XTTTXVU5      $ UR                  5       mUUUUUUUUUUU4S jn[        R*                  " U R                  5       U R                  5       UUS9nU(       a  [        R,                  R/                  U5      $ U$ )Nr   r$   r"   r  )r   r   r   r  c              3   *   #    U  H	  oS :g  v   M     g7fr  r   r  s     rW   rs   3max_pool2d_with_indices_backward.<locals>.<genexpr>O  s     
$8a68r   c           
   3   z   >#    U  H0  n[        UTS    -  [        S UTS    -
  TS    -  5      -
  S5      v   M2     g7fr   r$   Nr  rq   rO  rb  r$  s     rW   rs   r  \  H      *A 	ANSQQ%7F1I$EFFJJ*   8;c           
   3   z   >#    U  H0  n[        UTS    -  [        SUTS    -
  TS    -  5      -
  S 5      v   M2     g7fr$   r   Nr  rq   r  rb  r$  s     rW   rs   r  `  r  r  rh  c                   > U Gt pn[         R                  " UT-  U-   [        R                  5      nUTS   -   nUTS   -   n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R                  " [	        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n	[        T5       GHD  n
[        T5       GH0  n[         R                  " U[         R                  " U
[        R                  5      5      n[         R                  " U[         R                  " U[        R                  5      5      n/ UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TS   SS9PnT" U5      nT" U5      n[         R                  " X5      nU	c>  [         R                  " UU[         R                  " S[        R                  5      5      n	GM  [         R                  " [         R                  " [         R                   " X5      [         R                   " X5      5      U5      n[         R                  " U[         R                  " U	U5      U	5      n	GM3     GMG     U	c   eU	$ )Nr   r$   Fr  r.  r6  )rB   r7  rO   r  r   rX  r9  rY  r   rz   rn  rk  r  r  rV  r<  r;  )r  r[  rO  r  
index_testphstartpwstartphendpwendgradientph_pw_phpw
grad_indexindex_actual	grad_partr   rC  grad_loaderh_window_sizery  indices_sizerb  rM  pooled_heightpooled_widthr$  w_window_sizewidths                      rW   rK   ,max_pool2d_with_indices_backward.<locals>.fno  s+   A^^AIM5;;?

N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE))Bs||Au{{7S(TU$R(# ))Bs||Au{{7S(TU$R(#
  .j9'
3	|8#"yyy#,,sEMM*J H 88FF2-FF2- D  #yyswwx/KXVHE , (H ###rZ   r@  )r[   r5   r   r   r  r  rT  r2   r'   r  rT  r   r   decide_layoutr  r   )fallback_max_pool2d_with_indices_backwardrB  rv   r  r   r   r~  r  )grad_outputr~   rb  r$  rM  rk  rc  r  	gO_striderT  x_bufferx_strideis_channels_lastr  heightr   r  rl  rK   r   r  r  ry  r  r  r  r  r  s     ```               @@@@@@@@rW    max_pool2d_with_indices_backwardr  &  s    !|a&1}q6a####{q   v;!w<1x=Aqzz|&&& ,,.I!YJqvv{{I$F$Fvv{{$$$$(nn&]]_
 
 	 &&(%%' ,A!1A 3)A,!"3  
$8
$$$8K(w
 	
 JJLUE&1&:&:&<#Q|((*N))+KAJJL!H {1~)* M  {1~)* M
  -/KR8K(w
 	
 ##%L9 9v 

%%'##%	C 44S99
rZ   c                 ^   ^^ U R                  5       Gt p#nU R                  5       mUU4S jnU$ )Nc           
        >^ ^^^	^
 Uu  mm	Uu  mm
Uu  pE[         R                  " [         R                  " [         R                  " TT-   [        R
                  5      [         R                  " U[        R
                  5      5      [         R                  " [         R                  " T
T	-   [        R
                  5      [         R                  " U[        R
                  5      5      5      n[         R                  " UUUU	U U
U4S jT5      $ )Nc                  ,   > T" / TQT T-   PTT-   P5      $ ra   r   )h_start_indexrY  r  r[  w_start_indexr  s   rW   r4  3pad_adaptive_loader.<locals>.load.<locals>.<lambda>  s$    HNvN}r'9N=2;MNOrZ   )rB   r<  r;  r7  rO   r8  r=  )r[  
incrementsstart_indicesend_indicesh_end_indexw_end_indexrC  r  rY  r  r  pad_valr  s   `      @@@@rW   r5  !pad_adaptive_loader.<locals>.load  s    B'4$}#. xxFF}r15;;?{EKK8 FF}r15;;?{EKK8	
 zzOO
 	
rZ   r`  )r~   r  r   rO  r  r5  r  s    `    @rW   pad_adaptive_loaderr    s)    zz|HQ1}}H
, KrZ   c                     [         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n[         R                  " XUS9n	XgX4$ )N)out_diminp_dim)r   r   )
start_index	end_indexh_inr  r  r  r  r  r  r  s
             rW    compute_indices_adaptive_poolingr    sU    %%k$OM##IdKK%%k$OM##IdKK}AArZ   c                 \   ^^^^^^ Uu  pgUu  p[        XXgX5      u  mmmmUUUUUU4S jn
U
$ )Nc                    > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	[         R                  " [        TS   5      [        TS   5      5       H"  u  pU" UX/XW/Xh/5      nU	c  Un	M  T" X5      n	M$     U	$ r  )r   rv  r   )r  r  r[  ry  rz  r  r  r  r  r-  rY  r  r  h_end_index_fnh_start_index_fnkernel_maxes
pooling_fnw_end_index_fnw_start_index_fns                rW   rK    _adaptive_pooling_fn.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C ~#C0 X rZ   r  )r  r  r  in_sizes	out_sizesr  r  r  r  r  rK   r  r  r  r  s     `  `     @@@@rW   _adaptive_pooling_fnr    sF     JDLE 	)E	
 . IrZ   c                 d   ^^^
^^^^ Uu  nmUu  px[        XUTXx5      u  mm
mmU
UUUUUU4S jn	U	$ )Nc                   > U Gt p#nT" U5      nT" U5      nT" U5      nT" U5      nS n	S n
[         R                  " [        TS   5      [        TS   5      5       H  u  pU" UX/XW/Xh/5      n[        R                  " X[-   T-  U-   U-   [
        R                  5      nU
c  Un
O+[        R                  " [        R                  " X5      X5      n
U	c  Un	M{  T" X5      n	M     U
$ r  )	r   rv  r   rB   r7  rO   r8  r  rw  )r  r  r[  ry  rz  r  r  r  r  r{  r|  rY  r  r  r  r  r  r  r  r  r  r  s                  rW   rK   )_adaptive_pooling_fn_with_idx.<locals>.fn  s    R(,$R((,$R(''l1o(>lSTo@VWFB.*	C NN#t+m;b@%++E  99SVVC%8%J~#C0) X, rZ   r  )r  r  r  r  r  r  r  r  r  rK   r  r  r  r  r  s     `  `    @@@@@rW   _adaptive_pooling_fn_with_idxr    sK     JD$LE 	)dE	
! !F IrZ   c           	        ^ ^^ [        T [        5      (       d   e[        U5      S:X  d   eT R                  5         T R	                  5       Gt p#n[
        R                  R                  R                  U5      n[
        R                  R                  R                  U5      nUu  pVX5:X  a  XF:X  a  [        T 5      $ US:X  d  US:X  a/  / UQUPUPn[        UT R                  5       T R                  5       S9$ X5-  S:X  a  XF-  S:X  a  X5-  XF-  /n[        T U5      $ [        X5-   S-
  U5      n	[        XF-   S-
  U5      n
[        U5      XV/-   nT R                  5       nX-  nUS:  a  [!        T U5      $ S nS n[#        UUX/X4/XV/[$        R&                  S9m[)        [+        T 5      5      mUUU 4S	 jn[,        R.                  " T R                  5       UUUS
9nU$ )Nr"   r   r   r$   rh  c                     [        X-  U5      $ ra   r   r  r  r  s      rW   r  )_adaptive_avg_pool2d.<locals>.start_indexc      733rZ   c                 2    [        U S-   U-  U-   S-
  U5      $ r  r  r  s      rW   r  '_adaptive_avg_pool2d.<locals>.end_indexf  "    g-7!;WEErZ   r  r  r  r  r  r  c                 `   > [         R                  " T" U [        T5      5      T" U T5      5      $ ra   )rB   truedivr  )r  fn_sumones_loaderr~   s    rW   rK    _adaptive_avg_pool2d.<locals>.fnt  s-    {{3+A./[1I
 	
rZ   r@  )r[   r5   r   r  r   rC   r  r  r  ry  r
  r   r   
avg_pool2dr8   rv   fallback_adaptive_avg_pool2dr  rB   rz   r  	ones_liker2   r   )r~   r  r  r  r  r  r  o_sizerb  h_kernel_maxw_kernel_maxr  r   rl  r  r  rK   rvr  r  s   `                 @@rW   _adaptive_avg_pool2dr  @  s   a####{q   NNU$7711$7D7711$7DLE }QxzUaZ'5'%''V1;;=HH|qT\Q.}dm4![))DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ""1.77F &il3K

 
		||~	
B IrZ   c           	      D  ^ ^^ [        T [        5      (       d   e[        U5      S:X  d   eT R                  5         T R	                  5       Gt p#n[
        R                  R                  R                  U5      n[
        R                  R                  R                  U5      nUu  pVUS:X  d  US:X  aV  / UQUPUPn[        UT R                  5       T R                  5       S9[        U[        R                  T R                  5       S94$ X5-  S:X  a  XF-  S:X  a  [        e[        X5-   S-
  U5      n[        XF-   S-
  U5      n	[!        U5      XV/-   n
T R                  5       nX-  nUS:  a  [#        T U5      $ S nS n[%        UUX/X4/XV/[&        R(                  S9m[+        UUX/X4/XV/[&        R(                  S9mUU 4S	 jnUU 4S
 jn[,        R.                  " T R                  5       UUU
S9n[,        R.                  " T R                  5       [        R                  UU
S9nUU4$ )Nr"   r   r   r$   rh  c                     [        X-  U5      $ ra   r  r  s      rW   r  (adaptive_max_pool2d.<locals>.start_index  r  rZ   c                 2    [        U S-   U-  U-   S-
  U5      $ r  r  r  s      rW   r  &adaptive_max_pool2d.<locals>.end_index  r  rZ   r  c           	      <   > T" U [        T[        S5      5      5      $ Nrt  r  r  )r  inner_func_max_valr~   s    rW   inner_fn_max_val-adaptive_max_pool2d.<locals>.inner_fn_max_val      !#':1eFm'LMMrZ   c           	      <   > T" U [        T[        S5      5      5      $ r	  r
  )r  inner_func_max_idxr~   s    rW   inner_fn_max_idx-adaptive_max_pool2d.<locals>.inner_fn_max_idx  r  rZ   r@  )r[   r5   r   r  r   rC   r  r  r  r
  r   r   rO   r8  
ValueErrorr8   rv   fallback_adaptive_max_pool2dr  rB   rX  r  r2   r   )r~   r  r  r  r  r  r  r  r  r   r  r   rl  r  r  r  r  r  rir  r  s   `                  @@rW   adaptive_max_pool2dr    s   a####{q   NNU$7711$7D7711$7DLEzUaZ'5'%''V1;;=H%%++allnK
 
 	
 |qT\Q.DL1,u5LDL1,u5LE{e^+HKKME-KR+A{;;4F ."1.;; 7"1.;;NN 
		||~!	
B 
		||~kk!	
B r6MrZ   c                    ^ ^^^^^^ TT   mTT   mTT   m[        TT-
  TS-
  5      mT R                  5       mUUUUUU U4S jnU$ )Nr$   c                 V  > T" / U QTP5      n[         R                  " UTR                  5       5      n[         R                  " TTR                  5       5      n[         R                  " X2-   U-  5      [         R                  " X$-  5      -
  n[         R                  " U[
        R                  5      n[         R                  " U[         R                  " TS-
  [
        R                  5      5      n[         R                  " Xe[         R                  " T	T
-
  [
        R                  5      5      $ r  )	rB   r7  r   r  r   rO   r8  r;  r  )r[  r   samplei_expr
alpha_exprseq_irC  r(  r   in_sz	kernel_szout_szsamplessamples_loaders          rW   r5  )_fractional_pooling_offsets.<locals>.load  s    &#/7#4#4#67^^E7+<+<+>?
		6?j89CII=
 
 UEKK0vvNN6A:u{{3
 yycnnUY5F&TUUrZ   )r    rB  )r   r  r  r  r   r5  r(  r!  s   ````` @@rW   _fractional_pooling_offsetsr#    sT    C[F#JE#Iuy(&1*5E((*NV V KrZ   c           	      V  ^^^^^^ U R                  5         U R                  5       Gt nmmTu  pVUu  pxXV-  S:  a  [        U TX#5      $ [        R                  " [
        UTT/UTS9n	U	" SS9mU	" SS9mU R                  5       mUUUUUU4S jn
[        U5      Xx/-   n[        R                  " U R                  5       U R                  5       [        R                  " U
SS9US	9n[        R                  " U R                  5       [        R                  [        R                  " U
S
S9US	9nX4$ )Nrh  )r   r  r  r  r   rW  r$   c           	        > U Gt p#n[         R                  " T" X#5      T5      n[         R                  " T" X$5      T5      nS nS n[        R                  " [	        TS   5      [	        TS   5      5       H  u  pT" / UQXY-   PXj-   P5      nU(       a  [         R
                  " XY-   T-  U-   U
-   [        R                  5      nUc  UnOT[         R                  " [         R                  " [         R                  " X5      [         R                  " U5      5      X5      nUc  UnM  [         R                  " X5      nM     U(       a  U$ U$ r  )rB   rn  r   rv  r   r7  rO   r8  r  or_rw  r  rX  )r  rx  r[  ry  rz  r  r  r{  r|  rY  r  r  r  
h_index_fninp_hinp_wrb  
w_index_fnr  s                rW   rK   !fractional_max_pool2d.<locals>.fn  s"   R--j.DeL--j.DeL''k!n(=u[QR^?TUFBLVL]%7L9KLMC"'50=@2Eu{{ #$H"yys 3SYYs^De H ~S1 V  OMrZ   Fr  r@  T)r  r   fallback_fractional_max_pool2dr   r   r#  rB  rv   r2   r   r   r   rO   r8  )r~   rb  r  random_samplesr  kernel_hkernel_wr  r  gen_offsets_for_dimrK   r  r  r  r'  r(  r)  r*  r  s    `            @@@@@rW   fractional_max_pool2dr1    s#   NN::<UE5$HLEb -{K
 	
 $++#en %+J$+J}}H : E{e^+H			||~kkm""2E:	
B 
		||~kk""2D9	
B 6MrZ   c           	      D  ^ ^^ T R                  5         T R                  5       Gt pVn[        R                  R                  R                  U5      n[        R                  R                  R                  U5      nUGt pXn	Xh-  S:X  a  Xy-  S:X  a  [        T Xh-  Xy-  /SS9$ [        Xh5      n
[        Xy5      nS mU4S jn[        TUX/Xg/X/[        R                  S9mUU 4S jn[        R                  " T R                  5       T R                  5       U[        U5      S9nU$ )	Nr   r$   )divisor_overridec                 F    [        X-  [        R                  " U5      5      $ ra   )r   r   r  r  s      rW   r  0upsample_nearest2d_backward.<locals>.start_indexM  s    ug(>??rZ   c                    > T" U S-   X5      $ r  r   )r  r  r  r  s      rW   r  .upsample_nearest2d_backward.<locals>.end_indexP  s    EAI99rZ   r  c                 (   > T" U [        T5      5      $ ra   )r  )r  r  r~   s    rW   rK   'upsample_nearest2d_backward.<locals>.fn\  s    c.q122rZ   r@  )r  r   rC   r  r  r  r  r8   r  rB   rz   r2   r   r   r   rv   )r~   r  
input_sizer(  r)  r  r(  r)  out_hout_wr  r   r  rK   r  r  r  s   `              @@rW   upsample_nearest2d_backwardr=  ;  s    NN::<U5GG2259EGG2259E%U5}emq0!enen=PQRR5(L5(L@: ""1.77F3 
		||~kkmJ	
B IrZ   c                 "    [        U UUUUUUSS9$ )Nr"   rW  _avg_poolndr~   rb  r$  rM  rc  count_include_padr3  s          rW   r  r  q  )     		 	rZ   c                 "    [        U UUUUUUSS9$ )Nr   rW  r?  rA  s          rW   
avg_pool3drE    rC  rZ   c                   ^^^^^^^^^^ T(       d  TmT(       d  S/T-  m[        TT5      m[        TT5      m[        TT5      m[        U [        5      (       d   e[        T5      T:X  d   e[        T5      T:X  d   e[        T5      T:X  d   e[        U R	                  5       5      TS-   TS-   4;   d   eU R                  5         U R	                  5       S T*  nU R	                  5       T* S  m[        [        T5       V	s/ s H  n	[        TU	   U	TTTU5      PM     sn	6 u  p[        T5      (       d  [        U5      (       a  [        U STS9mSnOU R                  5       mSn[        U5      [        U
5      -   nU R                  5       m[        R                  " [         R"                  T5      nUS:  a6  TS:X  a  [$        nOTS	:X  a  [&        nO[)        S
T 35      eU" U TTTUTU5      $ UUUU4S jmU(       a  U(       a  U(       a  SU-  mOSU-  mUUUU4S jnOUUUUUUUU4S jn[*        R,                  " U R/                  5       TUUS9nU$ s  sn	f )Nr   r$   r"   r6  rW  TFrh  r   zUnknown dim: c           	      f  > U S T	*  nU T	* S  nS n[         R                  " [        T	5       Vs/ s H  n[        T
U   5      PM     sn6  HZ  n[        T	5       Vs/ s H  oSU   TU   -  Xe   -   TU   -
  PM     nnU" / UQUQ5      nUc  UnMD  [        R                  " X5      nM\     U$ s  snf s  snf ra   )r   rv  r   rB   rz   )r  r  r[  r  totalr   rY  r   r  r   rb  rM  r$  s            rW   r  _avg_poolnd.<locals>.fn_sum  s    UsdJ##U3Z%PZeKN&;Z%PQBBG*M*QQ4&)#be+gaj8*CM6C)C}+ R  &QMs   B)B.rq  c                 j   > [         R                  " T" U T5      [         R                  " TT5      5      $ ra   )rB   rA  r9  )r  r   r  r[  r  s    rW   rK   _avg_poolnd.<locals>.fn  s&    776#x0#,,ue2LMMrZ   c                   > U S T
*  nU T
* S  n/ n[        T
5       H  nX$   TU   -  TU   -
  n[        R                  " UTU   -   TU   TU   -   5      nT	(       d1  [        R                  " US5      n[        R                  " UTU   5      n[        R
                  " Xe-
  [        R                  5      nUR                  U5        M     [        R                  " [        R                  U5      n[        R                  " T" U T5      U5      $ r  )r   r   MinMaxrB   r7  rO   r  r   r   r  rA  r  )r  r[  ry  divide_factorsr   hstarthendfactordivide_factorrB  r   r  rO  rb  rM  r$  r  s            rW   rK   rK    s    3$ZFcTUBN3Z*WQZ7yy+a.!8!A$:KL("YYvq1F 99T1Q40Du{{C%%f-   &,,SWWnEM;;vc84mDDrZ   r@  )r>   r[   r5   r   r   r  r   r   rf  r   rX  rB  rv   r   r   r  r  rA  fallback_avg_pool2dfallback_avg_pool3dr  r2   r   r   )r~   rb  r$  rM  rc  rB  r3  r   r  r   r  
ceil_modeshad_paddingr  rl  fallbackrK   r  r   r  rO  r[  r  s    ``` ` `          @@@@@rW   r@  r@    sS    #){C0K&#&F7C(Ga####{s"""v;#w<3qzz|q#' 2222NNJJL3$E	

cTUA 3Z

 1q+vw	J

E 7||s:.q#3?==?E{T%[(HKKME""8<<=KR!8*HAX*H}SE233
 	
  *((E+%E	N 	N
	E 	E  
		||~	
B Ig

s    Ic                 (  ^^^^^^^^^^^^^^ Tb  TS:w  d   S5       eT(       d  TmT(       d  SS/m[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmm[        TSTTTU5      u  p[        TSTTTU5      u  pU R                  5       mTS   =(       d    TS   =(       d    U
=(       d    UmU R                  5       Gt nmm[        UR                  5       5      nUR                  5       n[        UU4S j[        TS   S-  5       5       5      m[        UU4S j[        TS   S-  5       5       5      mTT-  nUS:  a  [        U UTTTUTT5      $ UUUUU4S	 jmUUUUUUUUUUUU4S
 jn[        R                  " U R                  5       UUUS9nU$ )Nr   divisor must be not zeror"   r  r$   c           
   3   z   >#    U  H0  n[        UTS    -  [        S UTS    -
  TS    -  5      -
  S5      v   M2     g7fr  r  r  s     rW   rs   &avg_pool2d_backward.<locals>.<genexpr><  r  r  c           
   3   z   >#    U  H0  n[        UTS    -  [        SUTS    -
  TS    -  5      -
  S 5      v   M2     g7fr  r  r  s     rW   rs   r\  @  r  r  rh  c           	      n  > [         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " TS   [        R                  5      n[         R                  " [         R
                  " X5      U5      n[         R                  " [         R
                  " X5      U5      n	[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n
[         R                  " [         R                  " X5      [         R                  " [         R                  " T[        R                  5      U5      5      n[         R                  " U[         R                  " S[        R                  5      5      n[         R                  " U	[         R                  " S[        R                  5      5      n	[         R                  " U
[         R                  " T[        R                  5      5      n
[         R                  " U[         R                  " T[        R                  5      5      n[         R
                  " [         R                  " X5      [         R                  " X5      5      nU$ )zc
This computes the scaling factor that we will divide an element
by when `count_include_pad=False`
r   r$   )
rB   r9  rO   r  rk  rA  rY  rz   r7  rX  )r  r  stride_hstride_wpad_hpad_wr.  r/  rP  wstartrQ  wendrS  r  rb  rM  r$  r  s                rW   !compute_pool_size_without_padding>avg_pool2d_backward.<locals>.compute_pool_size_without_paddingS  s   
 <<q	5;;7<<q	5;;7WQZ5WQZ5<<A<<<A<.6.6{{GGF%GGCNN65;;7?
 {{GGF%GGCNN5%++6>
 VS\\!U[[%ABVS\\!U[[%AB{{4!DE{{4u{{!CD 5swwt7LMrZ   c                   > U Gt pnUTS   -   nUTS   -   n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   -
  TS   -   TS   5      [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R                  " [        UTS   5      S-   [        R                  5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R
                  " U[         R                  " S[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      n[         R                  " U[         R                  " T[        R                  5      5      nS n[        T5       GHD  n	[        T5       GH0  n
[         R                  " U[         R                  " U	[        R                  5      5      n[         R                  " U[         R                  " U
[        R                  5      5      nTb  TnO"T(       d  T(       d  TS   TS   -  nOT" X5      n[         R                  " T" / UQ[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P[         R                  " [         R                  " U[         R                  " U[         R                  " S[        R                  5      5      5      TSS9P5      U5      n[         R                  " [         R                  " X5      [         R                  " X5      5      nUc=  [         R                  " X[         R                  " S[        R                   5      5      nGM  [         R                  " U[         R                  " X5      U5      nGM3     GMG     Uc   eU$ )Nr   r$   Fr  r6  )rB   r7  r   rO   r  rX  r9  rY  r   rz   r  rn  rk  r<  r;  r  rV  )r  r[  rO  r  r  r  r  r  r  r  r  r  r  r[  partrC  re  rB  r3  r  r  rW  rb  rM  r  r  r$  r  s                   rW   rK   avg_pool2d_backward.<locals>.fno  s   A
N
N..QQ'&)3VAY?
 ..QQ'&)3VAY?
 x6!9595;;Gx6!9595;;G++gs||Au{{'CD++gs||Au{{'CDE3>>-#MNE3>>,#LM'C]+WWWcll3&DEWWWcll3&DE#/,E&k'N[^;E=bEE{{#11 #$&s||Au{{7S(T!" !.&+  11 #$&s||Au{{7S(T!" !-&+& ). xxFF2%FF2% #"yyS\\#u}}5UVH"yyswwx/FQHS , (V ###rZ   r@  )r[   r5   r   r   r  rf  rB  rv   r   r  r   fallback_avg_pool2d_backwardr2   r   r   )r  r~   rb  r$  rM  rc  rB  r3  r  r  r  r  r  r   r  r   rl  rK   r  re  r  r  rW  r  r  r  r  r  s     ``` ``           @@@@@@@@@rW   avg_pool2d_backwardrk    s    #'71'<X>XX<a&k9----a####{q   v;!w<1qzz|&&&JJLUFE$VQVWiXE$UA{FGYWE))+K!*F
FjFJK&1&:&:&<#Q|AJJL!HKKME {1~)* M  {1~)* M
  -/KR+	
 		
 8? ? ?B 
		%%'	
B IrZ   c                   ^^^^^^^^^^^^^^^^^  Tb  TS:w  d   S5       eT(       d  TmT(       d  / SQm[        U [        5      (       d   e[        U[        5      (       d   e[        T5      S:X  d   e[        T5      S:X  d   e[        T5      S:X  d   e[        UR                  5       5      S;   d   eU R	                  5         UR                  5       Gt nmmm [        TSTTTU5      u  p[        TSTTTU5      u  p[        T STTTU5      u  pU R                  5       m[        T5      =(       d    U
=(       d    U=(       d    UmU R                  5       Gt nmmm[        UR                  5       5      nUR                  5       nUU4S j[        S5       5       u  mmmTT-  T-  nUS	:  a  [        U UTTTUTT5      $ UUUUUU 4S
 jmUUUUUUUUUUUUUU4S jn[        R                  " U R                  5       UUUS9nU$ )Nr   rZ  )r   r   r   r   )r   r   r$   r"   c              3   r   >^#    U  H+  m[        UUU4S  j[        TT   S-  5       5       5      v   M-     g7f)c           
   3   z   >#    U  H0  n[        UTT   -  [        S UTT   -
  TT   -  5      -
  S5      v   M2     g7fr  r  )rq   r  r   rb  r$  s     rW   rs   0avg_pool3d_backward.<locals>.<genexpr>.<genexpr>  sH      
. VAYQ[^);q	(I!JJANN.r  r"   N)r  r   )rq   r   rb  r$  s    @rW   rs   &avg_pool3d_backward.<locals>.<genexpr>  sA      3
 A	 	 
;q>A-.
 	
 	
 s   37}   c           	        > S T 5       u  p4nS T 5       u  pgnS T 5       u  pnS [        XU/X4U/XgU/5       5       u  pnS [        XU/XU/TTT/XgU/5       5       u  nnnS XU4 5       u  pnS [        UUU/TTT/5       5       u  nnn[        R                  " [        R                  " [        R                  " X5      [        R                  " UU5      5      [        R                  " UU5      5      nU$ )Nc              3   l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fra   r  )rq   r  s     rW   rs   Qavg_pool3d_backward.<locals>.compute_pool_size_without_padding.<locals>.<genexpr>  s"     'UfQ(D(Dfr  c              3   l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fra   r  r>  s     rW   rs   rt    s"     MWs||Au{{;;Wr  c              3   l   #    U  H*  n[         R                  " U[        R                  5      v   M,     g 7fra   r  )rq   r   s     rW   rs   rt     s%      (
2=QCLLEKK((+r  c              3   ~   #    U  H3  u  pn[         R                  " [         R                  " X5      U5      v   M5     g 7fra   )rB   rk  rA  )rq   r]  r  pads       rW   rs   rt    s5      "
	c GGCGGAM3''s   ;=c           
   3      #    U  Hk  u  pp4[         R                  " [         R                  " X5      [         R                  " [         R                  " U[        R
                  5      U5      5      v   Mm     g 7fra   )rB   rY  rz   r7  rO   r  )rq   r  r   r   rx  s        rW   rs   rt  
  sV      

'"# KK!3773>>#u{{+KS#Q 's   A3A5c              3      #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7fr  rB   rX  r9  rO   r  )rq   r  s     rW   rs   rt    s4      "
1 KKs||Au{{;<<1   AA	c              3      #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fra   rB   rY  r7  rO   r  )rq   r  r   s      rW   rs   rt    s6      
K KKS^^C=>>K   A	A)r   rB   rA  rk  )pdr  r  stride_dr_  r`  pad_dra  rb  kernel_dr.  r/  dstartrP  rc  dendrQ  rd  rS  depthr  rb  rM  r$  r  s                      rW   re  >avg_pool3d_backward.<locals>.compute_pool_size_without_padding  s.   'Uf'U$HMWMe(
2=(
$H"
 x8<uU>S"


 '*(X.&u%	'	

dD"
 &1"

tT 2UFE4JK
dD GGCGGD)3774+@A3774QWCX
 rZ   c                 R  > U Gt pp4S [        X#U/T5       5       u  p#nS [        X#U/TT!5       5       u  pVnS [        X#U/T!5       5       u  pn
S XVU4 5       u  pVnS [        XU
/TTT /5       5       u  pn
S n[        T5       GH  n[        T5       GH  n[        T"5       GHo  nS [        XVU/XU/5       5       u  nnnTb  TnO*T(       d  T(       d  TS   TS   -  TS	   -  nO
T" UUU5      n[        R                  " T" / UQ[        R                  " [        R
                  " U[        R                  " U[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U	[        R                  " S[        R                  5      5      5      TS
S9P[        R                  " [        R
                  " U[        R                  " U
[        R                  " S[        R                  5      5      5      T S
S9P5      U5      n[        R                  " [        R                  " [        R                  " X5      [        R                  " UU	5      5      [        R                  " UU
5      5      nUc>  [        R                  " UU[        R                  " S[        R                  5      5      nGMB  [        R                  " U[        R                  " UU5      U5      nGMr     GM     GM     Uc   eU$ )Nc              3   .   #    U  H  u  pX-   v   M     g 7fra   r   )rq   r   rx  s      rW   rs   2avg_pool3d_backward.<locals>.fn.<locals>.<genexpr>$  s     A)@vq17)@r  c              3      #    U  H<  u  pn[         R                  " [        X-
  U-   U5      [        R                  5      v   M>     g 7fra   rB   r7  r   rO   r  )rq   r   r   r  s       rW   rs   r  &  s:      %
>a NN8AEAIq15;;??>s   AAc              3      #    U  H8  u  p[         R                  " [        X5      S -   [        R                  5      v   M:     g7fr  r  )rq   r   r  s      rW   rs   r  +  s4      
. NN8A>A-u{{;;.s   A Ac              3      #    U  H?  n[         R                  " U[         R                  " S [        R                  5      5      v   MA     g7fr  r{  )rq   pstarts     rW   rs   r  0  s4      %
5 KKQ <==5r|  c              3      #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fra   r~  )rq   pend
pooled_dims      rW   rs   r  4  s;      
%  KKcnnZEFF%r  c              3      #    U  HA  u  p[         R                  " U[         R                  " U[        R                  5      5      v   MC     g 7fra   )rB   rz   r9  rO   r  )rq   r  p_s      rW   rs   r  @  s:      "+JF R(EFF+r  r   r$   r"   Fr  r6  )r   r   rB   r  rn  rY  rk  r9  rO   r  r<  r;  r  rV  rz   )#r  r[  r  rO  r  pdstartr  r  pdendr  r  r  pd_r  r  r  r  r  r[  rh  rC  re  rB  d_window_sizer3  r  r  rW  rb  rM  pooled_depthr  r  r$  r  s#                        rW   rK   avg_pool3d_backward.<locals>.fn"  s   AAaAY)@Aa%
ay+v>%
!'

Q1Iv.
e
%
"W5%
!'
$'u%m\'R%
e 'C]+ /C"*-$w7#C+"JBB (3 0*+ +AQ ?+a. P A"b" M;;#!' # 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!" !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %2*/!"  !$ 5 5$'KK(*CGGE3<<5;;;W,X%& %1*/!"!4 7D< 88!2CFF2u4EFr5)D  '#&99 $S%--(H$ $'99T3778T3JH#Um 0 , (r ###rZ   r@  )r[   r5   r   r   r  rf  rB  r   rv   r   r   fallback_avg_pool3d_backwardr2   r   r   )!r  r~   rb  r$  rM  rc  rB  r3  r  d_outceil_mode_dr  ceil_mode_hr  ceil_mode_wr   r  r   rl  rK   r  re  r  r  r  r  rW  r  r  r  r  r  r  s!     ``` ``             @@@@@@@@@@@@rW   avg_pool3d_backwardr    s    #'71'<X>XX<k9----a####{q   v;!w<1qzz|&&&#$::< UE65%eQVWiXE%;E &eQVWiXE))+Kg,K+KKK4?4H4H4J1Qm\AJJL!HKKME3
 q3/M=-  -/-?KS+	
 		
# #JU U Un 
		%%'	
B IrZ   c                 F   U R                  5       n[        U[        5      (       a  U/nOU(       d  [        [	        U5      5      n[	        U5      S:X  a  [        U5      S;   d
   SU 35       e/ $ [        U5      n[        [	        U5      5       Hg  nX   S:  a'  X==   [	        U5      (       a  [	        U5      OS-  ss'   SX   s=::  a  [	        U5      :  a  ML  O  [	        U5      S:X  a
  X   S:X  a  Mg   e   [	        [        U5      5      [	        U5      :X  d   S5       eU$ )Nr   )r   r~  r.  zinvalid axis: r$   zreduction axis not unique)r   r[   r   r   r   rw   rv   rx   )r~   rK  r   r   s       rW   _validate_reduction_axisr    s    ::<D$vSY
4yA~T{//H>$1HH/	:D3t97Q;GCIIs4y14GDG'c$i'CINtw!|LL  s4y>SY&C(CC&KrZ   c          	        ^^
^^^ Ub  [        X5      n U R                  5       m[        [        X5      5      n/ n/ m/ n/ m[	        [        T5      5       HT  nXq;   a'  TR                  U5        UR                  TU   5        M/  TR                  U5        UR                  TU   5        MV     U
UUUU4S jnT(       a1  [        T5      n	T H  n[        R                  R                  X'   M!     OUn	U R                  5       m
[        U R                  5       U=(       d    U R                  5       U R                  5       UU	US9$ )Nc                   > [        U5      [        T	5      :X  d   eT(       a.  [        U 5      [        T
5      :X  d   eT Vs/ s H  o U   PM	     n n[        U 5      [        T5      :X  d   eS /[        U 5      [        U5      -   -  n[        R                  " [        TU 5      [        T	U5      5       H	  u  pEXSU'   M     T" U5      $ s  snf ra   )r   r   r   r   )r  reduction_indexr   r  r  varinner_loaderkeepdimskept_idxreduced_idxr   s         rW   r  %_make_reduction_inner.<locals>.loader  s    ?#s;'7777u:T***'/0x!1XxE05zS]***Fc%j3+??@	!% #k?"C
HC !cN
 I&& 1s   C)r   	dst_dtyperu  r>  r=  reduction_ranges)r   r   rx   r  r   r   r   rv   r   r  r  rB  r   r   r   )r~   rK  r  r   r#  
kept_sizesreduced_sizesr   r  r  r  r  r  r   s     `       @@@@rW   _make_reduction_innerr    s   Q::<D'01DJHMK3t99q!  a)OOAd1g& ' ' :A''++HK  ==?L||~'81;;=++-& rZ   r  c                 $   ^ ^ SS S.UU 4S jjjnU$ )Nr   c                   > [        U UUUTS9n[        R                  " STU S.UD6n[        UR                  R                  [        5      (       a  UR                  5         U$ )NrK  r  r   r#  )r  
input_noder   )r  r3   r   r[   rT  rb  )r~   rK  r  r   r   r-  r#  r  s         rW   rM  make_reduction.<locals>.inner  sf    &"7
 !!XAXQWXKKi
 
 NNrZ   r   r   )r  r#  rM  s   `` rW   make_reductionr    s    T   LrZ   c                    Ub  [        X5      n U R                  5       n[        X5      n[        U R	                  5       U R                  5       4U R                  5       4U R                  5       US9$ )N)r   dtypes	inner_fnsr   rK  )r   r   r  r   r   r   rB  )r~   rK  r   r   s       rW   _make_scan_innerr    sa    Q::<D!D||~==?$ZZ\ rZ   r   c                  ^ Ub  [        X5      n U R                  5       m[        X5      nU R                  5       nU[        R
                  [        R                  4;   a  [        U [        R                  5      n [        XU5      n[        U4S jU 5       5      n[        R                  " X`R                  5       U R                  5       S9n[        R                  " U[        UR                  5       5      5      n[        [!        XV5      U5      $ )Nc              3   .   >#    U  H
  nTU   v   M     g 7fra   r   rq   r   r   s     rW   rs   mean.<locals>.<genexpr>       04a$q'4r   r  )r   r   r  r   rO   rH  rG  r  sum_r@   r'   r-   r   r,   r   rv   div)r~   rK  keepdimr   output_dtype
sum_resultdenomr   s          @rW   r  r    s    Q::<D#A,D;;=Lu~~66Q$aw'J0400Ee;;=XEeT**=*=*?%@AEC
*L99rZ   c                 F  ^
 Uc  SnU R                  5       m
[        X5      n[        XSS9nU(       a  UR                  5         [	        [        X5      5      n[        XaU5      n[        U
4S jU 5       5      nU(       a  [        R                  " X-
  S5      n[        R                  " XR                  5       U R                  5       S9n[        R                  " U[!        UR                  5       5      5      n[#        Xx5      n	U(       d  U	4$ U(       a  UO
[%        XQ5      nX4$ )Nr$   T)r  c              3   .   >#    U  H
  nTU   v   M     g 7fra   r   r  s     rW   rs    var_mean_sum_.<locals>.<genexpr>  r  r   r   r  )r   r  r  rb  squarerk  r  r@   r   rN  r'   r-   r   r   r,   r   rv   r  r  )r~   rK  
correctionr  return_meanx_meandiffsr  r  x_varr   s             @rW   var_mean_sum_r    s    
::<D#A,D!4(F3q>"Ee7+J0400E		%,a0e;;=XEeT**=*=*?%@AE
"ExVGF$9F=rZ   c                     [        X5      n[        XUS S S9nUS   n[        US   5      n[        U[        R
                  5      =(       a1    [        U5      [        R                  :  =(       a    [        U5      S:g  $ )Nr  r=  r  r$   )	r  r  r@   r[   r   r$  r   r%   unroll_reductions_threshold)r~   rK  r  r   r=  reduction_numels         rW   use_two_step_variancer    sx    #A,D"	wd$F HF#F+=$>?O?EMM2 	' 6#E#EE	'&!Q&rZ   c                  ^^^^^ Tc  Sm[        XUS S S9nUR                  S5      nUR                  S5        UR                  S5        [        R                  R                  " SU4SU R                  5       S.UD6u  pxn	UR                  5         U R                  5       mU R                  5       m[        X5      n[        U4S jU 5       5      mS	 mUUUU4S
 jn
[        U
5      " U5      nU(       a  UR                  5         X4$ U4$ )Nr$   r  r>  r  ru  welford_reduce)r  r  r   c              3   .   >#    U  H
  nTU   v   M     g 7fra   r   r  s     rW   rs   $var_mean_welford_.<locals>.<genexpr>8  s     1Dq47Dr   c                    [        U [        R                  5      (       aK  U R                  (       d:  [        R
                  " [        R                  " U [        R                  5      U5      $ [        R                  " X5      $ ra   )
r[   r   r   	is_numberrB   r   r7  rO   r8  r9  r  s     rW   get_constant_or_index_expr5var_mean_welford_.<locals>.get_constant_or_index_expr:  sH    a$$Q[[<<q%++ >FF||A%%rZ   c                    > T" TT5      nT" TT5      n[         R                  " ST5      nU [         R                  " X2U-
  5      -  $ r  )rB   r9  rX  )rT  cNzeror  r   r  rnumels       rW   r  #var_mean_welford_.<locals>.scale_fn?  sC    &z59&vu5||Au%ckk$A...rZ   r   )r  ru  r'   WelfordReductionr   r   rb  r   r  r@   rO  )r~   rK  r  r  r  r   r  r  m2r   r  r  r   r  r  r   s     `         @@@@rW   var_mean_welford_r  "  s    
"	wd$F ZZ
#F
JJ{
JJ{%%,, )'kkm 	KDa JJLKKME::<D#A,D1D11F&
/ / 
"2
&Cy6MrZ   c                   ^ U R                  5       m[        T5      n[        XSS9n [        U UUUUS9n[	        XUS9(       a  [        S0 UD6O
[        S0 UD6n[        U4S jU 5       5      nU(       d  US   $ U$ )NFr  )r~   rK  r  r  r  )rK  r  c              3   :   >#    U  H  n[        UTS S9v   M     g7f)Fr  N)r   )rq   r~   	out_dtypes     rW   rs   #var_mean_helper_.<locals>.<genexpr>]  s     Fv!8Ayu5vr  r   r   )r   r   r   r   r  r  r  rw   )	r~   rK  r  r  r  compute_dtyper   r  r  s	           @rW   var_mean_helper_r  M  s    I))4M.A
F !w? 	(( 
 FvFFF'6!93V3rZ   )r  r  c                    [        XX#SS9$ )NFrK  r  r  r  r  r~   rK  r  r  s       rW   var_r  a  s    	% rZ   c                    [        XX#SS9$ )NTr  r  r  s       rW   var_meanr  h  s    	$ rZ   c                 (   US:  a"  [        [        R                  " U 5      U* U5      $ US:X  a  [        R                  " SU5      $ US:X  a  U $ [        XS-  U5      n[        R                  " X35      nUS-  S:X  a  [        R                  " X05      nU$ )Nr   r$   r"   )pow_recursiverB   rW  r9  rA  )r~   r  r   r-  s       rW   r  r  o  s    1uS^^A.E::Av||Au%%Av11fe,FWWV$F	A!|#MrZ   c                 .    [         R                  " X5      $ ra   )rB   powr   r  s     rW   
pow_nativer  ~  s    771=rZ   )r   c                    ^ ^^ [        T[        5      (       a$  T[        T5      :X  a  [        T [        T5      5      $ [        T[        5      (       a  TS:X  a  [	        T 5      $ [        T[        5      (       a  TS:X  a  [        T 5      $ [        S T T4 5       5      n[        U5      n[        T[        5      =(       a*    STs=:  =(       a    S:  Os  =(       d    U=(       a    TS:  nU(       aY  T R                  5       mU UU4S jn[        R                  " T R                  5       T R                  5       UT R                  5       S9$ [        T [        5      (       aA  T S:X  a  [        TS5      $ T S	:X  a)  [!        TR                  5       5      (       a  [#        T5      $ U(       aN  [        T [        5      (       a  [%        T T5      $ [        T[        5      (       a  ['        T T5      $ [)        T T5      $ [+        T T5      $ )
Nr  r$   c              3      #    U  H6  n[        U[        R                  5      (       d  M$  UR                  5       v   M8     g 7fra   )r[   r'   r5   r   r   s     rW   rs   pow.<locals>.<genexpr>  s'     N1*Q2Ms
   #A A i    r   c                 F   > [        T" U 5      TTR                  5       5      $ ra   )r  r   )r  r   r  r  s    rW   rK   pow.<locals>.fn  s     a??rZ   r@  r"   )r[   r  r   r  sqrtry  r   r   rB  r2   r   r   r   r   r   r  r   exp2fallback_pow_scalarfallback_pow_tensor_scalarfallback_pow_tensor_tensorr  )r   r  r   is_integer_powembed_exponentrK   r  s   ``    @rW   r  r    s   !USV1c!f~	Au		!s(Aw	As		QQx NANNE%e,N  3' a"32AF  	@ <<>++-::<	
 	
 !V6Q?"6nQ[[]337Na  &q!,,6""-a33-a33arZ   c                 (   [        U [        5      (       a  U R                  nOU n[        U[        5      (       a  UR                  n[        U[        R                  5      (       dz  [
        R                  " U R                  5       U R                  5       UR                  5       U R                  5       S9R                  n[        U[        R                  5      (       d   e[        U[        R                  5      (       av  UR                  5       (       da  UR                  5       (       dL  [        UR                  [        R                  5      (       d#  UR                  5         UR                  Ul        U $ [        R                  R!                  XUS9  U $ )Nr@  unsafe_alias)r[   r5   rT  r'   r  r2   r   r   r   rB  r   is_input_bufferis_module_buffer	NopKernelrb  r  realize_into)changedr  r  changed_datas       rW   r  r    s,   '9%%||#y!!hhc2==))%%'##%__&##%	

 $ 	 #r}}----,..$$&&((**l''66 	HH!!.. /  NrZ   c                 ,    [        U [        X5      5      $ ra   )r  r  )r~   rB  s     rW   fill_r    s    Q	!011rZ   c                     XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ ra   r  r   r   r   r  r   r  )r  r  r  s      rW   r  r    sH    
z

C)
*C
3
(C
lln
%CSrZ   c                 .    [         R                  " X5      $ ra   )rB   floordivr  s     rW   r  r        <<rZ   c                 .    [         R                  " X5      $ ra   )rB   truncdivr  s     rW   r  r    r  rZ   c                 t   [        U 5      =(       a    [        U5      n[        U 5      =(       a    [        U5      nUS:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [	        X5      5      $ US:X  a4  U(       a   S5       eU(       a  [        X5      $ [        [	        X5      5      $ [	        X5      $ )Nr  z5floordiv operands can not be boolean at the same timer  z5truncdiv operands can not be boolean at the same time)r   r   r  r  r  r  r  )r   r  rounding_modeboth_integerboth_booleans        rW   div_moder    s    "1%</!*<L"1%</!*<L X!XX!-x~C5Q3CCX!XX!-x~C5Q3CCq9rZ   c                     [        U 5      =(       a    [        U5      nU(       a  [        X5      $ [        [        R                  R
                  5      n[        U5      " X5      $ ra   )r   logical_andr0   r  rA  r  rO  )r   r  	both_boolrK   s       rW   rA  rA  	  sH    "9q'9I1  **+b!!''rZ   c                    [        U [        R                  5      (       a  [        U R                  5      $ [        U [        R
                  5      (       a  [        U R                  5       5      $ [        U [        R                  5      (       a  U $ [        U [        R                  5      (       d  g[        R                  R                  R                  U R                  5       5      n[        R                  " U5         [         R"                  " [        R$                  SS5         U R&                  " U R)                  5       6 nSSS5        SSS5        [        W[        R                  R*                  R,                  5      (       d   e[        UR.                  [        R                  5      (       a  UR.                  $ g! , (       d  f       N= f! , (       d  f       N= f)z:Try convert an arbitrary IR node into an ir.Constant valueNallow_indexingT)r[   r'   
MutableBoxget_constant_valuerT  r  r  r   LoopsrO   	_inductorops_handlerExtractConstantsHandlerr   rC   set_ops_handlerr   objectrT  r>  inner_fn_argsvirtualizedOpsValuer   )r~   r  r   s      rW   r#  r#    s1   
 !R]]##!!&&))!R[[!!!!--/22!R[[!! a""oo))AA!,,.QG	
		7	#U\\
+T& jj!//+,&	#
 c5??66??@@@@#))R[[))yy& &	#	#s$   ;'G"F; G;
G		G
Gc                 :   [        S X4 5       5      nU(       a  [        X5      $ [        U5      =nbU  UR                  S:X  a+  [        R
                  " [        S5      UR                  5      nOSUR                  -  n[        X5      $ S n[        U5      " X5      $ )Nc              3   \   #    U  H"  n[        U5      =(       d    [        U5      v   M$     g 7fra   )r   r   r   s     rW   rs   div_prim.<locals>.<genexpr>4  s"     O1oa(>OA,>>s   *,r   infrq  c                  (    [         R                  " U 6 $ ra   )rB   r  r  s    rW   rK   div_prim.<locals>.fnA  s    {{D!!rZ   )	r   r  r#  r   mathcopysignr  rA  rO  )r   r  is_integraldivisorrW  rK   s         rW   div_primr7  2  s    OOOK~%a((5==AuU|W]]CJw}},J1!!" "a##rZ   c                 N    [        X4[        R                  S9u  p[        X5      $ r1  )r%  r   INT_TO_FLOATr7  r  s     rW   r  r  G  s*     	
$C$P$PDA A>rZ   c                 z    [        U 5      =(       d    [        U 5      nU(       a  S nOS n[        U5      " X5      $ )Nc                 .    [         R                  " X5      $ ra   )rB   modr  s     rW   rK   fmod.<locals>.fnY  s    771= rZ   c                 .    [         R                  " X5      $ ra   )rB   fmodr  s     rW   rK   r=  ^  s    88A>!rZ   )r   r   rO  )r   r  r5  rK   s       rW   r?  r?  S  s5    !!$:(:K	!
	" "a##rZ   c                    [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nr  rx  r   r   r   r   rO   r8  r  r~   rK  r  r   rK   s        rW   r  r  d  sM     	''+;AKKM+J+J
-	U	;Bax--rZ   c                    [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   r   r.  Tr  c                 @    U u  nUu  n[         R                  " X#5      4$ ra   )rB   rz   a_tupleb_tupler   r  s       rW   
combine_fncumsum.<locals>.combine_fn  !    rZ   rK  r   rI  r   r   r   )r   r   r   rO   r8  r   r   r   r  r'   Scanr   fallback_cumsumr~   rK  r   rI  r   r-  s         rW   cumsumrQ  v  s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~q%88MrZ   c                    [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        U R                  5       5      S:X  a+  US;   d   eU=(       d    U R                  5       n[        XSS9$ S n[        XUS9n[        R                  R                  " S	0 UDSU0D6u  nUc
  [        XUS9$ U$ )
Nr   rD  Tr  c                 @    U u  nUu  n[         R                  " X#5      4$ ra   )rB   rA  rF  s       rW   rI  cumprod.<locals>.combine_fn  rK  rZ   rL  rI  rM  r   )r   r   r   rO   r8  r   r   r   r  r'   rN  r   fallback_cumprodrP  s         rW   cumprodrV    s     	''+;AKKM+J+J
-
1::<Aw&t,, 
 a%8F??J?IV~599MrZ   c                    S nU R                  5       n[        U R                  5       5      S:X  a  US;   d   e[        U 5      $ [	        XUS9n[
        R                  R                  " S0 UDSU0D6u  nUc	  [        XS9$ U$ )Nc           	      .   U u  nUu  n[         R                  " X#5      n[         R                  " X#5      nXE:g  [         R                  " U5      ) -  n[         R                  " U[         R
                  " [         R                  " XE-
  5      5      U-   U5      4$ ra   )rB   rY  rX  r  r  log1pexp)rG  rH  r   r  min_vmax_vrC  s          rW   log_add_exp_helper(logcumsumexp.<locals>.log_add_exp_helper  ss    A!A!CIIe$4#45		$		#''%-*@ AE I1MOOrZ   r   rD  rL  rI  rW  r   )	r   r   r   ry  r  r'   rN  r   fallback_logcumsumexp)r~   r   r]  r   r   r-  s         rW   logcumsumexpr`    s{    P KKME
1::<Ag~~Qxa7FGG4FGIV~$Q00MrZ   c                 ~   [        U R                  5       5      S:X  a+  US;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9nU[        R                  L a  SOPUR                  (       a   [        R                  " U5      R                  O[        R                  " U5      R                  n[        XUS9nU[        R
                  4US'   U R                  5       S	 4US
'   [        R                   R"                  " S0 UDSU0D6u  pgUc	  [%        XS9$ Xg4$ )Nr   rD  r   argmaxFr   arg_break_ties_leftrL  r  c                     gNrindexr   rh  s    rW   r4  cummax.<locals>.<lambda>      hrZ   r  rI  rW  r   )r   r   ry  r  rO   r8  r   r'   get_reduction_combine_fnr   r  r  r  r  r  rB  rN  r   fallback_cummax)r~   rK  r   rI  r  r   r   r  s           rW   cummaxrl    s   
1::<AwQxAU[[999KKME,,5J EJJ 	 && KK""U#''  a%8Fu{{+F8==?,>?F;ggnnEvE*EOF~q++?rZ   c                 ~   [        U R                  5       5      S:X  a+  US;   d   e[        U 5      [        U [        R
                  S94$ U R                  5       n[        R                  " SUSS9nU[        R                  L a  SOPUR                  (       a   [        R                  " U5      R                  O[        R                  " U5      R                  n[        XUS9nU[        R
                  4US	'   U R                  5       S
 4US'   [        R                   R"                  " S0 UDSU0D6u  pgUc	  [%        XS9$ Xg4$ )Nr   rD  r   argminFrc  TrL  r  c                     grf  r   rh  s    rW   r4  cummin.<locals>.<lambda>  ri  rZ   r  rI  rW  r   )r   r   ry  r  rO   r8  r   r'   rj  r   r  r  r  r  r  rB  rN  r   fallback_cummin)r~   rK  r   rI  	max_valuer   r   r  s           rW   cumminrs    s   
1::<AwQxAU[[999KKME,,5J EJJ 	 && KK""U#''  a%8Fu{{+F8==?,>?F;ggnnEvE*EOF~q++?rZ   c                    [        U R                  5       5      (       d  [        U R                  5       5      (       a  Uc  [        R                  n[        SUS9nU" XX#S9$ )Nr  rx  r   rA  rB  s        rW   r  r    sM     	''+;AKKM+J+J
-	e	<Bax--rZ   c                 V    [        U [        R                  5      n [        S5      " XUS9$ )Nr   rK  r  )r   rO   r   r  r~   r   r  s      rW   
reduce_anyrx    s$    EJJA% w??rZ   c                 F    Ub  [        XUS9[        XUS94$ [        U S US9$ Nrv  )reduce_amaxreduce_argmaxrw  s      rW   
reduce_maxr}  	  6    
g6!8
 	

 qtg66rZ   c                 F    Ub  [        XUS9[        XUS94$ [        U S US9$ rz  )reduce_aminreduce_argminrw  s      rW   
reduce_minr    r~  rZ   xor_sumr  r  rb  rx  rn  
logical_or)rJ  r<  stabler   
descendingc          
         Uc  SnU R                  5       nU R                  5       n[        [        U5      U5      n[        U5      S:X  a'  [	        U 5      [        SU[        R                  U5      4$ [        U5      (       a  XB   OSn[        R                  R                  R                  U[        R                  " [        R                  5      R                  5      (       d
  [        XX#S9$ [!        USS[        R                  USS9nS/[        U5      -  n[        U5      (       a  XhU'   [#        Xx5      n[%        Xt5      n[&        R(                  R+                  UU R,                  UR,                  4U R/                  5       UR/                  5       4UUUUS9u  pU	c
  [        XX#S9$ Uc   eU	[1        U[        R                  5      4$ )NFr   r$   r  )r  r  r   r   r   )r   r  r  r   rK  r  r  )r   r   r   r   ry  rC  rO   r8  rC   r  r  statically_known_ltr  int16r  sort_fallbackr  r  r  r'   Sortr   r   rB  r   )
r~   r  r   r  r  r   r  r  
view_shaper   s
             rW   sort_stabler  0  s   ~JJLE\\^F
3u:s
+C
5zQQxq&%++u=== ZZuzQH77//%++ekk:R:V:VWWQ3NNVSXG s5z!J
5zz"37'GW$Gggnn'==?G$7$7$9: % OF ~Q3NN8GU[[111rZ   c                     [        U SXS9$ )NFr  )r  )r~   r   r  s      rW   sortr  X  s    qCGGrZ   c                 6    [        U U[        R                  US9$ )Nr   r   rL  r  r   r9  )r   r   rL  s      rW   register_pointwise_numericr  ]  s!    
;HH'	 rZ   c                 4    [        U [        R                  SS9$ )NT)r   r  r  r  s    rW    register_pointwise_numeric_ldf64r  f  s    
;HH" rZ   r  logical_not)r<  )r   r   r#  identity)r`  pointwise_overrides_datac              #   V  ^#    [         U   m[        U TR                  S 5      nUc  g U4S jn[        U[        R
                  R                  5      (       a:  UR                  5        H%  n[        X$5      nUTR                  U" U5      4v   M'     g UTR                  U" U5      4v   g 7f)Nc                 6   > TR                   c  [        U 5      $ g ra   )tritonr  )r   rT  s    rW   make_triton_fallback6_get_pointwise_overrides.<locals>.make_triton_fallback  s    ;;#B'' rZ   )	r  rf   r   r[   rO   r\   r{   r}   r   )nsr   r   r  olnamerC  rT  s         @rW   _get_pointwise_overridesr    s     #D)D	TYY	%B	z( "ejj1122llnF$Bd..0DR0HHH % $**,@,DDDs   B&B)r  c                 d   ^ U [         U'   [        R                  U 5        U4S jn[        X5        g )Nc            	      ~   > T" U 0 UD6n/ n[        U S   U5       H  u  pEUR                  [        XESS95        M      U$ )Nr   Tr  )r   r   r  )r   r   resultsmut_resultsr   r-  outplace_ops         rW   rK   $register_foreach_inplace.<locals>.fn  sK    t.v.tAw0KCy4HI 1 rZ   )rJ   rI   rz   r   )aten_opoutplace_aten_opr  rK   s     ` rW   register_foreach_inplacer    s,    07,-G$ w+rZ   c                 .   ^ [        U S S9U4S j5       nU$ )Nr{  c                  j   > T" U 0 UD6n[        X S   R                  5       5      n[        U S   U5      $ r  )r   r   r  )r   r   r-  r  s      rW   rK   register_inplace.<locals>.fn?  s:    d-f-&q'"3"3"56a&))rZ   )r   )r  r  rK   s    ` rW   register_inplacer  >  s#    wD9* :*
 IrZ   c                     g ra   r   r  s      rW   sym_constrain_ranger  h      rZ   c                     [         R                  R                  R                  S   n[	        U[
        R                  5      (       d   eUR                  R                  $ r  	rC   r  r0  rE  r[   rO   r1  rf  r4  r   r   r  s      rW   sym_sizer  m  sC    
''


#
#E
*C c5<<((((88==rZ   c                     [         R                  R                  R                  S   n[	        U[
        R                  5      (       d   eUR                  R                  $ r  r  r  s      rW   
sym_strider  ~  sA    
''


#
#E
*Cc5<<((((88==rZ   c                 "    U R                  5       $ ra   )rr  )r   s    rW   	sym_numelr    s    ;;=rZ   c                 (    [         R                  " U 6 $ ra   )r   Addr  s    rW   sym_sumr    s    99drZ   c                     [        S5      e)NzHelpful for debuggingrj   )r  r   r   s      rW   foobarr    s    
5
66rZ   c                 8    U R                  5         [        U 5      $ ra   )rb  ry  r   s    rW   _realizer    s    IIK8OrZ   c                 R    U R                  5         [        R                  " X5        U $ ra   )rb  r'   ResizeStorageBytes)variabler  s     rW   resize_storage_bytes_r    s!    (-OrZ   c                     U R                  5         UR                  5         [        R                  " [        R                  " X5      5      $ ra   )rb  r5   r   r'   SetSourceTensorKernel)r  source_tensors     rW   set__source_tensorr    s3    LLNB44TIJJrZ   r  c                     XL a  U $ [        XR                  5       5      n[        XR                  5       5      n[	        XR                  5       5      n[        X5      $ ra   r  )r  r  s     rW   
fsdp_copy_r    sH    :J^^-.sMMO,S,,.)""rZ   c                  ^	^
^^ [        U [        5      (       d   e[        U[        [        45      (       d   eUc  [        R
                  nU[        R                  :X  a  [        SU 35      eU[        R                  :X  a  [        U5      S:X  d   eU[        R                  :X  a  [        U5      S:X  d   eU R                  5       m
U R                  5       nU R                  5       n[        U R                  [        R                   5      (       a  U R                  R#                  5       U l        [        R$                  " 5       (       ay  [        R&                  R(                  R*                  (       aP  [-        U5      (       a  [/        S5      mO6[1        U5      (       a!  [        R2                  " U5      R4                  mOSmOSm[6        R8                  R:                  R=                  T
S5      (       a  [?        UTX4S9$ [A        U T
/S	/5      nURC                  5       m	[        RD                  RG                  X5      n[        RH                  " XCX5      RK                  5       mU	U
UU4S
 jn[L        RN                  " XCU[        U5      S9nU$ )Nzunsupported memory format: r   r   nanTr6  r   r   r$   c                   >^ T" U 5      m[         R                  " T[        R                  5      n[         R                  " T[        R                  5      n[         R                  " X5      n[         R
                  " UUU4S jT5      $ )Nc                     > T" T /5      $ ra   r   )
flat_indexflat_loaders   rW   r4  *resize.<locals>.inner_fn.<locals>.<lambda>  s    ZL(ArZ   )rB   r7  rO   r8  r;  r=  )	r  flat_index_exprlimitrC  r  r  	old_numelout_indexeruninitalized_vals	       @rW   r>  resize.<locals>.inner_fn  sX     %
..U[[Ay%++6vvo-zz$ ACSTTrZ   r@  )(r[   r5   rv   rw   rO   contiguous_formatpreserve_formatrA  channels_lastr   channels_last_3drr  r   r!  rT  r'   r  r  r  rd   r  fill_uninitialized_memoryr   r  r   r  r  rC   r  r  statically_known_equalsrk  r)  rB  rT   stride_ordered_for_memory_formatr"  rV  r2   r   )r~   r   r  r   r   x_flat
out_strider>  r   r  r  r  r  s            @@@@rW   resizer    s
   a####dT5M****//---8HII+++4yA~~...4yA~~IKKME""$F!&&"++&&##% 	2244KK%%??%  $U|e$$${{5155# ww//	1==D*%GG		
 	
F $$&K""CCDXJ..ANNPKU U 

Xd4jC JrZ   )auto_functionalizedc                     SSK Jn  UR                  U5      n[        R                  " U UU0 UEUES9  UR                  5        VVs0 s H  u  px[        U[        5      (       d  M  Xx_M      snn$ s  snnf )Nr   )kernel_side_table)
kernel_idxgridtma_descriptor_metadatakernel_args)*torch._higher_order_ops.triton_kernel_wrapr  get_constant_argsr'   UserDefinedTritonKernelr   r[   r5   )	r  constant_args_idxr  r  r   r  constant_argsr  r  s	            rW   triton_kernel_wrap_r    sl     M%778IJM 7/v//	 &,\\^R^z#y7QHCH^RRRs   A/%A/c                 n   [        S U /UQ 5       5      (       aY  Sn[        R                  R                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  R                  XX#5      n[        [        [        R                  U5      5      $ )Nc              3   f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fra   r+  r   s     rW   rs   cond.<locals>.<genexpr>  s%     
M;La:a 1Yq\1;Lr-  z"control flow operator: torch.cond.r  r  )r   rC   r  r0  rE  rF  r  r'   Conditionalr   rv   mapr5   )predtrue_fnfalse_fnoperandsrm   r  r-  s          rW   rl   rl     s    

MD;L8;L
MMM2''..3377tLL;LE*;-8C,/)^^""4(EFI$$f-..rZ   c                 l   [        S X#-    5       5      (       aY  Sn[        R                  R                  R                  R                  SS 5      =n(       a  U SU 3nU[        R                  l        [        R                  R                  XX#5      n[        [        [        R                  U5      5      $ )Nc              3   f   #    U  H'  n[        U[        5      =(       a    [        U5      v   M)     g 7fra   r+  r   s     rW   rs   while_loop.<locals>.<genexpr>$  s*      3A 	1f.)A,.3r-  z(control flow operator: torch.while_loop.r  r  )r   rC   r  r0  rE  rF  r  r'   	WhileLoopr   rv   r  r5   )cond_fnbody_fncarried_inputsadditional_inputsrm   r  r-  s          rW   
while_loopr   "  s    
 3   9''..3377tLL;LE*;-8C,/)\\  >UFI$$f-..rZ   subgraph_fn
identifierc                     [         R                  R                  X5      n[        [	        [
        R                  U5      5      $ ra   )r'   InvokeSubgraphr   rv   r  r5   )r  r  r  r-  s       rW   invoke_subgraphr  1  s0    %%k<FI$$f-..rZ   rI  c                   ^
 SSK JnJn  [        R                  " X5       Vs/ s H&  nU" UR                  5       UR                  5       S9PM(     nnU" X5      m
U
4S jn[        US   US S9n[        S U 5       5      US'   [        S	 U 5       5      US
'   [        R                  R                  " SUSS.UD6n	U	S   c  [        S5      eU	$ s  snf )Nr$   )InputDescriptorlower_pointwise_subgraphr   c                 h   > T" / [         R                  " U 5      Q[         R                  " U5      Q76 $ ra   )r  r#  )lhsrhslowered_combine_fns     rW   wrapped_combine_fn,associative_scan.<locals>.wrapped_combine_fnA  s6    ! 
$
$
 	
rZ   r   rL  c              3   @   #    U  H  oR                  5       v   M     g 7fra   )r   r   s     rW   rs   #associative_scan.<locals>.<genexpr>H  s     7Bq[[]]B   r  c              3   @   #    U  H  oR                  5       v   M     g 7fra   )rB  r   s     rW   rs   r  I  s     <Ar  r  F)rI  can_fallback_to_atenz/Unable to generate code for associative_scan opr   )subgraph_loweringr  r  r   r   r   r   r  rw   r'   rN  r   rA  )rI  xsr   r  r  r~   subgraph_inputsr  r   r-  r  s             @rW   associative_scanr  7  s    L ((A 	akkmALLNC(   2*N
 be#T:F7B77F8<<<F;WW^^ %" F
 ayLMMM-s   -Cc                     g ra   r   )tokenss    rW   _sink_tokensr  T  r  rZ   c                 h   [         R                  R                  " U/UQ70 UD6nSSKJn  U" XU5      nUc   e[
        R                  R                  U   nUc  U4$ [        R                  " [         R                  [        R                  U5      n[        U[        [        45      (       d  Xt4$ U/UQ7$ )Nr   )get_effect_key)r'   EffectfulKernelr   torch._higher_order_ops.effectsr  rC   r  effectful_opsr  r  MultiOutputr5   r[   rv   rw   )tokenr   r   r   r-  r  effect_typeeffectful_kernels           rW   with_effectsr$  Y  s    &&r;D;F;F> 62K"""ww,,[9~ ""!!"..)2B2BFKFftUm,, )) *6**rZ   )register_comm_lowerings)r  )quantized_lowerings)mkldnn_lowerings)jagged_lowerings)NN)NNNNFN)Fra   )r   r   l            r$   Tr~  )r   r   r$   r  )T)NTF)r   NNr$   )r.  FF)r"   F)NNN)Nrq  N)r6  )NNNN)r   r   FTNr   )r.  F(  ra  r   r   loggingr3  r  r6  r  collectionsr   typingr   r   r   r   r   r	   r
   r   r   unittest.mockr   r   rO   $torch.ao.quantization.fx._decomposedtorch.fxtorch.utils._pytreerd   _pytreer  (torch._higher_order_ops.associative_scanr   r  r   torch._prims_commonr   r   r   r   r   r   r   r   r   r   r   torch.fx.experimental.sym_noder   r   torch.utils._sympy.functionsr   r   r   r    r!   _dynamo.utilsr#    r%   r&   r'   r(   decompositionr)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   r@   rA   r+  rB   rC   FALLBACK_ALLOW_LIST	getLoggerr  r>  rD   r~  __annotations__rE   r\   r]   rx   rF   r  tr_c10dr  rG   rH   rI   rJ   quantized_decomposedr_   r^   rn   ry   r   r)  r.  r  rk  bmmconvolutionconvolution_backwardmax_pool2d_with_indicesr  mmr,  r.  _int_mmr  r  r  r  r8  rH  rV  r4  	complex32	complex64r   rG  r   r   r   r   r   r   r   r   r   r   r   r  r   r  r%  rO  rr  r   r   r  r  r  r  r  r   r  
device_putr  r  r  r  r  r   aliasdetachdetach_liftview_ofr  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  _unsafe_viewreshaper  slicer  r+  rH  quantize_per_channelrf  dequantize_per_channelrn  quantize_per_tensorr  ru  dequantize_per_tensorry  rG  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  	lru_cacher  rb  r  r  r!  r"  r-  rD  rI  rngprimsrZ  r\  ra  	bernoullir]  re  ri  rm  rp  rL  rv  rs  rt  r{  rz  ry  randintforce_stride_orderr  rW  r  r  r  lookup_seedr  randomr  r  r   r  r  r  r  r  r  r  rS   r  _adaptive_avg_pool3dadaptive_max_pool3dfractional_max_pool3dmax_pool3d_with_indicesuniformexponential_pdist_forwardsoft_margin_loss_backward_cdist_forward_cdist_backward
_trilinearsegment_reduce_segment_reduce_backwardhistc	histogrambin_ct_histogramdd_bin_edges_histogramdd_from_bin_ctsaddbmm_addmm_activation
_cudnn_rnn_cudnn_rnn_backward_embedding_bag_embedding_bag_forward_only_embedding_bag_backward*_embedding_bag_per_sample_weights_backward_fused_moving_avg_obs_fq_helper*_fused_moving_avg_obs_fq_helper_functional max_pool3d_with_indices_backward_adaptive_avg_pool2d_backward_adaptive_avg_pool3d_backwardadaptive_max_pool2d_backwardadaptive_max_pool3d_backwardfractional_max_pool2d_backwardfractional_max_pool3d_backwardreplication_pad1d_backwardreplication_pad2d_backwardupsample_linear1d_backwardupsample_bicubic2d_backwardupsample_trilinear3d_backwardgrid_sampler_2d_backward_pdist_backwardr  r  kthvaluetopkr  median	nanmedianrandpermresize_
resize_as__linalg_detlinalg_householder_productlinalg_inv_exlinalg_ldl_factor_exlinalg_ldl_solve	linalg_lulinalg_lu_factor_exlinalg_lu_solvelinalg_matrix_exp	linalg_qr_linalg_slogdet_linalg_solve_exlinalg_solve_triangular_linalg_svd	lu_unpackormqr_linalg_check_errorslinalg_pinvatol_rtol_tensor_linalg_eightriangular_solvelinalg_cholesky_excholesky_inversecholesky_solvegeqrf_fft_r2cnonzerogcd_thnn_fused_lstm_cell_prims	rng_primsrun_and_save_rng_staterun_with_rng_statemasked_scattermasked_scatter_backwardr)  angle_efficientzerotensor(_sparse_coo_tensor_with_dims_and_tensors	to_sparse
_to_sparser   rI  '_scaled_dot_product_efficient_attentionr  #_scaled_dot_product_flash_attention,_scaled_dot_product_flash_attention_backward#_scaled_dot_product_cudnn_attention,_scaled_dot_product_cudnn_attention_backward+_scaled_dot_product_flash_attention_for_cpu4_scaled_dot_product_flash_attention_for_cpu_backward_flash_attention_forward_flash_attention_backward_efficient_attention_forward_efficient_attention_backwardindex_reducerz  ry  r  r*  r  r  r  r  scalar_tensorr'  
LongTensorr)  r8  r:  rC  r  rG  r
  rF  r  r  r  
zeros_liker[  r_  rP  re  ri  rk  rt  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  fallback__unsafe_masked_indexr  ,fallback__unsafe_masked_index_put_accumulater  r  r  r  r  r  r  r  r  r  r$  r'  r2  r4  rU  r;  r  rC  rD  rS  rX  rf  rm  rr  r  r  r  r  r  r  r  r  r  r  r  r1  r,  r#  r=  rT  rE  rU  r@  rj  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  Tensor_Tensorr  Scalarr  Tensor_Scalarr  r  r  r  r  r  r  r  rA  r   r#  r7  true_divider9  r?  r  r  rQ  rO  rV  rU  r`  r_  rl  rk  rs  rq  r  r   rx  r  r}  r  r  r  r  r{  r   r  rb  r|  rn  r  rz   r  r  r  r  rsqrtrZ  r   expm1relur  r  r  rk  cossinabsbitwise_andbitwise_left_shiftbitwise_not
bitwise_orbitwise_right_shiftbitwise_xorlgammaerfspecial_erfrY  tantanhr  r  r  logical_xorrX  rY  	clamp_min	clamp_maxnegrW  	remaindersignsignbit	_neg_viewler;  r:  rw  r  necoshsinhacosacoshasinasinhatan2atanatanhr4  erfcerfinvhypotlog10log2	nextaftercodegen.commonr`  r  r  r   r   r   rL  _foreach_addforeach_add_listforeach_add_scalar_foreach_mulforeach_mul_listforeach_mul_scalar_foreach_sub_foreach_neg_foreach_abs_foreach_powScalarAndTensor_foreach_divforeach_div_listforeach_div_scalar_foreach_sqrt_foreach_rsqrt_foreach_maximum_foreach_minimum_foreach_clamp_min_foreach_clamp_max_foreach_reciprocal_foreach_sign_foreach_copyr  _foreach_add__foreach_mul__foreach_div_r  add_bitwise_and_bitwise_left_shift_bitwise_not_bitwise_or_bitwise_right_shift_bitwise_xor_mul_div_Tensor_modelogical_and_logical_not_logical_or_logical_xor_sub_relu_sigmoid___and__
__lshift____or__
__rshift____xor____iand____ilshift____ior____irshift____ixor__r  r  r  r  r   methodfuncr  r  _inductor_testrb  r  inductorr  set_source_Tensorr  fsdpr  r  *torch._higher_order_ops.auto_functionalizer  r  higher_orderrl   r   r  Subgraphr  r  r$  comm_loweringr%  r  r&  register_quantized_opsregister_woq_mm_opsr'  register_onednn_fusion_opsr(  register_jagged_opsr   rZ   rW   <module>r6	     sf6         	  # S S S    +  $ $ H U    M  - 8 8 =         
 !FH	4hsCx(#-.c0BBC H  4	JJ8HS#X$677  ),	3uzz$$% -yy~~
))

		47E s5::001 9*-%S&&' /25% S../ 7NP ejj33UZZ5J5JJK Pyy55 -#s(!3 -#s(AS8T -6
MF

3   !!$$--&&* {{zz{{{{{{}}}}}}

. "#3R  D
s)DcND D ""AB	D
  D 49d38n$%DN23 ""AB	3p  	(// "'(#40j  $!%YxM`E	 E%++ E 5--4H)Y )u{{ ) I) @E 5	 5%++ 5( 499??=&9 &U[[ & >& ;@e K KELL K 5##>F9 Fell F ?F 
7?? $1h>    4::DI J0 4))UPTU  V F DJJT\\499emmTU V 4doo&s+ 4<<T:B ;B2 4$$$?" @" DMM?# $ 4::C C 4::C C 499! ! 4::! ! 4::%%&% '% 4::! ! 4;;D9> :>2 5))tD
 E
 4>>t<# =# 4;;.  .b 4$$$?499$74<<T:1 ; 8 @1 4<<T:> ;> 4::48V 9V 4??=J >J$ 4##> ? 4''TB C
BJ '<<RVW/// / 	/
 / / ;;/ / X/d //T))) ) 	)
 ) ) ;;) ))X ,,44$!!! ! 	!
 ! ;;! !!H ..66D  	
  ;; < ,,33*** * 	*
 * ;;* **Z ..554&&& & 	&
 & ;;& &&R 488y: y:x 4==d;3EC 3E3 3E# 3E <3El 4%%4@6 6 6s 6 A6 4((dC  s  D 4;;D96 :6
 4::48 98 4((dC  D  4;;D9 : 4;;D9D :D4 4>>t< = 4??= > 488  T  3 3"< "<J/Nd	, 599%%11tL#+ M#+L 4&&DAG BG 4??= > 4>>##>' ?' 4<<  ! QB B/
 )):):; *499+>+>? )$***<*<= +DJJ,@,@A  dll  499J J 4::J J >44$OL PL
 >&&DAK%,, K BK >''TBJ CJ
 >--4H	 I	 >((dCRS $s) 9 C C  D6 >))tDLM	#Cy09FI E89 sEJJ

EJJ/V)W .y .U3

?%; . 4$$++F
 "&ZZ
Z 	Z
 Z 3-Z YZ Z GZz 4>>t<
 ,,, 	,
 , =,^$Mb d'' ( d&& ' d(( ) d** + dll ' d&&U 3 d!! " d,,5 9 d!! " d"" # doo  d!!)) * d++33 4 djj  dnn## $ d))11 2 d,,44 5 dkk  d$$5 1 d'')@ A doo} - d&&(: ; d!!#5 6 d..0B C d** + d== > d== > d22 3 d== > d33 4 d00- @ d00 1 d// 0 d// 0 d11 2 d11 2 d-- . d-- . d-- . d..0B C d00 1 d++] ; d"" # dii  dii  dmm  dii  dii  dkk  dnn  dmm  dll  doo  d  d-- . d   ! d'' ( d## $ dnn  d&& ' d"" # d$$ % dnn  d"" # d## $ d** + d  dnn  djj  d'' ( d// 0 d   d## $ d%% & d## $ d!! " djj  dmm  dll"" # dhhU + d((- 8 ell$$;; < ell$$77 8 d!! " d** + d""$6 7 djj  d'' ( d;; < dnn  doo  djj  0088	
 99AA	
 ,,44	
 55==	
 ,,44	
 55==	
 44<<	
 ==EE	
 d++33_ E d,,44o F d//77 I d0088/ J d  
 499$7
 8
 4::"  ( 4"##d**+E2 5:: ( 4&&DA C  B: 4%%4@@ A@F ELL$"4"456tDU 5 75p 5??#4 $4 5##$+ %+ 4++,+" -+"\ 4&&' (< 4>>t<K =K8 EKK,- 
 .$.> t/0B50IJ
1!45	 21 56
 4>>" $T$4  # 4%%&Td '> 4))*"4 +" 5%%--.A /A EJJ		*+: ,:
 4;;D9" :"J 4>>t< =8#B FR"1J 4::48
 9
 4%%4@/ A/ 4>>"= #= 4))*O +O;  4??=J >J >44$OK PKXv !1%%5!  0@,,44%0 ,
 4,,$G H, 4;;QUV	P W	P 1 1 4**E F 4<<T:9C 9 ;9 !&& 
 SM B 4==d;DH : :HSM : <:" 4##>3 3 ?3 4$$$?6 6 @6 4&&DAP3 P BP 4''TBPT ms m m Cmh , HUOS(), 	,
 ,^ 4**223>x > 4> 41199:Jhuo J ;J 4**223RVI&uoI@HI 4I 41199:RVU&uoU@HU ;U 4**223 !% $ $S uoS uo	S
 uoS 4S 41199: !% $ $	 uo	 uo		
 uo	 ;	7 599$$% &* 4''TB. C.bejj uUZZ_'= uzz  %** EJJ UZZ  :><,> CG@ @: 5<<RVW E XEP 	33B d** + -=))11- ) 488dSL TL^:B&R2j  0%%5  
 4,,-? .?D  0$$%  
 4++,K -K\ "2&&E" 
2 4--.B /BJ 433;;<BF* =*Z 'OO  'OO 
 4??=  >* 4??=  >*pf  0$$%  
 4++F ` G`F  0$$%  
 4++F @ G@F$.b3 & 499:t : : 4 (V4( DHHeii()T5  * 4==!u  "   .HH  'txxER -HH 
 488t,, -,^ F 4::2 2 4::48 9     488t, - DHH:.( /(")) (= > EII;$/$ 0$( 	txx'7DD

 DIIuzz*d;$ <$  DHHeii(). . *. #4;;#6#67#DLL$8$89 ():):)B)BC "4;;#6#67"4;;#6#67 4;;  . 4<<  !. 4$$% &* 4;;D9 :: 4;;D9 :: 499. . . 488@ @
 48867 77 48867 77 %--  	!: ;		*>%+@A		*>%+@A!$++.85;;? "$++.85;;? HH$L !!1!1uM 499##>!re $2 ?$2N 499$$$?H @H 	#4::.&txx0!$)),"4::.$))$
*4<<
8'		2	DKK	(t4   *   *" !1!12'(?(?@  -  0
()A)AB  !1!12 4;; ' * *I*V*V 4:: & 488 $ 499 %   * **	 !**	  OO**	
 !**	 T\\
*
T\\
* $.. !' * $.. !' *""'8
 4>> "$))L 499  4<<uzz B $.. !# & 477%** = 477%** = 477%** =uzzB 477%** = 477%** = 499 % 499 % 499 % 4:: & 499 % 4:: & 4:: & 499 % 4:: & 4== ) 499 % 4;; ' 4:: & 4:: & 499 % 4>> * D DE$ %D4Ld50 	 3+		
5 5Mt50 	 3+		
5 %, .CT  0ct  4,,33Sd K-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4,,113 7 4,,33S 9 4,,44c : 4,,44c : 4,,33S 9 4,,113 7 4,,<<c B-d.?.?.D.DcJ  4,,33S 9/0A0A0H0H#N  4--t 4 4.. 6 40055w ? 40077 A 40055w ? 40077 A 42277 A 422997 C 42277 A 422997 C 433Z @ 4--t 4 4--t 4, T..335E t00779K T..335E t00779K T..335E t00779K
 C   ""K 0 ))+= > ""K 0 !!: . **,? @ ""K 0 C   !!3 ' && 1 ""K 0 ""K 0 !!: . ""K 0 C   T "  ( $,,  , $// "#5 6 $++ z * $// "#6 7 $,,  ,  - !!4?? 3 t{{ + !!4?? 3  - 4++, - 4==$$% &  4??&&' ( 4>>" # "'')LFD(01$7 * 5==! " 4<< 7 !7 599++334 5
 599%%;;< = 599>>&&445K 6K 599>>7##uyy~~++334# 5# 599>>(()%) = *=@ K J ! " 12S 3S( 599))..// 0/ 599))445/ 6/ 599))99tT/ /# / U/
 &DA s  B8 599??//778 9 599))66DQ+ R+& 3 2       ! !  * * ,  ' ' )    + + -    $ $ &rZ   