
    RЦi                         S SK Jr  S SKrS SKJr  S SKJrJr  S SKJr  S r " S S\R                  5      r	 " S	 S
\R                  5      r
g)    )OptionalN)nn)r   Tensor)_get_activation_fnc                    [        U S5      (       aT  [        U S5      (       aC  [        R                  " 5       U l        U ?U R
                  nU R                  n[        XS9U l        O[        U S5      (       aT  [        U S5      (       aC  [        R                  " 5       U l        U ?U R
                  nU R                  n[        XS9U l        OgSU R                  5       ;   d  SU R                  5       ;   a)  U ?
U R
                  nU R                  n[        XS9U l
        O[        S5        [        S5        [        U S	5      (       a  S
U l        U $ )Nglobal_poolfc)num_classesinitial_num_features
classifierRegNetTResNetz;Model code-writing is not aligned currently with ml-decoder	drop_rater   )hasattrr   Identityr   r	   r
   num_features	MLDecoderr   	_get_nameheadprintexitr   )modelr
   r   s      U/var/www/html/ai-image-ml/venv/lib/python3.13/site-packages/timm/layers/ml_decoder.pyadd_ml_decoder_headr   	   s   um$$)=)=KKMH''))X		&	&75,+G+GKKM''))$`	U__&	&)u7H*HJ''));Z
KLRuk""L    c                      ^  \ rS rSr  S SU 4S jjjrU 4S jr    SS\S\S\\   S\\   S	\\   S
\\   S\4S jjrSr	U =r
$ )TransformerDecoderLayerOptimal#   returnc                 P  > [         TU ]  5         [        R                  " XS9U l        [        R
                  " U5      U l        [        R
                  " U5      U l        [        R
                  " U5      U l        [        R
                  " U5      U l	        [        R                  " XUS9U l        [        R                  " X5      U l        [        R                  " X15      U l        [        R                  " XS9U l        [        R                  " XS9U l        [#        U5      U l        g )N)eps)dropout)super__init__r   	LayerNormnorm1Dropoutr#   dropout1dropout2dropout3MultiheadAttentionmultihead_attnLinearlinear1linear2norm2norm3r   
activation)selfd_modelnheaddim_feedforwardr#   r3   layer_norm_eps	__class__s          r   r%   'TransformerDecoderLayerOptimal.__init__$   s    \\'>
zz'*

7+

7+

7+ 33GGT yy:yy:\\'>
\\'>
,Z8r   c                    > SU;  a'  [         R                  R                  R                  US'   [        [
        U ]  U5        g )Nr3   )torchr   
functionalrelur$   r   __setstate__)r4   stater9   s     r   r?   +TransformerDecoderLayerOptimal.__setstate__8   s5    u$"'(("5"5":":E,,d@Gr   tgtmemorytgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskc           	         XR                  U5      -   nU R                  U5      nU R                  XU5      S   nXR                  U5      -   nU R	                  U5      nU R                  U R                  U R                  U R                  U5      5      5      5      nXR                  U5      -   nU R                  U5      nU$ )Nr   )r)   r'   r-   r*   r1   r0   r#   r3   r/   r+   r2   )r4   rB   rC   rD   rE   rF   rG   tgt2s           r   forward&TransformerDecoderLayerOptimal.forward=   s     MM#&&jjo""37:MM$''jjo||DLLc9J)KLMMM$''jjo
r   )r3   r#   r)   r*   r+   r/   r0   r-   r'   r1   r2   )      皙?r>   gh㈵>)r    N)NNNN)__name__
__module____qualname____firstlineno__r%   r?   r   r   rJ   __static_attributes____classcell__r9   s   @r   r   r   #   s    W] $9)-9 9(H
 QU049=<@6 6 Xf=M %f-&.v&6 *2&)9 FL r   r   c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )r   Z   c                 R  > [         T
U ]  5         US:  a  SOUnXQ:  a  UnXPl        US:  a  SOUn[        R                  " XC5      U l        SnSnSn[        UXS9n	[        R                  " XS9U l        [        R                  " XS5      U l
        U R                  R                  S	5        Xl        [        X-  S
-   5      U l        [        R                  R!                  [        R"                  " XSU R                  5      5      U l        [        R                  R!                  [        R"                  " U5      5      U l        [        R                  R(                  R+                  U R$                  5        [        R                  R(                  R-                  U R&                  S5        g )Nr   d      rN      rM   )r5   r7   r#   )
num_layersFg+?)r$   r%   embed_len_decoderr   r.   embed_standartr   TransformerDecoderdecoder	Embeddingquery_embedrequires_grad_r
   intduplicate_factorr<   	Parameterr   duplicate_poolingduplicate_pooling_biasinitxavier_normal_	constant_)r4   r
   num_of_groupsdecoder_embeddingr   r]   decoder_dropoutnum_layers_decoderr7   layer_decoder9   s             r   r%   MLDecoder.__init__[   sT   #01#4C-* +!2 $5q#8C>O ii(<P 5>OFUp,,\Y <<(9M''. ' #K$Ce$K L!&!3!3LL*t?T?TU"W&+hh&8&8k9R&S#$$T%;%;< ; ;Q?r   c                    [        UR                  5      S:X  a"  UR                  S5      R                  SS5      nOUnU R	                  U5      n[
        R                  R                  R                  USS9nUR                  S   nU R                  R                  nUR                  S5      R                  SUS5      nU R                  XcR                  SS5      5      nUR                  SS5      n[
        R                  " UR                  S   UR                  S   U R                  UR                   UR"                  S9n[%        U R&                  5       HE  n	US S 2U	S S 24   n
U R(                  U	S S 2S S 24   n[
        R*                  " X5      US S 2U	S S 24'   MG     UR                  S5      S S 2S U R,                  24   nXR.                  -  nUnU$ )	N      r[   T)inplacer   r   )devicedtype)lenshapeflatten	transposer^   r<   r   r=   r>   rb   weight	unsqueezeexpandr`   zerosre   rv   rw   ranger]   rg   matmulr
   rh   )r4   xembedding_spatialembedding_spatial_786bsrb   rB   h
out_extrapih_iw_ih_outlogitss                 r   rJ   MLDecoder.forward{   s   qww<1 !		! 6 6q! < ! $ 3 34E F % 3 3 8 89NX\ 8 ]"((+&&--##A&--b"b9LL==aCDKK1[[QWWQZ9N9NWXW_W_ghgngno
t--.AAq!G*C((Aq1C"',,s"8Jq!Qw / ""1%a):$*:*:):&:;,,,r   )r`   re   rg   rh   r]   r^   r
   rb   )r   rZ   rM   )rO   rP   rQ   rR   r%   rJ   rS   rT   rU   s   @r   r   r   Z   s    @@ r   r   )typingr   r<   r   r   torch.nn.modules.transformerr   r   Moduler   r    r   r   <module>r      s:        ;4&RYY &n8		 8r   