
    bCi                        S r SSKrSSKJr  SSKJrJrJr  SSK	r	SSK	J
r
  SSKJr  SSKJr  SS	KJrJrJrJr  SS
KJrJr  SSKJr  SSKJrJr  SSKJrJrJrJ r J!r!  SSK"J#r#J$r$  SSK%J&r&  \ RN                  " \(5      r) " S S\
RT                  5      r+ " S S\
RT                  5      r, S?S\
RT                  S\	RZ                  S\	RZ                  S\	RZ                  S\\	RZ                     S\.S\.4S jjr/ " S S\
RT                  5      r0 " S S \
RT                  5      r1 " S! S"\
RT                  5      r2 " S# S$\
RT                  5      r3 " S% S&\
RT                  5      r4 " S' S(\5      r5 " S) S*\
RT                  5      r6\ " S+ S,\5      5       r7\ " S- S.\75      5       r8 " S/ S0\
RT                  5      r9\" S1S29 " S3 S4\75      5       r:\" S5S29 " S6 S7\75      5       r;\\" S8S29 " S9 S:\5      5       5       r<\" S;S29 " S< S=\75      5       r=/ S>Qr>g)@zPyTorch DeiT model.    N)	dataclass)CallableOptionalUnion)nn   )ACT2FN)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPoolingImageClassifierOutputMaskedImageModelingOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack) find_pruneable_heads_and_indicesprune_linear_layer)ModelOutputTransformersKwargsauto_docstringlogging	torch_int)can_return_tuplecheck_model_inputs   )
DeiTConfigc            	          ^  \ rS rSrSrSS\S\SS4U 4S jjjrS\R                  S	\
S
\
S\R                  4S jr  SS\R                  S\\R                     S\S\R                  4S jjrSrU =r$ )DeiTEmbeddings+   zn
Construct the CLS token, distillation token, position and patch embeddings. Optionally, also the mask token.
configuse_mask_tokenreturnNc                   > [         TU ]  5         [        R                  " [        R
                  " SSUR                  5      5      U l        [        R                  " [        R
                  " SSUR                  5      5      U l        U(       a6  [        R                  " [        R
                  " SSUR                  5      5      OS U l	        [        U5      U l        U R                  R                  n[        R                  " [        R
                  " SUS-   UR                  5      5      U l        [        R                  " UR                  5      U l        UR"                  U l        g )Nr      )super__init__r   	Parametertorchzeroshidden_size	cls_tokendistillation_token
mask_tokenDeiTPatchEmbeddingspatch_embeddingsnum_patchesposition_embeddingsDropouthidden_dropout_probdropout
patch_size)selfr    r!   r0   	__class__s       `/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/deit/modeling_deit.pyr&   DeiTEmbeddings.__init__0   s    ekk!Q8J8J&KL"$,,u{{1aASAS/T"UQ_",,u{{1a9K9K'LMei 3F ;++77#%<<A{QPVPbPb0c#d zz&"<"<= ++    
embeddingsheightwidthc                    UR                   S   S-
  nU R                  R                   S   S-
  n[        R                  R	                  5       (       d  XE:X  a  X#:X  a  U R                  $ U R                  SS2SS24   nU R                  SS2SS24   nUR                   S   nX R
                  -  n	X0R
                  -  n
[        US-  5      nUR                  SXU5      nUR                  SSSS5      n[        R                  R                  UX4SS	S
9nUR                  SSSS5      R                  SSU5      n[        R                  " Xg4SS9$ )a  
This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
images. This method is also adapted to support torch.jit tracing and 2 class embeddings.

Adapted from:
- https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
- https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
r   r$   N      ?r   r   bicubicF)sizemodealign_cornersdim)shaper1   r(   jit
is_tracingr5   r   reshapepermuter   
functionalinterpolateviewcat)r6   r;   r<   r=   r0   num_positionsclass_and_dist_pos_embedpatch_pos_embedrF   
new_height	new_widthsqrt_num_positionss               r8   interpolate_pos_encoding'DeiTEmbeddings.interpolate_pos_encoding<   sU    !&&q)A-0066q9A= yy##%%+*F6?+++#'#;#;ArrE#B 221ab59r".
__,	&}c'9:)11!5G]`a)11!Q1=--33(	 4 
 *11!Q1=BB1b#Nyy2D!LLr:   pixel_valuesbool_masked_posrV   c                    UR                   u    pEnU R                  U5      nUR                  5       u  pnUbI  U R                  R	                  XS5      n
UR                  S5      R                  U
5      nUSU-
  -  X-  -   nU R                  R	                  USS5      nU R                  R	                  USS5      n[        R                  " XU4SS9nU R                  nU(       a  U R                  XuU5      nX~-   nU R                  U5      nU$ )Nr?         ?r   rE   )rG   r/   rB   r-   expand	unsqueezetype_asr+   r,   r(   rO   r1   rV   r4   )r6   rX   rY   rV   _r<   r=   r;   
batch_size
seq_lengthmask_tokensmask
cls_tokensdistillation_tokensposition_embeddings                  r8   forwardDeiTEmbeddings.forwardd   s    +001e**<8
$.OO$5!
&//00LK",,R088ED#sTz2[5GGJ^^**:r2>
"55<<ZRPYY
LRST
!55#!%!>!>zSX!Y4
\\*-
r:   )r+   r,   r4   r-   r/   r5   r1   )FNF)__name__
__module____qualname____firstlineno____doc__r   boolr&   r(   TensorintrV   r   
BoolTensorrg   __static_attributes____classcell__r7   s   @r8   r   r   +   s    
,z 
,4 
,D 
, 
,&M5<< &M &MUX &M]b]i]i &MV 7;).	ll "%"2"23 #'	
 
 r:   r   c                   f   ^  \ rS rSrSrU 4S jrS\R                  S\R                  4S jrSr	U =r
$ )r.      z
This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
`hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
Transformer.
c                   > [         TU ]  5         UR                  UR                  p2UR                  UR
                  pT[        U[        R                  R                  5      (       a  UOX"4n[        U[        R                  R                  5      (       a  UOX34nUS   US   -  US   US   -  -  nX l        X0l        X@l        X`l
        [        R                  " XEX3S9U l        g )Nr   r   )kernel_sizestride)r%   r&   
image_sizer5   num_channelsr*   
isinstancecollectionsabcIterabler0   r   Conv2d
projection)r6   r    r{   r5   r|   r*   r0   r7   s          r8   r&   DeiTPatchEmbeddings.__init__   s    !'!2!2F4E4EJ$*$7$79K9Kk#-j+//:R:R#S#SZZdYq
#-j+//:R:R#S#SZZdYq
!!}
15*Q-:VW=:XY$$(&))L:ir:   rX   r"   c                     UR                   u  p#pEX0R                  :w  a  [        S5      eU R                  U5      R	                  S5      R                  SS5      nU$ )NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.r$   r   )rG   r|   
ValueErrorr   flatten	transpose)r6   rX   r`   r|   r<   r=   xs          r8   rg   DeiTPatchEmbeddings.forward   s[    2>2D2D/
&,,,w  OOL)11!4>>q!Dr:   )r{   r|   r0   r5   r   )rj   rk   rl   rm   rn   r&   r(   rp   rg   rs   rt   ru   s   @r8   r.   r.      s.    jELL U\\  r:   r.   modulequerykeyvalueattention_maskscalingr4   c                    [         R                  " XR                  SS5      5      U-  n[        R                  R                  US[         R                  S9R                  UR                  5      n[        R                  R                  XU R                  S9nUb  X-  n[         R                  " X5      n	U	R                  SS5      R                  5       n	X4$ )Nr?   )rF   dtype)ptrainingr   r$   )r(   matmulr   r   rL   softmaxfloat32tor   r4   r   
contiguous)
r   r   r   r   r   r   r4   kwargsattn_weightsattn_outputs
             r8   eager_attention_forwardr      s     <<}}R'<=GL ==((2U]](SVVW\WbWbcL ==((6??([L !#4,,|3K''1-88:K$$r:   c            	          ^  \ rS rSrS\4U 4S jjr S	S\R                  S\\R                     S\	\R                  \R                  4   4S jjr
SrU =r$ )
DeiTSelfAttention   r    c                 0  > [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eXl        UR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        UR                  U l        U R                  S-  U l        SU l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        [        R                  " UR                  U R                  UR                   S9U l        g )	Nr   embedding_sizezThe hidden size z4 is not a multiple of the number of attention heads .g      F)bias)r%   r&   r*   num_attention_headshasattrr   r    rq   attention_head_sizeall_head_sizeattention_probs_dropout_probdropout_probr   	is_causalr   Linearqkv_biasr   r   r   r6   r    r7   s     r8   r&   DeiTSelfAttention.__init__   sG    : ::a?PVXhHiHi"6#5#5"6 7334A7 
 #)#=#= #&v'9'9F<V<V'V#W !558P8PP"??//5YYv1143E3EFOO\
99V//1C1C&//ZYYv1143E3EFOO\
r:   hidden_states	head_maskr"   c                    UR                   S   nUSU R                  U R                  4nU R                  U5      R                  " U6 R                  SS5      nU R                  U5      R                  " U6 R                  SS5      nU R                  U5      R                  " U6 R                  SS5      n[        nU R                  R                  S:w  a  [        U R                  R                     nU" U UUUUU R                  U R                  U R                  (       d  SOU R                  S9u  pU	R!                  5       S S U R"                  4-   nU	R%                  U5      n	X4$ )	Nr   r?   r   r$   eager        )r   r   r4   r   )rG   r   r   r   rN   r   r   r   r   r    _attn_implementationr   r   r   r   r   rB   r   rJ   )r6   r   r   r`   	new_shape	key_layervalue_layerquery_layerattention_interfacecontext_layerattention_probsnew_context_layer_shapes               r8   rg   DeiTSelfAttention.forward   sH    #((+
D$<$<d>V>VV	HH]+00)<FFq!L	jj/44i@JJ1aPjj/44i@JJ1aP(?;;++w6"9$++:Z:Z"[)<nnLL#}}C$2C2C	*
& #0"4"4"6s";t?Q?Q>S"S%--.EF--r:   )
r   r   r    r   r   r   r   r   r   r   N)rj   rk   rl   rm   r   r&   r(   rp   r   tuplerg   rs   rt   ru   s   @r8   r   r      sY    ]z ]* PT."\\.6>u||6L.	u||U\\)	*. .r:   r   c                      ^  \ rS rSrSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jr	S	r
U =r$ )
DeiTSelfOutput   z
The residual connection is defined in DeiTLayer instead of here (as is the case with other models), due to the
layernorm applied before each block.
r    c                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g r   )	r%   r&   r   r   r*   denser2   r3   r4   r   s     r8   r&   DeiTSelfOutput.__init__   sB    YYv1163E3EF
zz&"<"<=r:   r   input_tensorr"   c                 J    U R                  U5      nU R                  U5      nU$ r   r   r4   r6   r   r   s      r8   rg   DeiTSelfOutput.forward  s$    

=1]3r:   r   )rj   rk   rl   rm   rn   r   r&   r(   rp   rg   rs   rt   ru   s   @r8   r   r      sB    
>z >
U\\  RWR^R^  r:   r   c                      ^  \ rS rSrS\4U 4S jjrS\\   4S jrSS\	R                  S\\	R                     S\	R                  4S	 jjrS
rU =r$ )DeiTAttentioni  r    c                    > [         TU ]  5         [        U5      U l        [	        U5      U l        [        5       U l        g r   )r%   r&   r   	attentionr   outputsetpruned_headsr   s     r8   r&   DeiTAttention.__init__  s0    *62$V,Er:   headsc                 6   [        U5      S:X  a  g [        XR                  R                  U R                  R                  U R
                  5      u  p[        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l	        [        U R                  R                  USS9U R                  l        U R                  R                  [        U5      -
  U R                  l        U R                  R                  U R                  R                  -  U R                  l        U R
                  R                  U5      U l        g )Nr   r   rE   )lenr   r   r   r   r   r   r   r   r   r   r   r   union)r6   r   indexs      r8   prune_headsDeiTAttention.prune_heads  s   u:?7>>55t~~7Y7Y[_[l[l

  2$..2F2FN/0B0BEJ1$..2F2FN.t{{/@/@%QO .2^^-O-ORUV[R\-\*'+~~'I'IDNNLnLn'n$ --33E:r:   r   r   r"   c                 N    U R                  X5      u  p4U R                  X15      nU$ r   )r   r   )r6   r   r   self_attn_outputr_   r   s         r8   rg   DeiTAttention.forward$  s(    "nn]F-=r:   )r   r   r   r   )rj   rk   rl   rm   r   r&   r   rq   r   r(   rp   r   rg   rs   rt   ru   s   @r8   r   r     sR    "z ";S ;$U\\ hu||>T `e`l`l  r:   r   c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )DeiTIntermediatei+  r    c                   > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                  [        5      (       a  [        UR                     U l        g UR                  U l        g r   )r%   r&   r   r   r*   intermediate_sizer   r}   
hidden_actstrr	   intermediate_act_fnr   s     r8   r&   DeiTIntermediate.__init__,  s`    YYv1163K3KL
f''--'-f.?.?'@D$'-'8'8D$r:   r   r"   c                 J    U R                  U5      nU R                  U5      nU$ r   r   r   )r6   r   s     r8   rg   DeiTIntermediate.forward4  s&    

=100?r:   r   rj   rk   rl   rm   r   r&   r(   rp   rg   rs   rt   ru   s   @r8   r   r   +  s/    9z 9U\\ ell  r:   r   c                      ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  S\R                  4S jrSr	U =r
$ )	
DeiTOutputi;  r    c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g r   )
r%   r&   r   r   r   r*   r   r2   r3   r4   r   s     r8   r&   DeiTOutput.__init__<  sB    YYv779K9KL
zz&"<"<=r:   r   r   r"   c                 R    U R                  U5      nU R                  U5      nX-   nU$ r   r   r   s      r8   rg   DeiTOutput.forwardA  s,    

=1]3%4r:   r   r   ru   s   @r8   r   r   ;  s=    >z >
U\\  RWR^R^  r:   r   c                      ^  \ rS rSrSrS\4U 4S jjrS
S\R                  S\	\R                     S\R                  4S jjr
S	rU =r$ )	DeiTLayeriI  z?This corresponds to the Block class in the timm implementation.r    c                 j  > [         TU ]  5         UR                  U l        SU l        [	        U5      U l        [        U5      U l        [        U5      U l	        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  S9U l        g )Nr   eps)r%   r&   chunk_size_feed_forwardseq_len_dimr   r   r   intermediater   r   r   	LayerNormr*   layer_norm_epslayernorm_beforelayernorm_afterr   s     r8   r&   DeiTLayer.__init__L  s    '-'E'E$&v.,V4 ( "V-?-?VEZEZ [!||F,>,>FDYDYZr:   r   r   r"   c                     U R                  U5      nU R                  X25      nXA-   nU R                  U5      nU R                  U5      nU R	                  XQ5      nU$ r   )r   r   r   r   r   )r6   r   r   hidden_states_normattention_outputlayer_outputs         r8   rg   DeiTLayer.forwardV  se    !22=A>>*<H )8 ++M:((6 {{<?r:   )r   r   r   r   r   r   r   r   )rj   rk   rl   rm   rn   r   r&   r(   rp   r   rg   rs   rt   ru   s   @r8   r   r   I  sG    I[z [U\\ hu||>T `e`l`l  r:   r   c                   x   ^  \ rS rSrS\4U 4S jjrS	S\R                  S\\R                     S\	4S jjr
SrU =r$ )
DeiTEncoderih  r    c                    > [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        SU l	        g s  snf ri   )
r%   r&   r    r   
ModuleListrangenum_hidden_layersr   layergradient_checkpointing)r6   r    r_   r7   s      r8   r&   DeiTEncoder.__init__i  sR    ]]uVE]E]?^#_?^!If$5?^#_`
&+# $`s   A&r   r   r"   c                 r    [        U R                  5       H  u  p4Ub  X#   OS nU" X5      nM     [        US9$ )N)last_hidden_state)	enumerater  r   )r6   r   r   ilayer_modulelayer_head_masks         r8   rg   DeiTEncoder.forwardo  s<    (4OA.7.CilO(HM  5 ??r:   )r    r	  r  r   )rj   rk   rl   rm   r   r&   r(   rp   r   r   rg   rs   rt   ru   s   @r8   r  r  h  sA    ,z ,@U\\ @hu||>T @`o @ @r:   r  c                       \ rS rSr% \\S'   SrSrSrS/r	Sr
SrSrSr\\S.rS\\R&                  \R(                  \R*                  4   S	S
4S jrSrg
)DeiTPreTrainedModeliw  r    deitrX   Tr   )r   
attentionsr   r"   Nc                 .   [        U[        R                  [        R                  45      (       a  [        R                  R                  UR                  R                  R                  [        R                  5      SU R                  R                  S9R                  UR                  R                  5      UR                  l        UR                  b%  UR                  R                  R                  5         gg[        U[        R                   5      (       aJ  UR                  R                  R                  5         UR                  R                  R#                  S5        g[        U[$        5      (       a  UR&                  R                  R                  5         UR(                  R                  R                  5         UR*                  R                  R                  5         UR,                  b%  UR,                  R                  R                  5         ggg)zInitialize the weightsr   )meanstdNr[   )r}   r   r   r   inittrunc_normal_weightdatar   r(   r   r    initializer_ranger   r   zero_r   fill_r   r+   r1   r,   r-   )r6   r   s     r8   _init_weights!DeiTPreTrainedModel._init_weights  sh   fryy"))455 "$!6!6""%%emm43DKKDaDa "7 "b$$% MM {{&  &&( '--KK""$MM$$S)//!!'')&&++113%%**002  ,!!&&,,. -	 0r:    )rj   rk   rl   rm   r   __annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_sdpa_supports_flash_attn_supports_flex_attn_supports_attention_backendr   r   _can_record_outputsr   r   r   r   r   r   rs   r"  r:   r8   r  r  w  sr    $O&*#$N"&"'
/E"))RYY*L$M /RV /r:   r  c                      ^  \ rS rSrSS\S\S\SS4U 4S jjjrS\4S	 jrS
 r	\
" SS9\    SS\\R                     S\\R                     S\\R                     S\S\\   S\4S jj5       5       rSrU =r$ )	DeiTModeli  Fr    add_pooling_layerr!   r"   Nc                   > [         TU ]  U5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  UR                  S9U l        U(       a  [        U5      OSU l        U R                  5         g)z
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
use_mask_token (`bool`, *optional*, defaults to `False`):
    Whether to use a mask token for masked image modeling.
)r!   r   N)r%   r&   r    r   r;   r  encoderr   r   r*   r   	layernorm
DeiTPoolerpooler	post_init)r6   r    r/  r!   r7   s       r8   r&   DeiTModel.__init__  si     	 (O"6*f&8&8f>S>ST,=j(4 	r:   c                 .    U R                   R                  $ r   )r;   r/   )r6   s    r8   get_input_embeddingsDeiTModel.get_input_embeddings  s    ///r:   c                     UR                  5        H7  u  p#U R                  R                  U   R                  R	                  U5        M9     g)z
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
class PreTrainedModel
N)itemsr1  r  r   r   )r6   heads_to_pruner  r   s       r8   _prune_headsDeiTModel._prune_heads  s<    
 +002LELLu%//;;EB 3r:   )tie_last_hidden_statesrX   rY   r   rV   r   c                    Uc  [        S5      eU R                  X0R                  R                  5      nU R                  R
                  R                  R                  R                  nUR                  U:w  a  UR                  U5      nU R	                  XUS9nU R                  XsS9nUR                  n	U R                  U	5      n	U R                  b  U R                  U	5      OSn
[        U	U
S9$ )z
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
Nz You have to specify pixel_values)rY   rV   )r   )r  pooler_output)r   get_head_maskr    r  r;   r/   r   r  r   r   r1  r  r2  r4  r   )r6   rX   rY   r   rV   r   expected_dtypeembedding_outputencoder_outputssequence_outputpooled_outputs              r8   rg   DeiTModel.forward  s     ?@@ &&y++2O2OP	 99DDKKQQ/'??>:L??Tl + 
 ,0<<8H<+^);;..98<8OO4UY)-'
 	
r:   )r    r;   r1  r2  r4  )TFNNNF)rj   rk   rl   rm   r   ro   r&   r.   r8  r=  r   r   r   r(   rp   rr   r   r   r   rg   rs   rt   ru   s   @r8   r.  r.    s    z d [_ lp  &0&9 0C u5 046:,0).(
u||,(
 "%"2"23(
 ELL)	(

 #'(
 +,(
 
$(
  6(
r:   r.  c                   j   ^  \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )r3  i  r    c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        UR                     U l	        g r   )
r%   r&   r   r   r*   pooler_output_sizer   r	   
pooler_act
activationr   s     r8   r&   DeiTPooler.__init__  s>    YYv1163L3LM
 !2!23r:   r   r"   c                 \    US S 2S4   nU R                  U5      nU R                  U5      nU$ )Nr   )r   rN  )r6   r   first_token_tensorrG  s       r8   rg   DeiTPooler.forward  s6     +1a40

#566r:   )rN  r   r   ru   s   @r8   r3  r3    s/    4z 4
U\\ ell  r:   r3  ad  
    DeiT Model with a decoder on top for masked image modeling, as proposed in [SimMIM](https://huggingface.co/papers/2111.09886).

    <Tip>

    Note that we provide a script to pre-train this model on custom data in our [examples
    directory](https://github.com/huggingface/transformers/tree/main/examples/pytorch/image-pretraining).

    </Tip>
    )custom_introc                      ^  \ rS rSrS\SS4U 4S jjr\\    SS\\	R                     S\\	R                     S\\	R                     S	\S
\\   S\4S jj5       5       rSrU =r$ )DeiTForMaskedImageModelingi  r    r"   Nc                 H  > [         TU ]  U5        [        USSS9U l        [        R
                  " [        R                  " UR                  UR                  S-  UR                  -  SS9[        R                  " UR                  5      5      U l        U R                  5         g )NFT)r/  r!   r$   r   )in_channelsout_channelsry   )r%   r&   r.  r  r   
Sequentialr   r*   encoder_strider|   PixelShuffledecoderr5  r   s     r8   r&   #DeiTForMaskedImageModeling.__init__  s     fdS	}}II"..#22A58K8KK
 OOF112
 	r:   rX   rY   r   rV   r   c                 R   U R                   " U4UUUS.UD6nUR                  nUSS2SS24   nUR                  u  pn
[        U	S-  5      =pUR	                  SSS5      R                  XX5      nU R                  U5      nSnUGb  U R                  R                  U R                  R                  -  nUR                  SX5      nUR                  U R                  R                  S5      R                  U R                  R                  S5      R                  S5      R                  5       n[        R                  R                  XSS	9nUU-  R!                  5       UR!                  5       S
-   -  U R                  R"                  -  n[%        UUUR&                  UR(                  S9$ )a  
bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`):
    Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).

Examples:
```python
>>> from transformers import AutoImageProcessor, DeiTForMaskedImageModeling
>>> import torch
>>> from PIL import Image
>>> import requests

>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
>>> model = DeiTForMaskedImageModeling.from_pretrained("facebook/deit-base-distilled-patch16-224")

>>> num_patches = (model.config.image_size // model.config.patch_size) ** 2
>>> pixel_values = image_processor(images=image, return_tensors="pt").pixel_values
>>> # create random boolean mask of shape (batch_size, num_patches)
>>> bool_masked_pos = torch.randint(low=0, high=2, size=(1, num_patches)).bool()

>>> outputs = model(pixel_values, bool_masked_pos=bool_masked_pos)
>>> loss, reconstructed_pixel_values = outputs.loss, outputs.reconstruction
>>> list(reconstructed_pixel_values.shape)
[1, 3, 224, 224]
```)rY   r   rV   Nr   r?   r@   r   r$   none)	reductiongh㈵>)lossreconstructionr   r  )r  r  rG   rq   rK   rJ   r\  r    r{   r5   repeat_interleaver]   r   r   rL   l1_losssumr|   r   r   r  )r6   rX   rY   r   rV   r   outputsrF  r`   sequence_lengthr|   r<   r=   reconstructed_pixel_valuesmasked_im_lossrB   rc   reconstruction_losss                     r8   rg   "DeiTForMaskedImageModeling.forward  s   L /3ii/
+%=	/

 /
 "33 *!QrT'24C4I4I1
\_c122)11!Q:BB:]ck &*\\/%B"&;;))T[[-C-CCD-55b$EO11$++2H2H!L""4;;#9#91=1	  #%--"7"7lr"7"s1D8==?488:PTCTUX\XcXcXpXppN(5!//))	
 	
r:   )r\  r  rI  )rj   rk   rl   rm   r   r&   r   r   r   r(   rp   rr   ro   r   r   r   rg   rs   rt   ru   s   @r8   rU  rU    s    z d "  046:,0).I
u||,I
 "%"2"23I
 ELL)	I

 #'I
 +,I
 
#I
  I
r:   rU  z
    DeiT Model transformer with an image classification head on top (a linear layer on top of the final hidden state of
    the [CLS] token) e.g. for ImageNet.
    c                      ^  \ rS rSrS\SS4U 4S jjr\\    SS\\	R                     S\\	R                     S\\	R                     S	\S
\\   S\4S jj5       5       rSrU =r$ )DeiTForImageClassificationie  r    r"   Nc                 .  > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        U R                  5         g NF)r/  r   )r%   r&   
num_labelsr.  r  r   r   r*   Identity
classifierr5  r   s     r8   r&   #DeiTForImageClassification.__init__l  ss      ++f>	 OUN_N_bcNc"))F$6$68I8IJikititiv 	r:   rX   r   labelsrV   r   c                    U R                   " U4UUS.UD6nUR                  nU R                  USS2SSS24   5      nSn	Ub  U R                  " X8U R                  40 UD6n	[        U	UUR                  UR                  S9$ )a  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the image classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).

Examples:

```python
>>> from transformers import AutoImageProcessor, DeiTForImageClassification
>>> import torch
>>> from PIL import Image
>>> import requests

>>> torch.manual_seed(3)  # doctest: +IGNORE_RESULT
>>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> # note: we are loading a DeiTForImageClassificationWithTeacher from the hub here,
>>> # so the head will be randomly initialized, hence the predictions will be random
>>> image_processor = AutoImageProcessor.from_pretrained("facebook/deit-base-distilled-patch16-224")
>>> model = DeiTForImageClassification.from_pretrained("facebook/deit-base-distilled-patch16-224")

>>> inputs = image_processor(images=image, return_tensors="pt")
>>> outputs = model(**inputs)
>>> logits = outputs.logits
>>> # model predicts one of the 1000 ImageNet classes
>>> predicted_class_idx = logits.argmax(-1).item()
>>> print("Predicted class:", model.config.id2label[predicted_class_idx])
Predicted class: Polaroid camera, Polaroid Land camera
```r   rV   Nr   )ra  logitsr   r  )r  r  rr  loss_functionr    r   r   r  )
r6   rX   r   rt  rV   r   rf  rF  rw  ra  s
             r8   rg   "DeiTForImageClassification.forwardx  s    T /3ii/
%=/
 	/
 "33Aq!9: %%fdkkLVLD$!//))	
 	
r:   )rr  r  rp  rI  )rj   rk   rl   rm   r   r&   r   r   r   r(   rp   ro   r   r   r   rg   rs   rt   ru   s   @r8   rm  rm  e  s    
z 
d 
  04,0)-).=
u||,=
 ELL)=
 &	=

 #'=
 +,=
 
=
  =
r:   rm  zC
    Output type of [`DeiTForImageClassificationWithTeacher`].
    c                       \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\R                     \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   S	rg)
+DeiTForImageClassificationWithTeacherOutputi  aF  
logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
    Prediction scores as the average of the cls_logits and distillation logits.
cls_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
    Prediction scores of the classification head (i.e. the linear layer on top of the final hidden state of the
    class token).
distillation_logits (`torch.FloatTensor` of shape `(batch_size, config.num_labels)`):
    Prediction scores of the distillation head (i.e. the linear layer on top of the final hidden state of the
    distillation token).
Nrw  
cls_logitsdistillation_logitsr   r  r"  )rj   rk   rl   rm   rn   rw  r   r(   FloatTensorr#  r|  r}  r   r   r  rs   r"  r:   r8   r{  r{    s}    	 +/FHU&&'..2J**+27;%"3"34;8<M8E%"3"345<59Ju00129r:   r{  a  
    DeiT Model transformer with image classification heads on top (a linear layer on top of the final hidden state of
    the [CLS] token and a linear layer on top of the final hidden state of the distillation token) e.g. for ImageNet.

    .. warning::

           This model supports inference-only. Fine-tuning with distillation (i.e. with a teacher) is not yet
           supported.
    c                      ^  \ rS rSrS\SS4U 4S jjr\\   SS\\	R                     S\\	R                     S\S	\\   S\4
S
 jj5       5       rSrU =r$ )%DeiTForImageClassificationWithTeacheri  r    r"   Nc                   > [         TU ]  U5        UR                  U l        [        USS9U l        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l	        UR                  S:  a+  [
        R                  " UR                  UR                  5      O[
        R                  " 5       U l
        U R                  5         g ro  )r%   r&   rp  r.  r  r   r   r*   rq  cls_classifierdistillation_classifierr5  r   s     r8   r&   .DeiTForImageClassificationWithTeacher.__init__  s      ++f>	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	 AG@Q@QTU@UBIIf((&*;*;<[][f[f[h 	$
 	r:   rX   r   rV   r   c                    U R                   " U4UUS.UD6nUR                  nU R                  US S 2SS S 24   5      nU R                  US S 2SS S 24   5      nXx-   S-  n	[	        U	UUUR
                  UR                  S9$ )Nrv  r   r   r$   )rw  r|  r}  r   r  )r  r  r  r  r{  r   r  )
r6   rX   r   rV   r   rf  rF  r|  r}  rw  s
             r8   rg   -DeiTForImageClassificationWithTeacher.forward  s     /3ii/
%=/
 	/
 "33((Aq)AB
"::?1aQR7;ST 2a7:! 3!//))
 	
r:   )r  r  r  rp  )NNF)rj   rk   rl   rm   r   r&   r   r   r   r(   rp   ro   r   r   r{  rg   rs   rt   ru   s   @r8   r  r    s    z d "  04,0).	
u||,
 ELL)
 #'	

 +,
 
5
  
r:   r  )rm  r  rU  r.  r  )r   )?rn   collections.abcr~   dataclassesr   typingr   r   r   r(   r   activationsr	   modeling_layersr
   modeling_outputsr   r   r   r   modeling_utilsr   r   processing_utilsr   pytorch_utilsr   r   utilsr   r   r   r   r   utils.genericr   r   configuration_deitr   
get_loggerrj   loggerModuler   r.   rp   floatr   r   r   r   r   r   r   r  r  r.  r3  rU  rm  r{  r  __all__r"  r:   r8   <module>r     sK     ! , ,   ! 9  G & Q X X A * 
		H	%VRYY Vr")) P %II%<<% 
% <<	%
 U\\*% % %>1.		 1.jRYY $BII @ryy  
 
* >@")) @ !// !/ !/H I
# I
 I
Z  	]
!4 ]
]
@ L
!4 L
L
^ 
:+ : :& 
0
,? 0

0
fr:   