
    cCij                        S r SSKJr  SSKJrJr  SSKrSSKJr  SSKJ	r	J
r
  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJrJrJr  SSKJr  SSKJr  \R>                  " \ 5      r!\\" SS9 " S S\5      5       5       r"\\" SS9 " S S\5      5       5       r# " S S\RH                  5      r%\ " S S\5      5       r&\" SS9 " S S\&5      5       r'\" SS9 " S  S!\&\5      5       r(/ S"Qr)g)#zPyTorch PaliGemmamodel.    )	dataclass)OptionalUnionN)nn   )CacheStaticCache)GenerationMixin)FlashAttentionKwargs)BaseModelOutputWithPast)PreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging   )	AutoModel   )PaliGemmaConfigzN
    Base class for Paligemma outputs, with hidden states and attentions.
    )custom_introc                   B    \ rS rSr% SrSr\\R                     \	S'   Sr
g)PaligemmaModelOutputWithPast+   a  
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
Nimage_hidden_states )__name__
__module____qualname____firstlineno____doc__r   r   torchFloatTensor__annotations____static_attributes__r       j/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/paligemma/modeling_paligemma.pyr   r   +   s     8<%"3"34;r'   r   zU
    Base class for PaliGemma causal language model (or autoregressive) outputs.
    c                      \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\   \	S'   Sr\\\R                        \	S'   Sr\\\R                        \	S'   Sr\\R                     \	S	'   S
rg)PaliGemmaCausalLMOutputWithPast;   a  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.text_config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
image_hidden_states (`torch.FloatTensor`, *optional*):
    A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
    image_hidden_states of the model produced by the vision encoder after projecting last hidden state.
Nlosslogitspast_key_valueshidden_states
attentionsr   r   )r   r   r    r!   r"   r,   r   r#   r$   r%   r-   r.   r   r/   tupler0   r   r&   r   r'   r(   r*   r*   ;   s     )-D(5$$
%,*.FHU&&'.'+OXe_+8<M8E%"3"345<59Ju001297;%"3"34;r'   r*   c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )PaliGemmaMultiModalProjectorY   configc                    > [         TU ]  5         [        R                  " UR                  R
                  UR                  R                  SS9U l        g )NTbias)super__init__r   Linearvision_confighidden_sizeprojection_dimlinearselfr5   	__class__s     r(   r:   %PaliGemmaMultiModalProjector.__init__Z   s;    ii 4 4 @ @&BVBVBeBelpqr'   c                 (    U R                  U5      nU$ Nr?   )rA   image_featuresr/   s      r(   forward$PaliGemmaMultiModalProjector.forward^   s    N3r'   rF   )	r   r   r    r!   r   r:   rH   r&   __classcell__rB   s   @r(   r3   r3   Y   s    r r r'   r3   c                   L    \ rS rSr% \\S'   SrSrS/rSr	Sr
SrSrSrSrS rS	rg
)PaliGemmaPreTrainedModeld   r5    Tr3   r.   Fc                 b   [        U R                  SU R                  R                  5       R                  5      n[	        U[
        R                  5      (       aW  UR                  R                  R                  SUS9  UR                  b%  UR                  R                  R                  5         g g g )Ninitializer_range        )meanstd)getattrr5   get_text_configrQ   
isinstancer   r;   weightdatanormal_r8   zero_)rA   modulerT   s      r(   _init_weights&PaliGemmaPreTrainedModel._init_weightsr   s     dkk#68S8S8U8g8ghfbii((MM&&CS&9{{&  &&( ' )r'   r   N)r   r   r    r!   r   r%   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_can_compile_fullgraph_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendr]   r&   r   r'   r(   rM   rM   d   sE    &*#78"3"N"&)r'   rM   z|
    The Base Paligemma model which consists of a vision backbone and a language model without language modeling head.,
    c            #       `  ^  \ rS rSrSS0rSrS\4U 4S jjrS rS r	S	 r
S
 r     S!S\\   4S jjrS\R                   4S jrS\R$                  S\R                   S\R                   4S jr\\             S"S\\R$                     S\\R                      S\\R,                     S\\R$                     S\\   S\\R$                     S\\R$                     S\\R                      S\\R$                     S\\   S\\   S\\   S\\   S\\   S\\\4   4S jj5       5       rS rU =r$ )#PaliGemmaModel}   zlanguage_model.modellanguage_modelFr5   c                   > [         TU ]  U5        [        R                  " UR                  S9U l        [        U5      U l        UR                  R                  U l	        [        R                  " UR                  S9nX l
        U R                  R                  b  U R                  R                  OSU l        U R                  R                  5       R                  =(       d    U R                  U l        U R!                  5         g )N)r5   )r9   r:   r   from_configr<   vision_towerr3   multi_modal_projectortext_config
vocab_sizerk   r5   pad_token_idrV   dtypetext_config_dtype	post_init)rA   r5   rk   rB   s      r(   r:   PaliGemmaModel.__init__   s     %119M9MN%A&%I" ,,77"..f6H6HI,8<8P8P8\DKK44bd!%!<!<!>!D!D!R

r'   c                 6    U R                   R                  5       $ rE   )rk   get_input_embeddingsrA   s    r(   ry   #PaliGemmaModel.get_input_embeddings   s    ""7799r'   c                 :    U R                   R                  U5        g rE   )rk   set_input_embeddingsrA   values     r(   r}   #PaliGemmaModel.set_input_embeddings   s    007r'   c                     Xl         g rE   rk   rA   decoders     r(   set_decoderPaliGemmaModel.set_decoder   s    %r'   c                     U R                   $ rE   r   rz   s    r(   get_decoderPaliGemmaModel.get_decoder   s    """r'   is_trainingc                 J   U R                   R                  R                  S:X  a  Ub  SU;   a  U$ g Ub  UOU R                  n[	        U[
        5      n[        R                  " U R                  5      R                  nUc  UnUR                  S S u  pU(       a  UR                  5       nO9[	        U[        R                  5      (       a  UR                  S   O
US   U
-   S-   nUb  UR                  5       S:X  a  U$ [        R                  " X4UU R                  UR                  S9nU
S:w  a(  U(       a  [        R                   " USS	9nOSUS S 2S U
24'   U[        R"                  " XR                  S
9UR%                  SS5      :  -  nUS S S S 2S S 24   R'                  U	SSS5      nUb  UR)                  5       nUR                  S   nU(       ae  Uc  [+        S5      eUS S 2S S 2S S 2S U24   R-                  US S 2S S S S 24   R/                  UR                  5      S:H  S5      US S 2S S 2S S 2S U24'   US S 2S S 2S S 2S U24   US S 2S S S S 24   R/                  UR                  5      -   nUS:H  nUS S 2S S 2S S 2S U24   R-                  X5      US S 2S S 2S S 2S U24'   U$ )Nflash_attention_2rR   r   rm   r   r      
fill_valuert   devicediagonalr   z/Token type ids must be provided during training)r5   rq   _attn_implementationtrainingrW   r	   r#   finforu   minshapeget_max_cache_shapeTensordimfullr   triuarangereshapeexpandclone
ValueErrormasked_fillto)rA   attention_masktoken_type_idsr.   cache_positioninput_tensorr   using_static_cache	min_dtypeinputs_lead_dimsequence_lengthtarget_lengthcausal_maskmask_lengthpadding_masks                  r(   _update_causal_mask"PaliGemmaModel._update_causal_mask   s    ;;""77;NN)c^.C%%%0%<k$--'EKK 6 67;;	)L+7+=+=bq+A(+??AM nell;; $$R(#A&81<  %.*<*<*>!*C!!jj, ((!((	
 a#jjqA36A///0u||M:O:OPSaSiSijlnoSppp!$a"23::?ArSUV%%++-K(..r2K !)$%VWW5@Aq,;,AV5W5c5c"1dD!#34778J8JKqPRS6Aq!\k\12
 'q!Q'<=qRVX\^_O_@`@c@cdodvdv@wwL'1,L1<Q1l{l=R1S1_1_2K1a+-. r'   pixel_valuesc                     U R                  U5      nUR                  nU R                  U5      nX@R                  R                  R
                  S-  -  nU$ )ae  
Obtains image last hidden states from the vision tower and apply multimodal projection.

Args:
    pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
       The tensors corresponding to the input images.
Returns:
    image_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`).
g      ?)ro   last_hidden_staterp   r5   rq   r=   )rA   r   image_outputsselected_image_featurerG   s        r(   get_image_features!PaliGemmaModel.get_image_features   sU     )),7!.!@!@334JK';;+B+B+N+NPS+STr'   	input_idsinputs_embedsrG   c           	      J   Ucj  X R                  5       " [        R                  " U R                  R                  [        R
                  UR                  S95      :H  nUR                  S5      nOXR                  R                  :H  nUR                  5       nUR                  S5      R                  U5      R                  UR                  5      nUR                  S   UR                  S   -  nX$   R                  5       UR                  5       :w  a  [        SU SU 35      eU$ )z
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
equal to the length of multimodal features. If the lengths are different, an error is raised.
)rt   r   rm   r   r   z6Image features and image tokens do not match: tokens: z, features )ry   r#   tensorr5   image_token_idlongr   allsum	unsqueeze	expand_asr   r   numelr   )rA   r   r   rG   special_image_maskn_image_tokensn_image_featuress          r(   get_placeholder_mask#PaliGemmaModel.get_placeholder_mask   s    !.2K2K2MT[[77uzzR_RfRfg3 " "4!7!7!;!*kk.H.H!H+//1/99"=GGVYYZgZnZno)//2^5I5I!5LL,2248L8L8NNHHXXcdtcuv  "!r'   r   position_idsr.   r   r   labels	use_cacheoutput_attentionsoutput_hidden_statesreturn_dictkwargsreturnc                     USL USL-  (       a  [        S5      eUb  UOU R                  R                  nUb  UOU R                  R                  nUb  UOU R                  R                  nUSL=(       a    U	SLnUbR  U R                  R
                  U R                  :  a.  XR                  R
                  :H  nUR                  5       nSUU'   OUnUc  U R                  5       " U5      nUcE  Ub  UR                  5       OSn[        R                  " UUUR                  S   -   UR                  S9nUc  UR                  S5      S-   nUbY  U R                  U5      nUR!                  UR                  UR"                  5      nU R%                  XUS9nUR'                  UU5      nU R)                  X6XWX5      nU R*                  " S
UUUUU
UUSUS.	UD6n[-        UR.                  UR0                  UR2                  UR4                  Ub  WS	9$ SS	9$ )  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.text_config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.text_config.vocab_size]`.

Example:

```python
>>> from PIL import Image
>>> import requests
>>> from transformers import AutoProcessor, PaliGemmaForConditionalGeneration

>>> model = PaliGemmaForConditionalGeneration.from_pretrained("google/paligemma2-3b-mix-224")
>>> processor = AutoProcessor.from_pretrained("google/paligemma2-3b-mix-224")

>>> prompt = "Where is the cat standing?"
>>> url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/pipeline-cat-chonk.jpeg"
>>> image = Image.open(requests.get(url, stream=True).raw)

>>> inputs = processor(images=image, text=prompt,  return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(**inputs,)
>>> processor.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Where is the cat standing?\nsnow"
```Nz:You must specify exactly one of input_ids or inputs_embedsr   r   r   )r   rG   T)	r   r   r.   r   r   r   r   r   r   )r   r.   r/   r0   r   r   )r   r5   r   r   use_return_dictr   rr   r   ry   get_seq_lengthr#   r   r   r   r   r   r   rt   r   masked_scatterr   rk   r   r   r.   r/   r0   )rA   r   r   r   r   r.   r   r   r   r   r   r   r   r   r   r   r   llm_input_idspast_seen_tokensrG   r   outputss                         r(   rH   PaliGemmaModel.forward  sU   ^ -t";<YZZ1B1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]$D0GV45G  T[[%?%?4??%R!*kk.H.H!H%OO-M01M,-%M  557FM!CRC^==?de"\\ "2]5H5H5K"KTaThThN )33A6:L #!44\BN+..}/C/C]EXEXYN!%!:!:~ "; " *889K^\M..O]
 %% 
&%+'/!5)
 
 ,%77#33!//))2>2J
 	

 QU
 	
r'   )rk   rp   rs   ru   ro   rr   )NNNNN)NNNNNNNNNNNNN) r   r   r    r!   _checkpoint_conversion_mappingaccepts_loss_kwargsr   r:   ry   r}   r   r   r   boolr   r#   r$   r   
LongTensorr   r   r   r   r   r   r   r   r1   r   rH   r&   rJ   rK   s   @r(   ri   ri   }   s    '=>N%O" :8&# &*C d^CJu/@/@  "))":?:K:K"]b]n]n"0  15481537+/595959-1$(,0/3&*k
E,,-k
 u001k
 !.	k

 u//0k
 "%k
 !!1!12k
 !!1!12k
   1 12k
 ))*k
 D>k
 $D>k
 'tnk
 d^k
 -.k
  
u22	3!k
  k
r'   ri   c            %         ^  \ rS rSrSSSSS.rS/rS\4U 4S	 jjrS
 rS r	S r
S rS r\S 5       r\S 5       r\S 5       r\\              S*S\\R*                     S\\R,                     S\\R.                     S\\R*                     S\\   S\\R*                     S\\R*                     S\\R,                     S\\R*                     S\\   S\\   S\\   S\\   S\\\R.                  4   S \\   S!\\\4   4 S" jj5       5       r           S+U 4S# jjr!\"S\R.                  S$\S%\S&\RF                  S\R.                  S'\4S( j5       r$S)r%U =r&$ ),!PaliGemmaForConditionalGenerationi  zmodel.language_modelzmodel.vision_towerzmodel.multi_modal_projectorlm_head)z^language_model.modelz^vision_towerz^multi_modal_projectorz^language_model.lm_headzlm_head.weightr5   c                    > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  R                  UR                  R                  SS9U l	        U R                  5         g )NFr7   )r9   r:   ri   modelr   r;   rq   r=   rr   r   rv   r@   s     r(   r:   *PaliGemmaForConditionalGeneration.__init__  sS     #F+
yy!3!3!?!?ASASA^A^ejkr'   c                 6    U R                   R                  5       $ rE   )r   ry   rz   s    r(   ry   6PaliGemmaForConditionalGeneration.get_input_embeddings  s    zz..00r'   c                 :    U R                   R                  U5        g rE   )r   r}   r~   s     r(   r}   6PaliGemmaForConditionalGeneration.set_input_embeddings  s    

''.r'   c                 :    U R                   R                  U5        g rE   )r   r   r   s     r(   r   -PaliGemmaForConditionalGeneration.set_decoder  s    

w'r'   c                 6    U R                   R                  5       $ rE   )r   r   rz   s    r(   r   -PaliGemmaForConditionalGeneration.get_decoder  s    zz%%''r'   c                 8    U R                   R                  U5      $ rE   )r   r   )rA   r   s     r(   r   4PaliGemmaForConditionalGeneration.get_image_features  s    zz,,\::r'   c                 .    U R                   R                  $ rE   )r   rk   rz   s    r(   rk   0PaliGemmaForConditionalGeneration.language_model  s    zz(((r'   c                 .    U R                   R                  $ rE   )r   ro   rz   s    r(   ro   .PaliGemmaForConditionalGeneration.vision_tower  s    zz&&&r'   c                 .    U R                   R                  $ rE   )r   rp   rz   s    r(   rp   7PaliGemmaForConditionalGeneration.multi_modal_projector  s    zz///r'   r   r   r   r   r.   r   r   r   r   r   r   r   r   logits_to_keepr   r   c                 H   Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nU R                  " SUUUUUUUU
U	UUSUS.UD6nUS   n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nSnU	b3  U R                  " SUXR                   R                  R                  S.UD6n[        UUUR                  UR                  UR                  UR                   S9$ )r   NT)r   r   r   r   r   r.   r   r   r   r   r   r   r   r   )r-   r   rr   )r,   r-   r.   r/   r0   r   r   )r5   r   r   r   r   rW   intslicer   loss_functionrq   rr   r*   r.   r/   r0   r   )rA   r   r   r   r   r.   r   r   r   r   r   r   r   r   r   r   r   r/   slice_indicesr-   r,   s                        r(   rH   )PaliGemmaForConditionalGeneration.forward  sP   ^ 2C1N-TXT_T_TqTq$8$D $++JjJj 	 &1%<k$++B]B]** 
%))%+'/!5)
 
"  
8B>SV8W8W~ot4]kmA}a,?@A%% f9P9P9[9[_eD /#33!//)) ' ; ;
 	
r'   c                 ~  > [         TU ]  " U4UUUUUU	U
US.UD6nUR                  S5      b  US==   S-  ss'   US   S:X  a  XmS'   US L=(       a    US Ln[        U[        5      =(       a    [        UR                  5      nUS   S:X  a1  U(       a*  Ub  UOUnU R                  R                  XxX$UU5      nUUS'   U$ )N)r.   r   r   r   r   r   r   r   r   r   r   r   r   )	r9   prepare_inputs_for_generationgetrW   r	   any
is_slidingr   r   )rA   r   r.   r   r   r   r   r   r   r   r   r   r   model_inputsr   is_static_hybrid_cacher   r   rB   s                     r(   r   ?PaliGemmaForConditionalGeneration.prepare_inputs_for_generation	  s      w<
+')%)))
 
 N+7(A-( !!+7($D0GV45G!+O[!I!mcRaRlRlNm!!&<,9,E=9L**88Q]_jK .9L)*r'   r   r   rt   
batch_sizec                    U b  U R                  5       S:X  a  U nU$ [        R                  " U5      R                  n[        R                  " X4XUR
                  S9nUS:w  a  [        R                  " USS9nU[        R                  " X$R
                  S9UR                  SS5      :  -  nUSSSS2SS24   R                  USSS5      nU b  UR                  5       nU R                  S   n	USS2SS2SS2SU	24   U SS2SSSS24   R                  UR
                  5      -   n
U
S:H  n
USS2SS2SS2SU	24   R                  X5      USS2SS2SS2SU	24'   U$ )	a  
Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
`(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

Args:
    attention_mask (`torch.Tensor`):
        A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
        `(batch_size, 1, query_length, key_value_length)`.
    sequence_length (`int`):
        The sequence length being processed.
    target_length (`int`):
        The target length: when generating with static cache, the mask should be as long as the static cache,
        to account for the 0 padding, the part of the cache that is not filled yet.
    dtype (`torch.dtype`):
        The dtype to use for the 4D attention mask.
    cache_position (`torch.Tensor`):
        Indices depicting the position of the input sequence tokens in the sequence.
    batch_size (`torch.Tensor`):
        Batch size.
Nr   r   r   r   r   rm   r   )r   r#   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   rt   r   r  r   r   r   r   r   s              r(   5_prepare_4d_causal_attention_mask_with_cache_positionWPaliGemmaForConditionalGeneration._prepare_4d_causal_attention_mask_with_cache_position8  s}   > %.*<*<*>!*C(K* ' E*..I** 0Y\j\q\qK !##jjqA5<<>S>STWeWmWmnprsWtttK%dD!Q&67>>z1bRTUK))//1,2226*1aL[L+@ANSTVZ\`bcScDdDgDg&&E    ,q05@Aq,;,AV5W5c5c 6Aq!\k\12 r'   )r   r   )NNNNNNNNNNNNNr   )
NNNNNNNTNN)'r   r   r    r!   r   _tied_weights_keysr   r:   ry   r}   r   r   r   propertyrk   ro   rp   r   r   r   r#   r   r$   r   r   r   r   r   r   r   r1   r*   rH   r   staticmethodrt   r  r&   rJ   rK   s   @r(   r   r     s    "8-"?#,	&" ++ 1/((; ) ) ' ' 0 0  15481537+/595959-1$(,0/3&*34V
E,,-V
 u001V
 !.	V

 u//0V
 "%V
 !!1!12V
 !!1!12V
   1 12V
 ))*V
 D>V
 $D>V
 'tnV
 d^V
 c5<</0V
  +,!V
" 
u55	6#V
  V
v -^ 444 4 {{	4
 4 4 4r'   r   )r   rM   ri   )*r"   dataclassesr   typingr   r   r#   r   cache_utilsr   r	   
generationr
   modeling_flash_attention_utilsr   modeling_outputsr   modeling_utilsr   processing_utilsr   utilsr   r   r   r   r   autor   configuration_paligemmar   
get_loggerr   loggerr   r*   Moduler3   rM   ri   r   __all__r   r'   r(   <module>r     s0    ! "   - ) B 7 - &   4 
		H	% 
<#: < < 
<k < <0299  ) ) )0 
z
- z

z
z 
j(@/ j
jZ ^r'   