
    cCiJa                        S SK JrJrJr  S SKrS SKJr  SSKJr  SSK	J
r
Jr  SSKJrJr  SSKJrJr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJr  SSKJrJr  SSK J!r!  SSK"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,J-r-  \R\                  " \/5      r0 " S S\5      r1 " S S\*5      r2 " S S\'5      r3 " S S\+5      r4   S2S\Rj                  S\Rl                  S\Rl                  S\Rl                  S\\Rl                     S\7S\\7   S \\7   S!\8\Rl                  \Rl                  4   4S" jjr9 " S# S$\#5      r: " S% S&\5      r; " S' S(\)5      r< " S) S*\(5      r= " S+ S,\$5      r> " S- S.\%5      r? " S/ S0\&5      r@/ S1QrAg)3    )CallableOptionalUnionN   )ACT2FN)CacheDynamicCache)PretrainedConfiglayer_type_validation)create_causal_mask!create_sliding_window_causal_mask)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ALL_ATTENTION_FUNCTIONS)Unpack)TransformersKwargslogging)deprecate_kwarg   )GemmaAttentionGemmaForCausalLMGemmaForSequenceClassificationGemmaForTokenClassificationGemmaMLP
GemmaModelGemmaPreTrainedModelGemmaRMSNormGemmaRotaryEmbeddingapply_rotary_pos_emb	repeat_kvc                      ^  \ rS rSrSrSrS/rSSSSSSSS.rS/S	/4S
S/S
/4S
/S
/4S.r                        SU 4S jjr	Sr
U =r$ )Gemma2Config2   a  
This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the Gemma2-7B.
e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.
Args:
    vocab_size (`int`, *optional*, defaults to 256000):
        Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`Gemma2Model`]
    hidden_size (`int`, *optional*, defaults to 2304):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 9216):
        Dimension of the MLP representations.
    num_hidden_layers (`int`, *optional*, defaults to 26):
        Number of hidden layers in the Transformer decoder.
    num_attention_heads (`int`, *optional*, defaults to 8):
        Number of attention heads for each attention layer in the Transformer decoder.
    num_key_value_heads (`int`, *optional*, defaults to 4):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `num_attention_heads`.
    head_dim (`int`, *optional*, defaults to 256):
        The attention head dimension.
    hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
        The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
        if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
    max_position_embeddings (`int`, *optional*, defaults to 8192):
        The maximum sequence length that this model might ever be used with.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    rms_norm_eps (`float`, *optional*, defaults to 1e-06):
        The epsilon used by the rms normalization layers.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models). Only
        relevant if `config.is_decoder=True`.
    pad_token_id (`int`, *optional*, defaults to 0):
        Padding token id.
    eos_token_id (`int`, *optional*, defaults to 1):
        End of stream token id.
    bos_token_id (`int`, *optional*, defaults to 2):
        Beginning of stream token id.
    tie_word_embeddings (`bool`, *optional*, defaults to `True`):
        Whether to tie weight embeddings
    rope_theta (`float`, *optional*, defaults to 10000.0):
        The base period of the RoPE embeddings.
    attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
        Whether to use a bias in the query, key, value and output projection layers during self-attention.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    query_pre_attn_scalar (`float`, *optional*, defaults to 256):
        scaling factor used on the attention scores
    sliding_window (`int`, *optional*, defaults to 4096):
        in Gemma2, every other layer uses sliding window attention. This is the size of the sliding window.
    layer_types (`list`, *optional*):
        Attention pattern for each layer.
    final_logit_softcapping (`float`, *optional*, defaults to 30.0):
        scaling factor when applying tanh softcapping on the logits.
    attn_logit_softcapping (`float`, *optional*, defaults to 50.0):
        scaling factor when applying tanh softcapping on the attention scores.

```python
>>> from transformers import Gemma2Model, Gemma2Config
>>> # Initializing a Gemma2 gemma2-7b style configuration
>>> configuration = Gemma2Config()
>>> # Initializing a model from the gemma2-7b style configuration
>>> model = Gemma2Model(configuration)
>>> # Accessing the model configuration
>>> configuration = model.config
```gemma2past_key_valuescolwiserowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_proj	input_idsinputs_embedshidden_statesattention_mask)embed_tokenslayersnormc                   > [         TU ]  " SUUUUS.UD6  Xl        Xl        X l        X0l        X@l        XPl        Xpl        X`l	        Xl
        Xl        Xl        UU l        UU l        UU l        Xl        UU l        UU l        UU l        UU l        UU l        U R*                  cC  [-        U R                  5       Vs/ s H  n[/        US-   S-  5      (       a  SOSPM     snU l        [1        U R*                  U R                  5        g s  snf )N)pad_token_idbos_token_ideos_token_idtie_word_embeddings   r   sliding_attentionfull_attention )super__init__
vocab_sizemax_position_embeddingshidden_sizeintermediate_sizenum_hidden_layersnum_attention_headshead_dimnum_key_value_headsinitializer_rangerms_norm_eps	use_cache
rope_thetaattention_biasattention_dropouthidden_activationquery_pre_attn_scalarsliding_windowfinal_logit_softcappingattn_logit_softcappinglayer_typesrangeboolr   )selfr<   r>   r?   r@   rA   rC   rB   rJ   r=   rD   rE   rF   r2   r4   r3   r5   rG   rH   rI   rK   rL   rO   rM   rN   kwargsi	__class__s                              c/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/gemma2/modular_gemma2.pyr;   Gemma2Config.__init__   s   8 	 	
%%% 3		

 	
 %'>$&!2!2#6  #6 !2("$,!2!2%:",'>$&<#&#X]^b^t^tXu XuSTtQUaK'8'8#>NNXu D 	d..0F0FG s   ;$D)rH   rI   rN   rM   rB   rJ   r>   rD   r?   rO   r=   rA   r@   rC   rK   rE   rG   rL   rF   r<   )i  i 	  i $              gelu_pytorch_tanhi    g{Gz?gư>Tr   r6   r   Tg     @F        r[   i   Ng      >@g      I@)__name__
__module____qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferencebase_model_tp_planbase_model_pp_planr;   __static_attributes____classcell__rU   s   @rV   r$   r$   2   s    JX J#4"5%.%.%.%."+ )"+ &(9:#%568IJ!"_$56 - $ ! $#3<H <H    r$   c                       \ rS rSrSrg)Gemma2RMSNorm   r9   Nr^   r_   r`   ra   rg   r9   rj   rV   rl   rl          rj   rl   c                   (   ^  \ rS rSrU 4S jrSrU =r$ )	Gemma2MLP   c                 T   > [         TU ]  U5        [        UR                     U l        g N)r:   r;   r   rJ   act_fnrR   configrU   s     rV   r;   Gemma2MLP.__init__   s"     V556rj   )ru   )r^   r_   r`   ra   r;   rg   rh   ri   s   @rV   rq   rq      s    7 7rj   rq   c                       \ rS rSrSrg)Gemma2RotaryEmbedding   r9   Nrn   r9   rj   rV   rz   rz      ro   rj   rz   modulequerykeyvaluer-   dropoutscalingsoftcapreturnc                    Uc  U R                   S-  n[        X R                  5      n	[        X0R                  5      n
[        R                  " XR                  SS5      5      U-  nUb  X-  n[        R                  " U5      nX-  nUb"  US S 2S S 2S S 2S U	R                  S   24   nX-   n[        R                  R                  US[        R                  S9R                  UR                  5      n[        R                  R                  XU R                  S9n[        R                  " X5      nUR                  SS5      R!                  5       nX4$ )	N      r   r   )dimdtype)ptrainingr6   )rB   r"   num_key_value_groupstorchmatmul	transposetanhshapenn
functionalsoftmaxfloat32tor   r   r   
contiguous)r|   r}   r~   r   r-   r   r   r   rS   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                 rV   eager_attention_forwardr      s/    //4'3 ; ;<JU$?$?@L<<';';Aq'ABWLL#-zz,/#-!$Q1.D
0@0@0D.D%DE#1 ==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$rj   c                   X  ^  \ rS rSrS\S\4U 4S jjr\" SSSS9  SS	\R                  S
\
\R                  \R                  4   S\\R                     S\\   S\\R                     S\\   S\
\R                  \\R                     \\
\R                        4   4S jj5       rSrU =r$ )Gemma2Attentionr[   rw   	layer_idxc                   > [         TU ]  X5        U R                  R                  U l        U R                  R                  U l        SU l        UR                  S-  U l        UR                  U   S:X  a  UR                  U l	        g S U l	        g )NTr   r7   )
r:   r;   rw   rN   rI   	is_causalrK   r   rO   rL   rR   rw   r   rU   s      rV   r;   Gemma2Attention.__init__  sv    +&*kk&H&H#!%!>!>33T97=7I7I)7TXk7kf33qurj   past_key_valuer'   4.58new_nameversionr,   position_embeddingsr-   cache_positionrS   r   c                 `   UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      R                  U5      R	                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	U
UU4U R                  (       a  U R                  OSU R                   U R"                  U R$                  S.UD6u  nnUR&                  " / UQSP76 R)                  5       nU R+                  U5      nUU4$ )Nr   r6   r   )sincosr   eagerr]   )r   r   rL   r   )r   rB   q_projviewr   k_projv_projr!   updater   r   rw   _attn_implementationr   r   rI   r   rL   rN   reshaper   o_proj)rR   r,   r   r-   r'   r   rS   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   s                     rV   forwardGemma2Attention.forward	  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&#7RU#[ &#&nUL'6'='=jX\XfXfht'u$J(?;;++w6"9$++:Z:Z"[$7%
 /3mmD**LL..//%
 %
!\ "));;;;FFHkk+.L((rj   )rI   rN   r   r   rL   )NN)r^   r_   r`   ra   r$   intr;   r   r   Tensortupler   r   
LongTensorr   r   r   rg   rh   ri   s   @rV   r   r      s    v| v v %0A6R ,059+)||+) #5<<#=>+) !.	+)
 "%+) !!1!12+) -.+) 
u||Xell3XeELL>Q5RR	S+) S+)rj   r   c                     ^  \ rS rSrS\S\4U 4S jjr\" SSSS9      SS	\R                  S
\
\R                  \R                  4   S\\R                     S\\R                     S\\   S\\   S\\   S\\R                     S\
\R                  \\
\R                  \R                  4      4   4S jj5       rSrU =r$ )Gemma2DecoderLayeri8  rw   r   c                   > [         TU ]  5         UR                  U l        Xl        UR                  U   U l        [        XS9U l        [        U5      U l	        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        g )N)rw   r   )eps)r:   r;   r>   rw   rO   attention_typer   	self_attnrq   mlprl   rE   input_layernormpost_attention_layernormpre_feedforward_layernormpost_feedforward_layernormr   s      rV   r;   Gemma2DecoderLayer.__init__9  s    !--$00;(LV$,V-?-?VEXEXY(5f6H6HfNaNa(b%)6v7I7IvObOb)c&*78J8JPVPcPc*d'rj   r   r'   r   r   r,   r   r-   position_idsoutput_attentionsrF   r   r   c	                     Un
U R                  U5      nU R                  " SUUUUUUUUS.U	D6u  pU R                  U5      nX-   nUn
U R                  U5      nU R	                  U5      nU R                  U5      nX-   nU4nU(       a  X4-  nU$ )N)r,   r   r-   r   r'   r   rF   r   r9   )r   r   r   r   r   r   )rR   r,   r   r-   r   r'   r   rF   r   rS   residualself_attn_weightsoutputss                rV   r   Gemma2DecoderLayer.forwardF  s     !,,]; ,0>> 
,
' 3)%+/)
,
 
,
( 55mD 0 66}E/77F 0 "++Grj   )	r   rw   r>   r   r   r   r   r   r   )NNNFFN)r^   r_   r`   ra   r$   r   r;   r   r   r   r   r   r   r   rQ   FloatTensorr   rg   rh   ri   s   @rV   r   r   8  s   e| e e %0A6R
 2637+/,1$)59*||* #5<<#=>* !.	*
 u//0* "%* $D>* D>* !!1!12* 
u  (51B1BEDUDU1U+V"WW	X* S*rj   r   c                       \ rS rSrSrg)Gemma2PreTrainedModelit  r9   Nrn   r9   rj   rV   r   r   t  ro   rj   r   c                     ^  \ rS rSrS\4U 4S jjr         SS\\R                     S\\R                     S\\R                     S\\
   S\\R                     S	\\   S
\\   S\\   S\\R                     S\\   S\4S jjrSrU =r$ )Gemma2Modelix  rw   c           	         > [         TU ]  U5        [        R                  " [	        UR
                  5       Vs/ s H  n[        X5      PM     sn5      U l        g s  snf rt   )r:   r;   r   
ModuleListrP   r@   r   r/   r   s      rV   r;   Gemma2Model.__init__y  sH     mmDI&JbJbDcdDcy2Dcd
ds   Ar*   r-   r   r'   r+   rF   r   output_hidden_statesr   rS   r   c
                    Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUS L US L-  (       a  [	        S5      eU R
                  (       a/  U R                  (       a  U(       a  [        R                  S5        SnUc  U R                  U5      nU(       a'  Uc$  U R                  (       d  [        U R                   S9nU	cD  Ub  UR                  5       OSn[        R                  " XUR                  S   -   UR                  S9n	Uc  U	R!                  S5      n[#        U=n[$        5      (       d*  U R                   UUU	UUS.n['        S0 UD6[)        S0 UD6S	.nUnU R+                  X5      n[        R,                  " U R                   R.                  S
-  UR0                  S9nUU-  nU(       a  SOS nU(       a  SOS nU R2                  S U R                   R4                    HE  nU(       a  UU4-  nU" U4UUUR6                     UUUUU	S.U
D6nUS   nU(       d  M<  UUS   4-  nMG     U R9                  U5      nU(       a  UU4-  n[;        UUUUS9$ )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)rw   r   r6   )device)rw   input_embedsr-   r   r'   r   )r8   r7   g      ?)r   r9   )r   r-   r   r'   r   rF   r   )last_hidden_stater'   r,   
attentions)rw   r   r   rF   
ValueErrorgradient_checkpointingr   loggerwarning_oncer.   r	   get_seq_lengthr   aranger   r   	unsqueeze
isinstancedictr   r   
rotary_embtensorr>   r   r/   r@   r   r0   r   )rR   r*   r-   r   r'   r+   rF   r   r   r   rS   past_seen_tokenscausal_mask_mappingmask_kwargsr,   r   
normalizerall_hidden_statesall_self_attnsdecoder_layerlayer_outputss                        rV   r   Gemma2Model.forward  s    2C1N-TXT_T_TqTq$8$D $++JjJj 	 "+!6IDKK<Q<Q	-t";<YZZ&&4==Yj I  --i8M0*$++>O!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L ?-FF ++ -"0"0#2 ,K #5"C{"C%F%U%U# & #oomJ
 \\$++"9"93">mFYFYZ
%
2 #7BD0d![[)H4;;+H+HIM#!m%55!)
$72=3O3OP) /"3#-
 
M *!,M  =#3"55' J* 		-0-!11&+++%	
 	
rj   )r/   )	NNNNNNNNN)r^   r_   r`   ra   r$   r;   r   r   r   r   r   r   rQ   r   r   r   r   rg   rh   ri   s   @rV   r   r   x  s    
| 
 151537+/59$(,0/359k
E,,-k
 !.k
 u//0	k

 "%k
   1 12k
 D>k
 $D>k
 'tnk
 !!1!12k
 +,k
 
!k
 k
rj   r   c                   N  ^  \ rS rSrU 4S jr           SS\\R                     S\\R                     S\\R                     S\\	   S\\R                     S\\R                     S	\\   S
\\   S\\   S\\R                     S\\\R                  4   S\4S jjrSrU =r$ )Gemma2ForCausalLMi  c                 d   > [         TU ]  U5        [        U5      U l        U R	                  5         g rt   )r:   r;   r   model	post_initrv   s     rV   r;   Gemma2ForCausalLM.__init__  s&      (
rj   r*   r-   r   r'   r+   labelsrF   r   r   r   logits_to_keepr   c                    Ub  UOU R                   R                  nU	b  U	OU R                   R                  n	U R                  " SUUUUUUUU	U
S.	UD6nUR                  n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nU R                   R                  bH  UU R                   R                  -  n[        R                  " U5      nUU R                   R                  -  nSnUb  U R                  " UX`R                  40 UD6n[        UUUR                  UR                   UR"                  S9$ )a"  
Example:

```python
>>> from transformers import AutoTokenizer, Gemma2ForCausalLM

>>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```N)	r*   r-   r   r'   r+   rF   r   r   r   )losslogitsr'   r,   r   r9   )rw   r   r   r   r   r   r   slicelm_headrM   r   r   loss_functionr<   r   r'   r,   r   )rR   r*   r-   r   r'   r+   r   rF   r   r   r   r   rS   r   r,   slice_indicesr  r  s                     rV   r   Gemma2ForCausalLM.forward  sT   B 2C1N-TXT_T_TqTq$8$D $++JjJj 	 ,0:: ,
)%+'/!5),
 ,
  118B>SV8W8W~ot4]kmA}a,?@A;;..:dkkAAAFZZ'FdkkAAAF%%ffooPPD%#33!//))
 	
rj   )r   )NNNNNNNNNNr   )r^   r_   r`   ra   r;   r   r   r   r   r   r   rQ   r   r   r   r   rg   rh   ri   s   @rV   r   r     s    151537+/59-1$(,0/35934F
E,,-F
 !.F
 u//0	F

 "%F
   1 12F
 ))*F
 D>F
 $D>F
 'tnF
 !!1!12F
 c5<</0F
 
 F
 F
rj   r   c                       \ rS rSrSrg)Gemma2ForSequenceClassificationi<  r9   Nrn   r9   rj   rV   r	  r	  <  ro   rj   r	  c                       \ rS rSrSrg)Gemma2ForTokenClassificationi@  r9   Nrn   r9   rj   rV   r  r  @  ro   rj   r  )r$   r   r   r   r	  r  )r]   NN)Btypingr   r   r   r   torch.nnr   activationsr   cache_utilsr   r	   configuration_utilsr
   r   masking_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   modeling_utilsr   processing_utilsr   utilsr   r   utils.deprecationr   gemma.modeling_gemmar   r   r   r   r   r   r   r   r    r!   r"   
get_loggerr^   r   r$   rl   rq   rz   Moduler   floatr   r   r   r   r   r   r   r	  r  __all__r9   rj   rV   <module>r     s    - ,   ! . J R B 9 O 5 & 0 0    
		H	%ZH# ZHz	L 	7 7	0 	 ## %II %<< % 
 % <<	 %
 U\\* %  % e_ % e_ % 5<<%& %F5)n 5)p93 9x	0 	r
* r
jL
( L
^	&D 		#> 	rj   