
    cCij                     F   S SK r S SKJrJr  S SKrS SKJr  SSKJr  SSKJ	r	J
r
Jr  SSKJr  SSKJr  SS	KJr  SS
KJrJr  SSKJrJrJrJr  SSKJrJr  SSKJrJr  SSK J!r!  SSK"J#r#  SSK$J%r%J&r&J'r'J(r(  SSK)J*r*  SSK+J,r,  SSK-J.r.  \(R^                  " \05      r1 " S S\Rd                  5      r3S r4S9S jr5S\Rl                  S\7S\Rl                  4S jr8S r9 " S S\Rd                  5      r: " S  S!\:5      r; " S" S#\:5      r<\" S$5       " S% S&\Rd                  5      5       r=\:\;\<S'.r> " S( S)\5      r?\& " S* S+\!5      5       r@ " S, S-\Rd                  5      rA\& " S. S/\@5      5       rB\& " S0 S1\@\5      5       rC " S2 S3\\@5      rD " S4 S5\\@5      rE " S6 S7\\@5      rF/ S8QrGg):    N)OptionalUnion)nn   )ACT2FN)CacheDynamicCacheStaticCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)_flash_attention_forward!flash_attn_supports_top_left_mask)GenericForQuestionAnswering GenericForSequenceClassificationGenericForTokenClassificationGradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)PreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg)check_model_inputs   )DiffLlamaConfigc                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )DiffLlamaMLP7   c                   > [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g NFbias)super__init__confighidden_sizeintermediate_sizer   Linear	gate_projup_proj	down_projr   
hidden_actact_fnselfr+   	__class__s     j/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/diffllama/modeling_diffllama.pyr*   DiffLlamaMLP.__init__8   s    !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV4#9#94;K;KRWXV../    c                     U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ N)r1   r3   r/   r0   )r5   xr1   s      r7   forwardDiffLlamaMLP.forwardB   s6    NN4;;t~~a/@#ADLLQRO#ST	r9   )r3   r+   r1   r/   r,   r-   r0   )__name__
__module____qualname____firstlineno__r*   r=   __static_attributes____classcell__r6   s   @r7   r#   r#   7   s    0 r9   r#   c                     U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..N   dim)shapetorchcat)r<   x1x2s      r7   rotate_halfrP   G   sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r9   c                     UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXg4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)	unsqueezerP   )qkcossinposition_idsunsqueeze_dimq_embedk_embeds           r7   apply_rotary_pos_embr[   N   sS    ( --
&C
--
&Cw;q>C/0Gw;q>C/0Gr9   hidden_statesn_repreturnc                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r    N)rK   expandreshape)r\   r]   batchnum_key_value_headsslenhead_dims         r7   	repeat_kvrf   i   s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr9   c                 @    SS[         R                  " SU -  5      -  -
  $ )Ng?g333333?g333333ӿ)mathexp)	layer_idxs    r7   lambda_init_fnrk   u   s     txxy 01111r9   c                     ^  \ rS rSrSrSS\S\\   4U 4S jjjr\	" SSSS	9     SS
\
R                  S\\
R                  \
R                  4   S\\
R                     S\\
R                     S\\   S\S\\
R                     S\\
R                  \\
R                     \\\
R                        4   4S jj5       rSrU =r$ )DiffLlamaAttentiony   z=Multi-headed attention from 'Attention Is All You Need' paperr+   rj   c                   > [         TU ]  5         Xl        X l        Uc-  [        R                  SU R                  R                   S35        UR                  U l        UR                  U l	        UR                  U l        [        USU R                  U R                  -  5      U l        UR                  U l        U R                  U R                  -  U l        UR                   U l        UR"                  U l        SU l        [&        R(                  " U R                  U R                  U R                  -  UR*                  S9U l        [&        R(                  " U R                  U R                  U R                  -  UR*                  S9U l        [&        R(                  " U R                  U R                  U R                  -  UR*                  S9U l        [&        R(                  " U R                  U R                  -  U R                  UR*                  S9U l        [5        U5      U l        [&        R8                  " [:        R<                  " SUR>                  U R                  4S95      U l         [&        R8                  " [:        R<                  " SUR>                  U R                  4S95      U l!        [&        R8                  " [:        R<                  " SUR>                  U R                  4S95      U l"        [&        R8                  " [:        R<                  " SUR>                  U R                  4S95      U l#        [&        RH                  " SU R                  -  URJ                  S	S
9U l&        g )NzInstantiating z without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.re   Tr'   r   )sizerH   F)epselementwise_affine)'r)   r*   r+   rj   loggerwarning_oncer6   r?   attention_dropoutr,   num_attention_heads	num_headsgetattrre   rc   num_key_value_groupsmax_position_embeddings
rope_theta	is_causalr   r.   attention_biasq_projk_projv_projo_projrk   lambda_init	ParameterrL   normallambda_std_dev	lambda_q1	lambda_k1	lambda_q2	lambda_k2RMSNormrms_norm_eps	groupnormr5   r+   rj   r6   s      r7   r*   DiffLlamaAttention.__init__|   sz   " !8!8 9 :, , "(!9!9!--33
D4D4D4VW#)#=#= $(NNd6N6N$N!'-'E'E$ ++ii 0 0$..4==2PW]WlWlmii 0 0$2J2JT]]2Zagavavwii 0 0$2J2JT]]2Zagavavwii >@P@PW]WlWlm))4ell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdell1f6K6KSWS`S`Rb&cdA$56;N;Nchir9   past_key_valuepast_key_values4.58new_nameversionr\   position_embeddingsattention_maskrW   	use_cachecache_positionr^   c                    UR                  5       u  pnU
nU R                  U5      nU R                  U5      nU R                  U5      nUR	                  XU R
                  U R                  5      R                  SS5      nUR	                  XU R                  U R                  5      R                  SS5      nUR	                  XU R                  U R                  5      R                  SS5      nUu  nn[        XUU5      u  pUb%  UUUS.nUR                  XU R                  U5      u  p[        XR                  5      n[        XR                  5      n[        R                  " [        R                   " USSS9SS9nUR#                  SSSS5      n[        R$                  " XR                  SS5      5      [&        R(                  " U R                  5      -  nUb#  US S 2S S 2S S 2S UR*                  S   24   nUU-   n[,        R.                  R1                  US[        R2                  S9R5                  UR6                  5      n[,        R.                  R9                  UU R:                  U R<                  S	9n[        R>                  " [        R@                  " U RB                  U RD                  -  S[        R2                  S95      R5                  UR6                  5      n[        R>                  " [        R@                  " U RF                  U RH                  -  S[        R2                  S95      R5                  UR6                  5      nUU-
  U RJ                  -   n[        R$                  " UU5      n[        R                   " USSS9u  nnUUU-  -
  nSU RJ                  -
  U RM                  U5      -  nUR                  SS5      RO                  5       nURQ                  XS5      nU RS                  U5      nUU4$ )
Nr    rH   rV   rU   r   rI   rG   r   rJ   dtype)ptraining)*rp   r~   r   r   viewrw   re   	transposerc   r[   updaterj   rf   ry   rL   rM   chunkrepeatmatmulrh   sqrtrK   r   
functionalsoftmaxfloat32tor   dropoutru   r   ri   sumr   r   r   r   r   r   
contiguousra   r   )r5   r\   r   r   rW   r   r   r   kwargsbsz
target_len_q_lenquery_states
key_statesvalue_statesrU   rV   cache_kwargsattn_weightscausal_masklambda_1lambda_2lambda_fullattn_outputattn_output1attn_output2s                              r7   r=   DiffLlamaAttention.forward   sd    +//1{{=1[[/
{{=1#((T^^T]]S]]^_abc__S1I1I4==Yccdeghi
#((T5M5Mt}}]gghiklm&S#7RUWZ#[ &#&snUL'6'='=jX\XfXfht'u$Jz+D+DE
 /H/HIyy\1!!D"M#**1aA6||L2F2Fq!2LMPTPYPYZ^ZgZgPhh%(Aq2HJ4D4DR4H2H)HIK'+5L }},,\r,WZZ[g[m[mn}},,\T=S=S^b^k^k,l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<ll<>%*[[aQ%G"l"[<%??4+++t~~k/JJ!++Aq1<<>!))#b9kk+.L((r9   )ru   r+   r   re   r,   r|   r   r   r   r   r   r   rj   rz   rw   ry   rc   r   r~   r{   r   r;   NNNFN)r?   r@   rA   rB   __doc__r!   r   intr*   r   rL   Tensortuple
LongTensorr   boolr=   rC   rD   rE   s   @r7   rm   rm   y   s   G j  j8C=  j  jD %0A6R
 2637+/59<)||<) #5<<#=><) !.	<)
 u//0<) "%<) <) !!1!12<) 
u||Xell3XeELL>Q5RR	S<) S<)r9   rm   c                   2  ^  \ rS rSrSrU 4S jr\" SSSS9     SS	\R                  S
\	\R                  \R                  4   S\
\R                     S\
\R                     S\
\   S\S\
\R                     S\	\R                  S4   4S jj5       rSrU =r$ )DiffLlamaFlashAttention2   a>  
DiffLlama flash attention module. This module inherits from `DiffLlamaAttention` as the weights of the module stays
untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
flash attention and deal with padding tokens in case the input contains any of them.
c                 D   > [         TU ]  " U0 UD6  [        5       U l        g r;   )r)   r*   r   _flash_attn_uses_top_left_mask)r5   argsr   r6   s      r7   r*   !DiffLlamaFlashAttention2.__init__   s#    $)&)
 /P.Q+r9   r   r   r   r   Nr\   r   r   rW   r   r   r^   c                 H
   [        U[        5      (       a  [        S5      eUR                  5       u  pn
U R	                  U5      nU R                  U5      nU R                  U5      nUR                  XU R                  U R                  5      R                  SS5      nUR                  XU R                  U R                  5      R                  SS5      nUR                  XU R                  U R                  5      R                  SS5      nUc)  [        R                  S5        U R                  X5      u  pOUu  p[        XX5      u  pUb$  XUS.nUR!                  XU R"                  U5      u  pUR                  SS5      nUR                  SS5      nUR                  SS5      nU R$                  (       a  U R&                  OSnUR(                  nUR*                  R,                  S:w  a  UR*                  R,                  OSnU[.        R0                  :X  a  [.        R2                  " 5       (       aA  [5        [.        S	5      (       a  [.        R6                  " U5      O[.        R8                  " 5       nOR[5        U R:                  S
5      (       a  U R:                  R<                  nO U R                  R>                  R(                  n[        R                  SU S35        URA                  U5      nURA                  U5      nURA                  U5      n[.        RB                  " USSS9u  nnURE                  SSSS5      nURE                  SSSS5      n[G        UUUUU	UU[I        U SS 5      U RJ                  U RL                  S9
n[G        UUUUU	UU[I        U SS 5      U RJ                  U RL                  S9
n[.        RN                  " UU/SS9n[.        RB                  " USSS9u  nn[.        RP                  " [.        RR                  " U RT                  U RV                  -  S[.        R0                  S95      RA                  UR(                  5      n[.        RP                  " [.        RR                  " U RX                  U RZ                  -  S[.        R0                  S95      RA                  UR(                  5      nUU-
  U R\                  -   nUUU-  -
  nSU R\                  -
  U R_                  U5      -  nURa                  XS5      Rc                  5       nU Re                  U5      nUS 4$ )Nz`static` cache implementation is not compatible with `attn_implementation==flash_attention_2` make sure to use `sdpa` in the mean time, and open an issue at https://github.com/huggingface/transformersr    rH   aY  The attention layers in this model are transitioning from computing the RoPE embeddings internally through `position_ids` (2D tensor with the indexes of the tokens), to using externally computed `position_embeddings` (Tuple of tensors, containing cos and sin). In v4.46 `position_ids` will be removed and `position_embeddings` will be mandatory.r           mpscpuget_autocast_dtype_pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .rI   sliding_window)rW   r   r   use_top_left_maskr|   rG   r   )3
isinstancer
   
ValueErrorrp   r~   r   r   r   rw   re   r   rc   rs   rt   
rotary_embr[   r   rj   r   ru   r   devicetyperL   r   is_autocast_enabledhasattrr   get_autocast_gpu_dtyper+   r   weightr   r   r   r   rx   r   r|   rM   ri   r   r   r   r   r   r   r   ra   r   r   )r5   r\   r   r   rW   r   r   r   r   r   r   r   r   r   rU   rV   r   dropout_rateinput_dtypedevice_typetarget_dtypevalue_states1value_states2r   r   r   r   r   r   s                                r7   r=    DiffLlamaFlashAttention2.forward   s    o{33} 
 &**,A{{=1[[/
{{=1
 $((T^^T]]S]]^_abc__S1I1I4==Yccdeghi
#((T5M5Mt}}]gghiklm&G |BHC*HC#7RU#[ &#&nUL'6'='=jX\XfXfht'u$J $--a3))!Q/
#--a315t--C #((2>2E2E2J2Je2Sl))..Y^%--'((** u&:;; ,,[9557  &?@@#{{BB#{{1177 >$ (??<8L#|4J'??<8L',{{<'J$}%,,Q1a8%,,Q1a8/% "4)94@"AAnn
 0% "4)94@"AAnn
 ii| <"E%*[[aQ%G"l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<"[<%??4+++t~~k/JJ!))#b9DDFkk+.D  r9   )r   r   )r?   r@   rA   rB   r   r*   r   rL   r   r   r   r   r   r   r=   rC   rD   rE   s   @r7   r   r      s    R %0A6R
 6:37+/59B!||B! #5<<#=>B! !!1!12	B!
 u//0B! "%B! B! !!1!12B! 
u||T!	"B! SB!r9   r   c                   Z   \ rS rSrSr\" SSSS9     SS\R                  S	\\R                  \R                  4   S
\	\R                     S\	\R                     S\	\   S\S\	\R                     S\\R                  \	\R                     \	\\R                        4   4S jj5       rSrg)DiffLlamaSdpaAttentionis  z
DiffLlama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from
`DiffLlamaAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to
SDPA API.
r   r   r   r   Nr\   r   r   rW   r   r   r^   c           	         UR                  5       u  pnU R                  U5      nU R                  U5      nU R                  U5      nUR	                  XU R
                  U R                  5      R                  SS5      nUR	                  XU R                  U R                  5      R                  SS5      nUR	                  XU R                  U R                  5      R                  SS5      nUu  nn[        XUU5      u  pUb$  UXS.nUR                  XU R                  U5      u  p[        XR                  5      n[        XR                  5      n[        R                  " [        R                   " USSS9SS9nUR#                  SSSS5      nUnUb  US S 2S S 2S S 2S UR$                  S   24   nUR&                  R(                  S:X  a3  Ub0  UR+                  5       nUR+                  5       nUR+                  5       nUS L =(       a    U
S:  n[        R,                  R.                  R1                  UUUUU R2                  (       a  U R4                  OSUS	9n[        R                   " USSS9u  nn[        R6                  " [        R8                  " U R:                  U R<                  -  S[        R>                  S
95      RA                  URB                  5      n[        R6                  " [        R8                  " U RD                  U RF                  -  S[        R>                  S
95      RA                  URB                  5      nUU-
  U RH                  -   nUUU-  -
  nSU RH                  -
  U RK                  U5      -  nUR                  SS5      R+                  5       nUR	                  XS5      nU RM                  U5      nUS 4$ )Nr    rH   r   rI   rG   r   cudar   )	attn_mask	dropout_pr|   r   )'rp   r~   r   r   r   rw   re   r   rc   r[   r   rj   rf   ry   rL   rM   r   r   rK   r   r   r   r   r   scaled_dot_product_attentionr   ru   ri   r   r   r   r   r   r   r   r   r   r   r   )r5   r\   r   r   rW   r   r   r   r   r   r   r   r   r   r   rU   rV   r   r   r|   r   r   r   r   r   r   s                             r7   r=   DiffLlamaSdpaAttention.forward{  sU    &**,A{{=1[[/
{{=1#((T^^T]]S]]^_abc__S1I1I4==Yccdeghi
#((T5M5Mt}}]gghiklm&S#7RUWZ#[ &#&sUL'6'='=jX\XfXfht'u$Jz+D+DE
 /H/HIyy\1!!D"M#**1aA6$%%aA/E1A1A"1E/E&EFK ##v-+2I'224L#..0J'224L  4'5EAI	hh))FF!04d,,3 G 
 &+[[aQ%G"l99UYYt~~'FBV[VcVcdehh
 99UYYt~~'FBV[VcVcdehh
 )D,<,<<"[<%??4+++t~~k/JJ!++Aq1<<>!&&s26kk+.D  r9    r   )r?   r@   rA   rB   r   r   rL   r   r   r   r   r   r   r=   rC   r   r9   r7   r   r   s  s     %0A6R
 2637+/59I!||I! #5<<#=>I! !.	I!
 u//0I! "%I! I! !!1!12I! 
u||Xell3XeELL>Q5RR	SI! SI!r9   r   r   c                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )DiffLlamaRMSNormi  c                    > [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z/
DiffLlamaRMSNorm is equivalent to T5LayerNorm
N)r)   r*   r   r   rL   onesr   variance_epsilon)r5   r,   rq   r6   s      r7   r*   DiffLlamaRMSNorm.__init__  s/     	ll5::k#:; #r9   c                    UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )NrH   rG   T)keepdim)	r   r   rL   r   powmeanrsqrtr   r   )r5   r\   r   variances       r7   r=   DiffLlamaRMSNorm.forward  sw    #))%((7 $$Q',,R,>%H?T?T4T(UU{{]--k:::r9   c                 ^    [        U R                  R                  5       SU R                   3$ )Nz, eps=)r   r   rK   r   )r5   s    r7   
extra_reprDiffLlamaRMSNorm.extra_repr  s*    ))*+6$2G2G1HIIr9   )r   r   )gư>)	r?   r@   rA   rB   r*   r=   r   rC   rD   rE   s   @r7   r   r     s    $;J Jr9   r   )eagerflash_attention_2sdpac                   H  ^  \ rS rSrS\S\4U 4S jjr\" SSSS9      SS	\R                  S
\
\R                     S\
\R                     S\
\   S\
\   S\
\R                     S\
\\R                  \R                  4      S\\   S\R                  4S jj5       rSrU =r$ )DiffLlamaDecoderLayeri  r+   rj   c                 (  > [         TU ]  5         UR                  U l        [        UR                     " XS9U l        [        U5      U l        [        UR                  UR                  S9U l
        [        UR                  UR                  S9U l        g )N)r+   rj   rq   )r)   r*   r,   DIFFLLAMA_ATTENTION_CLASSES_attn_implementation	self_attnr#   mlpr   r   input_layernormpost_attention_layernormr   s      r7   r*   DiffLlamaDecoderLayer.__init__  sw    !--4V5P5PQY_u'/0B0BH[H[\(89K9KQWQdQd(e%r9   r   r   r   r   r\   r   rW   r   r   r   r   r^   c                     Un	U R                  U5      nU R                  " SUUUUUUUS.UD6u  pX-   nUn	U R                  U5      nU R                  U5      nX-   nU$ )N)r\   r   rW   r   r   r   r   r   )r  r	  r  r
  )r5   r\   r   rW   r   r   r   r   r   residualr   s              r7   r=   DiffLlamaDecoderLayer.forward  s     !,,];>> 	
')%+) 3	
 	
 !0 !55mD/ 0r9   )r,   r  r
  r  r	  )NNNFNN)r?   r@   rA   rB   r!   r   r*   r   rL   r   r   r   r   r   r   r   r   r=   rC   rD   rE   s   @r7   r  r    s    f f3 f %0A6R 2637+/$)59KO|| !. u//0	
 "% D> !!1!12 &eELL%,,,F&GH +, 
 Sr9   r  c                   f   ^  \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rU 4S	 jrS
rU =r$ )DiffLlamaPreTrainedModeli  r+   modelTr  r   F)r\   
attentionsc                    > [         TU ]  U5        [        U[        5      (       a  UR                  R
                  R                  SU R                  R                  5        UR                  R
                  R                  SU R                  R                  5        UR                  R
                  R                  SU R                  R                  5        UR                  R
                  R                  SU R                  R                  5        g g )Nr   )r)   _init_weightsr   rm   r   datanormal_r+   r   r   r   r   )r5   moduler6   s     r7   r  &DiffLlamaPreTrainedModel._init_weights$  s    f%f011!!))!T[[-G-GH!!))!T[[-G-GH!!))!T[[-G-GH!!))!T[[-G-GH	 2r9   r   )r?   r@   rA   rB   r!   __annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_supports_flex_attn_can_compile_fullgraph_supports_attention_backendr  rm   _can_record_outputsr  rC   rD   rE   s   @r7   r  r    s^    &*#01#4"5N!"'.(
I Ir9   r  c                      ^  \ rS rSr% \R
                  \S'   SS\4U 4S jjjr\R                  " 5       \
S 5       5       rSrU =r$ )DiffLlamaRotaryEmbeddingi-  inv_freqr+   c                   > [         TU ]  5         [        US5      (       aZ  [        UR                  [
        5      (       a;  UR                  R                  SUR                  R                  S5      5      U l        OSU l        UR                  U l	        UR                  U l
        Xl        [        U R                     U l        U R                  U R                  U5      u  o0l        U R                  SUSS9  U R                   U l        g )Nrope_scaling	rope_typer   defaultr(  F)
persistent)r)   r*   r   r   r*  dictgetr+  rz   max_seq_len_cachedoriginal_max_seq_lenr+   r   rope_init_fnattention_scalingregister_bufferr(  original_inv_freq)r5   r+   r   r(  r6   s       r7   r*   !DiffLlamaRotaryEmbedding.__init__0  s    6>**z&:M:Mt/T/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q((ZeD!%r9   c                 b   U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      R	                  UR
                  5      nUS S 2S S S 24   R                  5       n[        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        R                  " USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " Xf4SS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR	                  UR                   S
9W	R	                  UR                   S
94$ ! , (       d  f       N@= f)Nr   rG   r    r   r   F)r   enabledrH   rI   )r   )r(  floatr`   rK   r   r   r   r   strrL   autocastr   rM   rU   r3  rV   r   )
r5   r<   rW   inv_freq_expandedposition_ids_expandedr   freqsembrU   rV   s
             r7   r=    DiffLlamaRotaryEmbedding.forwardA  sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk^^UC&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d444C'')d444C	 D vvAGGv$cff177f&;;; DCs   $BF  
F.)r3  r+   r0  r5  r1  r2  r+  r;   )r?   r@   rA   rB   rL   r   r  r!   r*   no_gradr   r=   rC   rD   rE   s   @r7   r'  r'  -  s@    ll/ / /" ]]_<  <r9   r'  c                   "  ^  \ rS rSrS\4U 4S jjr\" 5       \       SS\\	R                     S\\	R                     S\\	R                     S\\   S\\	R                     S	\\	R                     S
\\   S\\   S\4S jj5       5       rSrU =r$ )DiffLlamaModeliQ  r+   c           	        > [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr  r+   F)r)   r*   pad_token_idpadding_idx
vocab_sizer   	Embeddingr,   embed_tokens
ModuleListrangenum_hidden_layersr  layersr   r   normr'  r   gradient_checkpointing	post_initr   s      r7   r*   DiffLlamaModel.__init__S  s     !.. ++LL):):F<N<NPTP`P`ammGLVMeMeGfgGf)"65Gfg
 %V%7%7V=P=PQ	2&A&+# 	 hs   C?	input_idsr   rW   r   inputs_embedsr   r   r   r^   c           
      J   US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcD  Ub  UR	                  5       OSn	[
        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      n[        U R                  UUUUUS9n
UnU R                  X5      nU R                  S U R                  R                    H  nU" U4U
UUUUS.UD6nM     U R                  U5      n[        UUS9$ )	Nz:You must specify exactly one of input_ids or inputs_embedsrE  r   r    )r   )r+   input_embedsr   r   r   rW   )r   rW   r   r   r   )last_hidden_stater   )r   rJ  r	   r+   get_seq_lengthrL   arangerK   r   rR   r   r   rN  rM  rO  r   )r5   rS  r   rW   r   rT  r   r   r   past_seen_tokensr   r\   r   decoder_layers                 r7   r=   DiffLlamaModel.forwardc  sR    -t";<YZZ *.*;*;I*FM0*$++>O!CRC^==?de+0<< ]5H5H5K"KTaThTh,N )33A6L(;;&))+%
 &"oomJ![[)H4;;+H+HIM)*) /-$7 M J 		-0&++
 	
r9   )rJ  rP  rN  rO  rG  r   rH  )NNNNNNN)r?   r@   rA   rB   r!   r*   r   r   r   rL   r   r   r   FloatTensorr   r   r   r   r=   rC   rD   rE   s   @r7   rC  rC  Q  s        151537+/5959$(8
E,,-8
 !.8
 u//0	8

 "%8
   1 128
 !!1!128
 D>8
 +,8
 
!8
  8
r9   rC  c                   r  ^  \ rS rSrS/rSS0rSS/S/40rU 4S jr\\	         SS\
\R                     S	\
\R                     S
\
\R                     S\
\   S\
\R                     S\
\R                     S\
\   S\
\R                     S\\\R                  4   S\\   S\4S jj5       5       rSrU =r$ )DiffLlamaForCausalLMi  zlm_head.weightlm_headcolwise_repr\   logitsc                    > [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g r&   )
r)   r*   rC  r  rH  r   r.   r,   r`  rQ  r4   s     r7   r*   DiffLlamaForCausalLM.__init__  sU     #F+
 ++yy!3!3V5F5FUS 	r9   rS  r   rW   r   rT  labelsr   r   logits_to_keepr   r^   c
                 ~   U R                   " SUUUUUUUS.U
D6nUR                  n[        U	[        5      (       a  [	        U	* S5      OU	nU R                  USS2USS24   5      nSnUb)  U R                  " SXU R                  R                  S.U
D6n[        UUUR                  UR                  UR                  S9$ )a,  
Example:

```python
>>> from transformers import AutoTokenizer, DiffLlamaForCausalLM

>>> model = DiffLlamaForCausalLM.from_pretrained("google/diffllama-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/diffllama-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```)rS  r   rW   r   rT  r   r   N)rb  re  rH  )lossrb  r   r\   r  r   )r  rW  r   r   slicer`  loss_functionr+   rH  r   r   r\   r  )r5   rS  r   rW   r   rT  re  r   r   rf  r   outputsr\   slice_indicesrb  rh  s                   r7   r=   DiffLlamaForCausalLM.forward  s    @ ,0:: 	,
)%+')	,
 	,
  118B>SV8W8W~ot4]kmA}a,?@A%%pVt{{OeOepiopD%#33!//))
 	
r9   )r`  r  rH  )	NNNNNNNNr   )r?   r@   rA   rB   _tied_weights_keys_tp_plan_pp_planr*   r   r   r   rL   r   r   r   r]  r   r   r   r   r   r   r=   rC   rD   rE   s   @r7   r_  r_    s0   *+=)H_-z:;H  151537+/59-1$(59348
E,,-8
 !.8
 u//0	8

 "%8
   1 128
 ))*8
 D>8
 !!1!128
 c5<</08
 +,8
 
 8
  8
r9   r_  c                       \ rS rSrSrg)"DiffLlamaForSequenceClassificationi  r   Nr?   r@   rA   rB   rC   r   r9   r7   rr  rr        r9   rr  c                       \ rS rSrSrSrg)DiffLlamaForQuestionAnsweringi  transformerr   N)r?   r@   rA   rB   r  rC   r   r9   r7   rv  rv    s    %r9   rv  c                       \ rS rSrSrg)DiffLlamaForTokenClassificationi  r   Nrs  r   r9   r7   ry  ry    rt  r9   ry  )r  rC  r_  rr  rv  ry  )Nr    )Hrh   typingr   r   rL   r   activationsr   cache_utilsr   r	   r
   
generationr   integrationsr   masking_utilsr   modeling_flash_attention_utilsr   r   modeling_layersr   r   r   r   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   processing_utilsr   utilsr   r   r   r   utils.deprecationr   utils.genericr   configuration_diffllamar!   
get_loggerr?   rs   Moduler#   rP   r[   r   r   rf   rk   rm   r   r   r   r  r  r  r'  rC  r_  rr  rv  ry  __all__r   r9   r7   <module>r     s  0  "   ! ; ; ) 7 / i  P K - & R R 0 / 4 
		H	%299  (6	UU\\ 	U# 	U%,, 	U2b) b)JR!1 R!jR!/ R!j Y'Jryy J (J*  1" +6 +\ I I I4!<ryy !<H K
- K
 K
\ H
3_ H
 H
V	)IKc 	&$?AY &	&CE] 	r9   