
    cCi|+                        S r SSKJrJr  SSKrSSKJr  SSKJr  SSKJ	r	  SSK
Jr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  SSKJrJrJrJrJrJrJr  SSKJr  \R@                  " \!5      r"Sr#Sr$ " S S\RJ                  5      r&S$S jr' " S S\RJ                  5      r( " S S\5      r) " S S\5      r* " S S\5      r+ " S S \5      r, " S! S"\5      r-/ S#Qr.g)%zPyTorch Phi-3 model.    )CallableOptionalN)nn   )ACT2FN)Cache)GenerationMixin)FlashAttentionKwargs)ALL_ATTENTION_FUNCTIONS)Unpack)logging)deprecate_kwarg   )MistralDecoderLayerMistralForCausalLM MistralForSequenceClassificationMistralForTokenClassificationMistralPreTrainedModeleager_attention_forwardrotate_half   )
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )Phi3MLP1   c                    > [         TU ]  5         Xl        [        R                  " UR
                  SUR                  -  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        UR                     U l        g )Nr   Fbias)super__init__configr   Linearhidden_sizeintermediate_sizegate_up_proj	down_projr   
hidden_actactivation_fn)selfr!   	__class__s     _/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/phi3/modular_phi3.pyr    Phi3MLP.__init__2   sn    IIf&8&8!f>V>V:V]bc6#;#;V=O=OV[\#F$5$56    hidden_statesreturnc                     U R                  U5      nUR                  SSS9u  p2X R                  U5      -  nU R                  U5      $ )Nr   dim)r%   chunkr(   r&   )r)   r.   	up_statesgates       r+   forwardPhi3MLP.forward:   sH    %%m4	#//!/4 2 24 88	~~i((r-   )r(   r!   r&   r%   )
__name__
__module____qualname____firstlineno__r    torchFloatTensorr7   __static_attributes____classcell__r*   s   @r+   r   r   1   s,    7)U%6%6 )5;L;L ) )r-   r   c                 N   UR                  U5      nUR                  U5      nUR                  S   nU SSU24   U SUS24   pUSSU24   USUS24   p[        R                  " Xr-  [	        U5      U-  -   U/SS9n[        R                  " X-  [	        U	5      U-  -   U
/SS9nX4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
r1   .Nr2   )	unsqueezeshaper=   catr   )qkcossinposition_idsunsqueeze_dim
rotary_dimq_rotq_passk_rotk_passq_embedk_embeds                r+   apply_rotary_pos_embrS   C   s    ( --
&C
--
&C2Jc;J;&'3
+;)<6c;J;&'3
+;)<6ii%++e*<s*BCVLRTUGii%++e*<s*BCVLRTUGr-   c                   f  ^  \ rS rSrSrSS\S\\   4U 4S jjjr\	" SSSS	9  SS
\
R                  S\\
R                  \
R                  4   S\\
R                     S\\   S\\
R                     S\\   S\\
R                  \\
R                     \\\
R                        4   4S jj5       rSrU =r$ )Phi3Attentionc   z=Multi-headed attention from 'Attention Is All You Need' paperr!   	layer_idxc                 p  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        UR                  U l        U R                  S-  U l
        UR                  U l        SU l        UR                  U R                  -  SUR                  U R                  -  -  -   n[        R                  " UR                  U R                  -  UR
                  SS9U l        [        R                  " UR
                  USS9U l        g )Nhead_dimg      Tr   Fr   )r   r    r!   rW   getattrr#   num_attention_headsrY   num_key_value_headsnum_key_value_groupsscalingattention_dropout	is_causalr   r"   o_projqkv_proj)r)   r!   rW   op_sizer*   s       r+   r    Phi3Attention.__init__f   s    "
F4F4F&JdJd4de$*$>$>&B\B\$\!#)#=#= }}d*!'!9!9,,t}}<qFD^D^aeananDn?ooii : :T]] JFL^L^ejk		&"4"4gEJr-   past_key_valuepast_key_values4.58new_nameversionr.   position_embeddingsattention_maskcache_positionkwargsr/   c           
         UR                   S S n/ UQSPU R                  P7nU R                  U5      n	U R                  R                  U R                  -  n
U	SS U
24   nU	SXU R
                  U R                  -  -   24   nU	SXR
                  U R                  -  -   S 24   nUR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U UUUU4U R                  (       d  SOU R                  U R                   [#        U R                  SS 5      S	.UD6u  nnUR$                  " / UQSP76 R'                  5       nU R)                  U5      nUU4$ )
Nr1   .r   r   )rI   rH   rm   eagerg        sliding_window)dropoutr^   rq   )rD   rY   rb   r!   r[   r\   view	transposerS   updaterW   r   _attn_implementationr   trainingr_   r^   rZ   reshape
contiguousra   )r)   r.   rk   rl   rf   rm   rn   input_shapehidden_shapeqkv	query_posquery_states
key_statesvalue_statesrH   rI   cache_kwargsattention_interfaceattn_outputattn_weightss                       r+   r7   Phi3Attention.forwardu   s    $))#2.88b8$--8mmM*KK33dmmC	3

?+id6N6NQUQ^Q^6^*^^^_
3	,D,Dt}},T T VVW#((6@@AF__\2<<QB
#((6@@AF&#7RU#[ &#&nUL'6'='=jX\XfXfht'u$J(?;;++w6"9$++:Z:Z"[$7
%
  $}}C$2H2HLL"4;;0@$G
%
 
%
!\ "));;;;FFHkk+.L((r-   )
r_   r!   rY   r`   rW   r]   r\   ra   rb   r^   )N)NN)r9   r:   r;   r<   __doc__r   r   intr    r   r=   Tensortupler   
LongTensorr   r
   r7   r?   r@   rA   s   @r+   rU   rU   c   s    GKz Khsm K K %0A6R ,0590)||0) #5<<#=>0) !.	0)
 "%0) !!1!120) -.0) 
u||Xell3XeELL>Q5RR	S0) S0)r-   rU   c                     ^  \ rS rSrS\S\4U 4S jjr\" SSSS9      SS	\R                  S
\
\R                     S\
\R                     S\
\   S\
\   S\
\R                     S\
\\R                  \R                  4      S\\   S\\R"                  \
\\R"                  \R"                  4      4   4S jj5       rSrU =r$ )Phi3DecoderLayer   r!   rW   c                    > [         TU ]  X5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  5      U l
        [        R                  " UR                  5      U l        g )N)r!   rW   )r   r    r!   rU   	self_attnr   mlpr   Dropoutresid_pdropresid_attn_dropoutresid_mlp_dropout)r)   r!   rW   r*   s      r+   r    Phi3DecoderLayer.__init__   sZ    +&fJ6?"$**V-?-?"@!#F,>,>!?r-   re   rf   rg   rh   r.   rl   rJ   	use_cacherm   rk   rn   r/   c                     Un	U R                  U5      nU R                  " SUUUUUUUS.UD6u  pXR                  U5      -   nUn	U R                  U5      nU R	                  U5      nXR                  U5      -   nU$ )N)r.   rl   rJ   rf   r   rm   rk    )input_layernormr   r   post_attention_layernormr   r   )r)   r.   rl   rJ   rf   r   rm   rk   rn   residualself_attn_weightss              r+   r7   Phi3DecoderLayer.forward   s     !,,];+/>> 	,
')%+) 3	,
 	,
( !#:#:=#II 55mD/ #9#9-#HHr-   )r!   r   r   r   r   )NNNFNN)r9   r:   r;   r<   r   r   r    r   r=   r   r   r   r   boolr   r   r
   r>   r7   r?   r@   rA   s   @r+   r   r      s   @z @c @ %0A6R 2637+/$)59KO|| !. u//0	
 "% D> !!1!12 &eELL%,,,F&GH -. 
u  (51B1BEDUDU1U+V"WW	X Sr-   r   c                       \ rS rSrSrSrg)Phi3PreTrainedModel   z0.0.5r   N)r9   r:   r;   r<   _versionr?   r   r-   r+   r   r      s    Hr-   r   c                   ,    \ rS rSr       SS jrSrg)Phi3ForCausalLM   Nc	                    U(       ae  U R                   R                  (       aJ  UR                  S   U R                   R                  S-   :  a   US   n
XR                   R                  ::  a  S n[        R
                  " U 4UUUUUUUUS.U	D6nU$ )Nr   r   )	input_idsrf   rl   inputs_embedsrm   rJ   r   logits_to_keep)r!   rope_scalingrD    original_max_position_embeddingsr	   prepare_inputs_for_generation)r)   r   rf   rl   r   rm   rJ   r   r   rn   past_lengthmodel_inputss               r+   r   -Phi3ForCausalLM.prepare_inputs_for_generation   s    $ (("dkk&R&RUV&VV(+KkkJJJ"&&DD
+)')%)
 
 r-   r   )NNNNNTN)r9   r:   r;   r<   r   r?   r   r-   r+   r   r      s     &r-   r   c                       \ rS rSrSrg)Phi3ForSequenceClassificationi  r   Nr9   r:   r;   r<   r?   r   r-   r+   r   r         r-   r   c                       \ rS rSrSrg)Phi3ForTokenClassificationi  r   Nr   r   r-   r+   r   r     r   r-   r   )r   	Phi3Modelr   r   r   )Nr   )/r   typingr   r   r=   r   activationsr   cache_utilsr   
generationr	   modeling_flash_attention_utilsr
   modeling_utilsr   processing_utilsr   utilsr   utils.deprecationr   mistral.modeling_mistralr   r   r   r   r   r   r   configuration_phi3r   
get_loggerr9   logger_CHECKPOINT_FOR_DOC_CONFIG_FOR_DOCModuler   rS   rU   r   r   r   r   r   __all__r   r-   r+   <module>r      s      %   !   ) B 5 &  0   + 
		H	%8 )bii )$@C)BII C)L(* (V0 '( 'T	$D 		!> 	r-   