
    cCi                    J   S SK r S SKrS SKJr  S SKJrJrJr  S SKrS SKJ	r	J
r
  SSKJr  SSKJrJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJrJrJrJrJr  SSKJrJ r   SSK!J"r"J#r#J$r$J%r%  SSK&J'r'  SSK(J)r)J*r*  SSK+J,r,J-r-J.r.  SSK/J0r0  SSK1J2r2J3r3  SSK4J5r5J6r6  S r7 " S S\
Rp                  5      r9S r:S r; " S S\
Rp                  5      r<  S`S\
Rp                  S\R                  S\R                  S \R                  S!\\R                     S"\=S#\=S$\\R                     S%\'\,   4S& jjr> " S' S(\
Rp                  5      r? " S) S*\
Rp                  5      r@ " S+ S,\
Rp                  5      rAS- rB " S. S/\
Rp                  5      rC " S0 S1\
Rp                  5      rD " S2 S3\5      rE " S4 S5\
Rp                  5      rF " S6 S7\
Rp                  5      rG\- " S8 S9\$5      5       rH " S: S;\H5      rI " S< S=\
Rp                  5      rJ " S> S?\
Rp                  5      rK " S@ SA\
Rp                  5      rL\\- " SB SC\5      5       5       rM " SD SE\
Rp                  5      rN " SF SG\
Rp                  5      rO\" SH5       " SI SJ\
Rp                  5      5       rP " SK SL\
Rp                  5      rQ " SM SN\
Rp                  5      rRSO rSSaSP jrTSQ\R                  SR\USS\R                  4ST jrV " SU SV\
Rp                  5      rW " SW SX\5      rX\- " SY SZ\$5      5       rY " S[ S\\Y5      rZ " S] S^\Y\5      r[/ S_Qr\g)b    N)	dataclass)CallableOptionalUnion)Tensornn   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentionsBaseModelOutputWithPast,BaseModelOutputWithPoolingAndCrossAttentionsCausalLMOutputWithPastModelOutput)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSModuleUtilsMixinPreTrainedModelget_parameter_dtype)Unpack) find_pruneable_heads_and_indicesprune_linear_layer)TransformersKwargsauto_docstringcan_return_tuple)deprecate_kwarg)OutputRecordercheck_model_inputs   )EvollaConfigSaProtConfigc                     U R                  U5      R                  5       n[        R                  " USS9R	                  U5      U-  nUR                  5       U-   $ )z
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's `utils.make_positions`.

Args:
    x: torch.Tensor x:

Returns: torch.Tensor
r%   dim)neinttorchcumsumtype_aslong)	input_idspadding_idxmaskincremental_indicess       d/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/evolla/modeling_evolla.py"create_position_ids_from_input_idsr6   5   sP     <<$((*D,,t3;;DADH##%33    c                   D   ^  \ rS rSrSrU 4S jr    SS jrS rSrU =r	$ )EvollaSaProtEmbeddingsE   zN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                   > [         TU ]  5         [        R                  " UR                  UR
                  UR                  S9U l        UR                  (       a/  [        R                  " UR
                  UR                  S9U l        OS U l        [        R                  " UR                  5      U l        [        USS5      U l        U R#                  S[$        R&                  " UR(                  5      R+                  S5      SS9  UR                  U l        U R                   S:X  a9  [        R                  " UR(                  UR
                  U R,                  S9U l        UR0                  U l        UR2                  U l        S U l        g )	N)r2   epsposition_embedding_typeabsoluteposition_ids)r%   F
persistent)super__init__r   	Embedding
vocab_sizehidden_sizepad_token_idword_embeddingsemb_layer_norm_before	LayerNormlayer_norm_eps
layer_normDropouthidden_dropout_probdropoutgetattrr>   register_bufferr-   arangemax_position_embeddingsexpandr2   position_embeddingstoken_dropoutmask_token_idr@   selfconfig	__class__s     r5   rE   EvollaSaProtEmbeddings.__init__J   s2   !||F,=,=v?Q?Q_e_r_rs'' ll6+=+=6CXCXYDO"DOzz&"<"<='.v7PR\']$ELL)G)GHOOPWXej 	 	
 "..'':5')||..0B0BPTP`P`(D$ $11#11 r7   c                    Uc*  Ub  [        XR                  5      nOU R                  U5      nUc  U R                  U5      nUnU R                  (       a  Ub  UR                  XR                  :H  R                  S5      S5      nSnUb  UR                  S5      OUR                  S   nXR                  :H  R                  S5      R                  5       U-  nUSU-
  -  SU-
  S S 2S S 4   -  R                  UR                  5      nU R                  S:X  a  U R                  U5      n	XY-   nU R                  b  U R                  U5      nUb,  XRR                  S5      -  R                  UR                  5      nU$ )NrA           gQ?r%   r?   )r6   r2   &create_position_ids_from_inputs_embedsrJ   rX   masked_fillrY   	unsqueezesumshapefloattodtyper>   rW   rN   )
r[   r1   attention_maskr@   inputs_embeds
embeddingsmask_ratio_trainsrc_lengthsmask_ratio_observedrW   s
             r5   forwardEvollaSaProtEmbeddings.forwardc   s    $A)M]M]^#JJ=Y  00;M #
 )"7#//>P>P1P0[0[\^0_adeJ)4B4N.,,R0T]TcTcdeTfK#,0B0B#B"G"G"K"Q"Q"SVa"a$,<(<=EXAXZ[]acgZg@hhll  J '':5"&":":<"H#9J??&4J%$'?'?'CCGG
HXHXYJ r7   c                    UR                  5       SS nUS   n[        R                  " U R                  S-   X0R                  -   S-   [        R                  UR
                  S9nUR                  S5      R                  U5      $ )z
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
NrA   r%   rh   devicer   )sizer-   rT   r2   r0   rs   rc   rV   )r[   rj   input_shapesequence_lengthr@   s        r5   ra   =EvollaSaProtEmbeddings.create_position_ids_from_inputs_embeds   s~     $((*3B/%a.||q /4D4D"Dq"HPUPZPZcpcwcw
 %%a(//<<r7   )	rQ   rN   rY   r2   r>   rW   r@   rX   rJ   NNNN)
__name__
__module____qualname____firstlineno____doc__rE   ro   ra   __static_attributes____classcell__r]   s   @r5   r9   r9   E   s+    !6 /b= =r7   r9   c                 V    U R                  SSS9u  p[        R                  " U* U4SS9$ )N   rA   r)   )chunkr-   catxx1x2s      r5   rotate_half_esmr      s-    WWQBWFB99rc2YB''r7   c                     US S 2S S 2S U R                   S   2S S 24   nUS S 2S S 2S U R                   S   2S S 24   nX-  [        U 5      U-  -   $ )N)re   r   )r   cossins      r5   apply_rotary_pos_emb_esmr      sW    
aMaggbkM1$
%C
aMaggbkM1$
%CG*S011r7   c                      ^  \ rS rSr% Sr\R                  \S'   S\4U 4S jjr	SS jr
S\R                  S\R                  S	\\R                  \R                  4   4S
 jrSrU =r$ )EvollaSaProtRotaryEmbedding   z
Rotary position embeddings based on those in
[RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation
matrices which depend on their relative positions.
inv_freqr*   c           	         > [         TU ]  5         SS[        R                  " SUS[        R                  S9R                  5       U-  -  -  nU R                  SU5        S U l        S U l        S U l	        g )N      ?i'  r   r   rh   r   )
rD   rE   r-   rT   int64rf   rS   _seq_len_cached_cos_cached_sin_cached)r[   r*   r   r]   s      r5   rE   $EvollaSaProtRotaryEmbedding.__init__   sg    %ELLC%++$N$T$T$VY\$\]^Z2#r7   c                 j   UR                   U   nX0R                  :w  d$  U R                  R                  UR                  :w  a  X0l        [        R
                  " UR                   U   UR                  S9R                  U R                  5      n[        R                  " X@R                  5      n[        R                  " XU4SS9R                  UR                  5      nUR                  5       S S S S 2S S 24   U l        UR                  5       S S S S 2S S 24   U l        U R                  U R                  4$ )Nrs   rA   r)   )re   r   r   rs   r-   rT   r/   r   outerr   rg   r   r   r   )r[   r   seq_dimensionseq_lentfreqsembs          r5   _update_cos_sin_tables2EvollaSaProtRotaryEmbedding._update_cos_sin_tables   s    ''-( ***d.>.>.E.E.Q#* QWW]3AHHEMMdmm\AKK==1E))UN366qxx@C"wwytQ)9:D"wwytQ)9:D!1!111r7   qkreturnc                    U R                  USS9u  U l        U l        [        XR                  U R                  5      R	                  UR
                  S9[        X R                  U R                  5      R	                  UR
                  S94$ )Nr   )r   r   )r   r   r   r   rg   rh   )r[   r   r   s      r5   ro   #EvollaSaProtRotaryEmbedding.forward   s    -1-H-HZ\-H-]*$* %Q(8(8$:J:JKNNUVU\U\N]$Q(8(8$:J:JKNNUVU\U\N]
 	
r7   )r   r   r   )r   )ry   rz   r{   r|   r}   r-   r   __annotations__r,   rE   r   tuplero   r~   r   r   s   @r5   r   r      s^     ll C  2 
 
%,, 
5u||A[;\ 
 
r7   r   modulequerykeyvalueri   scalingrQ   	head_maskkwargsc                    [         R                  " XR                  SS5      5      U-  n	[        U S5      (       GaI  U R                  S;   Ga8  UR
                  S   n
[         R                  " U
[         R                  U	R                  S9R                  SS5      n[         R                  " U
[         R                  U	R                  S9R                  SS5      nX-
  nU R                  XR                  -   S-
  5      nUR                  UR                  S9nU R                  S	:X  a  [         R                  " S
X5      nOCU R                  S:X  a3  [         R                  " S
X5      n[         R                  " SX.5      nUU-   nU	W-   n	Ub#  US S 2S S 2S S 2S UR
                  S   24   nU	U-   n	[        R                   R#                  U	S[         R$                  S9R                  UR                  5      n	[        R                   R'                  XU R(                  S9n	Ub  X-  n	[         R                  " X5      nUR                  SS5      R+                  5       nUU	4$ )Nr   r	   r>   relative_keyrelative_key_queryrr   rA   r%   r   r   zbhld,lrd->bhlrr   zbhrd,lrd->bhlrr   )r*   rh   )ptraining)r-   matmul	transposehasattrr>   re   rT   r0   rs   viewdistance_embeddingrU   rg   rh   einsumr   
functionalsoftmaxfloat32rQ   r   
contiguous)r   r   r   r   ri   r   rQ   r   r   attn_weights
seq_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keycausal_maskattn_outputs                       r5   eager_attention_forwardr      s    <<}}Q':;gELv011f6T6T Y 7 [[^
j

<K^K^_ddegijkj

<K^K^_ddefhjk!2%88DbDb9bef9fg366U[[6I))^;',||4De'b$++/CC-2\\:JE-h*+0<<8H#+d('EHd'd$#&>>!$Q1o		"o%=>#k1==((2U]](SVVW\WbWbcL==((6??([L#/,,|3K''1-88:K$$r7   c                      ^  \ rS rSrSU 4S jjr    SS\R                  S\\R                     S\\R                     S\\R                     S\\R                     S\	\
   S	\\R                     4S
 jjrSrU =r$ )EvollaSaProtSelfAttentioni  c                   > [         TU ]  5         Xl        UR                  UR                  -  S:w  a7  [        US5      (       d&  [        SUR                   SUR                   S35      eUR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l	        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  5      U l        [        R                  " UR                  U R                  5      U l        UR                  U l        U=(       d    [#        USS5      U l        S U l        U R$                  S:X  d  U R$                  S	:X  aH  UR(                  U l        [        R*                  " S
UR(                  -  S-
  U R                  5      U l        O(U R$                  S:X  a  [/        U R                  S9U l        UR0                  U l        X0l        SU l        U R0                  =(       a    U(       + U l        g )Nr   embedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()r>   r?   r   r   r   r%   rotaryr)   r   )rD   rE   r\   rH   num_attention_headsr   
ValueErrorr,   attention_head_sizeall_head_sizer   Linearr   r   r   attention_probs_dropout_probrQ   rR   r>   rotary_embeddingsrU   rF   r   r   
is_decoder	layer_idxr   	is_causal)r[   r\   r>   r   is_cross_attentionr]   s        r5   rE   "EvollaSaProtSelfAttention.__init__  s    : ::a?PVXhHiHi#F$6$6#7 8 445Q8 
 $*#=#= #&v'9'9F<V<V'V#W !558P8PPYYv1143E3EF
99V//1C1CDYYv1143E3EF
::'> (
'-zC
$ "&''>9T=Y=Y]q=q+1+I+ID(&(ll1v7U7U3UXY3Y[_[s[s&tD#))X5%@TE]E]%^D" ++"C1C-Cr7   hidden_statesri   r   encoder_hidden_statesencoder_attention_maskr   r   c                    UR                   S S u  pxXxSU R                  4n	U R                  U5      R                  U	5      R	                  SS5      n
US LnU(       a  UOUnU(       a  UOUnU R                  U5      R                  U	5      R	                  SS5      nU R                  U5      R                  U	5      R	                  SS5      nXR                  S-  -  n
U R                  S:X  a  U R                  X5      u  p[        nU R                  R                  S:w  a]  U R                  S;   a0  [        SU R                  R                   S	U R                   S
35      e[        U R                  R                     nU" U U
UUU4U R                  (       d  SOU R                  U R                   US.UD6u  nnUR#                  XxS5      R%                  5       nUU4$ )NrA   r%   r         r   eagerr   zESM z attention does not support z^ embeddings. Set attention explicitly to 'eager' with `model.set_attn_implementation('eager')`r`   )rQ   r   r   )re   r   r   r   r   r   r   r>   r   r   r\   _attn_implementationr   r   r   rQ   r   reshaper   )r[   r   ri   r   r   r   r   
batch_sizer   hidden_shapequery_layerr   current_states	key_layervalue_layerattention_interfacer   r   s                     r5   ro   !EvollaSaProtSelfAttention.forward3  s    "/!4!4Sb!9
"D4L4LMjj/44\BLLQPQR2$>2D.-3E/>HH^,11,?II!QO	jj055lCMMaQRS "$<$<d$BB''83%)%;%;K%S"K(?;;++w6++/UU 4;;;;<<XY]YuYuXv wh h  #:$++:Z:Z"[$7
%
  $}}C$,,LL
%
 
%
!\ "))*"EPPRL((r7   )r   r   r\   r   rQ   r   r   r   r   rU   r   r>   r   r   r   r   )NNFrx   )ry   rz   r{   r|   rE   r-   r   r   FloatTensorr   r   r   ro   r~   r   r   s   @r5   r   r     s     DJ 7;15=A>B3)||3) !!2!233) E--.	3)
  ((9(9:3) !)):): ;3) +,3) 
u||	3) 3)r7   r   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )EvollaSaProtSelfOutputii  c                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        g N)	rD   rE   r   r   rH   denserO   rP   rQ   rZ   s     r5   rE   EvollaSaProtSelfOutput.__init__j  sB    YYv1163E3EF
zz&"<"<=r7   c                 R    U R                  U5      nU R                  U5      nX-   nU$ r   r   rQ   r[   r   input_tensors      r5   ro   EvollaSaProtSelfOutput.forwardo  ,    

=1]3%4r7   r   ry   rz   r{   r|   rE   ro   r~   r   r   s   @r5   r   r   i      >
 r7   r   c                   R   ^  \ rS rSrSU 4S jjrS r    SS\\   4S jjrSr	U =r
$ )	EvollaSaProtAttentioniv  c                    > [         TU ]  5         [        XUS9U l        [	        U5      U l        [        5       U l        [        R                  " UR                  UR                  S9U l	        g )N)r   r   r<   )rD   rE   r   r[   r   outputsetpruned_headsr   rL   rH   rM   )r[   r\   r   r   r]   s       r5   rE   EvollaSaProtAttention.__init__w  sQ    -f^pq	,V4Ef&8&8f>S>STr7   c                 6   [        U5      S:X  a  g [        XR                  R                  U R                  R                  U R
                  5      u  p[        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l        [        U R                  R                  U5      U R                  l	        [        U R                  R                  USS9U R                  l        U R                  R                  [        U5      -
  U R                  l        U R                  R                  U R                  R                  -  U R                  l        U R
                  R                  U5      U l        g )Nr   r%   r)   )lenr   r[   r   r   r  r   r   r   r   r  r   r   union)r[   headsindexs      r5   prune_heads!EvollaSaProtAttention.prune_heads~  s   u:?79900$))2O2OQUQbQb

 -TYY__eD		*499==%@		,TYY__eD		.t{{/@/@%QO )-		(E(EE
(R		%"&))"?"?$))B_B_"_		 --33E:r7   r   c                 ~    U R                  U5      nU R                  " U4UUUUS.UD6u  pU R                  X5      nU$ )Nri   r   r   r   )rL   r[   r  )
r[   r   ri   r   r   r   r   hidden_states_lnr   _s
             r5   ro   EvollaSaProtAttention.forward  sW      >>-8
)"7#9
 
 kk+=r7   )rL   r  r  r[   )NFrx   )ry   rz   r{   r|   rE   r  r   r   ro   r~   r   r   s   @r5   r  r  v  s6    U;* "# +, r7   r  c                 n    U S-  S[         R                  " U [        R                  " S5      -  5      -   -  $ )zr
This is the gelu implementation from the original EVOLLA_SA_PROT repo. Using F.gelu yields subtly wrong results.
g      ?r   g       @)r-   erfmathsqrt)r   s    r5   gelur    s.     s7cEIIa$))C.&899::r7   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )EvollaSaProtIntermediatei  c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        g r   )rD   rE   r   r   rH   intermediate_sizer   rZ   s     r5   rE   !EvollaSaProtIntermediate.__init__  s,    YYv1163K3KL
r7   r   r   c                 >    U R                  U5      n[        U5      nU$ r   )r   r  )r[   r   s     r5   ro    EvollaSaProtIntermediate.forward  s     

=1]+r7   )r   
ry   rz   r{   r|   rE   r-   r   ro   r~   r   r   s   @r5   r  r    s)    MU\\ ell  r7   r  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )EvollaSaProtOutputi  c                    > [         TU ]  5         [        R                  " UR                  UR
                  5      U l        [        R                  " UR                  5      U l	        g r   )
rD   rE   r   r   r  rH   r   rO   rP   rQ   rZ   s     r5   rE   EvollaSaProtOutput.__init__  sB    YYv779K9KL
zz&"<"<=r7   c                 R    U R                  U5      nU R                  U5      nX-   nU$ r   r   r   s      r5   ro   EvollaSaProtOutput.forward  r   r7   r   r   r   s   @r5   r!  r!    r   r7   r!  c                   N   ^  \ rS rSrU 4S jr    SS\\   4S jjrS rSr	U =r
$ )EvollaSaProtLayeri  c                   > [         TU ]  5         UR                  U l        SU l        [	        U5      U l        UR                  U l        UR                  U l        U R                  (       a.  U R                  (       d  [        U  S35      e[	        USS9U l	        [        U5      U l        [        U5      U l        [        R                  " UR                   UR"                  S9U l        g )Nr%   z> should be used as a decoder model if cross attention is addedT)r   r<   )rD   rE   chunk_size_feed_forwardseq_len_dimr  	attentionr   add_cross_attentionRuntimeErrorcrossattentionr  intermediater!  r  r   rL   rH   rM   rZ   s     r5   rE   EvollaSaProtLayer.__init__  s    '-'E'E$.v6 ++#)#=#= ##??"dV+i#jkk"7SW"XD4V<(0f&8&8f>S>STr7   r   c                     U R                   " U4UUS.UD6nU R                  (       a;  Ub8  [        U S5      (       d  [        SU  S35      eU R                  " U4UUUUS.UD6nU R                  U5      nU$ )N)ri   r   r.  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r  )r+  r   r   AttributeErrorr.  feed_forward_chunk)	r[   r   ri   r   r   r   r   attention_outputlayer_outputs	            r5   ro   EvollaSaProtLayer.forward  s      >>
)
 	
 ??4@4!122$=dV D` ` 
  $22  -#&;'=    ../?@r7   c                 l    U R                  U5      nU R                  U5      nU R                  X15      nU$ r   )rL   r/  r  )r[   r4  attention_output_lnintermediate_outputr5  s        r5   r3  $EvollaSaProtLayer.feed_forward_chunk  s9    "nn-=>"//0CD{{#6Ir7   )	rL   r,  r+  r)  r.  r/  r   r  r*  rx   )ry   rz   r{   r|   rE   r   r   ro   r3  r~   r   r   s   @r5   r'  r'    s7    U$ "#! +,!F r7   r'  c                   R   ^  \ rS rSrU 4S jr\    SS\\   4S jj5       rSr	U =r
$ )EvollaSaProtEncoderi  c                 2  > [         TU ]  5         Xl        [        R                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l        [        R                  " UR                  UR                  S9U l        SU l        g s  snf )Nr<   F)rD   rE   r\   r   
ModuleListrangenum_hidden_layersr'  layerrL   rH   rM   emb_layer_norm_aftergradient_checkpointing)r[   r\   r  r]   s      r5   rE   EvollaSaProtEncoder.__init__   sr    ]]uVMeMeGf#gGf!$5f$=Gf#gh
$&LL1C1CI^I^$_!&+# $hs   Br   c           	          [        U R                  5       H  u  pxUb  X7   OS n	U" U4UU	UUS.UD6nM     U R                  (       a  U R                  U5      n[        US9$ )Nr  )last_hidden_state)	enumeraterA  rB  r   )
r[   r   ri   r   r   r   r   ilayer_modulelayer_head_masks
             r5   ro   EvollaSaProtEncoder.forward  su      )4OA.7.CilO(-)&;'= M  5 $$ 55mDM1MRRr7   )r\   rB  rC  rA  rx   )ry   rz   r{   r|   rE   r!   r   r   ro   r~   r   r   s   @r5   r<  r<    s=    ,  "#S +,S Sr7   r<  c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )EvollaSaProtPooleri"  c                    > [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " 5       U l        g r   )rD   rE   r   r   rH   r   Tanh
activationrZ   s     r5   rE   EvollaSaProtPooler.__init__#  s9    YYv1163E3EF
'')r7   r   r   c                 \    US S 2S4   nU R                  U5      nU R                  U5      nU$ )Nr   )r   rP  )r[   r   first_token_tensorpooled_outputs       r5   ro   EvollaSaProtPooler.forward(  s6     +1a40

#566r7   )rP  r   r  r   s   @r5   rM  rM  "  s(    $
U\\ ell  r7   rM  c                   `    \ rS rSr% \\S'   S/rSrSrSr	\
\" \SSS9/\" \SSS9/S	.rS
 rSrg)EvollaSaProtPreTrainedModeli1  r\   r'  Tr%   r+  )r  
layer_namer.  )r   
attentionscross_attentionsc                    U R                   R                  n[        U[        R                  5      (       aW  UR
                  R                  R                  SUS9  UR                  b%  UR                  R                  R                  5         gg[        U[        R                  5      (       ad  UR
                  R                  R                  SUS9  UR                  b2  UR
                  R                  UR                     R                  5         gg[        U[        R                  5      (       aJ  UR                  R                  R                  5         UR
                  R                  R                  S5        gg)zInitialize the weightsr`   meanstdNr   )r\   initializer_range
isinstancer   r   weightdatanormal_biaszero_rF   r2   rL   fill_)r[   r   r^  s      r5   _init_weights)EvollaSaProtPreTrainedModel._init_weightsA  s   kk++fbii((MM&&CS&9{{&  &&( '--MM&&CS&9!!-""6#5#56<<> .--KK""$MM$$S) .r7    N)ry   rz   r{   r|   r'   r   _no_split_modules_supports_flash_attn_supports_sdpa_supports_attention_backendr'  r#   r   _can_record_outputsrg  r~   ri  r7   r5   rW  rW  1  sX    ,-N"& +%&?qU`ab4AJZ[
*r7   rW  c                   *  ^  \ rS rSrS\4U 4S jjrS rS rS r\	" 5        SS\
\R                     S\
\R                     S	\\\R                     \4   4S
 jj5       r  SS\S\\   S\
\R$                     S\
\R&                     S	\4
S jjrSrU =r$ )EvollaSaProtProteinEncoderiQ  r\   c                 d   > [         TU ]  U5        [        U5      U l        [	        U5      U l        g r   )rD   rE   r9   rk   r<  encoderrZ   s     r5   rE   #EvollaSaProtProteinEncoder.__init__R  s(     08*62r7   c                 .    U R                   R                  $ r   rk   rJ   r[   s    r5   get_input_embeddings/EvollaSaProtProteinEncoder.get_input_embeddingsW  s    ...r7   c                 $    XR                   l        g r   ru  r[   r   s     r5   set_input_embeddings/EvollaSaProtProteinEncoder.set_input_embeddingsZ  s    */'r7   c                     UR                  5        H7  u  p#U R                  R                  U   R                  R	                  U5        M9     g)z
Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
class PreTrainedModel
N)itemsrr  rA  r+  r  )r[   heads_to_prunerA  r
  s       r5   _prune_heads'EvollaSaProtProteinEncoder._prune_heads]  s<    
 +002LELLu%//;;EB 3r7   r1   ri   r   c                 0   UR                  5       nUu  pEUR                  nUc  [        R                  " XE4US9nU R	                  XS9nU R                  X#5      nU R                  XxS9n	U	S   n
[        U
U	R                  U	R                  U	R                  S9$ )Nr   r1   ri   )ri   r   )rF  r   rY  rZ  )rt   rs   r-   onesrk   get_extended_attention_maskrr  r   r   rY  rZ  )r[   r1   ri   ru   r   r   rs   rj   extended_attention_maskencoder_outputssequence_outputs              r5   ro   "EvollaSaProtProteinEncoder.forwarde  s      nn&!,
!!!"ZZ*)A6RN)["&"B"B>"_,,},])!,;-)77&11,==	
 	
r7   ru   rs   rh   c                 P   Uc  [        U 5      nUR                  5       S:X  a  U R                  R                  (       d  Ub  [        R
                  " S[        5        UR                  5       S:X  a  USS2SSS2SS24   nOqUR                  5       S:X  aA  U R                  R                  (       a  [        R                  " X!U5      nO*USS2SSSS24   nO[        SU SUR                   S35      eUR                  US9nS	U-
  [        R                  " U5      R                  -  nU$ )
a  
Makes broadcastable attention and causal masks so that future and masked tokens are ignored.

Arguments:
    attention_mask (`torch.Tensor`):
        Mask with ones indicating tokens to attend to, zeros for tokens to ignore.
    input_shape (`Tuple[int]`):
        The shape of the input to the model.

Returns:
    `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`.
Nr   zNThe `device` argument is deprecated and will be removed in v5 of Transformers.r	   z!Wrong shape for input_ids (shape z) or attention_mask (shape r   r   r   )r   r*   r\   r   warningswarnFutureWarningr   *create_extended_attention_mask_for_decoderr   re   rg   r-   finfomin)r[   ri   ru   rs   rh   r  s         r5   r  6EvollaSaProtProteinEncoder.get_extended_attention_mask~  s    & ='-E""$)dkk.D.D!dfs
 1$&4Qa]&C#!Q& {{%%*:*e*e+' +9D$9I*J'3K=@[\j\p\p[qqrs  #:"<"<5"<"I#&)@#@EKKPUDVDZDZ"Z&&r7   )rk   rr  r   NN)ry   rz   r{   r|   r'   rE   rw  r{  r  r$   r   r-   r   r   r   r   ro   r,   rs   rh   r  r~   r   r   s   @r5   rp  rp  Q  s    3| 3
/0C  26
ELL)
 !.
 
uU\\"$PP	Q	
 
8 *.'+6'6' 3Z6' &	6'
 $6' 
6' 6'r7   rp  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )!EvollaSequenceCompressorAttentioni  c                 X  > [         TU ]  5         US-  U l        X0l        X#-  n[        R
                  " U5      U l        [        R
                  " U5      U l        [        R                  " XSS9U l	        [        R                  " XS-  SS9U l
        [        R                  " XASS9U l        g )Nr   Frd  r   )rD   rE   scaler
  r   rL   
norm_medianorm_latentsr   to_qto_kvto_out)r[   r*   dim_headr
  	inner_dimr]   s        r5   rE   *EvollaSequenceCompressorAttention.__init__  s    t^

$	,,s+LL-IIc59	YYsM>
ii	U;r7   c                 &   U R                  U5      nU R                  U5      nU R                  nU R                  U5      n[        R
                  " X4SS9nU R                  U5      R                  SSS9u  pxUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nXPR                  -  n[        R                  " XWR                  SS5      5      n	XR                  SSS	9R                  5       -
  n	U	R                   u  pp[        R"                  " X5      R%                  UR&                  5      nUS
S
2S
S
S
S
24   nUS
S
S
2S
S
2S
4   nUU-  nU	R)                  SU-
  R+                  5       S5      n	U	R-                  SS9n[        R                  " UU5      nUR                  SSSS5      nUR/                  UR                  S5      UR                  S5      S5      nU R1                  U5      $ )z
Args:
    x (torch.Tensor): image features
        shape (b, n1, D)
    latent (torch.Tensor): latent features
        shape (b, n2, D);  n2: num of latent tokens
r   r)   r   rA   r   r%   r	   Tr*   keepdimNg     )r  r  r
  r  r-   r   r  r   r   rt   permuter  r   r   amaxdetachre   r  rg   rs   rb   boolr   r   r  )r[   r   latentsr3   hr   kv_inputr   vsimbsnhskdokdr  mask_expones_expattnouts                      r5   ro   )EvollaSequenceCompressorAttention.forward  s2    OOA##G,JJIIg99a\r2zz(#))2 * 
 FF166!9affQiB/771aCFF166!9affQiB/771aCFF166!9affQiB/771aC

N ll1kk"b12HHTH299;;99zz""%%dkk24q()aD()("ooq4xoo/6{{r{"ll4#kk!Q1% kk#((1+sxx{B7{{3r7   )r
  r  r  r  r  r  r  )@      r   r   s   @r5   r  r    s    <)  ) r7   r  c                   2   ^  \ rS rSrSU 4S jjrS rSrU =r$ )EvollaFeedForwardi  c                   > [         TU ]  5         [        X-  5      n[        R                  " U5      U l        [        R                  " XSS9U l        [        R                  " 5       U l	        [        R                  " X1SS9U l
        g NFr  )rD   rE   r,   r   rL   normr   fc1GELUrP  fc2)r[   r*   multr  r]   s       r5   rE   EvollaFeedForward.__init__  sZ    
O	LL%	99S%8'')99Y%8r7   c           	      ~    U R                  U R                  U R                  U R                  U5      5      5      5      $ r   )r  rP  r  r  )r[   r   s     r5   ro   EvollaFeedForward.forward  s+    xx1(>?@@r7   )rP  r  r  r  )   r   r   s   @r5   r  r    s    9A Ar7   r  c                   6   ^  \ rS rSrS\4U 4S jjrS rSrU =r$ )!EvollaSequenceCompressorResampleri  r\   c                   > [         TU ]  5         UR                  R                  nUR                  U l        [        R                  " [        R                  " U R
                  U5      SS9U l
        [        R                  " / 5      U l        [        UR                  5       Ha  nU R                  R                  [        R                  " [!        X!R"                  UR$                  S9['        X!R(                  S9/5      5        Mc     [        R*                  " UR                  5      U l        [        R.                  " X!R                  5      U l        g )NT)requires_grad)r*   r  r
  )r*   r  )rD   rE   protein_encoder_configrH   resampler_num_latentsnum_latentsr   	Parameterr-   randnr  r>  layersr?  resampler_depthappendr  resampler_dim_headresampler_headsr  resampler_ff_multrL   r  r   protein_projector)r[   r\   protein_repr_dimr  r]   s       r5   rE   *EvollaSequenceCompressorResampler.__init__   s    !88DD!77||EKK0@0@BR$ScghmmB'v--.AKK9 0;T;T\b\r\r *.>E]E]^		 / LL!3!34	!#+;=O=O!Pr7   c                 d   UR                   S   nUR                   u  pE[        R                  " X@R                  5      R	                  UR
                  5      n[        R                  " X&4SS9n[        R                  " U5      R	                  U R                  R
                  5      nU R                  S    UR                  SSS5      -  nUR	                  UR                  5      nU R                   H  u  pU	" XU5      U-   nU
" U5      U-   nM     U R                  U5      nU R                  U5      $ )Nr   r%   r)   rA   )re   r-   r  r  rg   rs   r   r  r   rh   r  r  r  )r[   embedsr3   br  r  latent_maskr  r  r  fftransformed_features               r5   ro   )EvollaSequenceCompressorResampler.forward  s    LLO

jj%5%5699$++Fyy$,!4 zz!} 3 34,,t$tyyQ'::**V\\*HD6D1G;GkG+G $ #44W=yy,--r7   )r  r  r  r  r  )	ry   rz   r{   r|   r&   rE   ro   r~   r   r   s   @r5   r  r    s    Q| Q*. .r7   r  c                       \ rS rSr% Sr\\R                     \S'   Sr	\\R                     \S'   Sr
\\\R                  S4      \S'   Sr\\\R                  S4      \S'   Srg)	EvollaProteinEncoderModelOutputi)  Nsequence_compressor_outputrF  .r   rY  ri  )ry   rz   r{   r|   r  r   r-   r   r   rF  r   r   rY  r~   ri  r7   r5   r  r  )  so     ?C):): ;B59x 1 129=AM8E%"3"3S"89:A:>Ju00#567>r7   r  c                   t   ^  \ rS rSrS\4U 4S jjr\S\R                  S\R                  4S j5       r
SrU =r$ )EvollaProteinEncoderi2  r\   c                 n   > [         TU ]  5         [        UR                  S9U l        [        US9U l        g )Nr\   )rD   rE   rp  r  modelr  sequence_compressor_resamplerrZ   s     r5   rE   EvollaProteinEncoder.__init__3  s.    /v7T7TU
-NV\-]*r7   r1   ri   c                     U R                  XS9nUR                  nU R                  XR5      n[        UUR                  S9$ )Nr  )r  rF  )r  rF  r  r  )r[   r1   ri   r   protein_outputprotein_embedssequence_reprs          r5   ro   EvollaProteinEncoder.forward8  sF    iW'99::>Z.'4,>>
 	
r7   )r  r  )ry   rz   r{   r|   r&   rE   r!   r-   
LongTensorr   ro   r~   r   r   s   @r5   r  r  2  s?    ^| ^
 
!1!1 
5CTCT 
 
r7   r  c                      ^  \ rS rSr   SS\\   S\\   S\\   4U 4S jjjrS r\" SSS	S
9       SS j5       r	Sr
U =r$ )#EvollaSequenceAlignerCrossAttentioniD  protein_encoder_dimstructure_encoder_dimmsa_encoder_dimc                   > [         TU ]  5         UR                  U l        UR                  U l        U R                  S-  U l        [        U R                  U R                  -  5      U l        U R                  U R                  -  U l        UR                  nUR                  nUR                  n[        R                  " U R                  U R                  5      U l        UbK  [        R                  " X R                  5      U l        [        R                  " X R                  5      U l        OS U l        S U l        UbK  [        R                  " X0R                  5      U l        [        R                  " X0R                  5      U l        OS U l        S U l        UbK  [        R                  " X@R                  5      U l        [        R                  " X@R                  5      U l        OS U l        S U l        [)        U R                  5      U l        [        R,                  " U5      U l        [        R                  " U R                  U R                  US9U l        [3        U R                  U5      U l        [        R6                  " [8        R:                  " S/5      5      U l        [        R6                  " [8        R:                  " S/5      5      U l        g )Nr   r  r`   ) rD   rE   rH   r   r  r,   r   r   $aligner_attention_probs_dropout_probaligner_enable_biasaligner_ffn_multr   r   r   key_proteinvalue_proteinkey_structurevalue_structurekey_msa	value_msaEvollaRMSNormattention_normrO   rQ   out_projr  r  r  r-   tensorgate_attentiongate_ffw)	r[   r\   r  r  r  r   enable_biasffn_multr]   s	           r5   rE   ,EvollaSequenceAlignerCrossAttention.__init__E  s    	!--#)#=#= --t3
#&t'7'7$:R:R'R#S !558P8PP'-'R'R$00**YYt//1C1CD
*!yy)<>P>PQD!#+>@R@R!SD#D!%D ,!#+@BTBT!UD#%99-BDVDV#WD !%D#'D &99_6H6HIDLYY8J8JKDNDL!DN+D,<,<=zz">?		$"2"2D4D4D;W#D$4$4h? ll5<<+>?U\\3%%89r7   c	                    XgU/n	U	 V
s/ s H	  oc  M  U
PM     n	n
U	(       d  [        S5      e[        R                  " U	SS9n	U R                  U5      nU R	                  U5      nU R
                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnXU/nU V
s/ s H	  oc  M  U
PM     nn
[        R                  " USS9nXU/nU V
s/ s H	  oc  M  U
PM     nn
[        R                  " USS9nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nXR"                  -  nUcN  [        R$                  " UR                  S5      UR                  S5      5      R                  UR&                  5      nUSS2SSS2S4   U	SS2SSSS24   -  n[        R(                  " UUR+                  SS	5      5      nUUR-                  SS
S9R/                  5       -
  nUR1                  SU-
  R3                  5       [        R4                  " UR6                  5      R8                  5      n[:        R<                  " SS9" U5      n[        R(                  " UU5      nUR!                  SSSS5      R?                  5       nUR                  5       SS	 U R@                  4-   nUR                  " U6 nU RC                  U5      nU$ s  sn
f s  sn
f s  sn
f )z
query_states: text
key_value_states: protein
query_states: [bs, query_seq_len, dim]
key_value_states: [bs, kv_seq_len, dim]
query_attn_mask: [bs, query_seq_len]
kv_attn_mask: [bs, kv_seq_len]
Nz=At least one modality should be provided for cross attention.r%   r)   rA   r   r   r	   r   Tr  )"r   r-   r   r  r   r  r  rg   r  r  r  r  rt   r   r   r   r  r  r  rs   r   r   r  r  rb   r  r  rh   r  r   Softmaxr   r   r  )r[   query_statesprotein_key_value_statesstructure_key_value_statesmsa_key_value_statesquery_attn_maskprotein_kv_attn_maskstructure_kv_attn_maskmsa_kv_attn_maskkv_attn_maskr  r   key_layer_proteinvalue_layer_proteinkey_layer_structurevalue_layer_structurekey_layer_msavalue_layer_msar   r   new_query_layer_shapenew_key_layer_shapenew_value_layer_shaperi   r   attention_scoresattention_probscontext_layernew_context_layer_shapes                                r5   cross_attention3EvollaSequenceAlignerCrossAttention.cross_attentionx  sK   * -FVW#/A<a<A\]]yy15)),7 jj-'D,>,>,J'?'B'B<'P$ $ 0 01I J"&"4"45M"N $"&)d.B.B.N)C)F)F|)T&"&"4"45O"P$($8$89S$T!"&$(!<<#(B#7#:#:<#H  LL)=>M"nn-ABO M"O&]K	 );	1Q		;IIiQ/	*?S"-?+Qq+?ii3 + 0 0 23B 7$$$$;
 !
 "&&(=>FFq!QPQR'nn.s3$$$$7
 
 NN$78@@Aq!L	 + 0 0 23B 7$$$$;
 !
 "&&(=>FFq!QPQR!JJ. "#jj):):1)=|?P?PQR?STWWXdXkXklO(D!T)9:\!TSWYZJZ=[[||K1D1DR1LM#l&7&7B&7&M&T&T&VV'33%%'\5G5G)H)L)L
 **,-=> _kB%--aAq9DDF"/"4"4"6s";t?Q?Q>S"S%**,CDm4q BL < @s"   QQ"Q,QQQpast_key_valuepast_key_values4.58new_nameversionc                 z   Ubv  UR                   u  pnUcc  [        R                  " X5      R                  U	R                  5      U	R                  X4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      U
R                  UU4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      UR                  UU4S9R                  -  R                  UR                  5      nOS nUnUb  UR                  5       (       d0  Ub  UR                  5       (       d  Ub  UR                  5       (       ay  UnU R                  UUUUUUUUS9n[        R                  " U R                  5      U-  nUU-   nUnU R                  U5      [        R                  " U R                  5      -  nUU-   nU$ )N)rt   )r  r  r	  r
  r  r  r  r  )re   r-   r  rg   rs   rV   Tanyr  tanhr   r  r  )r[   r  protein_kv_statesstructure_kv_statesmsa_kv_statesr  r  r  r  protein_batch_maskstructure_batch_maskmsa_batch_maskr   r  protein_kv_seq_lenr*   structure_kv_seq_lenmsa_kv_seq_lenr   residuals                       r5   ro   +EvollaSequenceAlignerCrossAttention.forward  sP     (*;*A*A'BC#+JJr699:L:S:ST(//6H5M/NPPQ"&--. %
 $( *,?,E,E)B$c%-JJr#78;;<N<U<UV*118Lb7Q1RTTU"(//0 '
 &*"$&3&9&9#B'JJr>2556H6O6OP$++."1E+FHHI"]))* !
  $$ */C/G/G/I/I#/4J4N4N4P4P).>.B.B.D.D$H 00*):+>%2 /%9'=!1 1 	M "JJt':':;mKM$}4M$H GGM2UZZ5NNM$}4Mr7   )r   r   r  rQ   r  r   r  rH   r  r  r  r   r  r   r  r  r  r  )NNNNNNNNNN)ry   rz   r{   r|   r   r,   rE   r  r"   ro   r~   r   r   s   @r5   r  r  D  s     .2/3)-1: &c]1:  (}	1:
 "#1: 1:fn` %0A6R "#!G SGr7   r  RMSNormc                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )r  i3  c                    > [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z,
EvollaRMSNorm is equivalent to T5LayerNorm
N)rD   rE   r   r  r-   r  ra  variance_epsilon)r[   rH   r=   r]   s      r5   rE   EvollaRMSNorm.__init__5  s/     	ll5::k#:; #r7   c                    UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr   rA   T)r  )	rh   rg   r-   r   powr]  rsqrtr8  ra  )r[   r   input_dtypevariances       r5   ro   EvollaRMSNorm.forward=  sw    #))%((7 $$Q',,R,>%H?T?T4T(UU{{]--k:::r7   c                 ^    [        U R                  R                  5       SU R                   3$ )Nz, eps=)r   ra  re   r8  rv  s    r5   
extra_reprEvollaRMSNorm.extra_reprD  s*    ))*+6$2G2G1HIIr7   )r8  ra  )gư>)	ry   rz   r{   r|   rE   ro   rA  r~   r   r   s   @r5   r  r  3  s    $;J Jr7   r  c                      ^  \ rS rSr% \R
                  \S'   SS\4U 4S jjjr\R                  " 5       \
S 5       5       rSrU =r$ )EvollaRotaryEmbeddingiH  r   r\   c                   > [         TU ]  5         [        US5      (       aZ  [        UR                  [
        5      (       a;  UR                  R                  SUR                  R                  S5      5      U l        OSU l        UR                  U l	        UR                  U l
        Xl        [        U R                     U l        U R                  U R                  U5      u  o0l        U R                  SUSS9  U R                   U l        g )Nrope_scaling	rope_typetypedefaultr   FrB   )rD   rE   r   r`  rF  dictgetrG  rU   max_seq_len_cachedoriginal_max_seq_lenr\   r   rope_init_fnattention_scalingrS   r   original_inv_freq)r[   r\   rs   r   r]   s       r5   rE   EvollaRotaryEmbedding.__init__K  s    6>**z&:M:Mt/T/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q((ZeD!%r7   c                 b   U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      R	                  UR
                  5      nUS S 2S S S 24   R                  5       n[        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        R                  " USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " Xf4SS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR	                  UR                   S
9W	R	                  UR                   S
94$ ! , (       d  f       N@= f)Nr   rA   r%   mpscpuF)device_typeenabledr   r)   r   )r   rf   rV   re   rg   rs   r`  rH  strr-   autocastr   r   r   rO  r   rh   )
r[   r   r@   inv_freq_expandedposition_ids_expandedrU  r   r   r   r   s
             r5   ro   EvollaRotaryEmbedding.forward\  sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk^^UC&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d444C'')d444C	 D vvAGGv$cff177f&;;; DCs   $BF  
F.)rO  r\   rL  rP  rM  rN  rG  r   )ry   rz   r{   r|   r-   r   r   r&   rE   no_gradr   ro   r~   r   r   s   @r5   rD  rD  H  s@    ll/| / /" ]]_<  <r7   rD  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )	EvollaMLPil  c                   > [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  UR                  S9U l        [
        R                  " U R                  U R                  UR                  S9U l	        [
        R                  " U R                  U R                  UR                  S9U l
        [        UR                     U l        g )Nr  )rD   rE   r\   rH   r  r   r   mlp_bias	gate_projup_proj	down_projr
   
hidden_actact_fnrZ   s     r5   rE   EvollaMLP.__init__m  s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r7   c                     U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r   )rc  re  ra  rb  )r[   r   rc  s      r5   ro   EvollaMLP.forwardw  s6    NN4;;t~~a/@#ADLLQRO#ST	r7   )re  r\   rc  ra  rH   r  rb  r   r   s   @r5   r^  r^  l  s    0 r7   r^  c                     U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrA   r   r)   )re   r-   r   r   s      r5   rotate_halfrj  |  sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r7   c                     UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXg4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)rc   rj  )r   r   r   r   r@   unsqueeze_dimq_embedk_embeds           r5   apply_rotary_pos_embro    sS    ( --
&C
--
&Cw;q>C/0Gw;q>C/0Gr7   r   n_repr   c                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r%   N)re   rV   r   )r   rp  batchnum_key_value_headsslenhead_dims         r5   	repeat_kvrv    s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr7   c                   4  ^  \ rS rSrSrS\S\4U 4S jjr\" SSSS	9  SS
\	R                  S\\	R                  \	R                  4   S\\	R                     S\\   S\\	R                     S\\   S\\	R                  \	R                  4   4S jj5       rSrU =r$ )EvollaAttentioni  z=Multi-headed attention from 'Attention Is All You Need' paperr\   r   c                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nru  r   Tr  )rD   rE   r\   r   rR   rH   r   ru  rs  num_key_value_groupsr   attention_dropoutr   r   r   attention_biasq_projk_projv_projo_projr[   r\   r   r]   s      r5   rE   EvollaAttention.__init__  sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r7   r  r   r!  r"  r   rW   ri   cache_positionr   r   c                 4   UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      R                  U5      R	                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	U
UU4U R                  (       d  SOU R                  U R                   S.UD6u  nnUR"                  " / UQSP76 R%                  5       nU R'                  U5      nUU4$ )NrA   r%   r   )r   r   r  r   r`   )rQ   r   )re   ru  r}  r   r   r~  r  ro  updater   r   r\   r   r   r   r{  r   r   r   r  )r[   r   rW   ri   r   r  r   ru   r   r  
key_statesvalue_statesr   r   cache_kwargsr   r   r   s                     r5   ro   EvollaAttention.forward  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&#7RU#[ &#&nUL'6'='=jX\XfXfht'u$J(?;;++w6"9$++:Z:Z"[$7	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r7   )r{  r\   ru  r   r~  r   rz  r  r}  r   r  r  )ry   rz   r{   r|   r}   r&   r,   rE   r"   r-   r   r   r   r   r  r   r   ro   r~   r   r   s   @r5   rx  rx    s    G
| 
 
. %0A6R ,059))||)) #5<<#=>)) !.	))
 "%)) !!1!12)) +,)) 
u||U\\)	*)) S))r7   rx  c                      ^  \ rS rSrS\S\4U 4S jjr\" SSSS9            SS	\R                  S
\
\R                  \R                  4   S\\R                     S\\R                     S\\   S\\   S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\\R                     S\R                  4S jj5       rSrU =r$ )EvollaDecoderLayeri  r\   r   c                   > [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        US-   [        UR                  UR                  -  S5      -  S:X  a  [        UUR                  S9U l        g g )Nr\   r   r<   r%   r   )r  )rD   rE   rH   rx  	self_attnr^  mlpr  rms_norm_epsinput_layernormpost_attention_layernormmaxr@  aligner_num_add_layersr  adapterr  s      r5   rE   EvollaDecoderLayer.__init__  s    !--(LV$,V-?-?VEXEXY(5f6H6HfNaNa(b%MS!9!9V=Z=Z!Z\]^^bcc>$*$6$6DL dr7   r  r   r!  r"  r   rW   ri   r@   	use_cacher  r)  r*  r+  r,  r-  r.  r  r   c                    UnU R                  U5      nU R                  " SUUUUUUUS.UD6u  nnUU-   nUnU R                  U5      nU R                  U5      nUU-   n[	        U S5      (       a  U R                  UUU	U
UUUUS9nU$ )N)r   ri   r@   r   r  r  rW   r  )r  r)  r*  r+  r  r,  r-  r.  ri  )r  r  r  r  r   r  )r[   r   rW   ri   r@   r   r  r  r)  r*  r+  r,  r-  r.  r  r   r2  r  s                     r5   ro   EvollaDecoderLayer.forward  s    & !,,];  >> 	
')%+) 3	
 	
q !=0 !55mD/ =04## LL*"3$7+ /#5%9- ) 	M r7   )r  rH   r  r  r  r  )NNNFNNNNNNNN)ry   rz   r{   r|   r&   r,   rE   r"   r-   r   r   r   r  r   r  ro   r~   r   r   s   @r5   r  r    sp   |   %0A6R
 2637+/$)59486:04597;15265||5 #5<<#=>5 !.	5
 u//05 "%5 D>5 !!1!125 $ELL15 &ell35  -5 %U\\25 'u||45 !.5 "%,,/5" 
#5 S5r7   r  c                   h   ^  \ rS rSr% \\S'   SrSr/ SQrS/r	Sr
SrSrSrSr\\S.rU 4S	 jrS
rU =r$ )EvollaPreTrainedModeli:  r\   r  T)r  r  r  r   F)r   rY  c                   > U R                   R                  n[        TU ]  U5        [	        U[
        5      (       ad  UR                  R                  5         UR                  R                  5         UR                  R                  R                  R                  S5        g [	        U[        5      (       a%  UR                  R                  R                  SUS9  g g )Nr   r`   r\  )r\   r_  rD   rg  r`  r  r   re  r  r  ra  rb  rf  r  r  rc  )r[   r   r^  r]   s      r5   rg  #EvollaPreTrainedModel._init_weightsP  s    kk++f%fABB!!'')OO!!#!!((--33C8 ABBNN''Sc': Cr7   ri  )ry   rz   r{   r|   r&   r   base_model_prefixsupports_gradient_checkpointingrj  _skip_keys_device_placementrk  rl  _supports_flex_attn_can_compile_fullgraphrm  r  rx  rn  rg  r~   r   r   s   @r5   r  r  :  s]    &*#
 $5"5 N!"'+%
; ;r7   r  c            !         ^  \ rS rSrS\4U 4S jjrS rS r\\	" 5                    SS\
\R                     S\
\R                     S\
\R                     S	\
\   S
\
\R                     S\
\   S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\R                     S\\\4   4S jj5       5       rSrU =r$ )EvollaModeli[  r\   c           
      8  > [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " U R                  UR                  U R                  5      U l        [        US9U l
        [
        R                  " [        UR                  5       Vs/ s H  n[        UUS9PM     sn5      U l        [!        UR                  UR"                  S9U l        ['        US9U l        [+        USS5      U l        U R/                  5         g s  snf )Nr  r  r<   rC  F)rD   rE   rI   r2   rG   r   rF   rH   embed_tokensr  protein_encoderr>  r?  r@  r  r  r  r  r  rD  
rotary_embrR   rC  	post_initr  s      r5   rE   EvollaModel.__init__\  s     !.. ++LL&:L:LdN^N^_36Bmm "'v'?'?!@
 "AI	 #!' "A
 "&"4"4&:M:MN	/v>&-f6NPU&V#s   #Dc                     U R                   $ r   r  rv  s    r5   rw   EvollaModel.get_input_embeddingsq  s       r7   c                     Xl         g r   r  rz  s     r5   r{   EvollaModel.set_input_embeddingst  s    !r7   r1   ri   r@   r   rj   r  r  protein_input_idsprotein_attention_maskstructure_feats	msa_featsr-  r.  r   c                    USL USL-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcD  Ub  UR	                  5       OSn[
        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      nSnSnUbO  U	bL  U R                  UU	S9nUR                  n[
        R                  " S/UR                  S   -  UR                  S9n[        U R                  UUUUS	9nUnU R                  UU5      nU R                   H  nU" U4UUUUUUUU
UUUUUS
.UD6nM     U R!                  U5      n[#        UUS9nU$ )a  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
structure_feats (torch.FloatTensor):
    The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
msa_feats (torch.FloatTensor):
    The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
structure_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummpy input for now.
msa_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummpy input for now.
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r%   r   r  T)r\   input_embedsri   r  r   )ri   r@   r   r  r  rW   r)  r*  r+  r,  r-  r.  r  )rF  r   )r   r  r   r\   get_seq_lengthr-   rT   re   rs   rc   r  r  r  r   r  r  r  r   )r[   r1   ri   r@   r   rj   r  r  r  r  r  r  r-  r.  r   past_seen_tokensprotein_featsr,  protein_outputsr   r   rW   decoder_layerr  s                           r5   ro   EvollaModel.forwardw  s   B -t";<YZZ  --i8M0*$++>O!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L!(-C-O"22+5 3 O ,FFM!&tf7H7N7Nq7Q.QZkZrZr!s(;;&))+
 & #oom\J![[M)*) /#-$7"/$3'#5%9- . M )& 		-0(++
 r7   )r  rC  r  r  r2   r  r  rG   )NNNNNNNNNNNNN)ry   rz   r{   r|   r&   rE   rw  r{  r    r$   r   r-   r  r   r   r   r  r   r   r   ro   r~   r   r   s   @r5   r  r  [  s   | *!"  151537+/59$(598<9=7;157;15bE,,-b !.b u//0	b
 "%b   1 12b D>b !!1!12b $E$4$45b !) 6b "%"3"34b E--.b 'u||4b !.b  
u--	.!b  br7   r  c                   "  ^  \ rS rSrU 4S jrS rS r\\       SS\	\
R                     S\	\
R                     S\	\
R                     S\	\
R                     S	\	\
R                     S
\	\
R                     S\	\   4S jj5       5       rSrU =r$ )EvollaForProteinText2Texti  c                    > [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  U R                  SS9U l        U R                  5         g r  )
rD   rE   r  r  rG   r   r   rH   lm_headr  rZ   s     r5   rE   "EvollaForProteinText2Text.__init__  sQ      (
 ++yy!3!3T__5Qr7   c                 6    U R                   R                  5       $ r   )r  rw  rv  s    r5   rw  .EvollaForProteinText2Text.get_input_embeddings  s    zz..00r7   c                 8    U R                   R                  U5      $ r   )r  r{  rz  s     r5   r{  .EvollaForProteinText2Text.set_input_embeddings  s    zz..u55r7   r1   ri   rj   labelsr  r  r  c           
         U R                   " SUUUUUUS.UD6n	U	S   n
U R                  U
5      nSnUb  U R                  " SXU R                  S.UD6n[	        UUU	R
                  U	R                  U	R                  S9nU$ )a|  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

Example:

```python
>>> from transformers import EvollaProcessor, EvollaForProteinText2Text
>>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
>>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

>>> protein_information = {
    "aa_seq": "your amino acid sequence",
    "foldseek": "your foldseek sequence",
}
>>> question = "What is the function of this protein?"
>>> message = [
    {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
    {"role": "user", "content": question},
]

>>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
>>> outputs = model.generate(**inputs)

>>> print(processor.batch_decode(outputs, skip_special_tokens=True))
```)r1   ri   rj   r  r  r  r   N)logitsr  rG   )lossr  r   r   rY  ri  )r  r  loss_functionrG   r   r   r   rY  )r[   r1   ri   rj   r  r  r  r  r   outputsr   r  r  
lm_outputss                 r5   ro   !EvollaForProteinText2Text.forward  s    T ** 
)'/#9
 
  
m,%%iVtibhiD+#33!//))

 r7   )r  r  rG   r4  )ry   rz   r{   r|   rE   rw  r{  r!   r    r   r-   r  r   r   r  ro   r~   r   r   s   @r5   r  r    s    16  151559-18<9=$(?E,,-? !.?   1 12	?
 ))*? $E$4$45? !) 6? D>?  ?r7   r  )r  r  r  )r`   N)Nr%   )]r  r  dataclassesr   typingr   r   r   r-   r   r   activationsr
   cache_utilsr   r   
generationr   integrationsr   masking_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   r   modeling_utilsr   r   r   r   processing_utilsr   pytorch_utilsr   r   utilsr   r    r!   utils.deprecationr"   utils.genericr#   r$   configuration_evollar&   r'   r6   Moduler9   r   r   r   rf   r   r   r   r  r  r  r!  r'  r<  rM  rW  rp  r  r  r  r  r  r  r  rD  r^  rj  ro  r,   rv  rx  r  r  r  r  __all__ri  r7   r5   <module>r     s(  ,   ! , ,   ! . ) 7 / 9  L m m & Q I I 0 ? <4 ^=RYY ^=B(
2)
")) )
f (,/%II/%<</% 
/% <<	/%
 U\\*/% /% /% %/% '(/%dV)		 V)r
RYY 
-BII -`;ryy 
 
72 7t S"))  SF  */ * *>c'!< c'L7 		 7 tA		 A'.		 '.T ?k ?  ?
299 
$l")) l^ Y'JBII J (J(!<BII !<H		  (6	UU\\ 	U# 	U%,, 	UD)bii D)NF3 FR ;O ; ;@@' @FP 5 Pf Pr7   