
    C1iB                     x    S SK r S SKrS SKJrJr   " S S\ R                  5      r " S S\R                  5      r	g)    N)common_spec
model_specc                   $    \ rS rSrSrSrSrSrSrg)RotaryScalingType	   zRoPE scaling type.r          N)	__name__
__module____qualname____firstlineno____doc__LinearSuLlama3__static_attributes__r
       Z/home/james-whalen/.local/lib/python3.13/site-packages/ctranslate2/specs/attention_spec.pyr   r   	   s    F	
BFr   r   c                   B    \ rS rSr                  SS jrSrg)MultiHeadAttentionSpec   Nc                    [         R                  U l        U(       a  [        R                  " US9U l        [        U(       a  SOS5       Vs/ s H  n[        R                  " 5       PM     snU l        U(       a2  [        R                  " US9U l	        [        R                  " US9U l
        U(       a  S U l        S U l        U(       a  S U l        S U l        U(       a  S U l        S U l        S U l        US:w  a*  [$        R&                  " S5      R)                  U5      U l        US:w  a*  [$        R&                  " S5      R)                  U5      U l        UGb  [$        R&                  " S5      R)                  U5      U l        Xpl        [$        R&                  " S5      R)                  U
5      U l        Ub*  [$        R&                  " S5      R)                  U5      U l        U[6        R8                  L a+  [$        R&                  " S5      R)                  U	5      U l        OCU[6        R<                  L a  S U l        S U l         O!U[6        RB                  L a  S U l"        S U l#        Ub*  [$        R&                  " S5      R)                  U5      U l$        Ub*  [$        R&                  " S5      R)                  U5      U l%        Ub+  [$        R&                  " S5      R)                  U5      U l&        g g s  snf )N)rms_normr	      r   int32float32int8)'r   OPTIONALqueries_scaler   LayerNormSpec
layer_normrange
LinearSpeclinearq_normk_normrelative_position_keysrelative_position_valuesrelative_attention_biasrelative_attention_max_distance!relative_asymmetric_position_keysrelative_left_max_positionrelative_right_max_positionnpdtypetype original_max_position_embeddingsmax_position_embeddings
rotary_dimrotary_interleaverotary_baserotary_scaling_typer   r   rotary_scaling_factorr   rotary_scaling_long_factorrotary_scaling_short_factorr   rotary_low_freq_factorrotary_high_freq_factornum_heads_kvhead_dimsliding_window)selfself_attentionrelative_positionrelative_asymmetric_positionr*   r   r4   r5   r7   r8   r6   r2   r3   r=   r>   r?   qk_normqk_norm_rmshas_norm_s                       r   __init__MultiHeadAttentionSpec.__init__   sU   * (00)77JDO.3AQ.O
.OK""$.O
 %33[IDK%33[IDK*.D',0D)"+/D(37D0'59D2.2D+/3D,+q046HHW4E4J4J05D1 #a'+-88G+<+A+A',D( ! hhw/44Z@DO%6"!xx	277DD".+-88F+;+@+@AT+U("&7&>&>>-/XXi-@-E-E).* %(9(<(<<26/370$(9(@(@@.2+/3,# " 1 6 6| DDHHW-228<DM%"$((7"3"8"8"HD &m
s   K')r>   r'   r"   r%   r3   r=   r2   r&   r    r,   r*   r+   r-   r(   r)   r.   r6   r4   r<   r5   r;   r8   r9   r:   r7   r?   )FFFFFNTNr   i'  r   r   NNNFTT)r   r   r   r   rH   r   r
   r   r   r   r      sA     %* % )* !'PIr   r   )
enumnumpyr/   ctranslate2.specsr   r   IntEnumr   	LayerSpecr   r
   r   r   <module>rO      s4      5 QIZ11 QIr   