
    cCi                        S SK JrJrJr  S SKrS SKJr  S SKJrJ	r	  SSK
Jr  SSKJrJrJr  SSKJr  SSKJr  SS	KJr  SS
KJrJr  SSKJr  SSKJr  SSKJrJrJ r J!r!J"r"  SSK#J$r$  SSK%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,J-r-J.r.  SSK/J0r0  SSK1J2r2J3r3J4r4  SSK5J6r6J7r7J8r8  SSK9J:r:J;r;  \.Rx                  " \=5      r> " S S\5      r? " S S\R                  5      rA " S S\R                  5      rB " S S\25      rC " S S \35      rD " S! S"\65      rE " S# S$\5      rF\, " S% S&\'5      5       rG " S' S(\G5      rH " S) S*\75      rI " S+ S,\:5      rJ\," S-S.9 " S/ S0\G\5      5       rK/ S1QrLg)2    )CallableOptionalUnionN)OutputRecordercheck_model_inputs   )ACT2FN)CacheDynamicCacheEncoderDecoderCache)PretrainedConfig)GenerationMixin)create_causal_mask)_prepare_4d_attention_mask#_prepare_4d_attention_mask_for_sdpa)FlashAttentionKwargs)GradientCheckpointingLayer)BaseModelOutputBaseModelOutputWithPast)BaseModelOutputWithPastAndCrossAttentionsSeq2SeqLMOutputSeq2SeqModelOutput)rope_config_validation)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)TransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg   )GlmAttentionGlmRotaryEmbeddingapply_rotary_pos_emb)LlamaDecoderLayer
LlamaModeleager_attention_forward)WhisperModelshift_tokens_rightc                   v   ^  \ rS rSrSrSrS/rSSSS.r                        SU 4S	 jjrS
r	U =r
$ )MoonshineConfig2   a!  
This is the configuration class to store the configuration of a [`MoonshineModel`]. It is used to instantiate a Moonshine
model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
defaults will yield a similar configuration to that of the Moonshine
[UsefulSensors/moonshine-tiny](https://huggingface.co/UsefulSensors/moonshine-tiny).

Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
documentation from [`PretrainedConfig`] for more information.

Args:
    vocab_size (`int`, *optional*, defaults to 32768):
        Vocabulary size of the Moonshine model. Defines the number of different tokens that can be represented by the
        `inputs_ids` passed when calling [`MoonshineModel`].
    hidden_size (`int`, *optional*, defaults to 288):
        Dimension of the hidden representations.
    intermediate_size (`int`, *optional*, defaults to 1152):
        Dimension of the MLP representations.
    encoder_num_hidden_layers (`int`, *optional*, defaults to 6):
        Number of hidden layers in the Transformer encoder.
    decoder_num_hidden_layers (`int`, *optional*, defaults to 6):
        Number of hidden layers in the Transformer decoder.
    encoder_num_attention_heads (`int`, *optional*, defaults to 8):
        Number of attention heads for each attention layer in the Transformer encoder.
    decoder_num_attention_heads (`int`, *optional*, defaults to 8):
        Number of attention heads for each attention layer in the Transformer decoder.
    encoder_num_key_value_heads (`int`, *optional*):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `encoder_num_key_value_heads=encoder_num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `encoder_num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `num_attention_heads`.
    decoder_num_key_value_heads (`int`, *optional*):
        This is the number of key_value heads that should be used to implement Grouped Query Attention. If
        `decoder_num_key_value_heads=decoder_num_attention_heads`, the model will use Multi Head Attention (MHA), if
        `decoder_num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
        converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
        by meanpooling all the original heads within that group. For more details, check out [this
        paper](https://huggingface.co/papers/2305.13245). If it is not specified, will default to
        `decoder_num_attention_heads`.
    pad_head_dim_to_multiple_of (`int`, *optional*):
        Pad head dimension in encoder and decoder to the next multiple of this value. Necessary for using certain
        optimized attention implementations.
    encoder_hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
        The non-linear activation function (function or string) in the encoder.
    decoder_hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
        The non-linear activation function (function or string) in the decoder.
    max_position_embeddings (`int`, *optional*, defaults to 512):
        The maximum sequence length that this model might ever be used with.
    initializer_range (`float`, *optional*, defaults to 0.02):
        The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
    decoder_start_token_id (`int`, *optional*, defaults to 1):
        Corresponds to the "<|startoftranscript|>" token, which is automatically used when no `decoder_input_ids`
        are provided to the `generate` function. It is used to guide the model`s generation process depending on
        the task.
    use_cache (`bool`, *optional*, defaults to `True`):
        Whether or not the model should return the last key/values attentions (not used by all models).
    rope_theta (`float`, *optional*, defaults to 10000.0):
        The base period of the RoPE embeddings.
    rope_scaling (`Dict`, *optional*):
        Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
        and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
        accordingly.
        Expected contents:
            `rope_type` (`str`):
                The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
                'llama3'], with 'default' being the original RoPE implementation.
            `factor` (`float`, *optional*):
                Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
                most scaling types, a `factor` of x will enable the model to handle sequences of length x *
                original maximum pre-trained length.
            `original_max_position_embeddings` (`int`, *optional*):
                Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
                pretraining.
            `attention_factor` (`float`, *optional*):
                Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
                computation. If unspecified, it defaults to value recommended by the implementation, using the
                `factor` field to infer the suggested value.
            `beta_fast` (`float`, *optional*):
                Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
                ramp function. If unspecified, it defaults to 32.
            `beta_slow` (`float`, *optional*):
                Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
                ramp function. If unspecified, it defaults to 1.
            `short_factor` (`list[float]`, *optional*):
                Only used with 'longrope'. The scaling factor to be applied to short contexts (<
                `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                size divided by the number of attention heads divided by 2
            `long_factor` (`list[float]`, *optional*):
                Only used with 'longrope'. The scaling factor to be applied to long contexts (<
                `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                size divided by the number of attention heads divided by 2
            `low_freq_factor` (`float`, *optional*):
                Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
            `high_freq_factor` (`float`, *optional*):
                Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
    partial_rotary_factor (`float`, *optional*, defaults to 0.9):
        Percentage of the query and keys which will have rotary embedding.
    is_encoder_decoder (`bool`, *optional*, defaults to `True`):
        Whether the model is used as an encoder/decoder or not.
    attention_bias (`bool`, *optional*, defaults to `False`):
        Whether to use a bias in the query, key, value and output projection layers during self-attention.
    attention_dropout (`float`, *optional*, defaults to 0.0):
        The dropout ratio for the attention probabilities.
    bos_token_id (`int`, *optional*, defaults to 1):
        Denotes beginning of sequences token id.
    eos_token_id (`int`, *optional*, defaults to 2):
        Denotes end of sequences token id.

Example:

```python
>>> from transformers import MoonshineModel, MoonshineConfig

>>> # Initializing a Moonshine style configuration
>>> configuration = MoonshineConfig().from_pretrained("UsefulSensors/moonshine-tiny")

>>> # Initializing a model from the configuration
>>> model = MoonshineModel(configuration)

>>> # Accessing the model configuration
>>> configuration = model.config
```	moonshinepast_key_valuesencoder_num_key_value_headsencoder_num_attention_headsencoder_num_hidden_layers)num_key_value_headsnum_attention_headsnum_hidden_layersc                 p  > Xl         X l        X0l        X@l        XPl        X`l        Xpl        Uc  UnXl        U	c  Un	Xl        Xl	        Xl
        Xl        Xl        Xl        Xl        UU l        UU l        UU l        UU l        UU l        UU l        UU l        [-        U 5        [.        TU ]`  " SUUUUS.UD6  g )N)bos_token_ideos_token_idis_encoder_decoderdecoder_start_token_id )
vocab_sizehidden_sizeintermediate_sizer2   decoder_num_hidden_layersr1   decoder_num_attention_headsr0   decoder_num_key_value_headspad_head_dim_to_multiple_ofencoder_hidden_actdecoder_hidden_actmax_position_embeddingsinitializer_ranger:   	use_cache
rope_thetarope_scalingpartial_rotary_factorr9   attention_biasattention_dropoutr   super__init__)selfr<   r=   r>   r2   r?   r1   r@   r0   rA   rB   rC   rD   rE   rF   r:   rG   rH   rI   rJ   r9   rK   rL   r7   r8   kwargs	__class__s                             i/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/moonshine/modular_moonshine.pyrN   MoonshineConfig.__init__   s    8 %&!2)B&)B&+F(+F(&.*E'+F(&.*E'+F(+F("4"4'>$!2&<#"$(%:""4,!2 	t$ 	
%%1#9		

 	
    )rK   rL   rD   r@   r?   rA   r:   rC   r1   r2   r0   r=   rF   r>   r9   rE   rB   rJ   rI   rH   rG   r<   )i   i   i     rU      rV   NNNgelusilui   g{Gz?   Tg     @Ng?TF        rY   r"   )__name__
__module____qualname____firstlineno____doc__
model_typekeys_to_ignore_at_inferenceattribute_maprN   __static_attributes____classcell__rQ   s   @rR   r,   r,   2   s    {z J#4"5<<8M "#"#$%$%$($($(!! # !3D
 D
rT   r,   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )MoonshineEncoderMLP   c                 
  > [         TU ]  5         Xl        [        U   U l        [
        R                  " UR                  UR                  5      U l	        [
        R                  " UR                  UR                  5      U l
        g NrM   rN   configr	   activation_fnnnLinearr=   r>   fc1fc2rO   rl   
hidden_actrQ   s      rR   rN   MoonshineEncoderMLP.__init__   s\    #J/99V//1I1IJ99V55v7I7IJrT   hidden_statesreturnc                 l    U R                  U5      nU R                  U5      nU R                  U5      nU$ rj   )rp   rm   rq   )rO   ru   s     rR   forwardMoonshineEncoderMLP.forward  s4    /**=9/rT   rm   rl   rp   rq   
r[   r\   r]   r^   rN   torchTensorrx   rc   rd   re   s   @rR   rg   rg      s)    KU\\ ell  rT   rg   c                   b   ^  \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )MoonshineDecoderMLPi  c                   > [         TU ]  5         Xl        [        U   U l        [
        R                  " UR                  UR                  S-  5      U l	        [
        R                  " UR                  UR                  5      U l
        g )Nr"   rk   rr   s      rR   rN   MoonshineDecoderMLP.__init__  sa    #J/99V//1I1IA1MN99V55v7I7IJrT   ru   rv   c                     U R                  U5      nUR                  SSS9u  pU R                  U5      U-  nU R                  U5      nU$ )Nr"   )dim)rp   chunkrm   rq   )rO   ru   gates      rR   rx   MoonshineDecoderMLP.forward  sQ    /+11!1<**40=@/rT   rz   r{   re   s   @rR   r   r     s)    KU\\ ell  rT   r   c                     ^  \ rS rSrS\S\S\S\S\4
U 4S jjr\" SS	S
S9     SS\	R                  S\\\	R                  \	R                  4      S\\	R                     S	\\   S\\	R                     S\\	R                     S\\   S\\	R                  \\	R                     \\\	R                        4   4S jj5       rSrU =r$ )MoonshineAttentioni  rl   	layer_idx	is_causalr4   r3   c                 f  > UR                  XES.5        [        TU ]	  X5        X0l        [	        USUR
                  UR                  -  5      U l        U R                  R                  bA  U R                  R                  nX`R                  U-   S-
  U-  -  nXpR                  -
  U l
        g SU l
        g )N)r4   r3   head_dimrY   r   )updaterM   rN   r   getattrr=   r4   r   rl   rB   head_dim_padding)	rO   rl   r   r   r4   r3   target_multipletarget_head_dimrQ   s	           rR   rN   MoonshineAttention.__init__  s     	.Ano+"
F4F4F&JdJd4de ;;22>"kkEEO---/2QTU2UZi1ijO$3mm$CD!$%D!rT   past_key_valuer/   4.58new_nameversionru   position_embeddingsattention_maskcache_positionkey_value_statesrP   rv   c                    UR                   S S u  pU R                  U5      R                  XU R                  R                  U R
                  5      R                  SS5      n
US LnUb^  UR                  R                  U R                  5      nU(       a&  SUR                  U R                  '   UR                  nOUR                  nUb  UOUnU(       aU  U(       aN  W(       aG  UR                  U R                     R                  nUR                  U R                     R                  nOU R                  U5      R                  USU R                  R                  U R
                  5      R                  SS5      nU R!                  U5      R                  USU R                  R                  U R
                  5      R                  SS5      nU(       a$  Ub!  UR#                  XU R                  SU05      u  pU(       d<  Uu  nn[%        XUU5      u  pUb%  UUUS.nUR#                  XU R                  U5      u  p[&        nU R                  R(                  S:w  a  [*        U R                  R(                     nU R,                  =(       a    US L =(       a    U	S:  nU R.                  S:  a  [0        R2                  R4                  R7                  U
SU R.                  45      n
[0        R2                  R4                  R7                  USU R.                  45      n[0        R2                  R4                  R7                  USU R.                  45      nU" U U
UUU4U R8                  (       d  S	OU R:                  U R<                  US
.UD6u  nnU R.                  S:  a  USS U R.                  * 24   nUR?                  XS5      RA                  5       nU RC                  U5      nUU4$ )Nr   rY   r"   Tr   )sincosr   eagerr   rZ   )dropoutscalingr   .)"shapeq_projviewrl   r3   r   	transpose
is_updatedgetr   cross_attention_cacheself_attention_cachelayerskeysvaluesk_projv_projr   r%   r(   _attn_implementationr   r   r   r|   rn   
functionalpadtrainingrL   r   reshape
contiguouso_proj)rO   ru   r   r   r/   r   r   rP   bszq_lenquery_statesis_cross_attentionr   current_states
key_statesvalue_statesr   r   cache_kwargsattention_interfacer   attn_outputattn_weightss                          rR   rx   MoonshineAttention.forward4  sq    #(("-
 KK&++C8W8WY]YfYfgqqrsuvw 	 .T9&(3377GJ!=A**4>>:"1"G"G"1"F"F .>-I)}/j(//?DDJ*11$..AHHL N+c2t{{>>N1a  N+c2t{{>>N1a 
 "o&A+:+A+Adnn?OQ_>`,(
 "*HC';LVY[^'_$L*'*3.Y+:+A+Adnnl,(
 )@;;++w6"9$++:Z:Z"[NNK~'=K%!)	  1$ 88..22<!TEZEZA[\L,,00aAVAV=WXJ 88..22<!TEZEZA[\L$7
%
  $}}C$2H2HLL
%
 
%
!\   1$%c+Cd.C.C-C+C&CDK!))#b9DDFkk+.L((rT   )r   r   r   )NNNNN)r[   r\   r]   r^   r,   intboolrN   r!   r|   r}   r   tupler
   
LongTensorr   r   rx   rc   rd   re   s   @rR   r   r     s,   && & 	&
 !& !&* %0A6R LP15+/5937U)||U) &eELL%,,,F&GHU) !.	U)
 "%U) !!1!12U) #5<<0U) -.U) 
u||Xell3XeELL>Q5RR	SU) SU)rT   r   c                       \ rS rSrSrg)MoonshineRotaryEmbeddingi  r;   N)r[   r\   r]   r^   rc   r;   rT   rR   r   r     s    rT   r   c                   4   ^  \ rS rSrS\S\4U 4S jjrSrU =r$ )MoonshineEncoderLayeri  rl   r   c                 4  > [         TU ]  X5        [        UUSUR                  UR                  S9U l        [        XR                  5      U l        [        R                  " UR                  SS9U l        [        R                  " UR                  SS9U l        g )NFrl   r   r   r4   r3   bias)rM   rN   r   r1   r0   	self_attnrg   rC   mlprn   	LayerNormr=   input_layernormpost_attention_layernormrO   rl   r   rQ   s      rR   rN   MoonshineEncoderLayer.__init__  s}    ++ & B B & B B
 'v/H/HI!||F,>,>UK(*V5G5Ge(T%rT   )r   r   r   r   )	r[   r\   r]   r^   r,   r   rN   rc   rd   re   s   @rR   r   r     s    U U3 U UrT   r   c            !       2  ^  \ rS rSrSS\S\\   4U 4S jjjr\" SSSS9          SS	\	R                  S
\\	R                     S\\	R                     S\\	R                     S\\	R                     S\\	R                     S\\   S\\   S\\	R                     S\\\	R                  \	R                  4      S\\\	R                  \	R                  4      S\\   S\\	R"                  \\\	R"                  \	R"                  4      4   4S jj5       rSrU =r$ )MoonshineDecoderLayeri  rl   r   c                   > [         TU ]  5         UR                  U l        [        UUSUR                  UR
                  S9U l        [        UUSUR                  UR
                  S9U l        [        XR                  5      U l
        [        R                  " UR                  SS9U l        [        R                  " UR                  SS9U l        [        R                  " UR                  SS9U l        g )NTr   Fr   )rM   rN   r=   r   r@   rA   r   encoder_attnr   rD   r   rn   r   r   r   final_layernormr   s      rR   rN   MoonshineDecoderLayer.__init__  s    !--+ & B B & B B
 / & B B & B B
 'v/H/HI!||F,>,>UK(*V5G5Ge(T%!||F,>,>UKrT   r   r/   r   r   ru   r   encoder_hidden_statesencoder_attention_maskposition_idsencoder_position_idsrG   r   r   encoder_position_embeddingsrP   rv   c                    UnU R                  U5      nU R                  " SUUUUUU	U
S.UD6u  pX-   nUb,  UnU R                  U5      nU R                  UUUUUS9u  pX-   nUnU R	                  U5      nU R                  U5      nX-   nU$ )N)ru   r   r   r/   rG   r   r   )ru   r   r   r/   rG   r;   )r   r   r   r   r   r   )rO   ru   r   r   r   r   r   r/   rG   r   r   r   rP   residual_s                  rR   rx   MoonshineDecoderLayer.forward  s      !,,];>> 	
')%+) 3	
 	
 !0 ,$H 99-HM#00+!65 /#  1  M %4M ,,];/ 0rT   )r   r   r=   r   r   r   r   rj   )
NNNNNNFNNN)r[   r\   r]   r^   r,   r   r   rN   r!   r|   r}   r   r
   r   r   r   r   FloatTensorrx   rc   rd   re   s   @rR   r   r     s   L L8C= L L0 %0A6R 268<9=37;?+/$)59KOSW.||. !..  (5	.
 !) 6. u//0. 'u'7'78. "%. D>. !!1!12. &eELL%,,,F&GH. &.eELL%,,4N.O%P. +,. 
u  (51B1BEDUDU1U+V"WW	X. S.rT   r   c                   b    \ rS rSr% \\S'   SrSrSrSS/r	Sr
SrSrS\R                  4S	 jrS
rg)MoonshinePreTrainedModeli  rl   modelinput_valuesTr   r   input_lengthsc                 ~    [        US-
  S-  S-   5      n[        US-
  S-  S-   5      n[        US-
  S-  S-   5      nU$ )z8
Computes the output length of the convolutional layers
   @   rY      r   r"   )r   )rO   r   output_conv1_lengthoutput_conv2_lengthoutput_conv3_lengths        rR    _get_feat_extract_output_lengths9MoonshinePreTrainedModel._get_feat_extract_output_lengths  sZ     "=3#6""<q"@A!#6#:a"?!"CD!#6#:a"?!"CD""rT   r;   N)r[   r\   r]   r^   r,   __annotations__base_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modules_supports_flash_attn_supports_sdpa_can_compile_fullgraphr|   r   r   rc   r;   rT   rR   r   r     sH    $O&*#02IJN!#e>N>N #rT   r   c            
          ^  \ rS rSrSrSr\\S.rS\	4U 4S jjr
S\R                  4S jrS	\R                  4S
 jr\" 5        SS\R"                  S\\R&                     S\\   S\4S jj5       rSrU =r$ )MoonshineEncoderi  z
Transformer encoder consisting of *config.num_hidden_layers* layers. Each layer is a [`MoonshineEncoderLayer`]

Args:
    config: MoonshineConfig
r   )
attentionsru   rl   c           	      L  > [         TU ]  U5        Xl        UR                  n[        R
                  " SUSSSS9U l        [        R
                  " USU-  SSS	9U l        [        R
                  " SU-  USSS	9U l        [        R                  " SUS
S9U l
        [        US9U l        [        R                  " [        UR                  5       Vs/ s H  n[!        X5      PM     sn5      U l        [        R$                  " USS9U l        SU l        U R+                  5         g s  snf )NrY   r   r   F)kernel_sizestrider   r"   r   r   )r   r  gh㈵>)
num_groupsnum_channelsepsrl   r   )rM   rN   rl   r=   rn   Conv1dconv1conv2conv3	GroupNorm	groupnormr   
rotary_emb
ModuleListranger2   r   r   r   
layer_normgradient_checkpointing	post_init)rO   rl   	embed_dimidxrQ   s       rR   rN   MoonshineEncoder.__init__  s     &&	YYq)ReT
YYy!i-QqQ
YYq9}iQqQ
PTU2&Amm;@AaAa;bc;bC"6/;bc
 ,,yu=&+#	 ds   D!rv   c                     U R                   $ rj   r  rO   s    rR   get_input_embeddings%MoonshineEncoder.get_input_embeddings%  s    zzrT   valuec                     Xl         g rj   r  )rO   r  s     rR   set_input_embeddings%MoonshineEncoder.set_input_embeddings(  s    
rT   r   rP   c                    UR                  S5      n[        R                  R                  U R	                  U5      5      nU R                  U5      n[        R                  R                  U R                  U5      5      n[        R                  R                  U R                  U5      5      nUR                  SSS5      nUb  U R                  UR                  S   5      nSnUSSSU24   SSU24   nU R                  R                  S:X  a  US	:H  R                  5       (       a  UOSnOEU R                  R                  S
:X  a  [        X$R                   5      nO[#        X$R                   5      n[$        R&                  " SUR                  S   UR(                  S9R                  S5      nU R+                  XG5      nU R,                   H  n	U	" U4UUUS.UD6nM     U R/                  U5      n[1        US9$ )a  
Args:
    input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
        Float values of the raw speech waveform. Raw speech waveform can be
        obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
        `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
        the soundfile library (`pip install soundfile`). To prepare the array into
        `input_values`, the [`AutoFeatureExtractor`] should be used for padding
        and conversion into a tensor of type `torch.FloatTensor`.
    attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
        Mask to avoid performing attention on padding indices in `input_values`. Mask values selected in `[0, 1]`:
        - 1 for tokens that are **not masked**,
        - 0 for tokens that are **masked**.
        [What are attention masks?](../glossary#attention-mask)
rY   r   r"   Nr     .flash_attention_2rZ   sdpadevice)r   r   r   )last_hidden_state)	unsqueezern   r   tanhr  r  rW   r  r	  permuter   r   rl   r   anyr   dtyper   r|   aranger#  r  r   r  r   )
rO   r   r   rP   ru   mask_lendownsample_strider   r   encoder_layers
             rR   rx   MoonshineEncoder.forward+  s   , $--a0**4::l+CD}5**4::m+DE**4::m+DE%--aA6 %<<^=Q=QRT=UVH *+C1D3D1D,DEc9H9nUN{{//3FF4Bc4I3N3N3P3PVZ11V;!D^UhUh!i!;NL_L_!`||A}':':1'=mFZFZ[eefgh"oomJ![[M)-)$7	
 M ) 6&+
 	
rT   )	rl   r  r  r	  r  r  r  r   r  rj   )r[   r\   r]   r^   r_   r   r   r   _can_record_outputsr,   rN   rn   Moduler  r  r   r|   r   r   r}   r   r   r   rx   rc   rd   re   s   @rR   r   r     s     %O(.
 $bii "))   268
''8
 !.8
 +,	8

 
!8
 8
rT   r   c                     ^  \ rS rSrSr\" \SSS9\\" \SSS9S.rS\	4U 4S	 jjr
\" 5                SS\\R                     S
\\R                     S\\R                     S\\   S\\R"                     S\\   S\\R                     S\\R"                     S\\R                     S\\   S\\\4   4S jj5       rSrU =r$ )MoonshineDecoderig  	input_idsrY   r   )index
layer_namer   )r   ru   cross_attentionsrl   c           	        > [         TU ]  U5        [        R                  " UR                  SS9U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l
        g s  snf NFr   )rM   rN   rn   r   r=   normr  r  r?   r   r   )rO   rl   r  rQ   s      rR   rN   MoonshineDecoder.__init__o  s`     LL!3!3%@	mm;@AaAa;bc;bC"6/;bc
cs   A>r   r   r/   inputs_embedsrG   r   r   r   rP   rv   c
                    USL USL-  (       a  [        S5      eUc  U R                  U5      nU(       a1  Uc.  [        [        U R                  S9[        U R                  S95      nUcD  Ub  UR                  5       OSn[        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      n[        U R                  UUUUUS9nUnU R                  X5      nU	b  UR                  S   nS	nU	S
SSU24   S
SU24   n	U R                  R                  S:X  a  U	S:H  R                  5       (       a  U	OSn	OaU R                  R                  S:X  a$  [        XR                   UR                  S   5      n	O#[#        XR                   UR                  S   5      n	U R$                   H  nU" UUU4U	UUUUUS.U
D6nM     U R'                  U5      n[)        UU(       a  US9$ SS9$ )a\  
encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*):
    Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention
    of the decoder.
encoder_attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
    Mask to avoid performing attention on padding indices in `encoder_hidden_states`. Mask values selected in `[0, 1]`:
    - 1 for tokens that are **not masked**,
    - 0 for tokens that are **masked**.
    [What are attention masks?](../glossary#attention-mask)
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   rY   r"  )rl   input_embedsr   r   r/   r   r  .r   rZ   r!  )r   r   r/   rG   r   r   )r$  r/   )
ValueErrorembed_tokensr   r   rl   get_seq_lengthr|   r*  r   r#  r%  r   r  r   r(  r   r)  r   r   r9  r   )rO   r3  r   r   r/   r;  rG   r   r   r   rP   past_seen_tokenscausal_maskru   r   r+  r,  decoder_layers                     rR   rx   MoonshineDecoder.forwardv  s>   0 -t";<YZZ  --i8M01,dkk2RT`hlhshsTtuO!CRC^==?de"\\ ]5H5H5K"KTaThThN )33A6L(;;&))+%
 &"oomJ!-,2226H *%;CATCTAT<T%UVY[d\d[dVd%e"{{//3FFDZ^aDaCfCfChCh)?nr&11V;)L*,?,?ATATUWAX*& *D*,?,?ATATUWAX*& "[[M)% (>) /#-$7 M ) 		-08+/8O
 	
>B
 	
rT   )r   r9  )	NNNNNNNNN)r[   r\   r]   r^   r   r   r   r   r/  r,   rN   r   r   r|   r   r}   r
   r   r   r   r   r   r   r   rx   rc   rd   re   s   @rR   r2  r2  g  sI   !O$%7q[Y.*+=QSab
 
  151537+/59$(59=A9=W
E,,-W
 !.W
 u//0	W

 "%W
   1 12W
 D>W
 !!1!12W
  ((9(9:W
 !) 6W
 +,W
 
u--	.W
 W
rT   r2  c                      \ rS rSr\\          SS\\R                     S\\R                     S\\R                     S\\R                     S\\
\
\R                           S\\\\
\R                     4      S	\\
\R                        S
\\
\R                        S\\   S\\R                     S\\   S\4S jj5       5       rSrg)MoonshineModeli  Nr   r   decoder_input_idsdecoder_attention_maskencoder_outputsr/   decoder_inputs_embedsdecoder_position_idsrG   r   rP   rv   c                 >   Uc  U R                   " U4SU0UD6nU R                  " SUUUUR                  UUUU	U
S.	UD6n[        UR                  UR                  UR
                  UR                  UR                  UR                  UR
                  UR                  S9$ )a:  
input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
    Float values of the raw speech waveform. Raw speech waveform can be
    obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
    `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
    the soundfile library (`pip install soundfile`). To prepare the array into
    `input_values`, the [`AutoFeatureExtractor`] should be used for padding
    and conversion into a tensor of type `torch.FloatTensor`.
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

Example:

```python
>>> import torch
>>> from transformers import AutoFeatureExtractor, MoonshineModel
>>> from datasets import load_dataset

>>> model = MoonshineModel.from_pretrained("UsefulSensors/moonshine-tiny")
>>> feature_extractor = AutoFeatureExtractor.from_pretrained("UsefulSensors/moonshine-tiny")
>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")
>>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_values = inputs.input_values
>>> decoder_input_ids = torch.tensor([[1, 1]]) * model.config.decoder_start_token_id
>>> last_hidden_state = model(input_values, decoder_input_ids=decoder_input_ids).last_hidden_state
>>> list(last_hidden_state.shape)
[1, 2, 288]
```
r   )	r3  r   r   r   r/   r;  r   rG   r   )r$  r/   decoder_hidden_statesdecoder_attentionsr6  encoder_last_hidden_stater   encoder_attentionsr;   )encoderdecoderr$  r   r/   ru   r   r6  )rO   r   r   rH  rI  rJ  r/   rK  rL  rG   r   rP   decoder_outputss                rR   rx   MoonshineModel.forward  s    \ "/3||L/rYg/rkq/rOEI\\ F
'1#1"1"C"C+/-)F
 F
 "-??+;;"1"?"?.99,==&5&G&G"1"?"?.99	
 		
rT   r;   )
NNNNNNNNNN)r[   r\   r]   r^   r   r   r   r|   r   r   r   r   r   r   r   r   r   rx   rc   r;   rT   rR   rG  rG    sQ    59598<=AEIZ^DHBF$(59E
u001E
 !!1!12E
 $E$4$45	E

 !))9)9 :E
 "%e.?.?(@"ABE
 "%(;U5CTCT=U(U"VWE
  (e.?.?(@AE
 'uU-=-='>?E
 D>E
 !!1!12E
 +,E
 
E
  E
rT   rG  zj
    The Moonshine Model with a language modeling head. Can be used for automatic speech recognition.
    )custom_introc                     ^  \ rS rSrS/rS\4U 4S jjrS rS rS r	S r
S	\R                  4S
 jr\\           SS\\R$                     S\\R&                     S\\R&                     S\\R&                     S\\\\R$                           S\\\\\R$                     4      S\\\R$                        S\\\R&                        S\\   S\\R&                     S\\R&                     S\\   S	\4S jj5       5       rSrU =r$ )!MoonshineForConditionalGenerationi  zproj_out.weightrl   c                    > [         TU ]  U5        [        U5      U l        [        R
                  " UR                  UR                  SS9U l        U R                  5         g r8  )
rM   rN   rG  r   rn   ro   r=   r<   proj_outr  )rO   rl   rQ   s     rR   rN   *MoonshineForConditionalGeneration.__init__$  sH     #F+
		&"4"4f6G6GeT 	rT   c                 6    U R                   R                  5       $ rj   )r   get_encoderr  s    rR   r]  -MoonshineForConditionalGeneration.get_encoder,      zz%%''rT   c                 6    U R                   R                  5       $ rj   )r   get_decoderr  s    rR   ra  -MoonshineForConditionalGeneration.get_decoder/  r_  rT   c                     U R                   $ rj   rZ  r  s    rR   get_output_embeddings7MoonshineForConditionalGeneration.get_output_embeddings2  s    }}rT   c                     Xl         g rj   rd  )rO   new_embeddingss     rR   set_output_embeddings7MoonshineForConditionalGeneration.set_output_embeddings5  s    &rT   rv   c                 6    U R                   R                  5       $ rj   )r   r  r  s    rR   r  6MoonshineForConditionalGeneration.get_input_embeddings8  s    zz..00rT   r   r   rH  rI  rJ  r/   rK  rL  rG   r   labelsrP   c                    Ub:  Uc7  Uc4  [        XR                  R                  U R                  R                  5      nU R                  " U4UUUUUUUU	U
S.	UD6nU R                  UR                  5      nSnUb$  U R                  XU R                  R                  S9n[        UUUR                  UR                  UR                  UR                  UR                  UR                  UR                   S9	$ )ah  
input_values (`torch.FloatTensor` of shape `(batch_size, audio_length)`):
    Float values of the raw speech waveform. Raw speech waveform can be
    obtained by loading a `.flac` or `.wav` audio file into an array of type `list[float]`, a
    `numpy.ndarray` or a `torch.Tensor`, *e.g.* via the torchcodec library (`pip install torchcodec`) or
    the soundfile library (`pip install soundfile`). To prepare the array into
    `input_values`, the [`AutoFeatureExtractor`] should be used for padding
    and conversion into a tensor of type `torch.FloatTensor`.
decoder_position_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`):
    Indices of positions of each input sequence tokens in the position embeddings.
    Used to calculate the position embeddings up to `config.decoder_config.max_position_embeddings`

Example:

```python
>>> import torch
>>> from transformers import AutoProcessor, MoonshineForConditionalGeneration
>>> from datasets import load_dataset

>>> processor = AutoProcessor.from_pretrained("UsefulSensors/moonshine-tiny")
>>> model = MoonshineForConditionalGeneration.from_pretrained("UsefulSensors/moonshine-tiny")

>>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation")

>>> inputs = processor(ds[0]["audio"]["array"], return_tensors="pt")
>>> input_values = inputs.input_values

>>> generated_ids = model.generate(input_values, max_new_tokens=100)

>>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
>>> transcription
'Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.'
```N)	r   rH  rJ  rI  r/   rK  rL  rG   r   )logitsrm  r<   )	lossro  r/   rN  rO  r6  rP  r   rQ  )r*   rl   pad_token_idr:   r   rZ  r$  loss_functionr<   r   r/   rN  rO  r6  rP  r   rQ  )rO   r   r   rH  rI  rJ  r/   rK  rL  rG   r   rm  rP   outputsro  rp  s                   rR   rx   )MoonshineForConditionalGeneration.forward;  s
   f  (-B-J$6KK44dkk6X6X%! '+jj'
)/+#9+"7!5)'
 '
 w889%%Vt{{OeOe%fD#33")"?"?&99$55&-&G&G")"?"?&99

 
	
rT   )r   rZ  )NNNNNNNNNNN)r[   r\   r]   r^   _tied_weights_keysr,   rN   r]  ra  re  ri  rn   r0  r  r   r   r   r|   r   r   r   r   r   r   r   r   r   rx   rc   rd   re   s   @rR   rX  rX    s    ,, (('1bii 1  59598<=AEIZ^DHBF$(59-1T
u001T
 !!1!12T
 $E$4$45	T

 !))9)9 :T
 "%e.?.?(@"ABT
 "%(;U5CTCT=U(U"VWT
  (e.?.?(@AT
 'uU-=-='>?T
 D>T
 !!1!12T
 ))*T
 +,T
 
T
  T
rT   rX  )r,   rG  r   rX  )Mtypingr   r   r   r|   torch.nnrn   transformers.utils.genericr   r   activationsr	   cache_utilsr
   r   r   configuration_utilsr   
generationr   masking_utilsr   modeling_attn_mask_utilsr   r   modeling_flash_attention_utilsr   modeling_layersr   modeling_outputsr   r   r   r   r   modeling_rope_utilsr   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r    utils.deprecationr!   glm.modeling_glmr#   r$   r%   llama.modeling_llamar&   r'   r(   whisper.modeling_whisperr)   r*   
get_loggerr[   loggerr,   r0  rg   r   r   r   r   r   r   r   r2  rG  rX  __all__r;   rT   rR   <module>r     sW   - ,   I ! C C 3 ) / g B 9  : F & R R 0 U U Y Y G 
		H	%J
& J
Z")) "))  l) l)^	1 	U- U"H6 HV # # #._
/ _
Dg
z g
TH
\ H
V 
p
(@/ p

p
frT   