
    bCi                     V   S SK Jr  S SKJrJrJr  S SKrS SKJr  S SK	J
r
  SSKJr  SSKJrJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJrJr  SSKJrJr  SSKJrJ r   SSK!J"r"  SSK#J$r$J%r%J&r&J'r'J(r(  SSK)J*r*  SSK+J,r,  SSK-J.r.J/r/  SSK0J1r1  \(Rd                  " \35      r4\\&" SS9 " S S\$5      5       5       r5\" S5       " S S\Rl                  5      5       r7 " S S\Rl                  5      r8 " S  S!\Rl                  5      r9S" r:SGS# jr;S$\Rx                  S%\=S&\Rx                  4S' jr> SHS(\Rl                  S)\Rx                  S*\Rx                  S+\Rx                  S,\\Rx                     S-\?S.\?S/\"\%   4S0 jjr@ " S1 S2\Rl                  5      rA " S3 S4\5      rB\&" S5S9\& " S6 S7\ 5      5       5       rC\& " S8 S9\C5      5       rD " S: S;\Rl                  5      rE\&" S<S9 " S= S>\C\5      5       rF " S? S@\Rl                  5      rG\& " SA SB\C5      5       rH\&" SCS9 " SD SE\C\15      5       rI/ SFQrJg)I    )	dataclass)CallableOptionalUnionN)check_model_inputs   )ACT2FN)CacheDynamicCache)GenerationMixin)use_kernel_forward_from_hub)create_causal_mask)GradientCheckpointingLayer)BaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ModelOutputTransformersKwargsauto_docstringcan_return_tuplelogging)deprecate_kwarg   )	AutoModel   )	CsmConfigCsmDepthDecoderConfig)CsmGenerationMixinz:
    Base class for the model autoregressive outputs.
    )custom_introc                      \ rS rSr% SrSr\\R                     \	S'   Sr
\\R                     \	S'   Sr\\   \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S	'   Sr\\R                     \	S
'   Sr\\R                     \	S'   Sr\\   \	S'   Sr\\\R                  S4      \	S'   Sr\\\R                  S4      \	S'   Sr\\R                     \	S'   Srg)CsmOutputWithPast2   a=	  
loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction).
logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).

    Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
    `past_key_values` input) to speed up sequential decoding.
depth_decoder_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction) of the depth decoder model.
depth_decoder_logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
    Prediction scores of the depth decoder (scores for each vocabulary token before SoftMax).
depth_decoder_past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
    It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
depth_decoder_hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
    Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
    one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

    Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
depth_decoder_attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
    Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
    sequence_length)`.
backbone_loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
    Language modeling loss (for next-token prediction) of the backbone model.
Nlosslogitspast_key_values.hidden_states
attentionsdepth_decoder_lossdepth_decoder_logitsdepth_decoder_past_key_valuesdepth_decoder_hidden_statesdepth_decoder_attentionsbackbone_loss )__name__
__module____qualname____firstlineno____doc__r'   r   torchFloatTensor__annotations__r(   r)   r
   r*   tupler+   r,   r-   r.   r/   r0   r1   __static_attributes__r2       ^/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/csm/modeling_csm.pyr%   r%   2   s   8 )-D(5$$
%,*.FHU&&'.'+OXe_+=AM8E%"3"3S"89:A:>Ju00#567>6:!2!23:8<(5#4#45<59!8E?9KO%0A0A30F*G!HOHLhuU->->-C'DEL15M8E--.5r=   r%   RMSNormc                   8   ^  \ rS rSrSU 4S jjrS rS rSrU =r$ )
CsmRMSNormb   c                    > [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z)
CsmRMSNorm is equivalent to T5LayerNorm
N)super__init__nn	Parameterr8   onesweightvariance_epsilon)selfhidden_sizeeps	__class__s      r>   rE   CsmRMSNorm.__init__d   s/     	ll5::k#:; #r=   c                    UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr   T)keepdim)	dtypetor8   float32powmeanrsqrtrJ   rI   )rK   r*   input_dtypevariances       r>   forwardCsmRMSNorm.forwardl   sw    #))%((7 $$Q',,R,>%H?T?T4T(UU{{]--k:::r=   c                 ^    [        U R                  R                  5       SU R                   3$ )Nz, eps=)r;   rI   shaperJ   rK   s    r>   
extra_reprCsmRMSNorm.extra_reprs   s*    ))*+6$2G2G1HIIr=   )rJ   rI   )gư>)	r3   r4   r5   r6   rE   r[   r`   r<   __classcell__rN   s   @r>   rA   rA   b   s    $;J Jr=   rA   c                      ^  \ rS rSr% \R
                  \S'   SS\4U 4S jjjr\R                  " 5       \
S 5       5       rSrU =r$ )CsmRotaryEmbeddingw   inv_freqconfigc                   > [         TU ]  5         [        US5      (       aZ  [        UR                  [
        5      (       a;  UR                  R                  SUR                  R                  S5      5      U l        OSU l        UR                  U l	        UR                  U l
        Xl        [        U R                     U l        U R                  U R                  U5      u  o0l        U R                  SUSS9  U R                   U l        g )Nrope_scaling	rope_typetypedefaultrg   F
persistent)rD   rE   hasattr
isinstancerj   dictgetrk   max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrh   r   rope_init_fnattention_scalingregister_bufferrg   original_inv_freq)rK   rh   devicerg   rN   s       r>   rE   CsmRotaryEmbedding.__init__z   s    6>**z&:M:Mt/T/T#0044[&BUBUBYBYZ`BabDN&DN"("@"@$*$B$B!/?+/+<+<T[[&+Q((ZeD!%r=   c                 b   U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      R	                  UR
                  5      nUS S 2S S S 24   R                  5       n[        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        R                  " USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " Xf4SS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR	                  UR                   S
9W	R	                  UR                   S
94$ ! , (       d  f       N@= f)Nr   rQ   r   mpscpuF)device_typeenabledr   dim)rS   )rg   floatexpandr^   rT   r{   rq   rl   strr8   autocast	transposecatcosrx   sinrS   )
rK   xposition_idsinv_freq_expandedposition_ids_expandedr   freqsembr   r   s
             r>   r[   CsmRotaryEmbedding.forward   sR    !MM$4-8>>@GGHZHZ[\H]_acdehhijiqiqr ,QaZ 8 > > @'1!((--'E'E!((--[`J`ahhmmfk^^UC&,,.1F1L1L1NNYYZ[]^_E))UN3C'')d444C'')d444C	 D vvAGGv$cff177f&;;; DCs   $BF  
F.)rx   rh   ru   rz   rv   rw   rk   N)r3   r4   r5   r6   r8   Tensorr:   r    rE   no_gradr   r[   r<   rb   rc   s   @r>   re   re   w   s@    ll/y / /" ]]_<  <r=   re   c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )CsmMLP   c                   > [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  UR                  S9U l        [
        R                  " U R                  U R                  UR                  S9U l	        [
        R                  " U R                  U R                  UR                  S9U l
        [        UR                     U l        g )Nbias)rD   rE   rh   rL   intermediate_sizerF   Linearmlp_bias	gate_projup_proj	down_projr	   
hidden_actact_fnrK   rh   rN   s     r>   rE   CsmMLP.__init__   s    !--!'!9!94#3#3T5K5KRXRaRabyy!1!143I3IPVP_P_`4#9#94;K;KRXRaRabV../r=   c                     U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r   )r   r   r   r   )rK   r   r   s      r>   r[   CsmMLP.forward   s6    NN4;;t~~a/@#ADLLQRO#ST	r=   )r   rh   r   r   rL   r   r   r3   r4   r5   r6   rE   r[   r<   rb   rc   s   @r>   r   r      s    0 r=   r   c                     U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrQ   r   r   )r^   r8   r   )r   x1x2s      r>   rotate_halfr      sZ    	
3"!''"+"""	#B	
3q ""	#B99rc2YB''r=   c                     UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXg4$ )a  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    position_ids (`torch.Tensor`, *optional*):
        Deprecated and unused.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)	unsqueezer   )qkr   r   r   unsqueeze_dimq_embedk_embeds           r>   apply_rotary_pos_embr      sS    ( --
&C
--
&Cw;q>C/0Gw;q>C/0Gr=   r*   n_repreturnc                     U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )z
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)r^   r   reshape)r*   r   batchnum_key_value_headsslenhead_dims         r>   	repeat_kvr      s_    
 2?1D1D.Ez!!Qa"23::5W\dlmM  e(CTTTr=   modulequerykeyvalueattention_maskscalingdropoutkwargsc                 @   [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub"  US S 2S S 2S S 2S UR
                  S   24   nX-   n
[        R                  R                  U
S[        R                  S9R                  UR                  5      n
[        R                  R                  XU R                  S9n
[        R                  " X5      nUR	                  SS5      R                  5       nX4$ )Nr   r   rQ   )r   rS   )ptrainingr   )r   num_key_value_groupsr8   matmulr   r^   rF   
functionalsoftmaxrU   rT   rS   r   r   
contiguous)r   r   r   r   r   r   r   r   
key_statesvalue_statesattn_weightscausal_maskattn_outputs                r>   eager_attention_forwardr      s     3 ; ;<JU$?$?@L<<';';Aq'ABWLL!$Q1.D
0@0@0D.D%DE#1==((2U]](SVVW\WbWbcL==((6??([L,,|:K''1-88:K$$r=   c                   4  ^  \ rS rSrSrS\S\4U 4S jjr\" SSSS	9  SS
\	R                  S\\	R                  \	R                  4   S\\	R                     S\\   S\\	R                     S\\   S\\	R                  \	R                  4   4S jj5       rSrU =r$ )CsmAttention   z=Multi-headed attention from 'Attention Is All You Need' paperrh   	layer_idxc                 P  > [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nr   g      Tr   )rD   rE   rh   r   getattrrL   num_attention_headsr   r   r   r   attention_dropout	is_causalrF   r   attention_biasq_projk_projv_projo_projrK   rh   r   rN   s      r>   rE   CsmAttention.__init__   sI   "
F4F4F&JdJd4de$*$>$>&B\B\$\!}}d*!'!9!9ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii : :T]] JQWQfQf
 ii&&68J8JQWQfQf
r=   past_key_valuer)   4.58new_nameversionr*   position_embeddingsr   cache_positionr   r   c                 4   UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  U5      R                  U5      R	                  SS5      nUu  p[        XX5      u  pUb$  XUS.nUR                  XU R                  U5      u  p[        nU R                  R                  S:w  a  [        U R                  R                     nU" U U	U
UU4U R                  (       d  SOU R                  U R                   S.UD6u  nnUR"                  " / UQSP76 R%                  5       nU R'                  U5      nUU4$ )NrQ   r   r   )r   r   r   eager        )r   r   )r^   r   r   viewr   r   r   r   updater   r   rh   _attn_implementationr   r   r   r   r   r   r   )rK   r*   r   r   r)   r   r   input_shapehidden_shapequery_statesr   r   r   r   cache_kwargsattention_interfacer   r   s                     r>   r[   CsmAttention.forward  s    $))#2.88b8$--8{{=166|DNNqRST[[/44\BLLQPQR
{{=166|DNNqRST&#7RU#[ &#&nUL'6'='=jX\XfXfht'u$J(?;;++w6"9$++:Z:Z"[$7	%
  $}}C$2H2HLL	%
 	%
!\ "));;;;FFHkk+.L((r=   )r   rh   r   r   r   r   r   r   r   r   r   )NN)r3   r4   r5   r6   r7   r    intrE   r   r8   r   r;   r   r
   
LongTensorr   r   r[   r<   rb   rc   s   @r>   r   r      s    G
y 
S 
. %0A6R ,059))||)) #5<<#=>)) !.	))
 "%)) !!1!12)) +,)) 
u||U\\)	*)) S))r=   r   c                   H  ^  \ rS rSrS\S\4U 4S jjr\" SSSS9      SS	\R                  S
\
\R                     S\
\R                     S\
\   S\
\   S\
\R                     S\
\\R                  \R                  4      S\\   S\R                  4S jj5       rSrU =r$ )CsmDecoderLayeri:  rh   r   c                   > [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N)rh   r   rM   )rD   rE   rL   r   	self_attnr   mlprA   rms_norm_epsinput_layernormpost_attention_layernormr   s      r>   rE   CsmDecoderLayer.__init__;  si    !--%VI&>)&*<*<&BUBUV(263E3E6K^K^(_%r=   r   r)   r   r   r*   r   r   	use_cacher   r   r   r   c                     Un	U R                  U5      nU R                  " SUUUUUUUS.UD6u  pX-   nUn	U R                  U5      nU R                  U5      nX-   nU$ )N)r*   r   r   r)   r  r   r   r2   )r  r   r  r   )rK   r*   r   r   r)   r  r   r   r   residual_s              r>   r[   CsmDecoderLayer.forwardE  s     !,,];>> 	
')%+) 3	
 	
 !0 !55mD/ 0r=   )rL   r  r   r  r   )NNNFNN)r3   r4   r5   r6   r    r   rE   r   r8   r   r   r   r
   boolr;   r   r   r[   r<   rb   rc   s   @r>   r   r   :  s    `y `S ` %0A6R 2637+/$)59KO|| !. u//0	
 "% D> !!1!12 &eELL%,,,F&GH +, 
 Sr=   r   z[
    The bare Csm Model outputting raw hidden-states without any specific head on top.
    c                   b   ^  \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSr\\S.rU 4S jrS	rU =r$ )
CsmPreTrainedModelih  rh   modelTr   r)   )r*   r+   c                   > [         TU ]  U5        [        U[        5      (       a]  UR                  n[        US-
  5       H>  nUR                  R                  U   R                  SU R                  R                  S9  M@     g g )Nr   r   )rW   std)rD   _init_weightsrq   CsmCodebooksHeadnum_codebooksrangerI   datanormal_rh   initializer_range)rK   r   r  irN   s       r>   r   CsmPreTrainedModel._init_weights  sn    f%f.//"00M=1,-""1%--3DKK<Y<Y-Z . 0r=   r2   )r3   r4   r5   r6   r    r:   base_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_supports_sdpa_can_compile_fullgraph_supports_attention_backendr   r   _can_record_outputsr  r<   rb   rc   s   @r>   r  r  h  s\     &*#*+#4"5N ""&("
[ [r=   r  c                   P  ^  \ rS rSr% \\S'   U 4S jr\" 5       \        SS\	\
R                     S\	\
R                     S\	\
R                     S\	\
R                     S\	\   S	\	\
R                     S
\	\   S\	\
R                     S\\   S\\\4   4S jj5       5       rSrU =r$ )CsmDepthDecoderModeli  rh   c           	      j  > [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  -  UR                  5      U l	        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                   UR"                  S9U l        ['        US9U l        SU l        [
        R,                  " UR                  UR                   SS9U l        U R1                  5         g s  snf )Nr   rh   Fr   )rD   rE   pad_token_idpadding_idx
vocab_sizerF   	Embeddingr  backbone_hidden_sizeembed_tokens
ModuleListr  num_hidden_layersr   layersrA   rL   r   normre   
rotary_embgradient_checkpointingr   inputs_embeds_projector	post_initr   s      r>   rE   CsmDepthDecoderModel.__init__  s     !.. ++LL&*>*>ARAR*RU[UpUpqmmAFvG_G_A`aA`I_V/A`a
 v11v7J7JK	,F;&+#')yy1L1LfN`N`gl'm$ 	 bs   D0	input_idsbackbone_last_hidden_stater   r   r)   inputs_embedsr  r   r   r   c	                 :   Ub:  [         R                  R                  5       (       d  [        R	                  S5        SnUSL USL-  (       a  [        S5      eU(       a  Uc  [        U R                  S9nUci  Ub  UR                  5       OSn
Ub  UR                  S   OUR                  S   nUb  UR                  OUR                  n[         R                  " XU-   US9nUc  [         R                  " US-
  SS9nXR                  -  nU R                  X-   5      nUS   S:H  nUb	  X&SS2S4'   O?[         R                  R                  5       (       d  U(       a  [        R                  S	5        U R!                  U5      n[#        U R                  UUUUUS
9nUnUR%                  S5      nU R'                  UU5      nU R(                  SU R                  R*                    H  nU" U4UUUUUUS.U	D6nM     U R-                  U5      n[/        UU(       a  US9$ SS9$ )a*  
backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*):
    The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
    is provided in the `input_ids` argument.
NzCustom `position_ids` were provided but will be ignored. CSM depth decoder automatically determines position_ids from `cache_position` and as it requires them to be identical across the batch, the provided position_ids will be ignored.z;You must specify exactly one of input_ids or inputs_embeds.r$  r   r   r{   )minzvWhen the first codebook token is provided, `backbone_last_hidden_state` should also be provided for correct inference.rh   input_embedsr   r   r)   r   )r   r   r)   r  r   r   last_hidden_stater)   )r8   compileris_compilingloggerwarning_once
ValueErrorr   rh   get_seq_lengthr^   r{   arangeclampr'  r*  warningr1  r   r   r/  r-  r,  r.  r   )rK   r4  r5  r   r   r)   r6  r  r   r   past_seen_tokensinputs_seq_lengthr{   codebook_idxsoffsetinput_ids_are_first_codebookr   r*   r   decoder_layers                       r>   r[   CsmDepthDecoderModel.forward  s>   & #ENN,G,G,I,IM  L-t";<Z[[0*$++>O!CRC^==?de:G:S 3 3A 6YbYhYhijYk-:-F]))IL\L\F"\\*:O`<`iopN !KK(:BM"__4F --i.@AM+9!+<+A()5&@ad#~~22449UNN Q 44]C(;;&))+%
 & &//2"oom\J![[)H4;;+H+HIM)	*) /#-$7	 	M J 		-0&+/8O
 	
>B
 	
r=   )r*  r0  r1  r-  r.  r&  r/  r'  )NNNNNNNN)r3   r4   r5   r6   r!   r:   rE   r   r   r   r8   r   r9   r   r
   r	  r   r   r   r;   r   r[   r<   rb   rc   s   @r>   r"  r"    s   !!   15BF1537+/59$(59R
E,,-R
 %-U->->$?R
 !.	R

 u//0R
 "%R
   1 12R
 D>R
 !!1!12R
 +,R
 
u--	.R
  R
r=   r"  c                   2   ^  \ rS rSrU 4S jrSS jrSrU =r$ )r  i  c                    > [         TU ]  5         X l        [        R                  " [
        R                  " U R                  S-
  X5      5      U l        g Nr   )rD   rE   r  rF   rG   r8   emptyrI   )rK   rL   r  r'  rN   s       r>   rE   CsmCodebooksHead.__init__  s:    *ll5;;t/A/AA/E{#_`r=   c           
         Uc3  UR                   S   nU R                  [        R                  " U5         nOUS-
  nU R                  U   n[	        UR                   S   5       Vs/ s H9  n[
        R                  R                  US S 2US S 24   XF   R                  5      PM;     nn[        R                  " USS9nU$ s  snf )Nr   r   r   )
r^   rI   r8   rD  r  rF   r   linearTstack)rK   r*   r   
seq_lengthcodebook_weightrI  codebook_idxs          r>   r[   CsmCodebooksHead.forward  s    !&,,Q/J"kk%,,z*BCO*Q.M"kk-8O !&o&;&;A&> ?
 ? MM  q,/A!BODaDcDcd ? 	 
 Mq9
s   %A B>)r  rI   r   r   rc   s   @r>   r  r    s    a
 r=   r  a$  
    The CsmDepthDecoder Model transformer, with a [`CsmCodebooksHead`] on top,
    which can be seen a position-specific language modeling head, allowing to use a different linear layer for each codebook
    (e.g. position 0 is the first codebook and uses the first codebook head, etc.)
    c                   F  ^  \ rS rSrSrSrSrU 4S jr\\	          SS\
\R                     S\
\R                     S\
\R                     S\
\R                     S\
\\\\R                     4      S	\
\R                     S
\
\R                     S\
\   S\
\R                     S\\\R                  4   S\\   S\\\4   4S jj5       5       r    SS\R                  S\
\   S\
\R                     S	\
\R                     S\
\R                     4
U 4S jjjrSrU =r$ )CsmDepthDecoderForCausalLMi
  Nc                    > [         TU ]  U5        [        U5      U l        UR                  U l        [        UR                  UR                  UR                  5      U l        U R                  5         g r   )
rD   rE   r"  r  r'  r  rL   r  codebooks_headr2  r   s     r>   rE   #CsmDepthDecoderForCausalLM.__init__  sY     )&1
 ++.v/A/A6CWCWY_YjYjk 	r=   r4  r5  r   r   r)   r6  labelsr  r   logits_to_keepr   r   c                    U R                   " SUUUUUUUU	S.UD6nUS   n[        U
[        5      (       a!  U
S:X  a  [        SS5      nO[        U
* S5      nOU
nU R	                  USS2USS24   U	b  X   OS5      nUR                  5       nSnUbB  USSS24   R                  5       nU R                  " SUSU R                  R                  US.UD6n[        UUUR                  UR                  UR                  S9$ )	a  
backbone_last_hidden_state (`torch.FloatTensor` of shape `(batch_size, backbone_hidden_size)`, *optional*):
    The last hidden state of the backbone model. Such input is required when the first codebook token (the one generated by the backbone model)
    is provided in the `input_ids` argument.
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
)r4  r5  r   r   r)   r6  r  r   r   r   N.)r(   r`  r'  shift_labels)r'   r(   r)   r*   r+   r2   )r  rq   r   slicer^  r   loss_functionrh   r'  r   r)   r*   r+   )rK   r4  r5  r   r   r)   r6  r`  r  r   ra  r   outputsr*   slice_indicesr(   r'   rc  s                     r>   r[   "CsmDepthDecoderForCausalLM.forward  s8   2 ** 

'A)%+')

 

  
nc**" %a %~ot <*M$$!]A-.Q_Qk0Mqu
 ""$!#qr'?557L%% dt{{7M7M\hlrD &#33!//))
 	
r=   c                    > [         T	U ]  " XX4U40 UD6nUS   S   S:H  nU(       d  UR                  S5        UR                  S5        U$ )Nr   r   r5  r   )rD   prepare_inputs_for_generationpop)
rK   r4  r)   r   r6  r   r   model_inputsis_first_generation_steprN   s
            r>   rj  8CsmDepthDecoderForCausalLM.prepare_inputs_for_generationc  sc     w<~
Y_
 $00@#A!#D#I '9: 	(r=   )r^  r  r'  )
NNNNNNNNNr   NNNN)r3   r4   r5   r6   _tied_weights_keys_tp_plan_pp_planrE   r   r   r   r8   r   r9   r   r   r
   listr	  r   r   r   r;   r   r[   rj  r<   rb   rc   s   @r>   r\  r\  
  s    HH  15BF1537KO59-1$(5934@
E,,-@
 %-U->->$?@
 !.	@

 u//0@
 "%tE4E4E/F(F"GH@
   1 12@
 ))*@
 D>@
 !!1!12@
 c5<</0@
 +,@
 
u,,	-@
  @
J ,0595959## "% !!1!12	
   1 12 !!1!12 r=   r\  c                   .   ^  \ rS rSrU 4S jrS rSrU =r$ )CsmBackboneModelEmbeddingsiz  c                   > [         TU ]  5         [        R                  " UR                  UR
                  -  UR                  5      U l        U R                  S[        R                  " UR                  5      UR
                  -  SS9  g )Naudio_tokens_offsetsFrn   )rD   rE   rF   r(  r  r'  rL   embed_audio_tokensry   r8   rD  r   s     r>   rE   #CsmBackboneModelEmbeddings.__init__{  sn    "$,,0D0DvGXGX0X[a[m[m"n"ELL1E1E$FIZIZ$Zgl 	 	
r=   c                 ^    U R                  XR                  -   5      nUR                  SS9nU$ )Nr   r   )rx  rw  sum)rK   r4  r;  s      r>   r[   "CsmBackboneModelEmbeddings.forward  s4    ..y;T;T/TU#''A'.r=   )rx  r   rc   s   @r>   ru  ru  z  s    
 r=   ru  c                     ^  \ rS rSrU 4S jr\" 5       \       SS\\R                     S\\R                     S\\R                     S\\   S\\R                     S\\R                     S	\\   S
\\   S\4S jj5       5       rSrU =r$ )CsmBackboneModeli  c           	        > [         TU ]  U5        UR                  U l        UR                  U l        [        U5      U l        [        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr   r$  F)rD   rE   r%  r&  r'  ru  r*  rF   r+  r  r,  r   r-  rA   rL   r   r.  re   r/  r0  r2  r   s      r>   rE   CsmBackboneModel.__init__  s     !.. ++6v>mmAFvG_G_A`aA`I_V/A`a
 v11v7J7JK	,F;&+# 	 bs   *Cr4  r   r   r)   r6  r   r  r   r   c           
      J   USL USL-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcD  Ub  UR	                  5       OSn	[
        R                  " XUR                  S   -   UR                  S9nUc  UR                  S5      n[        U R                  UUUUUS9n
UnU R                  X5      nU R                  SU R                  R                    H  nU" U4U
UUUUS.UD6nM     U R                  U5      n[        UUS	9$ )
a  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`):
    1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
    requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.

    2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
Nz:You must specify exactly one of input_ids or inputs_embedsr$  r   r   r8  r:  )r   r   r)   r   r   r<  )rB  r*  r   rh   rC  r8   rD  r^   r{   r   r   r/  r-  r,  r.  r   )rK   r4  r   r   r)   r6  r   r  r   rG  r   r*   r   rL  s                 r>   r[   CsmBackboneModel.forward  sR   2 -t";<YZZ *.*;*;I*FM0*$++>O!CRC^==?de+0<< ]5H5H5K"KTaThTh,N )33A6L(;;&))+%
 &"oomJ![[)H4;;+H+HIM)*) /-$7 M J 		-0&++
 	
r=   )r*  r0  r-  r.  r&  r/  r'  )NNNNNNN)r3   r4   r5   r6   rE   r   r   r   r8   r   r   r
   r9   r	  r   r   r   r[   r<   rb   rc   s   @r>   r~  r~    s      151537+/5959$(D
E,,-D
 !.D
 u//0	D

 "%D
   1 12D
 !!1!12D
 D>D
 +,D
 
!D
  D
r=   r~  z
    The Csm model consists of two llama-like auto-regressive transformer models: a backbone model that predicts the first codebook token and a depth decoder that predicts the other codebook tokens.
    c                   B  ^  \ rS rSrSS/rU 4S jrS rS rS r\	U 4S j5       r
U 4S	 jr    SS
\\R                     S\\R                     S\\R                     S\\R                     S\\R                     4
S jjr    SS
\R                   S\\   S\\R                      S\\R$                     S\\R                      4
U 4S jjjr\\           SS
\\R                      S\\R                     S\\R                     S\\R                     S\\R                      S\\\\\R$                     4      S\\R$                     S\\R                      S\\   S\\R                      S\\\R                  4   S\\   S\\\4   4S jj5       5       rSrU =r $ )CsmForConditionalGenerationi  z5backbone_model.embed_tokens.embed_audio_tokens.weightz'depth_decoder.model.embed_tokens.weightc                   > [         TU ]  U5        UR                  U l        [        R                  " UR
                  UR                  SS9U l        [        R                  " UR                  UR
                  5      U l	        [        R                  U5      U l        [        R                  UR                  5      U l        [         R"                  " UR$                  5      U l        U R)                  5         g )NFr   )rD   rE   r'  rF   r   rL   lm_headr(  text_vocab_sizeembed_text_tokensr~  _from_configbackbone_modelr\  depth_decoder_configdepth_decoderr   from_configcodec_configcodec_modelr2  r   s     r>   rE   $CsmForConditionalGeneration.__init__  s      ++yy!3!3V5F5FUS!#f.D.DfFXFX!Y.;;FC7DDVE`E`a$001D1DEr=   c                 .    U R                   R                  $ r   r  r*  r_   s    r>   get_input_embeddings0CsmForConditionalGeneration.get_input_embeddings  s    ""///r=   c                 $    XR                   l        g r   r  )rK   r   s     r>   set_input_embeddings0CsmForConditionalGeneration.set_input_embeddings  s    +0(r=   c                     U R                   R                  (       aO  U R                  U R                  R                  R
                  U R                  R                  R                  5        g g r   )rh   tie_codebooks_embeddings_tie_or_clone_weightsr  r*  rx  r  r  r_   s    r>   _tie_weights(CsmForConditionalGeneration._tie_weights  sL    ;;//&&##00CC""((55 0r=   c                    > UR                  SS5      (       a  [        T
U ]  " U0 UD6u  p4O[        T
U ]  " U0 UD6nSn[        U5      n[	        UR
                  5      R                  5        VVs0 s H"  u  pxUR                  U5      (       d  M  XvS  U_M$     n	nn[	        UR                  R
                  5      R                  SS0U	E5        U	 H  n[        UR
                  XW-   5        M     SU;   a  UW4$ U$ s  snnf )Noutput_loading_infoFdepth_decoder__from_model_config)rs   rD   from_pretrainedlenvarsgeneration_configitems
startswithr  r   delattr)clsargsr   r  loading_infoprefix
prefix_lenattrr   depth_decoder_attrsrN   s             r>   r  +CsmForConditionalGeneration.from_pretrained  s   ::+U33"''"94"J6"JE<G+T<V<E "[
  $E$;$;<BBD
Dv& %Du$D 	 
 	U  223::<PRW;o[n;op (DE++V]; ( !F*,&&L
s   /C:	C:c                    > SnU R                   R                  R                  5       nUR                  SS 5        UR	                  5        H  u  pV[        U R                  X5-   U5        M      [        TU ]  " U0 UD6  g )Nr  transformers_version)r  r  to_diff_dictrk  r  setattrrD   save_pretrained)rK   r  r   r  r  r  r   rN   s          r>   r  +CsmForConditionalGeneration.save_pretrained  sq    !"00BBOOQ 6=.446KDD**FM5A 7 	00r=   r4  input_valuesinput_values_cutoffsr`  r   c                    U R                  U5      nUGbN  [        R                  R                  US5      nX3S:     R	                  5       nXfS:     n[
        R                  " UR                  5       UR                  S9R                  [        U5      S5      nXvR                  S5      :  n[
        R                  " 5          / n[        X#5       H  u  pXS:     n
[        U
R                  S   S-
  5       Hp  nX   nXS-      nU	SX24   nU R                   R#                  UR                  S5      5      nUR$                  R'                  SS5      nUR)                  US   5        Mr     M     [        S U 5       5      n[
        R*                  " U Vs/ s H7  n[        R                  R                  USSSUUR                  S   -
  45      PM9     sn5      nU R                   R-                  U5      nSSS5        U R.                  R0                  nUU:H  nU R2                  R5                  W5      nUW   UU'   [
        R6                  " SSU R.                  R8                  4UR                  [
        R:                  S	9U R.                  R<                  -  nU R2                  R5                  U5      R?                  S5      nXR.                  R@                  :H  nURC                  URE                  5       S5      UU'   Ubg  UR                  S5      RC                  SSU R.                  R8                  5      nUU   UU'   UUU'   US
:H  RG                  SS9nSUUS   US   SS24'   UnXTS.$ s  snf ! , (       d  f       GN= f)a8  
Merges the input_ids and input_values to produce a single inputs_embeds tensor:
1 - Infers the codec model on the input_values to retrieve codebook token.
2 - Embeds codebook tokens and places them at the correct positions in the inputs_embeds tensor.
3 - If labels are provided, expands them to match codebook dimensions and position the target codebook tokens in the inputs_embeds tensor.

Args:
    input_ids (`torch.Tensor` of shape `(batch_size, sequence_length)`):
        The input ids to embed.
    input_values (`torch.Tensor` of shape `(batch_size, channels, audio_sequence_length)`):
        The audio input values to embed.
    input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`):
        The cutoffs of the audio input values relative to its batch index, padded with -1 when no audio.
Nr   r   r   r8  rQ   r   .c              3   >   #    U  H  oR                   S    v   M     g7f)r   N)r^   ).0els     r>   	<genexpr>QCsmForConditionalGeneration._merge_input_ids_with_input_values.<locals>.<genexpr>Y  s     &O=Nrxx{=Ns   )r{   rS   iTas_tuple)r6  r`  )$r  rF   r   paddiffr8   rD  maxr{   r   r  r   r   zipr  r^   r  encodeaudio_codesr   appendrV  get_audio_codes_maskrh   audio_token_idr  r*  rH   r  longcodebook_eos_token_idsqueezeaudio_eos_token_idrepeatr{  nonzero)rK   r4  r  r  r`  r6  audio_lengthsinput_values_maskaudio_tokens_listbatch_input_valuesbatch_input_values_cutoffsr  	start_idxend_idxaudio_batchcodec_outputscodebook_idsmax_audio_framesr  batched_audio_token_idsaudio_codes_maskr  audio_token_maskaudio_embedsaudio_eos_frame_idsaudio_eos_embedsaudio_eos_token_masklabels_expanded depth_decoder_ignore_frames_idxss                                r>   "_merge_input_ids_with_input_values>CsmForConditionalGeneration._merge_input_ids_with_input_values)  su   * ..y9##%==#4#45I6#R 01JKPPRM)!*;<M %-A-E-E-GP\PcPc d k kM"B! !24K4KA4N N
 $&!FI,FmB&1KjkLk1l."#=#C#CA#F#JK$>$A	"<U"C&8i>O9O&P(,(8(8(?(?@U@UVW@X(Y'4'@'@'J'J1b'Q)00aA L Gn $'&O=N&O#O */++`qr`qZ\R]]&&rAq!5EQR5S+TU`qr+' $(#3#3#H#HIZ#[ ! !$ "[[77N(N:..;;<STL.:;K.LM*+ 

Aq$++";";<YEUEU]b]g]gh++334    $22??@ST\\]^_#,0N0N#N 2B2I2IJ^JbJbJdfg2hM./ !"("2"22"6"="=aDKKD]D]"^4KL\4] 018K 454:dN3K3KUY3K3Z0pt @ CEefgEhjkjl lm(!.AA= s !s    CM->M(
"M-(M--
M<r)   r   r6  r   c           	      2  > [         T	U ]  " S	UUUUUS.UD6nUb|  UR                  S:X  al  UR                  S5      cZ  U R	                  UUR                  S5      UR                  S5      UR                  S5      S9nUR                  US   US   S S.5        U$ )
N)r4  r)   r   r6  r   r   r6  r  r  r`  )r4  r  r  r`  )r6  r`  r4  r2   )rD   rj  ndimrs   r  r   )
rK   r4  r)   r   r6  r   r   rl  merged_inputsrN   s
            r>   rj  9CsmForConditionalGeneration.prepare_inputs_for_generation{  s     w< 
+)')
 
  Y^^q%8\=M=Mo=^=f CC##ZZ7%+ZZ0F%Gzz(+	 D M "/"@MZbLcrvw r=   r   r  ra  r   c                    Ub.  UR                   S:X  a  U R                  XXH5      nUS   nUS   nSnU R                  " SUUUUUU	U
S.UD6nUS   n[        U[        5      (       a  [        U* S5      OUnU R                  USS2USS24   5      nSnSnSnSnUb  USS2SS2S4   nU R                  " SUUU R                  R                  S.UD6nUSS2SS2SS24   S	:H  R                  S
S9) nUU   SSU R                  R                  S-
  24   n[        R                  R                  USSS9nUR                  SS9nUUS   US   S-
  SS24   nUU   nU R                   " SUUU	SUS.UD6nUR"                  nUU-   n[%        UUUUUR&                  UR(                  UR*                  Ub  UR,                  OSUb  UR&                  OSUb  UR(                  OSUb  UR*                  S9$ SS9$ )a`  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length, num_codebooks) or (batch_size, sequence_length)`):
    1. (batch_size, sequence_length): corresponds to the input sequence prepared with the processor from the text prompt. Such input
    requires `input_values` to be provided so that audio can be encoded in codebook tokens and then merged with the text tokens.

    2. (batch_size, sequence_length, num_codebooks): codebook tokens generated during the autoregressive decoding. Such input is not meant to be used by end users.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
input_values_cutoffs (`torch.Tensor` of shape `(batch_size, max_num_audio)`, *optional*):
    Specify the end positions of audio segments within each batch entry, relative to the concatenated audio input.
    If a batch entry has fewer segments than the maximum, it is padded with -1. For example, in a batch of 2 sequences
    where the first contains 2 audio segments of length l1, and the second contains 1 audio segment of length l2,
    the input_values_cutoffs would be: [[l1, 2 * l1], [l2, -1]].
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[config.audio_token_id, -100, -101]`.
    Requires targeted `input_values` to be provided as audio tokens will be inferred from it using the `codec_model`.
    - `config.audio_token_id` indicates an audio frames (considering sequence length elements as frames)
    - `-100` will be ignored in the loss computation
    - `-101` indicates the audio frame will be used only for the backbone model (using the first codebook token as labels)

    Such labels can be prepared using `output_labels=True` when calling [`CsmProcessor`].
logits_to_keep (`int` or `torch.Tensor`, *optional*):
    Kept for compatibility. Does not support another value than:
    1. `0`, which is equivalent to keeping all logits, used in the training regime
    2. `1`, which is equivalent to keeping only the last logit, used in the generation regime

Example:

```python
>>> import torch
>>> from transformers import CsmForConditionalGeneration, AutoProcessor
>>> from datasets import load_dataset, Audio

>>> model_id = "sesame/csm-1b"
>>> torch_device = "cuda" if torch.cuda.is_available() else "cpu"

>>> processor = AutoProcessor.from_pretrained(model_id)

>>> ds = load_dataset("hf-internal-testing/dailytalk-dummy", split="train")
>>> # ensure the audio is 24kHz
>>> ds = ds.cast_column("audio", Audio(sampling_rate=24000))

>>> conversation = []
>>> # prepare a conversation with text and corresponding audio
>>> for text, audio, speaker_id in zip(ds[:4]["text"], ds[:4]["audio"], ds[:4]["speaker_id"]):
...     conversation.append(
...         {
...             "role": f"{speaker_id}",
...             "content": [{"type": "text", "text": text}, {"type": "audio", "path": audio["array"]}],
...         }
...     )

>>> inputs = processor.apply_chat_template(
...     conversation,
...     tokenize=True,
...     return_dict=True,
...     output_labels=True,
... ).to(torch_device)

>>> model = CsmForConditionalGeneration.from_pretrained(model_id, device_map=torch_device)
>>> output = model(**inputs)
>>> output.loss.backward()
```Nr   r6  r`  )r4  r   r   r)   r6  r  r   r   )r(   r`  r'  r   r  rQ   r   .r  )r   Tr  )r4  r5  r  return_dictr`  )r'   r1   r,   r(   r)   r*   r+   r-   r.   r/   r0   r2   )r  r  r  rq   r   rd  r  re  rh   r'  allr  rF   r   r  r  r  r'   r%   r)   r*   r+   r(   )rK   r4  r  r   r  r   r)   r6  r`  r  r   ra  r   r  backbone_outputsbackbone_hidden_statesrg  backbone_logitsr'   r1   r,   depth_decoder_outputsbackbone_labels
train_maskdepth_decoder_input_ids
train_idxsbackbone_last_hidden_statesdepth_decoder_labelss                               r>   r[   #CsmForConditionalGeneration.forward  s   f  Y^^q%8 CC)=M */:M"8,FI.. 	
)%+')	
 	
 "2!!48B>SV8W8W~ot4]k,,'=aPQ>Q'RS! $$Q1WoO .. &4;;KaKaekM "!Q(+t388R8@@J&,Z&8>]@Y@Y\]@]>]9]&^#&(mm&7&78OQW_`&7&a##++T+:J*@APZ[\P]`aPacdAd*e'#)*#5 $($6$6 %1+F# +% %! "7!;!; #55D '1",<<*88'22AVAb!6!=!=hl$0 +@*O*O$0 )>(K(KI^Ij%:%E%E
 	
 qu
 	
r=   )r  r  r  r  r  r'  ro  )NNNNNNNNNNr   )!r3   r4   r5   r6   rp  rE   r  r  r  classmethodr  r  r   r8   r   r  r   r
   r9   rj  r   r   r   rs  r	  r   r   r   r;   r%   r[   r<   rb   rc   s   @r>   r  r    s    	@1
01  41 -1/37;)-PBELL)PB u||,PB 'u||4	PB
 &PB 
%,,	PBj ,0595959## "% !!1!12	
   1 12 !!1!12 >  15/3157;37KO59-1$(5934[
E,,-[
 u||,[
 !.	[

 'u||4[
 u//0[
 "%tE4E4E/F(F"GH[
   1 12[
 ))*[
 D>[
 !!1!12[
 c5<</0[
 +,[
 
u''	([
  [
r=   r  )r  r~  r"  r\  r  rP  )r   )Kdataclassesr   typingr   r   r   r8   torch.nnrF   transformers.utils.genericr   activationsr	   cache_utilsr
   r   
generationr   integrationsr   masking_utilsr   modeling_layersr   modeling_outputsr   r   modeling_rope_utilsr   r   modeling_utilsr   r   processing_utilsr   utilsr   r   r   r   r   utils.deprecationr   autor   configuration_csmr    r!   generation_csmr"   
get_loggerr3   r@  r%   ModulerA   re   r   r   r   r   r   r   r   r   r   r   r  r"  r  r\  ru  r~  r  __all__r2   r=   r>   <module>r     s  , " , ,   9 ! . ) 7 / 9 O K F & _ _ 0  ? . 
		H	% 
'6 '6 '6T Y'J J (J(!< !<HRYY  (6	UU\\ 	U# 	U%,, 	U& %II%<<% 
% <<	%
 U\\*% % % '(%4D)299 D)N+0 +\ 
 [ [ [4 g
- g
 g
Tryy . f!3_ ffR  V
) V
 V
r 
P
"46H P

P
f
r=   