cCibNSrSSKrSSKJrJr SSKrSSKJr SSKJrJ r J r J r SSKJ r SSKJrJr SS KJr SS KJr SS KJr SS KJrJrJrJrJr SS KJr SSKJ r J!r! SSK"J#r# SSK$J%r% \!RL"\'5r(S)Sjr)"SS\RT5r+"SS\RT5r,"SS\5r-\ "SS\55r.\ "SS\.55r/\ "SS9"SS \.\55r0\ "S!S9"S"S#\.55r1\ "S$S%\.55r2\ "S&S'\.55r3/S(Qr4g)*zPyTorch MPT model.N)OptionalUnion)nn)BCEWithLogitsLossCrossEntropyLoss LayerNormMSELoss) functional)Cache DynamicCache)GenerationMixin)!_prepare_4d_causal_attention_mask)GradientCheckpointingLayer))BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsQuestionAnsweringModelOutput SequenceClassifierOutputWithPastTokenClassifierOutput)PreTrainedModel)auto_docstringlogging)deprecate_kwarg) MptConfigcH[R"SU- S[RUS9RSSSU5nS[R "[R "U55-n[R"SUS-[RUS9R5nXbU- -nS[R"SU5- nURSUSS5nXP:wa7[R"USS2SSS2S4USS2SSS2S4/SS9SS2SU2S4nXG-nURS5$) a Link to paper: https://huggingface.co/papers/2108.12409 - Alibi tensor is not causal as the original paper mentions, it relies on a translation invariance of softmax for quick implementation. This implementation has been copied from the alibi implementation of MPT source code that led to slightly different results than the Bloom alibi: https://huggingface.co/mosaicml/mpt-7b/blob/main/attention.py#L292 r)dtypedevice?N.dimr) torcharangeint32viewmathceillog2int64floatpowconcatsqueeze) num_headssequence_lengthalibi_bias_maxralibinum_heads_power_of_2baseslopess ^/home/james-whalen/.local/lib/python3.13/site-packages/transformers/models/mpt/modeling_mpt.pybuild_mpt_alibi_tensorr7-s LL_,au{{6 R W WXY[\^_ap qE $))I*> ?? <</!35;;v V \ \ ^D $88 9D 599Q% %F [[0!Q 7F(vaAsl3VAssCK5HIqQRSU_V_U_adRde NE == c ^\rSrSrSrSS\S\\4U4Sjjjr\ "SSSS 9SS \ RS \ RS\\ S \\ RS \\ R4 Sjj5r SrU=r$) MptAttentionDzrMulti-head self attention. Using torch or triton attention implementation enables user to also use additive bias. config layer_idxc>[TU]5 URUlURUlURUlURUR-UlURRUlURc5S[R"URUR- 5- UlURRUl URRUl [R"URSUR-SS9Ul[R"URURSS9UlX lg)Nrr Fbias)super__init__ hidden_sizen_heads max_seq_lenmax_seq_lengthhead_dim attn_config softmax_scaler'sqrt attn_pdropattn_dropout_pclip_qkvrLinearWqkvout_projr=)selfr<r= __class__s r6rBMptAttention.__init__Is !--~~ $00((DLL8 #//==    %!"TYYt/?/?$,,/N%O!OD $00;;**33 IId..D4D4D0D5Q  $"2"2D4D4D5Q "r8past_key_valuepast_key_values4.58new_nameversion hidden_states position_biasattention_maskcache_positionc\URSSupgURU5nUR(a%URUR*URS9nUR SSS9upn U R XgUR UR5RSS5n U R XgUR UR5RSS5n U R XgUR UR5RSS5n Ub#SU0n URXURU 5up[R"XRSS55UR-n UcUOXsR5-nUb[UR5S:wa![!S [UR535eU RSn[#S UR%S5U- 5n[#S UR%S5U- 5nUSS2US2US24nX-n Ub:U R'U[R("U R*5R,5n [.R0R3U R55SS9R7U R*5n[.R0R9UUR:UR<S 9n[R"UU 5nUR?S SSS5RA5RCXgS5nUREU5nUU4$) Nr)minmaxr r!rr]z6Expecting position_bias shape to be 3 dimensions, got rptraining)#shaperOrMclampchunkreshaperDrG transposeupdater=r#matmulrIget_seq_lengthlen ValueErrorr`size masked_fillfinforr_rr softmaxr+todropoutrLrepermute contiguousr&rP)rQrZr[rUr\r] batch_size seq_length mixed_qkv query_states key_states value_states cache_kwargsattention_scores query_length key_lengthposition_bias_query_indexposition_bias_key_index attn_weightscontext_states attn_outputs r6forwardMptAttention.forwardYs"/!4!4Ra!8 IIm, ==!T]]N NI1:1J. ,#++JDLLRVR_R_`jjklnop ''  dmm\ffghjkl #++JDLLRVR_R_`jjklnop  &,n=L'6'='=jX\XfXfht'u $J << 6J6J2r6RSVZVhVhh%4%[TU]5 URn[R"USU-SS9Ul[R "SS9Ul[R"SU-USS9UlURRUl g)NFr?none) approximate) rArBrCrrNup_projGELUact down_projrHrKhidden_dropout)rQr<rCrRs r6rBMptMLP.__init__sm (( yya+oEJ 77v.1{?KeL$00;;r8rZresidualreturncURURU55nURU5n[R"X0R UR S9nXB-nU$)Nrc)rrrFrurre)rQrZrintermediate_outputoutputs r6rMptMLP.forwardsSm!<= "nn];.2E2EPTP]P]^" r8)rrrr) rrrrrrBr#rrrrrs@r6rrs:[TU]5 URn[X1RS9UlSUR lURUl[X5Ul [X1RS9Ul SURl[U5Ul URRUl["R$"UR 5Ulg)Neps)rArBrCrlayer_norm_epsilonnorm_1r@rDr/r:attnnorm_2rffnrHrK dropout_raterDropoutresid_attn_dropout)rQr<r=rCrRs r6rBMptBlock.__init__s ((  1J1JK   3  1J1JK  &>"..99"$**T->->"?r8rZr[r\ layer_past use_cacheoutput_attentionsr]cURU5nUn URUUUUUS9upURU 5U -nURU5nUn UR X5n X4$)N)r[r\rUr])rrrrr) rQrZr[r\rrrr]layernorm_outputr attn_outputsrrs r6rMptBlock.forwards ;;}5 &*YY ')&) &/& " // =H ;;}5!*5##r8)rrrrrr/rr)NFFN)rrrrrrrrBr#rr boolrrrrs@r6rrs@y@Xc]@@2'+"'15"$||"$||"$ "$ UO "$  "$ "$!."$"$r8rc ^\rSrSr%\\S'SrSrS/rS/r U4Sjr S\ R4S jr \\"S S S S 9S \\\R$\R$4S\\\R$\R$44Sj55rSrU=r$)MptPreTrainedModelr< transformerTrz lm_head.*.c&>[TU]"U0UD6 gr)rArB)rQinputskwargsrRs r6rBMptPreTrainedModel.__init__s &+F+r8modulec[U[R5(akURRR SUR RS9 URb%URRR5 gg[U[R5(axURRR SUR RS9 URb2URRURR5 gg[U[5(aWURb$URRR5 URRRS5 gg)zInitialize the weights.g)meanstdNr ) isinstancerrNweightdatanormal_r<initializer_ranger@zero_ Embedding padding_idxrfill_)rQrs r6 _init_weights MptPreTrainedModel._init_weightss fbii ( ( MM   & &CT[[5R5R & S{{&   &&('  - - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  * *{{&   &&( MM   $ $S )+r8rTrUrVrWrcj^^^USSRupmmX-m[UUU4SjU55$)zg Converts the cache to the format expected by Mpt, i.e. to tuple(tuple([batch_size * num_heads, ...])) rc3|># UH1nUSRTTT5USRTTT54v M3 g7f)rrN)ri).0rbatch_size_times_num_headsrGrys r6 ;MptPreTrainedModel._convert_to_mpt_cache..sL . 1 %%&@(JW1 %%&@*hW .s9<)rftuple)rUrxr/rrGrys @@@r6_convert_to_mpt_cache(MptPreTrainedModel._convert_to_mpt_cachesI7Fa6H6K6Q6Q3 x%/%;" .    r8)rrrrr__annotations__base_model_prefixsupports_gradient_checkpointing_no_split_modules_keys_to_ignore_on_load_missingrBrModuler staticmethodrrr#rrrrrs@r6rrs %&*## '4o#,*BII*"%0A6R uU\\5<<%?@A uU\\5<</0 1 S r8rcb^\rSrSrS\4U4SjjrSrSSjrS\R4Sjr \ SS\ \RS \ \S \ \RS \ \RS \ \S \ \S\ \S\ \S\ \RS\\\RS4\44Sjj5rSrU=r$)MptModelir<c >[TU]U5 URUlURUl[ R "URUR5Ul[ R"[UR5Vs/sH n[XS9PM sn5Ul [URURS9UlSUR lSUlUR'5 gs snf)N)r=rF)rArBrCrDr/rr vocab_sizewte ModuleListrangen_layersrblocksrrnorm_fr@gradient_checkpointing post_init)rQr<irRs r6rBMptModel.__init__s  !--<< 1 143C3CDmmERXRaRaLb$cLbqXf%BLb$cd   0 0f6O6OP  &+# %ds C5cUR$rr)rQs r6get_input_embeddingsMptModel.get_input_embeddings&s xxr8c[XX45$r)r7)rQr/r0r1rs r6r7MptModel.build_mpt_alibi_tensor)s%i.YYr8new_embeddingscXlgrrrQrs r6set_input_embeddingsMptModel.set_input_embeddings,s!r8 input_idsrUr\ inputs_embedsrroutput_hidden_states return_dictr]r.c rUbUOURRnUbUOURRnUbUOURRnUbUOURRnUbUb [ S5eUbUR upOUbUR upn O [ S5eUR(a/UR(aU(a[RS5 SnUcURU5nU(aUc[URS9nU(a@[U[5(a+[RS5 [R"U5nUnU(aSOSnU(aSOSnUbUR!5OS nU U-nUc"["R$"U U4UR&S 9nOUR)UR&5nUR+UR,URR.UR&S 9n[1X;U 4UU5nUR35nUR4H3nU(aUU4-nU"UUUUUUU S 9nUS nU(dM*UUS 4-nM5 UR7U5nU(aUU4-nU(d[S XUU455$[9UUUUS9$)j input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) NzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...F)r<zPassing a tuple of `past_key_values` is deprecated and will be removed in Transformers v4.58.0. You should pass an instance of `DynamicCache` instead, e.g. `past_key_values=DynamicCache.from_legacy_cache(past_key_values)`.rrr)rr\rrr[r]rc3.# UH ocMUv M g7frr)rvs r6r#MptModel.forward..scacs )last_hidden_staterUrZ attentions)r<rrruse_return_dictrorfrrelogger warning_oncerr rrfrom_legacy_cachermr#onesrrtr7r/rErrrrr)rQrrUr\rrrrrr]rrxry_rZall_self_attentionsall_hidden_statespast_key_values_lengthseq_length_with_pastr2 causal_maskblockoutputss r6rMptModel.forward/s62C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKcd d  "%.__ "J  &(5(;(; %JATU U  & &4==##p"   HHY/M 0*$++>O OU;;   U  +<<_MO% $5b4"6BDFUE`!?!?!Afg),BB  !"ZZ5I(JS`SgSghN+..}/C/CDN++DNNDKK t $D>t 'tnt d^t !.t  uU\\3&')RR St t r8rz The MPT Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). ) custom_introct^\rSrSrS/rS\4U4SjjrS\R4Sjr \ SS\ \RS\ \ S \ \RS \ \RS \ \RS \ \S \ \S\ \S\ \S\ \RS\\\R\44Sjj5rSrU=r$)MptForCausalLMizlm_head.weightr<c>[TU]U5 [U5Ul[R "UR URSS9UlUR5 gNFr?) rArBrrrrNrCrlm_headrrQr<rRs r6rBMptForCausalLM.__init__sI  #F+yy!3!3V5F5FUS  r8rcXlgr)r#rs r6set_output_embeddings$MptForCausalLM.set_output_embeddingss% r8rrUr\rlabelsrrrrr]rc U bU OURRn URUUUUUUUU U S9 n U Sn URU 5nSnUbEUR UR 5nUR "UU4SURR0U D6nU (dU4U SS-nUbU4U-$U$[UUU RU RU RS9$)a input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` N)rUr\rrrrrr]rrrlosslogitsrUrZr ) r<r rr#rtr loss_functionrrrUrZr )rQrrUr\rr)rrrrr]rtransformer_outputsrZ lm_logitsr,rs r6rMptForCausalLM.forwards@&1%F $D>F 'tnF d^F !.F  uU\\"$EE FF F r8r a The MPT Model transformer with a sequence classification head on top (linear layer). [`MptForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-1) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). c,^\rSrSrS\4U4Sjjr\SS\\RS\\ S\\RS\\RS\\RS \\ S \\ S \\ S \\ S \ \\R\44Sjj5rSrU=r$)MptForSequenceClassificationir<c>[TU]U5 URUl[U5Ul[ R "URURSS9UlUR5 gr") rArB num_labelsrrrrNrCscorerr$s r6rB%MptForSequenceClassification.__init__sV   ++#F+YYv1163D3D5Q  r8rrUr\rr)rrrrrc U bU OURRn URUUUUUUUU S9n U Sn URU 5n UbURSn OURSn URR cU S:wa [ S5eURR cSnOUbXRR :gRU R[R5n[R"URSU R[RS9nUU-RS5nO.Sn[RURR S35 U [R"XRS 9U4nSnUGbgURR"cUR$S:XaS URlOoUR$S:aNUR&[R(:XdUR&[R*:XaS URlOS URlURR"S :XaJ[-5nUR$S:Xa&U"UR/5UR/55nOeU"UU5nO[URR"S :Xa[15nU"UU5nO-URR"S :Xa[35nU"UU5nU (dU4U SS-nUbU4U-$U$[5UUU R6U R8U R:S 9$) input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values.get_seq_length()` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NrUr\rrrrrrrz=Cannot handle batch sizes > 1 if no padding token is defined.ra)rrz will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`r regressionsingle_label_classificationmulti_label_classificationr+)r<r rr7rf pad_token_idrortrr#r%r$argmaxr rrRr problem_typer6rlongrr r.rrrrUrZr )rQrrUr\rr)rrrrr/rZr-rxlast_non_pad_token non_pad_mask token_indices pooled_logitsr,loss_fctrs r6r$MptForSequenceClassification.forwards<&1%!F!Fr!J !#    >>**+,ZZ  u||J}}MOaab   {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- v6))-II,. v6#%(;AB(??F)-)9TGf$ Ev E/ /??-;;*55   r8)r6r7rr)rrrrrrBrrr#rr rrrrrrrrrs@r6r4r4sy15+/1504)-$(,0/3&*d E,,-d "%d !. d   - d & d D>d $D>d 'tnd d^d  uU\\"$DD Ed d r8r4c,^\rSrSrS\4U4Sjjr\SS\\RS\\ S\\RS\\RS\\RS \\ S \\ S \\ S \\ S \ \\R\44Sjj5rSrU=r$)MptForTokenClassificationir<c>[TU]U5 URUl[U5Ul[ US5(aUR b UR nO-[ US5(aURb URnOSn[R"U5Ul [R"URUR5Ul UR5 g)Nclassifier_dropoutrg?)rArBr6rrhasattrrLrrrrurNrC classifierr)rQr<rLrRs r6rB"MptForTokenClassification.__init__s   ++#F+ 6/ 0 0V5N5N5Z!'!:!:  V- . .63H3H3T!'!6!6 !$ zz"45 ))F$6$68I8IJ r8rrUr\rr)rrrrrc U bU OURRn URUUUUUUUU S9n U Sn URU 5n UR U 5n SnUblUR U R 5nURunn[5nU"U RUU-UR5URUU-55nU (dU 4U SS-nUbU4U-$U$[UU U RU RS9$)r:Nr;rr)r,r-rZr )r<r rrurNrtrrfrr&r6rrZr )rQrrUr\rr)rrrrdeprecated_argumentsr/rZr-r,rxryrGrs r6r!MptForTokenClassification.forwards+>&1%B $D>B 'tnB d^B  uU\\"$99 :B B r8rJc^\rSrSrU4Sjr\SS\\RS\\RS\\RS\\RS\\RS\\ S \\ S \\ S \ \ \ 44S jj5rS rU=r$)MptForQuestionAnsweringic>[TU]U5 [U5Ul[R "UR S5UlUR5 g)Nr) rArBrrrrNrC qa_outputsrr$s r6rB MptForQuestionAnswering.__init__sA  #F+))F$6$6: r8rr\rstart_positions end_positionsrrrrc UbUOURRnURUUUUUUS9n U Sn URU 5n U R SSS9upU R S5R 5n U R S5R 5n SnUbUb[UR55S:aUR S5n[UR55S:aUR S5nU RS5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- nU(dX4U SS-nUbU4U-$U$[UU U U RU RS 9$) rN)r\rrrrrrrar!) ignore_indexr)r, start_logits end_logitsrZr )r<r rrVsplitr.rwrnrprgrrrZr )rQrr\rrXrYrrrrsequence_outputr-r\r] total_loss ignored_indexrG start_lossend_lossrs r6rMptForQuestionAnswering.forwards2&1%E 'tnE d^E  u22 3E E r8rT)r rrr4rJrTr)5rr'typingrrr#rtorch.nnrrrr r r cache_utilsr r generationrmodeling_attn_mask_utilsrmodeling_layersrmodeling_outputsrrrrrmodeling_utilsrutilsrrutils.deprecationrconfiguration_mptr get_loggerrr r7rr:rrrrr r4rJrT__all__rr8r6rssd " LL$.)I9.,0(   H %.G)299G)TRYY*7$)7$t- - - `U !U U p U 'U  U p  o #5o  o dU 2U U pO 0O O d r8