h@~SSKrSSKrSSKJrJr SSKrSSKrSSKJ r SSK J r SSK J r Jr SSKJrJrJrJrJrJrJrJr SSKJr SSKJr S S KJr S S KJr S S K J!r!J"r"J#r# S S K$J%r% SSK&J'r' SSK(J)r) \"5(aSSK*J+r+J,r, SSK-J.r. SSK/J0r0 SSK1J2r2 \"5(aSSK3r3\Rh"\55r6S-S\7\8S\S\S\ S\\S\9S\7\84Sjjr:"SS\5r;"SS \5r"S%S&\5r?"S'S(\5r@"S)S*\5rA"S+S,\5rBg).N)OptionalUnion) Accelerator)AcceleratorState) gather_objectis_wandb_available)GenerationConfigPreTrainedModelPreTrainedTokenizerBaseTrainerTrainerCallbackTrainerControl TrainerStateTrainingArguments) has_length)is_rich_available)maybe_apply_chat_template)is_mergekit_available) MergeConfig merge_modelsupload_model_to_hf)unwrap_model_for_generation)BasePairwiseJudge)log_table_to_comet_experiment)ConsoleGroup)Live)Panel)Progresspromptsmodel tokenizer acceleratorgeneration_config batch_sizereturnc /n[X5n[S[U5U5HnXX-n U"U SSSS9RUR5n UR "S0U DSU0D6n [ U RU 5H4upU [U 5Sn URU SS9nURU5 M6 M SSS5 U$!,(df  U$=f) a Generates completions for a list of pre-formatted prompts from the given model. Args: prompts (list[str]): A list of input prompts for which completions are to be generated. model (PreTrainedModel): The pre-trained model to be used for generation. tokenizer (PreTrainedTokenizerBase): The tokenizer to be used for encoding and decoding. accelerator (Accelerator): The accelerator to be used for model execution. generation_config (GenerationConfig): Configuration for text generation. batch_size (int, optional): The number of prompts to process in each batch. Default is 1. Returns: list[str]: A list of generated text completions corresponding to the input prompts. rptT)return_tensorspadding truncationr&N)skip_special_tokens) rrangelentodevicegeneratezip input_idsdecodeappend)r"r#r$r%r&r' completionsunwrapped_modelidxbatchtokenized_batch generationsprompt generation completions O/home/james-whalen/.local/lib/python3.13/site-packages/trl/trainer/callbacks.py_generate_completionsrC;s,K $U 8OCL*5C#"23E'dD]abeefkfrfrsO)22!"3K'*/*C*C[&Q"'F 6 &--jd-S "":. 'R6 9  9 8 s B*C Cc\rSrSrSrS\\\RR4S\ \ 4Sjr \ S5r\ S5rSrS rg ) SyncRefModelCallbackbz; Callback to synchronize the model with a reference model. ref_modelr%cX lXlgNr%rG)selfrGr%s rB__init__SyncRefModelCallback.__init__gs '"c[UR5UR55H;up4URRSU- 5R URUS9 M= g)N?alpha)r5 parametersdatamul_add_)r# target_modelrR target_param copy_params rB_sync_target_model'SyncRefModelCallback._sync_target_modelosU(+L,C,C,EuGWGWGY(Z $L    " "3; / 4 4Z__E 4 R)[rNc[5RnUbURS:XaSSKnURR [ UR55[ UR55-SS9 URR5S:Xa[RXU5 SSS5 g[RXU5 g!,(df  g=f)Nr) modifier_rank) rdeepspeed_plugin zero_stage deepspeedzeroGatheredParameterslistrScommget_rankrErZ)r#rWrRr_ras rBsync_target_model&SyncRefModelCallback.sync_target_modelts+->>  ',<,G,G1,L 22U%%'(4 0G0G0I+JJZ[3>>**,1(;;EQVW  ! 3 3E O s 75C  Cc USnURbqURUR-S:XaSUR(aURR U5nUR XPRUR 5 ggg)Nr#r)rG global_stepref_model_sync_stepsr% unwrap_modelrgref_model_mixup_alpha)rKargsstatecontrolkwargsr#s rB on_step_end SyncRefModelCallback.on_step_endsp!' >> %%*;*;d>W>W*W[\*\((55e<  " "5..$:T:T U+] %rNrJN)__name__ __module__ __qualname____firstlineno____doc__rr torchnnModulerrrL staticmethodrZrgrr__static_attributes__r/rNrBrErEbsg#%((//9:#k*#SS P PVrNrEcP\rSrSrSrSrSrSrS SjrSr S r S S jr S r S r g)RichProgressCallbackzX A [`TrainerCallback`] that displays the progress of training or evaluation using Rich. c[5(d [S5eSUlSUlSUlSUlSUlSUlSUlSUl g)NzSRichProgressCallback requires the `rich` extra. To install, run `pip install rich`.) r ImportError training_barprediction_bartraining_task_idprediction_task_id rich_group rich_consoletraining_status current_step)rKs rBrLRichProgressCallback.__init__sV ""st t " $"& # rNc UR(a[5Ul[5Ul[ 5UlUR R S5Ul[[[URURUR555Ul URR5 URRSURS9UlSUlgg)NzNothing to log yet ...z[blue]Training the modeltotalr)is_world_process_zeror!rrrrstatusrrr rrstartadd_task max_stepsrrrKrnrorprqs rBon_train_begin#RichProgressCallback.on_train_begins  & & ( D "**D  ' D #'#4#4#;#;$>?Yafapap$>$qD ! !D  'rNc UR(aNURRURURUR - SS9 URUlgg)NTadvanceupdate)rrrrrjrrs rBrr RichProgressCallback.on_step_endsR  & &    $ $T%:%:EDUDUX\XiXiDirv $ w % 1 1D  'rNNc UR(al[U5(a[URc(URR S[ U5S9UlURR URSSS9 ggg)Nz*[blue]Predicting on the evaluation datasetrrTr)rrrrrr1r)rKrnrorpeval_dataloaderrqs rBon_prediction_step'RichProgressCallback.on_prediction_stepsw  & &:o+F+F&&.*.*=*=*F*F@OH\+G+'    & &t'>'>RV & W ,G &rNc UR(a;URb-URRUR5 SUlgggrIrrr remove_taskrs rB on_evaluate RichProgressCallback.on_evaluateC  & &&&2##//0G0GH*.'3 'rNc UR(a;URb-URRUR5 SUlgggrIrrs rB on_predictRichProgressCallback.on_predictrrNc UR(aHURb:URSS5nURR S[ U535 ggg)N total_flosz[bold green]Status = )rrpoprrstr)rKrnrorplogsrq_s rBon_logRichProgressCallback.on_logsO  & &4+<+<+Ht,A  ' '*?D {(K L,I &rNc UR(aSURR5 SUlSUlSUlSUlSUlSUlSUlSUl ggrI) rrstoprrrrrrrrs rB on_train_end!RichProgressCallback.on_train_ends_  & & OO " $D "&D $(D !&*D #"DO $D #'D $D  'rN)rrrrrrrrrI)rtrurvrwrxrLrrrrrrrrr}r/rNrBrrs1! "2 X/ / M %rNrror9winner_indicesc [UR5/[U5-n[[ XAX#55nUVs/sHofSUSUSSUSSUS4PM nn[ R "U/SQS9$s snf)Nrrrr])stepr?reference_modelpolicy winner_index)columns)rrjr1rdr5pd DataFrame)ror"r9rrjrTitem split_datas rB_win_rate_completions_dfrsu(()*S\9K K+F GDRVWRV$7DGT!WQZaT!WERVJW << ,k llXs%A:c ~\rSrSrSrSS\S\S\\S\\ S\ S \ 4 S jjr S \ S \ S \4SjrS \ S \ S \4SjrSrg)WinRateCallbacka| A [`~transformers.TrainerCallback`] that computes the win rate of a model based on a reference. It generates completions using prompts from the evaluation dataset and compares the trained model's outputs against a reference. The reference is either the initial version of the model (before training) or the reference model, if available in the trainer. During each evaluation step, a judge determines how often the trained model's completions win against the reference using a judge. The win rate is then logged in the trainer's logs under the key `"eval_win_rate"`. Usage: ```python trainer = DPOTrainer(...) judge = PairRMJudge() win_rate_callback = WinRateCallback(judge=judge, trainer=trainer) trainer.add_callback(win_rate_callback) ``` Args: judge (`BasePairwiseJudge`): The judge to use for comparing completions. trainer (`Trainer`): Trainer to which the callback will be attached. The trainer's evaluation dataset must include a `"prompt"` column containing the prompts for generating completions. If the `Trainer` has a reference model (via the `ref_model` attribute), it will use this reference model for generating the reference completions; otherwise, it defaults to using the initial model. generation_config (`GenerationConfig`, *optional*): The generation config to use for generating completions. num_prompts (`int` or `None`, *optional*, defaults to `None`): The number of prompts to generate completions for. If not provided, defaults to the number of examples in the evaluation dataset. shuffle_order (`bool`, *optional*, defaults to `True`): Whether to shuffle the order of the completions before judging. use_soft_judge (`bool`, *optional*, defaults to `False`): Whether to use a soft judge that returns a win probability between 0 and 1 for the first completion vs the second. Njudgetrainerr& num_prompts shuffle_orderuse_soft_judgec"XlX lXPlX0l/UlX`lURR c [S5eURR UlUb*UR R[U55Ulgg)NzCTrainer must have an evaluation dataset to use the WinRateCallback.) rrrr&ref_completionsr eval_dataset ValueErrorselectr0)rKrrr&rrrs rBrLWinRateCallback.__init__ s  *!2!, << $ $ ,bc c $ 9 9D   " $ 1 1 8 8{9K LD  #rNrnrorpc HUSnSUlURRn[URSS5nUcURRnUR UR S5n[UUUUURURS9Ul [[URUR55n UR(aJURRXURSS9n U V s/sH oS:aS OS PM n n [!U 5n O&URRXUR5n [!U5n[!U 5n [!U 5n SSS5 URRR"(a[%S W 55['U 5- n UR(a8S [%W 5['U 5- - nURR)XS .5 OURR)SU 05 SUR*;a?S SKnUR.b.[1UWW U S9nUR("SUR2"US905 SUR*;a[1UWW U S9n[5SUS9 gggs sn f!,(df  GN5=f)Nprocessing_classleftrGr?r#r$r%r&r'T return_scores?rrc3*# UH oS:Hv M g7frNr/.0 winner_idxs rB 1WinRateCallback.on_train_begin..JL^z?^rPeval_avg_win_prob eval_win_raterwandbror"r9rwin_rate_completions dataframecomet_mlwin_rate_completions.csvnametable) padding_siderr%getattr model_wrappedsplit_between_processesrrCr&per_device_eval_batch_sizerrdr5rrrris_main_processsumr1log report_torrunrTablerrKrnrorprqr$r%r#r"r9 ref_win_probsscorerwin_rate avg_win_probrdfs rBrWinRateCallback.on_train_begin#sZ-. !' ll..  k48 =LL..E  0 01B1B81L MQX#8#'"&"8"8:: $D s4#7#79M9MNOK"" $ 0 0tGYGYim 0 n GT!U}es{!"9}!U -m < !%!1!1'HZHZ![#G,G' 4K*>:N'N, << # # 3 3L^LLsSaObbH"""S%7#m:L%LL    |!_`   /8!<=$..(99(1# '$/'5 B II5u{{R7PQRT^^+-# +#1  .3,) 4"VN Ms 1B J<J AJ J J!c USnSUlURRnURRnUR UR S5n[ UUUUURURS9n [[URU 55n UR(aJURRXURSS9n U V s/sH oS:aSOS PM n n [U 5n O&URRXUR5n [U5n[U 5n [U 5n SSS5 URRR (a[#S W 55[%U 5- n UR(a8S [#W 5[%U 5- - nURR'XS .5 OURR'S U 05 SUR(;a?SSKnUR,b.[/UWW U S9nUR&"SUR0"US905 SUR(;a[/UWW U S9n[3SUS9 gggs sn f!,(df  GN5=f)Nrrr?rTrrrrc3*# UH oS:Hv M g7frr/rs rBr.WinRateCallback.on_evaluate..rrrPrrrrrrrrr)rrr%rrrrCr&rrdr5rrrrrrrr1rrrrrrrrs rBrWinRateCallback.on_evaluateis:-. !' ll..  **  0 01B1B81L MQX/#'"&"8"8:: Ks4#7#7EFK"" $ 0 0tGYGYim 0 n GT!U}es{!"9}!U -m < !%!1!1'HZHZ![#G,G' 4K*>:N)N. << # # 3 3L^LLsSaObbH"""S%7#m:L%LL    |!_`   /8!<=$..(99(1# '$/'5 B II5u{{R7PQRT^^+-# +#1  .3,) 4"VN Ms A?D JJ  d #LL)IPTPZPZ[E$..( =%01T^^+-* "'!2!2;qN Ms%G8*G3:G83G88 H)r rrr&rr)NNN) rtrurvrwrxr rr rrLrrr}r/rNrBrrsU49=%)" MM$$45Mc] M sm M*,3rNrcX\rSrSrSrS S\SS\S\4SjjrS rSS jr SS jr S r g)MergeModelCallbackia$ A [`~transformers.TrainerCallback`] that merges the policy model (the model being trained) with another model based on a merge configuration. Args: merge_config ([`MergeConfig`], *optional*, defaults to `None`): Configuration used for the merging process. If not provided, the default [`MergeConfig`] is used. merge_at_every_checkpoint (`bool`, *optional*, defaults to `False`): Whether to merge the model at every checkpoint. push_to_hub (`bool`, *optional*, defaults to `False`): Whether to push the merged model to the Hub after merging. Example: ```python from trl.mergekit_utils import MergeConfig from trl import MergeModelCallback config = MergeConfig() merge_callback = MergeModelCallback(config) trainer = DPOTrainer(..., callbacks=[merge_callback]) ``` N merge_configrmerge_at_every_checkpoint push_to_hubc[5(d [S5eU=(d [5UlX lX0lg)NzYMergeModelCallback requires the `mergekit` extra. To install, run `pip install mergekit`.)rrrrrr)rKrrrs rBrLMergeModelCallback.__init__s: %&&k )9KM)B&&rNc[RRUSU35nX@RlURR c%UR RURl[RRUS5n[URR5U5 UR(aUSUS3n[XV5 gg)Nz checkpoint-mergedz _checkpoint-_merged) ospathjoinrpolicy_model_pathtarget_model_pathconfig _name_or_pathrcreaterr)rK output_dirrjr#checkpoint_path merge_path repo_names rB_merge_and_maybe_push(MergeModelCallback._merge_and_maybe_push,s'',,z[ 3NO.=+    . . 627,,2L2LD   /WW\\/8< T&&--/<   %,l;-wGI z 5 rNc vUR(a(URURURU5 ggrIrr%r!rjrKrnrorpr#rqs rBon_saveMergeModelCallback.on_save9s,  ) )  & &t8I8I5 Q *rNc vUR(d(URURURU5 ggrIr(r)s rBrMergeModelCallback.on_train_end=s,--  & &t8I8I5 Q.rN)rrr)NFFrI) rtrurvrwrxrrrLr%r*rr}r/rNrBrrsI415*/! '}- '$( ' ' 6RRrNrcZ\rSrSrSrSS\S\S\S\S\S\S \S \4S jjr\ S 5r \ R"5S \ S\S\S\4Sj5rS\S\4SjrS\S\4SjrS\4Sjr\ R"5S \ S\S\S\4Sj5r\ R"5S \ S\S\4Sj5rSrg) BEMACallbackiBa A [`~transformers.TrainerCallback`] that implements [BEMA](https://huggingface.co/papers/2508.00180) (Bias-Corrected Exponential Moving Average) by [Adam Block](https://huggingface.co/abblock) and [Cyril Zhang](https://huggingface.co/cyrilzhang). Code from https://github.com/abblock/bema under MIT license. BEMA computes model weights that scale like: $$ \theta_t' = \alpha_t \cdot (\theta_t - \theta_0) + \text{EMA}_t $$ where \\( \theta_t \\) is the current model weights, \\( \theta_0 \\) is a snapshot of the model weights at the first `update_after` step, \\( \text{EMA}_t \\) is the exponential moving average of the model weights, and \\( \alpha_t \\) is a scaling factor that decays with the number of steps \\( t \\) as $$ \alpha_t = (\rho + \gamma \cdot t)^{-\eta}. $$ The EMA is computed as: $$ \text{EMA}_t = (1 - \beta_t) \cdot \text{EMA}_{t-1} + \beta_t \cdot \theta_t $$ where \\( \beta_t \\) is a decay factor that decays with the number of steps \\( t \\) as $$ \beta_t = (\rho + \gamma \cdot t)^{-\kappa}. $$ Args: update_freq (`int`, *optional*, defaults to `400`): Update the BEMA weights every X steps. Denoted this as \\( \phi \\) in the paper. ema_power (`float`, *optional*, defaults to `0.5`): Power for the EMA decay factor. Denoted \\( \kappa \\) in the paper. To disable EMA, set this to `0.0`. bias_power (`float`, *optional*, defaults to `0.2`): Power for the BEMA scaling factor. Denoted \\( \eta \\) in the paper. To disable BEMA, set this to `0.0`. lag (`int`, *optional*, defaults to `10`): Initial offset in the weight decay schedule that controls early-stage smoothness by acting as a virtual starting age for the updates. Denoted as \\( \rho \\) in the paper. update_after (`int`, *optional*, defaults to `0`): Burn-in time before starting to update the BEMA weights. Denoted \\( \tau \\) in the paper. multiplier (`float`, *optional*, defaults to `1.0`): Initial value for the EMA decay factor. Denoted as \\( \gamma \\) in the paper. min_ema_multiplier (`float`, *optional*, defaults to `0.0`): Minimum value for the EMA decay factor. device (`str`, *optional*, defaults to `"cpu"`): Device to use for the BEMA buffers, e.g. `"cpu"` or `"cuda"`. Note that in most cases, this device SHOULD BE DIFFERENT from the device used for training in order to avoid OOM. Example: ```python from trl import BEMACallback trainer = Trainer(..., callbacks=[BEMACallback()]) ``` update_freq ema_power bias_powerlag update_after multipliermin_ema_multiplierr3c XlX lX0lX@lXPlX`lXplXl/Ul/Ul /Ul /Ul SUl grI) r0r1r2r3r4r5r6r3 param_names thetat_params theta0_params ema_params running_model) rKr0r1r2r3r4r5r6r3s rBrLBEMACallback.__init__sX'"$($"4 !rNc[US5(a[US5(a UR$[US5(a UR$[US5(a UR$U$)z} Helper function to unwrap model from various wrappers including DataParallel, DistributedDataParallel, DeepSpeed, and FSDP. moduleengine_fsdp_wrapped_module)hasattrr?rA)r#s rB _unwrap_modelBEMACallback._unwrap_modelsa 5( # #x(@(@<<  50 1 1-- - 5( # #<<  rNrnrorpr#c URU5n[U5"UR5RUR5UlUR R UR55 UR5HupgUR(dMURRU5 URRU5 UR5R5RUR5nURRU5 UR RUR55 M grI)rCtyperr2r3r<load_state_dict state_dictnamed_parameters requires_gradr8r8r9detachcloner:r;) rKrnrorpr#rqrparamtheta0s rBrBEMACallback.on_train_begins""5)"%[699$++F **5+;+;+=>!113KD&&    # #D )    % %e ,\\^))+..t{{;F    % %f - OO " "6<<> 24rNrr(cURURU--UR*-n[X R5$)uDCompute the EMA decay factor βₜ = (ρ + γ·t)⁻ᵏᵃᵖᵖᵃ.)r3r5r1maxr6)rKrbetas rB _ema_betaBEMACallback._ema_betas54??T11G40011rNcVURURU--UR*-$)uACompute the BEMA scaling factor αₜ = (ρ + γ·t)⁻ᵉᵗᵃ.)r3r5r2)rKrs rB _bema_alphaBEMACallback._bema_alphas&4??T117GHHrNcURU5nURU5n[URURUR UR R55HhupEpgUR5RUR5nURSU- 5RXBS9 URXcXE- --5 Mj g)NrrQ)rSrVr5r9r:r;r<rSrKr2r3rUrVcopy_)rKrrRrRthetatrNema run_params rB_update_bema_weights!BEMACallback._update_bema_weightss~~d#  &/2    2 2DOOTEWEWEbEbEd/ *FC]]_'' 4F HHQX  # #F # 7 OOC6?";; < / rNc URnX`R:agX`R:XaX[URURUR 5H(upxn UR U5 U R U5 M* gX`R- UR-S:Xa*URU5 [RSU35 gg)NrzUpdated BEMA weights at step ) rjr4r5r9r:r;rYr0r]loggerinfo) rKrnrorpr#rqr thetat_param theta0_param ema_params rBrrBEMACallback.on_step_ends   ## #  $$ $9 ?@rNc UR(aCURS3nURRU5 [R SU35 gg)Nz/bemazSaved BEMA model to )rr!r<save_pretrainedr`ra)rKrnrorprqsave_directorys rBrBEMACallback.on_train_endsJ  & & $06N    . .~ > KK.~.>? @ 'rN) r2r3r;r1r3r6r5r8r<r:r9r4r0N)irg? rrPgcpu)rtrurvrwrxrfloatrrLr|rCryno_gradrrrr rrSrVr]rrrr}r/rNrBr/r/Bsn:|$'""" "  "  """""8* ]]_3%3.:3ES3\k33*2c2e2 III = = ]]_@%@.:@ES@\k@@( ]]_A!2A<AR`AArNr/)r)Cloggingrtypingrrpandasrry accelerateraccelerate.stateraccelerate.utilsrr transformersr r r r r rrrtransformers.trainer_utilsrtransformers.utilsr data_utilsr import_utilsrmergekit_utilsrrr models.utilsrjudgesrutilsr rich.consolerr rich.liver rich.panelr rich.progressr!r getLoggerrtr`rdrrrCrErrrrrrr/r/rNrBrs{ " "->   2020JJ6%0+ &   8 $ $ #Y$ $'$ $  01 $  $ #Y$N&V?&VRM%?M%`m m"&s)m:>s)mUYZ]U^m\\mAoAHY3_Y3x:R:RzuA?uArN