
    h$              
       V   S SK JrJr  S SKJr  S SKrS SKJr  S SKJ	r	  S SK
J
r
  S SKJrJrJrJrJrJr  S SKJrJrJrJr  S S	KJr  \
R2                  " 5         \ " S
 S5      5       r\" \5      r\R9                  5       S    r\" \R<                  \R>                  \R@                  S\RB                  \RD                  \RF                  S9r$ S$S jr%Sr&Sr'\%" \$\&\'S9r(S r)\" \$RT                  5        \RV                  " \$R<                  \RX                  S9r-\RV                  " \-5      r-\" \-SS9r.\	" \/" S \-Ra                  5       5      \$R>                  S9r1\RV                  " \$R<                  5      r2\2Rf                  \2l4        \" \$\-\.\2\(\)\1S9r5Sr6\RV                  " \65      r7\RV                  " \6\Rp                  S9Rs                  \5Rt                  Rv                  5      r<SSSS\2Rz                  S.r>Sr?Sr@\" \?\@5      rA\R                  rB\
" \C" \5R                  5      5       GHw  u  rErF\FS   rG/ rH\G HE  rI\A" 5       rJ\J\>S '   \5R                  " \I40 \>D6rL\HR                  \LR                  5       \J* S 5        MG     \H V s/ s H"  n \2R                  U R                  5       5      PM$     sn \FS!'   \FS!   rP\7" \PSSS"S#9Rs                  \5Rt                  Rv                  5      rQ\<" S%0 \QD6R                  R                  5       rR\RSS2S 4   R                  5       rU\U Vs/ s H  n\R                  U5      PM     snrW\5R                  \G\H\W5      rY\5R                  \Y\F\W5        \ES-  S :X  d  GMH  \5Rt                  R                  (       d  GMf  \5R                  \B5        GMz     gs  sn f s  snf )&    )	dataclassfield)OptionalN)load_dataset)Adam)tqdm)AutoModelForCausalLMAutoTokenizerHfArgumentParser RobertaForSequenceClassificationRobertaTokenizerset_seed)!AutoModelForCausalLMWithValueHead	PPOConfig
PPOTrainercreate_reference_model)LengthSamplerc                      \ rS rSr% Sr\" SSS0S9r\\   \	S'   \" SSS	0S9r
\\   \	S
'   \" SSS0S9r\\   \	S'   \" SSS0S9r\\   \	S'   \" SSS0S9r\\   \	S'   \" SSS0S9r\\   \	S'   \" SSS0S9r\\   \	S'   Srg)ScriptArguments<   z?
The name of the Casual LM model we wish to fine-tune with PPO
zybelkada/gpt-j-6b-sharded-bf16helpzthe model name)defaultmetadata
model_nameNzuse 'wandb' to log with wandblog_withgM >zthe learning ratelearning_rate   zthe PPO minibatch sizemini_batch_size   zthe batch size
batch_size   z)the number of gradient accumulation stepsgradient_accumulation_stepsz3./gpt-j-6B-detoxified-long-context-26-shl-1e4-finalzthe path to save the modelmodel_save_path )__name__
__module____qualname____firstlineno____doc__r   r   r   str__annotations__r   r   floatr   intr    r"   r#   __static_attributes__r$       w/home/james-whalen/.local/lib/python3.13/site-packages/examples/research_projects/toxicity/scripts/gpt-j-6b-toxicity.pyr   r   <   s     !&.NZ`brYs tJt#DFDc;deHhsme%*=FTgKh%iM8E?i%*1H`?a%bOXc]b %bFDT;U VJV16V%PQ2#  &+E67&OXc] r/   r   d   )r   r   r   
ppo_epochsr   r    r"   c                 .  ^^ [         R                  " U R                  5      mTR                  Tl        [        USS9nS nUR                  USS9n[        X#5      mUU4S jnUR                  USS9nUR                  SS9  UR                  S	SS
9S   nU$ )aO  
Build dataset for training. This builds the dataset from `load_dataset`, one should
customize this function to train the model on its own dataset.

Args:
    config (`PPOConfig`):
        The configuration of the PPO training.
    dataset_name (`str`):
        The name of the dataset to be loaded.
    input_min_text_length (`int`, defaults to 5):
        The minimum length of the input text.
    input_max_text_length (`int`, defaults to 10):
        The maximum length of the input text.

Returns:
    dataloader (`torch.utils.data.DataLoader`):
        The dataloader for the dataset.
train)splitc                 2    U S   S   nUS L=(       a    US:  $ )Nprompttoxicityg333333?r$   )sampler8   s     r0   	filter_fn build_dataset.<locals>.filter_fn}   s%    (#J/t#636r/   F)batchedc                    > U S   S   nU S   S   nTR                  X-   5      S T" 5        U S'   TR                  U S   5      U S'   U $ )Nr7   textcontinuation	input_idsquery)encodedecode)r9   r7   r?   
input_size	tokenizers      r0   tokenizebuild_dataset.<locals>.tokenize   s`    !&)n-f5'..v/DEn
U{#**6++>?wr/   torch)typeg?)	test_sizeshuffle)r
   from_pretrainedr   	eos_token	pad_tokenr   filterr   map
set_formattrain_test_split)	configdataset_nameinput_min_text_lengthinput_max_text_lengthdsr:   rF   rD   rE   s	          @@r0   build_datasetrX   c   s    * --f.?.?@I#--I	l'	2B7 
9e	,B4LJ 
%	(BMMwM			sE		:7	CBIr/      (   )rU   rV   c           
      n    U S    VVs0 s H  oU  Vs/ s H  o"U   PM	     sn_M     snn$ s  snf s  snnf )Nr   r$   )datakeyds      r0   collatorr_      s5    377;7C$'$QC&$''7;;';s   
1,11)dtype   )num_shared_layersc                     U R                   $ )N)requires_grad)ps    r0   <lambda>rf      s    !//r/   )lr)	ref_modelrE   datasetdata_collator	optimizerz0facebook/roberta-hate-speech-dynabench-r4-targetg        g      ?T)
min_lengthtop_ktop_p	do_samplepad_token_idr@   max_new_tokensresponsept)padding
truncationreturn_tensors)zallenai/real-toxicity-prompts   
   r$   )]dataclassesr   r   typingr   rH   datasetsr   torch.optimr   r   transformersr	   r
   r   r   r   r   trlr   r   r   r   trl.corer   pandasr   parserparse_args_into_dataclassesscript_argsr   r   r   r   r    r"   rS   rX   min_input_lengthmax_input_lengthri   r_   seedrL   bfloat16modelrh   rO   
parametersrk   rE   rM   rN   ppo_trainertoxicity_model_idtoxicity_tokenizerfloat16toacceleratordevicetoxicity_modeleos_token_idgeneration_kwargsoutput_min_lengthoutput_max_lengthoutput_length_samplerr#   	enumerate
dataloaderepochbatchquery_tensorsresponse_tensorsrA   gen_lengeneraters   appendsqueezerC   textstoxicity_inputslogitsr,   tolisttoxicity_labelstensorrewardsstepstats	log_statsis_main_processsave_pretrained)routputs   00r0   <module>r      s   )   !    a ` " 2   * 
/	*00215	%%++!!//%% + G G
 jl/f   
6F^n
o<
 	  	,,V->->ennU)99%@ #5B?	 153C3C3EF6K_K_`	 ))&*;*;<	))	  
	 G %556GH 1AABS[`[h[hill"" **    %&79JK --;#9#9:;LE5+&M ').5*+''C1BC 0 0 2G89 =>	 
 AQQ@P1))!))+6@PQE* *E(^bcff&&O .o.55;;=Fad|++-O2AB/u||F#/BG ],<gFE%0 s{a""222''8; < R Cs   )N!N&