
    hW                     H   S SK JrJr  S SKrS SKJrJrJrJ	r	J
r
  S SKJr  \	" \	" \
" S5      \
" S5      S.5      \
" S5      S.5      r\ " S S	5      5       rS
 r\S:X  aD  \" \5      r\R%                  5       S    r\" \R(                  \R*                  \R,                  5        gg)    )	dataclassfieldN)DatasetFeaturesImageListValue)HfArgumentParserstring)texttype)contentrolec                   n    \ rS rSr% Sr\" SSS0S9r\\S'   \" SSS	0S9r	\
\S
'   \" SSS0S9r\\S'   Srg)ScriptArguments   a  
Arguments for the script.

Args:
    test_size (`float`, *optional*, defaults to `0.1`):
        Fraction of the dataset to include in the test split.
    push_to_hub (`bool`, *optional*, defaults to `False`):
        Whether to push the dataset to the Hugging Face Hub.
    repo_id (`str`, *optional*, defaults to `"trl-internal-testing/zen-multi-image"`):
        Hugging Face repository ID to push the dataset to.
g?helpz5Fraction of the dataset to include in the test split.)defaultmetadata	test_sizeFz4Whether to push the dataset to the Hugging Face Hub.push_to_hubz$trl-internal-testing/zen-multi-imagez2Hugging Face repository ID to push the dataset to.repo_id N)__name__
__module____qualname____firstlineno____doc__r   r   float__annotations__r   boolr   str__static_attributes__r       b/home/james-whalen/.local/lib/python3.13/site-packages/scripts/generate_zen_multi_image_dataset.pyr   r      sd    
 QRIu  PQK  6NOGS r$   r   c                    SSS0SSS./S.SSS	S./S./SSS
S.SS0/S.SSSS./S./SSS0SSS.SS0/S.SSSS./S./SSS0SS0SSS./S.SSSS./S./SSS0SSS./S.SSSS./S./SSSS.SS0/S.SSSS./S./SSS0SSS.SS0/S.SSSS./S./SSSS./S.SSSS./S./SSSS.SS0SS0/S.SSSS./S./SSS0SS0SSS.SS0/S.SSSS./S./SSS0SSS./S.SSSS./S./SSSS.SS0/S.SSSS./S./SSS0SS S.SS0SS0/S.SSS!S./S./SSS"S./S.SSS#S./S./SSS$S.SS0/S.SSS%S./S./SSS&S.SS0SS'S./S.SSS(S./S./SSS0SS)S./S.SSS*S./S./SSS+S.SS0/S.SSS,S./S./SSS-S./S.SSS.S./S.//nU Vs/ s H  n[        S/ US0   S1    5       5      PM     nnU Vs/ s H$  n[        R                  R                  S2S3US44S59PM&     nnU VV	V
s/ s HU  o V	V
s/ s HC  u  p[        R                  R	                  S6S7XS84S99R                  [        R                  5      PME     sn
n	PMW     nn	nn
[        R                  " X;S:.[        [        [        [        5       5      S:9S;9nUR                  U S<S=9nU(       a  UR                  US>S?9  SSS0SSS./S./SSS
S.SS0/S./SSS0SSS.SS0/S./SSS0SS0SSS./S./SSS0SSS./S./SSSS.SS0/S./SSS0SSS.SS0/S./SSSS./S./SSSS.SS0SS0/S./SSS0SS0SSS.SS0/S./SSS0SSS./S./SSSS.SS0/S./SSS0SS S.SS0SS0/S./SSS"S./S./SSS$S.SS0/S./SSS&S.SS0SS'S./S./SSS0SS)S./S./SSS+S.SS0/S./SSS-S./S.//nU Vs/ s H  n[        S@ US0   S1    5       5      PM     nnU Vs/ s H$  n[        R                  R                  S2S3US44S59PM&     nnU VV	V
s/ s HU  o V	V
s/ s HC  u  p[        R                  R	                  S6S7XS84S99R                  [        R                  5      PME     sn
n	PMW     nn	nn
[        R                  " XSA.[        [        [        [        5       5      SA9S;9nUR                  U S<S=9nU(       a  UR                  USBS?9  SSS0SSS./S./SSS
S.SS0/S./SSS0SSS.SS0/S./SSS0SS0SSS./S./SSS0SSS./S./SSSS.SS0/S./SSS0SSS.SS0/S./SSSS./S./SSSS.SS0SS0/S./SSS0SS0SSS.SS0/S./SSS0SSS./S./SSSS.SS0/S./SSS0SS S.SS0SS0/S./SSS"S./S./SSS$S.SS0/S./SSS&S.SS0SS'S./S./SSS0SS)S./S./SSS+S.SS0/S./SSS-S./S.//nSSS	S./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSS!S./S./SSS#S./S./SSS%S./S./SSS(S./S./SSS*S./S./SSS,S./S./SSS.S./S.//nU Vs/ s H  n[        SC US0   S1    5       5      PM     nnU Vs/ s H$  n[        R                  R                  S2S3US44S59PM&     nnU VV	V
s/ s HU  o V	V
s/ s HC  u  p[        R                  R	                  S6S7XS84S99R                  [        R                  5      PME     sn
n	PMW     nn	nn
[        R                  " XUSD.[        [        [        [        [        5       5      SD9S;9nUR                  U S<S=9nU(       a  UR                  USES?9  SSS0SSS./S./SSS
S.SS0/S./SSS0SSS.SS0/S./SSS0SS0SSS./S./SSS0SSS./S./SSSS.SS0/S./SSS0SSS.SS0/S./SSSS./S./SSSS.SS0SS0/S./SSS0SS0SSS.SS0/S./SSS0SSS./S./SSSS.SS0/S./SSS0SS S.SS0SS0/S./SSS"S./S./SSS$S.SS0/S./SSS&S.SS0SS'S./S./SSS0SS)S./S./SSS+S.SS0/S./SSS-S./S.//nSSS	S./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSSS./S./SSS!S./S./SSS#S./S./SSS%S./S./SSS(S./S./SSS*S./S./SSS,S./S./SSS.S./S.//nSSSFS./S./SSSGS./S./SSSHS./S./SSSIS./S./SSSJS./S./SSSKS./S./SSSLS./S./SSSMS./S./SSSNS./S./SSSOS./S./SSSPS./S./SSSQS./S./SSSRS./S./SSSSS./S./SSSTS./S./SSSUS./S./SSSVS./S./SSS*S./S./SSSWS./S.//4nU Vs/ s H  n[        SX US0   S1    5       5      PM     nnU Vs/ s H$  n[        R                  R                  S2S3US44S59PM&     nnU VV	V
s/ s HU  o V	V
s/ s HC  u  p[        R                  R	                  S6S7XS84S99R                  [        R                  5      PME     sn
n	PMW     nn	nn
[        R                  " UUUUSY.[        [        [        [        [        [        5       5      SY9S;9nUR                  U S<S=9nU(       a  UR                  USZS?9  g g s  snf s  snf s  sn
n	f s  sn
n	nf s  snf s  snf s  sn
n	f s  sn
n	nf s  snf s  snf s  sn
n	f s  sn
n	nf s  snf s  snf s  sn
n	f s  sn
n	nf )[Nuserr   imager   zWhat is better than ugly?)r   r   )r   r   	assistantz
Beautiful.zWhat is better than implicit?z	Explicit.zWhat is better than complex?zSimple.z What is better than complicated?zComplex.zWhat is better than nested?zFlat.zWhat is better than dense?zSparse.zWhat counts?zReadability.z,Are special cases enough to break the rules?z;No, special cases aren't special enough to break the rules.zWhat beats purity?zPracticality.z What should never pass silently?zErrors.zWhen can errors pass silently?zWhen explicitly silenced.z,What should you do in the face of ambiguity?zRefuse the temptation to guess.z'How many ways should there be to do it?zOne, and preferably only one.z-For whom may the way not be obvious at first?zDutch.zWhat is better than never?zNow is better than never.Isz never better than *right* now?zYes, often.z;What does it mean if the implementation is hard to explain?zIt means it's a bad idea.z;What does it mean if the implementation is easy to explain?zIt means it may be a good idea.zAny great ideas?z&Namespaces are one honking great idea.c              3   R   #    U  H  oR                  S 5      S:X  d  M  Sv   M     g7fr   r(      Nget.0parts     r%   	<genexpr>main.<locals>.<genexpr>M   #     [*;$xx?OSZ?ZAA*;   '	'r   r       @      )sizeg        g     o@   )lowhighr:   )messagesimages)featuresF)r   shuffle conversational_language_modeling)config_namec              3   R   #    U  H  oR                  S 5      S:X  d  M  Sv   M     g7fr,   r.   r0   s     r%   r3   r4   k   r5   r6   )promptr?   conversational_prompt_onlyc              3   R   #    U  H  oR                  S 5      S:X  d  M  Sv   M     g7fr,   r.   r0   s     r%   r3   r4      r5   r6   )rE   
completionr?    conversational_prompt_completionzAcceptable.z
Explained.zVery complex.zVery complicated.z	Circular.zHeavy.zLooking complicated.z9Yes, special cases are special enough to break the rules.zNothing.z	Warnings.zNever.zGive up.zAs many as possible.zFrench.z	Some day.z
No, never.zIt means it's a good idea.z
Recursion.c              3   R   #    U  H  oR                  S 5      S:X  d  M  Sv   M     g7fr,   r.   r0   s     r%   r3   r4      r5   r6   )rE   chosenrejectedr?   conversational_preference)sumnprandomrandintuniformastypeuint8r   	from_dictr   Messager   r   train_test_splitr   )r   r   r   r>   rownumber_of_images
num_imagessizesshwr?   (conversational_language_modeling_datasetrE   "conversational_prompt_only_datasetrH   (conversational_prompt_completion_datasetrK   rL   !conversational_preference_datasets                       r%   mainrc   5   s    vw&7&Rm9n%o	p  |G  ^d  nz  U{  T|  s}  	~v?^&_bhjqar%s	t  @K  bh  r}  Y~  X  w@  	Avw&7&Sq9ru{  ~E  uF  &G  
H  S^  u{  EN  lO  kP  JQ  	Rvw&7&'9JU[  fH  MI  &J  
K  Va  x~  HR  oS  nT  MU  	Vvw&7&Ro9p%q	r  ~I  `f  pw  Wx  Vy  uz  	{v?[&\_egn^o%p	q  }H  _e  ox  Vy  Uz  t{  	|vw&7&R`9adjlsct%u	v  BM  dj  tB  [C  ZD  yE  	Fv?m&n%o	p  |G  ^d  nk  Ul  Tm  sn  	ov?S&TW]_fVgjpryiz%{	|  HS  jp  zI  aJ  `K  L  	Mvw&7&'9JU[  fH  MI  LR  T[  K\  &]  
^  it  KQ  [d  Be  Af  `g  	hvw&7&Rr9s%t	u  AL  ci  sN  ZO  YP  xQ  	Rv?m&nqw  zA  qB  &C  
D  OZ  qw  Ab  hc  gd  Fe  	fvw&7&R{9|  @F  HO  P  SY  [b  Rc  &d  
e  p{  RX  bA  IB  HC  gD  	Ev?n&o%p	q  }H  _e  ow  Vx  Uy  tz  	{v?[&\_egn^o%p	q  }H  _e  oJ  VK  UL  tM  	Nvt&DvwFWbh  sT  ZU  &V  
W  bm  DJ  Ta  {b  zc  Yd  	evw&7&  SP  :Q  &R  
S  ^i  @F  Pk  wl  vm  Un  	ov?|&}  AG  IP  @Q  &R  
S  ^i  @F  Pq  wr  vs  Ut  	uv?Q&R%S	T_j  BH  Rz  y{  x|  W}  	~'H, hppgo`c[#a&*;[[gopO_`O_RYYr2ZO<O_E`otuotjkcdecd[_[\ryy  SuA!9 ELLRXXVcdeotFu/6/@/@hAit|  GN  W[  \a  \c  Wd  ue  0f,/W/h/hs|  GL/h  0M,0<<WRt<u vw&7&Rm9n%o	pqv?^&_bhjqar%s	tuvw&7&Sq9ru{  ~E  uF  &G  
H  	Ivw&7&'9JU[  fH  MI  &J  
K  	Lvw&7&Ro9p%q	rsv?[&\_egn^o%p	qrvw&7&R`9adjlsct%u	vwv?m&n%o	pqv?S&TW]_fVgjpryiz%{	|}vw&7&'9JU[  fH  MI  LR  T[  K\  &]  
^  	_vw&7&Rr9s%t	uvv?m&nqw  zA  qB  &C  
D  	Evw&7&R{9|  @F  HO  P  SY  [b  Rc  &d  
e  	fv?n&o%p	qrv?[&\_egn^o%p	qrvt&DvwFWbh  sT  ZU  &V  
W  	Xvw&7&  SP  :Q  &R  
S  	Tv?|&}  AG  IP  @Q  &R  
S  	Tv?Q&R%S	TU'F, hnngm`c[#a&*;[[gmnO_`O_RYYr2ZO<O_E`otuotjkcdecd[_[\ryy  SuA!9 ELLRXXVcdeotFu)0):):f;_jr  {B  KO  PU  PW  KX  kY  *Z&)K)\)\gpz)\  *A&*66wLh6i vw&7&Rm9n%o	pqv?^&_bhjqar%s	tuvw&7&Sq9ru{  ~E  uF  &G  
H  	Ivw&7&'9JU[  fH  MI  &J  
K  	Lvw&7&Ro9p%q	rsv?[&\_egn^o%p	qrvw&7&R`9adjlsct%u	vwv?m&n%o	pqv?S&TW]_fVgjpryiz%{	|}vw&7&'9JU[  fH  MI  LR  T[  K\  &]  
^  	_vw&7&Rr9s%t	uvv?m&nqw  zA  qB  &C  
D  	Evw&7&R{9|  @F  HO  P  SY  [b  Rc  &d  
e  	fv?n&o%p	qrv?[&\_egn^o%p	qrvt&DvwFWbh  sT  ZU  &V  
W  	Xvw&7&  SP  :Q  &R  
S  	Tv?|&}  AG  IP  @Q  &R  
S  	Tv?Q&R%S	TU'F, FL+Q*R	STFK+P*Q	RSFI+N*O	PQFJ+O*P	QRFG+L*M	NOFI+N*O	PQFN+S*T	UVF  EB  ,C  +D  
E  	FFO+T*U	VWFI+N*O	PQFD_+`*a	bcFDe+f*g	hiFDc+d*e	fgFH+M*N	OPFD_+`*a	bcFM+R*S	TUFD_+`*a	bcFDe+f*g	hiFDl+m*n	op'J, hnngm`c[#a&*;[[gmnO_`O_RYYr2ZO<O_E`otuotjkcdecd[_[\ryy  SuA!9 ELLRXXVcdeotFu/6/@/@Fx~A  KS  [b  ov  C  DI  DK  L  KM  0N,/W/h/hs|  GL/h  0M,0<<WRt<u vw&7&Rm9n%o	pqv?^&_bhjqar%s	tuvw&7&Sq9ru{  ~E  uF  &G  
H  	Ivw&7&'9JU[  fH  MI  &J  
K  	Lvw&7&Ro9p%q	rsv?[&\_egn^o%p	qrvw&7&R`9adjlsct%u	vwv?m&n%o	pqv?S&TW]_fVgjpryiz%{	|}vw&7&'9JU[  fH  MI  LR  T[  K\  &]  
^  	_vw&7&Rr9s%t	uvv?m&nqw  zA  qB  &C  
D  	Evw&7&R{9|  @F  HO  P  SY  [b  Rc  &d  
e  	fv?n&o%p	qrv?[&\_egn^o%p	qrvt&DvwFWbh  sT  ZU  &V  
W  	Xvw&7&  SP  :Q  &R  
S  	Tv?|&}  AG  IP  @Q  &R  
S  	Tv?Q&R%S	TU'F, FL+Q*R	STFK+P*Q	RSFI+N*O	PQFJ+O*P	QRFG+L*M	NOFI+N*O	PQFN+S*T	UVF  EB  ,C  +D  
E  	FFO+T*U	VWFI+N*O	PQFD_+`*a	bcFDe+f*g	hiFDc+d*e	fgFH+M*N	OPFD_+`*a	bcFM+R*S	TUFD_+`*a	bcFDe+f*g	hiFDl+m*n	op'F, FM+R*S	TUFL+Q*R	STFO+T*U	VWFDW+X*Y	Z[FK+P*Q	RSFH+M*N	OPFDZ+[*\	]^FD  ,A  +B  
C  	DFJ+O*P	QRFK+P*Q	RSFH+M*N	OPFJ+O*P	QRFDZ+[*\	]^FI+N*O	PQFK+P*Q	RSFL+Q*R	STFD`+a*b	cdFD_+`*a	bcFL+Q*R	ST' H, hnngm`c[#a&*;[[gmnO_`O_RYYr2ZO<O_E`otuotjkcdecd[_[\ryy  SuA!9 ELLRXXVcdeotFu(/(9(9VW]ks  @F  ;G  RZ  bi  ry  DK  TX  Y^  Y`  Ta  Rb  )c%(I(Z(Zenx}(Z(~%)55gKf5g } q`eu8 o`eub o`euL o`eus   "l>+l1
l;A
ll)"l+l
l)A
l#l))"l0+l5
m A
l:m "m+m8
mA
mml#l):m m__main__)dataclassesr   r   numpyrO   datasetsr   r   r   r   r	   transformersr
   rV   r   rc   r   parserparse_args_into_dataclassesscript_argsr   r   r   r   r$   r%   <module>rl      s    )  : : ) 4x%/ RS]bck]lm
n   6whv zo.F446q9K		 7 79L9LM r$   