
    bCi3T                        S r SSKrSSKJr  SSKrSSKJr  SSKrSSK	J
r
  SSKJrJr  SSKrSSKJrJr  SSKJrJr  \" 5       (       a  SSKr\" 5       (       a  SS	KJr  SS
KJr  \R6                  " \5      r SS jrS\\    S\RB                  S\"\ \RB                  4   S\ S\\\    \RB                  4   4
S jr#S r$S r%S r&S r'g)z!PyTorch - Flax general utilities.    N)UnpicklingError)
from_bytes)flatten_dictunflatten_dict   )is_safetensors_availableis_torch_available)check_torch_load_is_safelogging)	safe_open)	load_filec                    U(       d  [         R                  R                  U5      n[        R	                  SU 35        UR                  S5      (       a@  0 n[        USS9 nUR                  5        H  nUR                  U5      XW'   M     SSS5        OY SSK	n[        5         UR                  " USS	S
9n[        R	                  S[        S UR!                  5        5       5      S S35        [#        XP5      n	U	$ [%        X5      n	U	$ ! , (       d  f       N(= f! [        [        4 a    [        R                  S5        e f = f)z(Load pytorch checkpoints in a flax modelzLoading PyTorch weights from .safetensorsflax)	frameworkNr   zLoading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see https://pytorch.org/ and https://flax.readthedocs.io/en/latest/index.html#installation for installation instructions.cpuT)map_locationweights_onlyzPyTorch checkpoint contains c              3   @   #    U  H  oR                  5       v   M     g 7fN)numel).0ts     b/home/james-whalen/.local/lib/python3.13/site-packages/transformers/modeling_flax_pytorch_utils.py	<genexpr>=load_pytorch_checkpoint_in_flax_state_dict.<locals>.<genexpr>K   s     :eNd7799Nds   ,z parameters.)ospathabspathloggerinfoendswithr   keys
get_tensortorchImportErrorModuleNotFoundErrorerrorr
   loadsumvalues"convert_pytorch_state_dict_to_flax*convert_pytorch_sharded_state_dict_to_flax)

flax_modelpytorch_checkpoint_path
is_shardedallow_missing_keyspt_pathpt_state_dictfkr&   flax_state_dicts
             r   *load_pytorch_checkpoint_in_flax_state_dictr8   0   s%   
 ''//"9:3G9=>N++M7f5A'(||AM$ " 65 %&!JJwUQUVMKK6s:emNbNbNd:e7efg6hhtuv<]W  EE\i- 65  !45 %
 s   !+DD 
D'E pt_tuple_key	pt_tensorrandom_flax_state_dictmodel_prefixreturnc                   ^^ S[         [           S[        4UU4S jjnU SS S-   nU S   S;   a  U" U5      (       a  XQ4$ U SS S-   nU S   S	:X  a  U" U 5      (       d  XQ4$ U SS S
-   nU S   S:X  a  U" U 5      (       d  XQ4$ U SS S-   nU S   S:X  a  U" U5      (       a  XQ4$ U SS S-   nU S   S:X  a4  UR                  S:X  a$  U" U 5      (       d  UR	                  SSSS5      nXQ4$ U SS S-   nU S   S:X  a  U" U 5      (       d  UR
                  nXQ4$ U SS S-   nU S   S:X  a  XQ4$ U SS S-   nU S   S:X  a  XQ4$ SnU SSS2   S:X  a	  U S   S-   nOU SSS2   S:X  a  U S   S-   nUb  U SS U4-   nXQ4$ X4$ )zYRename PT weight names to corresponding Flax weight names and reshape tensor if necessarykeyr=   c                 D   > [        [        T5      U T4U -   1-  5      S:  $ )zAChecks if `key` of `(prefix,) + key` is in random_flax_state_dictr   )lenset)r?   r<   r;   s    r   is_key_or_prefix_key_in_dictCrename_key_and_reshape_tensor.<locals>.is_key_or_prefix_key_in_dict\   s+    3-.#7L1MMNQRRR    N)scale)weightgamma)meanrunning_mean)varrunning_var)	embeddingrH   )kernel         r   r   rH   rI   )biasbeta)parametrizations	original0_g)rW   	original1_v)tuplestrboolndim	transposeT)r9   r:   r;   r<   rC   renamed_pt_tuple_keynames     ``   r   rename_key_and_reshape_tensorre   T   s\   S%* S S S
 (,z9B..3OPd3e3e#.. (,y8B>)2N|2\2\#.. (,x7B=(1Ml1[1[#.. (,~=B8#(DEY(Z(Z#.. (,{:B8#	!(;D`amDnDn''1a3	#.. (,{:B8#,H,V,VKK	#.. (,{:B7"#.. (,y8B6!#.. DBEE??B$&	be!e	 A	AB$&+CR0D7:#..""rE   c           
      V   [        5       =(       a:    [        [        [        U R	                  5       5      5      [
        R                  5      nU(       a  [
        R                  OSnU R                  5        VVs0 s H  u  pEXER                  _M     nnnU(       aY  U R                  5        HE  u  pEUR                  U:X  a  UR                  5       nUR                  5       R                  5       X'   MG     UR                  nSUR                  ;   a  UR                  S   nOUR                  n[        U5      n	SUR                  ;   a)  [        UR                  S   5      n
U	R!                  U
5        0 nXx;  =(       a&    Xp Vs1 s H  oDR#                  S5      S   iM     sn;   nXx;   =(       a&    Xp Vs1 s H  oDR#                  S5      S   iM     sn;  nU R                  5        GH  u  p[%        UR#                  S5      5      nXn   U:H  nUS   U:H  nU(       a  U(       a  USS  n['        UXU5      u  nnU4U-   U	;   nU(       a  U(       a  U4U-   nUU	;   aI  UR(                  U	U   R(                  :w  a,  [+        SU SU	U   R(                   S	UR(                   S35      eSUR                  ;   a  S
US   ;   d	  SUS   ;   a  [,        R.                  " U5      USU-   '   M  SUS   ;   a  UR1                  US 5        GM  U(       d  [,        R.                  " U5      O"[,        R.                  " U[,        R                  S9USU-   '   GMe  U(       d  [,        R.                  " U5      O"[,        R.                  " U[,        R                  S9UU'   GM     [3        U5      $ s  snnf s  snf s  snf )Nbfloat16paramsbatch_stats.r   r   1PyTorch checkpoint seems to be incorrect. Weight  was expected to be of shape 	, but is rJ   rF   rL   ri   num_batches_trackeddtyperh   )r	   
isinstancenextiterr,   r&   Tensorrg   itemsrq   floatr   numpybase_model_prefixrh   r   updatesplitr]   re   shape
ValueErrorjnpasarraypopr   )r4   r/   from_binrg   r6   vweight_dtypesr<   flax_model_paramsr;   flax_batch_statsr7   $load_model_with_head_into_base_model$load_base_model_into_model_with_headpt_keyr:   r9   is_bfloat_16has_base_model_prefixflax_keyflax_tensorrequire_base_model_prefixs                         r   r-   r-      sk   !#d
4]=Q=Q=S8T3UW\WcWc(dH!)u~~zH,9,?,?,AB,ADAQZ,AMB!'')DAww("GGI uuw}}M	 * //L :$$$&--h7&--)*;< 
)))'
(9(9-(HI%%&67O,8,Q ,-@-Qa-@@ ) -9,M ,mDmWWS\!_mDD )
 +002V\\#./$,8 !-Q< ?/4I'+L !>)\!
+
 &2Oh$>BX$X!/4M$1H--  $:8$D$J$JJ GxOl-h7==>iHYHYGZZ[]  J---"%(2,)>?B{{;?W 08 ;<$4##Hd3 1=K(#++kadamamBn K(23 1=K(#++kadamamBn H%U 3\ /**_ C4 A Es   8N N!1N&c                 H   SS K n0 nU  GHu  n[        5         UR                  " USS9nUR                  5        VVs0 s H  u  pgXgR                  _M     nnnUR                  5        VVs0 s HM  u  pgXgR                  UR
                  :w  a  UR                  5       OUR                  5       R                  5       _MO     nnnUR                  n	SUR                  ;   aB  UR                  S   n
[        U
5      nUR                  [        UR                  S   5      5        OUR                  n
[        U
5      nX;  =(       a&    X Vs1 s H  ofR                  S5      S   iM     sn;   nX;   =(       a&    X Vs1 s H  ofR                  S5      S   iM     sn;  nUR                  5        GH  u  p[        UR                  S5      5      nX   UR
                  :H  nUS   U	:H  nU(       a  U(       a  USS  n[        UXU	5      u  nnU	4U-   U;   nU(       a  U(       a  U	4U-   nUU;   aI  UR                  UU   R                  :w  a,  [!        SU S	UU   R                   S
UR                   S35      eSUR                  ;   a  SUS   ;   a  ["        R$                  " U5      USU-   '   GM   SUS   ;   a  ["        R$                  " U5      USU-   '   GM(  SUS   ;   a  UR'                  US 5        GMF  U(       d  ["        R$                  " U5      O"["        R$                  " U["        R
                  S9USU-   '   GM  U(       d  ["        R$                  " U5      O"["        R$                  " U["        R
                  S9UU'   GM     GMx     [)        U5      $ s  snnf s  snnf s  snf s  snf )Nr   T)r   ri   rh   rj   r   rk   rl   rm   rJ   rF   rn   rL   ro   rp   rr   )r&   r
   r*   rw   rq   rg   ry   rx   rz   rh   r   r{   r|   r]   re   r}   r~   r   r   r   r   )shard_filenamesr/   r&   r7   
shard_filer4   r6   r   r   r<   r   r;   r   r   r   r:   r9   r   r   r   r   r   s                         r   r.   r.      s    O%
 "

:DA0=0C0C0EF0EGG0EFYfYlYlYn
YnQUQRAGGu~~5qwwy1779??;LLYn 	 
 "33 J--- * 1 1( ;%12C%D"")),z7H7H7W*XY * 1 1%12C%D"0<0U 0
mDmWWS\!_mDD 	- 1=0Q 0
- H-Qa- HH 	- "/!4!4!6F c!23L(0ENNBL %1O|$C!38M+AB/ %Bi%!Hk *6((BF\(\%38Q(?X511$$(>x(H(N(NN$KF8Sp1(;AAB)KL]L]K^^_a  
 1 11Xb\)CF;;{C[O$4x$?@HRL(CF;;{C[O$4x$?@(HRL8#''$7 5ACKK,ckkR]eheqeqFr  h 67 5ACKK,ckkR]eheqeqFr  )[ "79 &X /**Q G
" E !Is   N)ANN5Nc                    [         R                  R                  U5      n[        R	                  SU 35        [        [        SU R                  R                  -   5      nUR                  S5      (       a  [        U5      n[        USS9nO0[        US5       n [        X$R                  5       5      n S	S	S	5        [#        U W5      $ ! [         a    [!        SU S35      ef = f! , (       d  f       N7= f)
(Load flax checkpoints in a PyTorch modelzLoading Flax weights from Flaxr   rj   )seprbzUnable to convert z  to Flax deserializable object. N)r   r   r    r!   r"   getattrtransformers	__class____name__r#   safe_load_filer   openr   readr   OSError"load_flax_weights_in_pytorch_model)modelflax_checkpoint_pathflax_clsr7   state_fs        r   %load_flax_checkpoint_in_pytorch_modelr   L  s    77??+?@
KK,-A,BCD |Veoo.F.F%FGH $$^44()=>(cB&-k",X||~"F . .e_EE # k 23G2HHhijjk .-s   C$CC!!C$$
C2c                 
    SSK n[        [        R                  R                  S U5      5      R                  5       n[        U5      (       a6  [        R                  S5        [        R                  R                  S U5      n[        U5      nU R                  5       nU R                  U;   =(       a1    U R                  U Vs1 s H  ofR                  S5      S   iM     sn;  nU R                  U;  =(       a1    U R                  U Vs1 s H  ofR                  S5      S   iM     sn;   n/ n	[        UR!                  5       5      n
UR#                  5        GH  u  pUS   U R                  :H  nSR%                  U R                  4U-   5      U;   nU(       a  U(       a  USS nOU(       a  U(       a  U R                  4U-   nUS	   S
:X  aE  UR&                  S:X  a5  SR%                  U5      U;  a   USS	 S-   n[(        R*                  " US5      nOhUS	   S
:X  a*  SR%                  U5      U;  a  USS	 S-   nUR,                  nO5US	   S;   a	  USS	 S-   nO#SUS	   ;   a	  USS	 S-   nOSUS	   ;   a  USS	 S-   nSU;   a  SR%                  USS 5      nOSR%                  U5      n0 nU Hg  nUR                  S5      nSnUSSS2   SS/:X  a	  US   S-   nOUSSS2   SS/:X  a  US   S-   nUc  MH  USS U/-   nSR%                  U5      nUUU'   Mi     UU;   a  UU   nX;   a  UR.                  X_   R.                  :w  a+  [1        SU SX_   R.                   SUR.                   S35      e[3        U[4        R6                  5      (       d  [4        R8                  " U5      OUnUR:                  " U5      X_'   U
R=                  U5        GMs  U	R?                  U5        GM     U RA                  U5        [C        U
5      n
[E        U	5      S:  a_  [        R                  SU RF                  RH                   S U	 S!U RF                  RH                   S"U RF                  RH                   S#3	5        O-[        R                  S$U RF                  RH                   S%35        [E        U
5      S:  a2  [        R                  S&U RF                  RH                   S'U
 S(35        U $ [        R                  S)U RF                  RH                   S*U RF                  RH                   S+35        U $ ! [        [        4 a    [        R	                  S5        e f = fs  snf s  snf ),r   r   NzLoading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see https://pytorch.org/ and https://flax.readthedocs.io/en/latest/index.html#installation for installation instructions.c                 <    U R                   [        R                  :H  $ r   )rq   r   rg   )xs    r   <lambda>4load_flax_weights_in_pytorch_model.<locals>.<lambda>p  s    CLLAXrE   zFound ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` before loading those in PyTorch model.c                     U R                   [        R                  :X  a  U R                  [        R
                  5      $ U $ r   )rq   r   rg   astypenpfloat32rr   s    r   r   r   y  s)    8T6==4`Z``rE   rj   r   rF   rO   rP   rS   )rR   rQ   r   r   )rG   rN   rJ   )rK   rL   )rM   ri   rV   rQ   rW   rX   rY   rZ   r[   r\   z.Flax checkpoint seems to be incorrect. Weight rl   rm   zQSome weights of the Flax model were not used when initializing the PyTorch model z: z,
- This IS expected if you are initializing z from a Flax model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).
- This IS NOT expected if you are initializing z from a Flax model that you expect to be exactly identical (e.g. initializing a BertForSequenceClassification model from a FlaxBertForSequenceClassification model).z3All Flax model weights were used when initializing z.
zSome weights of zE were not initialized from the Flax model and are newly initialized: zo
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.zAll the weights of z were initialized from the Flax model.
If your task is similar to the task the model of the checkpoint was trained on, you can already use z* for predictions without further training.)%r&   r'   r(   r!   r)   r   jax	tree_utiltree_mapr,   anywarning
state_dictrz   r|   rB   r$   rw   joinr`   r   ra   rb   r}   r~   rs   r   ndarrayr   
from_numpyremoveappendload_state_dictlistrA   r   r   )pt_model
flax_stater&   is_type_bf16r7   pt_model_dictr6   r   r   unexpected_keysmissing_keysflax_key_tupler   r   r   r   special_pt_namesr?   key_componentsrd   key_to_checks                        r   r   r   b  s      6 67XZd efmmoL
< 	5	
 ]]++`bl

 #:.O'')M,4,F,F*,T ,""M*RMq773<?M*RR ) -5,F,Fj,X ,""&N1wws|A&NN )
 O}))+,L'6'<'<'># .q 1X5O5O O$'HHh.H.H-J^-[$\`m$m! 04I+AB/N16O&88:^KN ")k.>.>!.CQ_H`huHu+CR0;>N--\BKB8+0HP]0]+CR0;>N%--KB#99+CR0;>N ~b))+CR03DDNnR((+CR03CCNJ&xxqr 23Hxx/H  C YYs^NDbe!e$);[(II%b)D0A&+={*KK%b)D0!/!4v!="xx714 . ! '''1H$  M$;$A$AA D^DT U&&3&=&C&C%DIkN_N_M``ac  >HUWU_U_=`=`bjj5fq*/*:*:;*G'##H- ""8,C (?F ]+ %L
?a""++,B.? @""++, -88@8J8J8S8S7T U9	9	
 	LXM_M_MhMhLiilmn
<1x11::; <)N +55	
 O 	!("4"4"="=!> ?##+#5#5#>#>"??ik	
 OI ,- 	

 	2 +S 'Os   T U	U'U)F)(__doc__r   pickler   r   	jax.numpyry   r   r   flax.serializationr   flax.traverse_utilr   r   r    r   r	   utilsr
   r   r&   safetensorsr   safetensors.flaxr   r   
get_loggerr   r!   r8   r]   r^   r   dictre   r-   r.   r   r    rE   r   <module>r      s    ( 	 " 
   ) ;  : 4 %< 
		H	% IN!H@#*@#zz@# !ckk!12@# 	@#
 5:rzz!"@#FT+xQ+rF,IrE   