
    -jiE8                         S SK r S SKJr  S SKJr  S SKJr  S SKrS SKJ	s  J
r  S SKJrJr  S SKJr  S SKJrJrJrJrJr  S SKJr  S	S
KJr  S	SKJrJrJr  \ " S S5      5       r " S S5      rg)    N)suppress)	dataclass)Any)
LongTensorTensor)
ModuleList)AutoModelForCausalLMAutoTokenizerBatchEncodingPreTrainedTokenizerBaseTextStreamer)GenerateOutput   )Settings)batchifyempty_cacheprintc                   >    \ rS rSr% \\S'   \\S'   \\S'   \\S'   Srg)AbliterationParameters   
max_weightmax_weight_position
min_weightmin_weight_distance N)__name__
__module____qualname____firstlineno__float__annotations____static_attributes__r       G/home/james-whalen/.local/lib/python3.13/site-packages/heretic/model.pyr   r      s    r#   r   c            	          \ rS rSrS\4S jrS rS\4S jrS\	S\
\\\   4   4S jrS\\   4S	 jrS
\S\S-  S\
\\4   4S jrS\S\\
\\4      4S jrS\\   S\S\\\\-  4   4S jrS\\   S\\   4S jrS\\   S\\   4S jrS\\   S\4S jrS\\   S\4S jrS\\   S\4S jrS\\   S\4S jrS\\
\\4      S\4S jr Sr!g)Model"   settingsc           	      @   Xl         SU l        [        5         [        SUR                   S35        [        R
                  " UR                  UR                  S9U l        U R                  R                  c%  U R                  R                  U R                  l        SU R                  l
        S U l        UR                  UR                  0U l        U R                   R                  b#  UR                  U R                  UR                  '   UR                   H  n[        SU S3SS9   [        R
                  " UR                  UUR                  U R                  R!                  UR                  5      S	9U l        U R                  R!                  UR                  5      c  S
U R                  UR                  '   U R#                  S/SS9  [        S5          O   U R                  c  [%        S5      e[        S[)        U R+                  5       5       S35        [        S5        U R-                  S5      R/                  5        H   u  pE[        SU S[)        U5       S35        M"     g ! [$         a,  nS U l        ['        5         [        SU S35         S nAGM  S nAff = f)N zLoading model [bold]z[/]...)trust_remote_codeleftz* Trying dtype [bold]z[/]... )enddtype
device_mapr+   TTestr   max_new_tokensz[red]Failed[/] ()z[green]Ok[/]z0Failed to load model with all configured dtypes.z* Transformer model with [bold]z
[/] layersz* Abliterable components:r   z
  * [bold]z[/]: [bold]z[/] matrices per layer)r(   response_prefixr   modelr
   from_pretrainedr+   	tokenizer	pad_token	eos_tokenpadding_sidetrusted_modelsevaluate_modeldtypesr	   r0   getgenerate	Exceptionr   len
get_layersget_layer_matricesitems)selfr(   r/   error	componentmatricess         r$   __init__Model.__init__#   s7    !$X^^$4F;<2?2O2ONN&883
 >>##+'+~~'?'?DNN$
 '-#
'~~x/I/IJ==''3;C;U;UD 7 78__E)%8bA1AANN'22&*&9&9&=&=hnn&M	
 &&**8>>:B:>D''7
 vhq9 .!7 %: ::NOO/DOO4E0F/GzRS)*#'#:#:1#=#C#C#EIYK{3x=/AWX $F  !
(q12	s   (B#I''
J1 JJc                    U R                   R                  nS U l         [        5         [        R                  " U R
                  R                   UU R
                  R                  U R                  R                  U R
                  R                   5      S9U l         U R                  R                  U R
                  R                   5      c$  SU R                  U R
                  R                   '   g g )Nr.   T)	r6   r/   r   r	   r7   r(   r0   r<   r?   )rF   r/   s     r$   reload_modelModel.reload_modele   s    

   
)99MM}}//"1155dmm6I6IJ	

 ""4==#6#67?7;D 3 34 @r#   returnc                     [        [        5         U R                  R                  R                  R                  sS S S 5        $ ! , (       d  f       O= fU R                  R                  R                  $ N)r   rA   r6   language_modellayersrF   s    r$   rC   Model.get_layersv   sG    i ::##2299 !   zz&&&s   *A
Alayer_indexc                   ^ U R                  5       U   n0 mS[        S[        4U4S jjnU" SUR                  R                  R
                  5        [        [        5         U" SUR                  R                  R
                  5        S S S 5        [        [        5         UR                  R                   H   nU" SUR                  R
                  5        M"     S S S 5        [        [        5         UR                  R                   H   nU" SUR                  R
                  5        M"     S S S 5        [        [        5         U" SUR                  R                  R                  5        S S S 5        [        [        5         U" SUR                  R                  R
                  5        S S S 5        [        [        5         UR                  R                   H   nU" SUR                  R
                  5        M"     S S S 5        TS   (       d   eT$ ! , (       d  f       GN= f! , (       d  f       GNQ= f! , (       d  f       GN= f! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       Nt= f)NrH   matrixc                   > [        US5      (       a1  [        R                  " UR                  5      (       a  UR                  n[        R                  " U5      (       d   eU T;  a  / TU '   TU    R	                  U5        g )Ndata)hasattrtorch	is_tensorrZ   append)rH   rX   rI   s     r$   try_add)Model.get_layer_matrices.<locals>.try_add   sg     vv&&5??6;;+G+G??6****(&(#Y&&v.r#   zattn.o_projzmlp.down_proj)rC   strr   	self_attno_projweightr   rA   mlp	down_projexpertsblock_sparse_moew2
shared_mlpoutput_linearmoe)rF   rV   layerr_   expertrI   s        @r$   rD   Model.get_layer_matrices~   s   !+.	/s 	/C 	/ 	u55<<= i OUYY%8%8%?%?@ ! i ))++)9)9)@)@A , !
 i 0088)9)9: 9 !
 i  OUYY%6%6%@%@A	 ! i OU%5%5%C%C%J%JK ! i ))++)=)=)D)DE , !
 (((A !  ! 
 ! 
 !  !  ! sH   !(H ;H%2;H7(I	(I;I+
H"%
H47
I	
I
I(+
I9c                 R    [        U R                  S5      R                  5       5      $ )Nr   )listrD   keysrT   s    r$   get_abliterable_components Model.get_abliterable_components   s!    D++A.33566r#   refusal_directionsdirection_indexN
parametersc                 2   Uc  S nO\[         R                  " US-   5      u  pV[        R                  " U[	        U5         R                  U[	        U5      S-      U5      SSS9n[        [        U R                  5       5      5       GH  nU R                  U5      R                  5        H  u  pX8   n
[        XzR                  -
  5      nXR                  :  a  M1  U
R                  XR                  -  U
R                  U
R                  -
  -  -   nUc  XS-      nOUn[         R"                  " UU5      R%                  U R&                  R(                  5      nU	 H4  nUR%                  UR*                  5      nUR-                  X_U-  -  5        M6     M     GM     g )Nr      r   )pdim)mathmodfF	normalizeintlerprangerB   rC   rD   rE   absr   r   r   r   r\   outertor6   r/   devicesub_)rF   ru   rv   rw   refusal_directionrd   indexrV   rH   rI   paramsdistancelayer_refusal_direction	projectorrX   device_projectors                   r$   
abliterateModel.abliterate   s    " $ !IIo&9:MF !"3u:.33&s5zA~6 ! !T__%6!78K'+'>'>{'K'Q'Q'S#	#.{-G-GGH 888  **h9S9S.S%%(9(99.  %, /Aq.Q+.?+ "KK++ "TZZ%%& 
 'F'0||FMM'B$KKV*C DE	 '= (T 9r#   promptc                 >    SU R                   R                  S.SUS./$ )Nsystem)rolecontentuser)r(   system_prompt)rF   r   s     r$   get_chatModel.get_chat   s&    $--*E*EF/
 	
r#   promptskwargsc                    U Vs/ s H  o0R                  U5      PM     nnU R                  R                  USSS9nU R                  (       a  U Vs/ s H  o3U R                  -   PM     nnU R                  USSSS9R	                  U R
                  R                  5      nX`R
                  R                  " S0 UDUDU R                  R                  SS.D64$ s  snf s  snf )NTFadd_generation_prompttokenizept)return_tensorspaddingreturn_token_type_ids)pad_token_id	do_sampler   )	r   r8   apply_chat_templater5   r   r6   r   r@   r   )rF   r   r   r   chatschat_promptsinputss          r$   r@   Model.generate   s    
 6==W6v&W="&.."D"D"& #E #
  IUUfT%9%99LU"'	   

 "TZZ
 	 zz** 


 44	
 
 	
) > Vs   CCc                     U R                  UU R                  R                  S9u  p#U R                  R	                  US S 2US   R
                  S   S 24   5      $ )Nr2   	input_idsr   )r@   r(   max_response_lengthr8   batch_decodeshape)rF   r   r   outputss       r$   get_responsesModel.get_responses  s\    --==<< ( 
 ~~**71f[6I6O6OPQ6R6T3T+UVVr#   c                     / n[        XR                  R                  5       H,  nU R                  U5       H  nUR	                  U5        M     M.     U$ rQ   )r   r(   
batch_sizer   r^   )rF   r   	responsesbatchresponses        r$   get_responses_batchedModel.get_responses_batched%  sL    	g}}'?'?@E ..u5  * 6 A r#   c           
          U R                  USSSS9u  p#UR                  S   n[        R                  " U Vs/ s H  oUS S 2SS S 24   PM     snSS9nUR	                  [        R
                  5      $ s  snf )Nr   T)r3   output_hidden_statesreturn_dict_in_generater   r{   )r@   hidden_statesr\   stackr   float32)rF   r   _r   r   layer_hidden_states	residualss          r$   get_residualsModel.get_residuals.  s     ]]!%$(	 # 

  --a0 KK GTTm/BB*mT
	 ||EMM** Us   A2c                     / n[        XR                  R                  5       H#  nUR                  U R	                  U5      5        M%     [
        R                  " USS9$ Nr   r   )r   r(   r   r^   r   r\   cat)rF   r   r   r   s       r$   get_residuals_batchedModel.get_residuals_batchedH  sM    	g}}'?'?@ET//67 A yy**r#   c                 r    U R                  USSSS9u  p#UR                  S   n[        R                  " USS9$ )Nr   T)r3   output_scoresr   r   r   r   )r@   scoresr~   log_softmax)rF   r   r   r   logitss        r$   get_logprobsModel.get_logprobsR  sH     ]]$(	 # 

 " }}V,,r#   c                     / n[        XR                  R                  5       H#  nUR                  U R	                  U5      5        M%     [
        R                  " USS9$ r   )r   r(   r   r^   r   r\   r   )rF   r   logprobsr   s       r$   get_logprobs_batchedModel.get_logprobs_batchedb  sK    g}}'?'?@EOOD--e45 A yyq))r#   chatc                 n   U R                   R                  USSS9nU R                  USSS9R                  U R                  R                  5      n[        U R                   SSS9nU R                  R                  " S0 UDUSS.D6nU R                   R                  US	US
   R                  S   S 24   SS9$ )NTFr   r   )r   r   )skip_promptskip_special_tokensi   )streamerr3   r   r   r   )r   r   )	r8   r   r   r6   r   r   r@   decoder   )rF   r   chat_promptr   r   r   s         r$   stream_chat_responseModel.stream_chat_responsej  s    >>=="& > 
 "'   
 "TZZ
	 	  NN $
 **%% 


 ~~$$Avk*003556 $ % 
 	
r#   )r6   r5   r(   r8   r<   )"r   r   r   r   r   rJ   rM   r   rC   r   dictra   rq   r   rD   rs   r    r   r   r   r   tupler   r   r   r@   r   r   r   r   r   r   r   r"   r   r#   r$   r&   r&   "   s   @ @D<"'J '7c 7d3V;L6M 7r7DI 7:F":F :F 445	:Fx
s 
tDcN'; 

c
 
 
}nz99	:	
@WT#Y W49 WT#Y 49 +T#Y +6 +4+T#Y +6 +-DI -& - *DI *& *
d38n)= 
# 
r#   r&   ) r|   
contextlibr   dataclassesr   typingr   r\   torch.nn.functionalnn
functionalr~   r   r   torch.nnr   transformersr	   r
   r   r   r   transformers.generation.utilsr   configr   utilsr   r   r   r   r&   r   r#   r$   <module>r      s_      !     $   9  / /   d
 d
r#   