
    oi(                        S SK Jr  S SKrS SKrS SKJrJr  S SKrS SKJ	r	  S SK
JrJr  S SKJr  SSKJrJr  \	" 5       (       a&  S S	KJr   " S
 S\R(                  R*                  \5      rSS jrg)    )annotationsN)AnyOptional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge)	transpose   )	LoraLayerLoraVariant)	HQQLinearc                     ^  \ rS rSr       S                   SU 4S jjjrSS jrSSS jjrSS jrS r          SS jr	SS jr
SU 4S	 jjrS
rU =r$ )HqqLoraLinear    c
                   > U	(       a"  [        U R                  R                   S35      e[        TU ]  5         [
        R                  " X5        SU l        X l        U R                  UUUUUUUU	S9  g )Nz0 does not support lora_bias yet, set it to FalseF)
lora_alphalora_dropoutinit_lora_weights
use_rslorause_dora	lora_bias)	
ValueError	__class____name__super__init__r   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   kwargsr   s              N/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/lora/hqq.pyr   HqqLoraLinear.__init__"   sx      DNN$;$;#<<l!mnnGt0"'D#/ %)"3%!#  	    c               ,    U(       d  g SSK Jn  U" 5       $ )Nr
   )DoraLinearVariant)variantsr)   )r    r   r$   r)   s       r%   resolve_lora_variant"HqqLoraLinear.resolve_lora_variantB   s    3$&&r'   c                   [        X5      nU(       d  gU GH[  nX0R                  R                  5       ;  a  M#  U R                  5       n0 [        R
                  " UR                  5      ESUR                  0EnUR                  5       nX0R                  ;  a  U R                  U5      nXg-   nOU R                  U   R                  XU5      nU(       a8  [        R                  " U5      R                  5       (       d  [        SU S35      e[!        SXTR"                  UR$                  S9n	UR'                  SS5        U	R(                  " U40 UD6  Xl        U R,                  R/                  U5        GM^     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
Noffload_metaz1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr.   
dequantizelora_variantget_delta_weight
merge_safetorchisfiniteallr   r   r0   r1   popquantizer!   merged_adaptersappend)
r    
safe_mergeadapter_namesactive_adapterlayerr7   output	lora_dataw_datanew_hqq_layers
             r%   mergeHqqLoraLinear.mergeJ   sG    4DHM "/!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+!):):: $ 5 5n EI#/F!..~>II$`fgFennV&<&@&@&B&B$KNK[[no  !*$L_L_hmhtht u  6&&v>>"/$$++N;/ #0r'   c                V   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Gag  U R                  R                  5       nXR                  R                  5       ;  a  MS  U R                  5       n0 [        R                  " UR                  5      ESUR                  0EnUR                  5       nXR                  ;  aI  U R                  U5      nUR!                  UR"                  5      R!                  UR$                  5      U-
  nOU R                  U   R'                  XU5      n[)        SX2R*                  UR$                  S9nUR                  SS5        UR,                  " U40 UD6  Xpl        [        U R                  5      S:  a  GMf  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   r.   r/   )mergedwarningswarnlenrA   r?   r2   r3   r4   r5   r6   r7   r.   r8   r9   r:   todtyper1   unmerger   r0   r@   r!   )r    rE   rF   r7   rG   rH   rI   rJ   s           r%   rT   HqqLoraLinear.unmergeu   sX    ;;@Ad**+a/!%!5!5!9!9!;!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+!):):: $ 5 5n EI#YYy7::9;K;KLyXF!..~>FFt]cdF )$L_L_hmhtht u  6&&v>>"/% d**+a//r'   c                    [        U R                  U   R                  U R                  U   R                  -  S5      U R                  U   -  $ )NF)r	   lora_Bweightr2   scaling)r    adapters     r%   r:   HqqLoraLinear.get_delta_weight   sK    KK(//$++g2F2M2MM ,,w'	(r'   c          	        U R                   " U/UQ70 UD6n[        U5      n/ nU H9  nUR                  [        U5       V	V
s/ s H  u  pX:X  d  M  U	PM     sn
n	5        M;     [        U5       GH   u  pUS:X  a  M  XR                  R                  5       ;  a  M-  U R                  U   nU R                  U   nU R                  U   nU R                  U   n[        R                  " 5       (       + nU(       a1  UR                  nU R                  XR                  R                  5      nXU      nU" U" U" U5      5      5      U-  nU(       a  UR                  W5      nXWU   ==   U-  ss'   GM     U$ s  sn
n	f )N__base__)r!   setrB   	enumerater2   r3   rW   r   rY   r<   is_autocast_enabledrS   _cast_input_dtyperX   rR   )r    xrD   argsr$   resultunique_adapterssub_batch_indices_listrZ   indexitemirE   r2   rW   dropoutrY   requires_conversionexpected_dtype	sub_batchrG   s                        r%   _mixed_batch_forward"HqqLoraLinear._mixed_batch_forward   sd   
 __Q888F!-0O%'"*&--	R_H`.tH`dhdsuH`.tu + &/%?!!Z/!)9)9);;^4^4++N;,,~6*/*C*C*E&E#&%+\\N..q--2E2EFA Q78	wy'9 :;gE&#YY~6Fa01V;1- &@0 M5 /us   E5E5c           	        U R                   " U/UQ70 UD6  UR                  SS 5      nU R                  (       a9  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6nU$ Ub  U R                  " U/UQ7SU0UD6nU$ U R                  (       a  U R
                  " U/UQ70 UD6nU$ U R
                  " U/UQ70 UD6nU R                   GH  nX`R                  R                  5       ;  a  M#  U R                  U   nU R                  U   nU R                  U   n	U R                  U   n
[        R                  " 5       (       + nU(       a1  UR                  nU R!                  XR"                  R                  5      nX`R$                  ;  a  XX" U" U	" U5      5      5      U
-  -   nOU R$                  U   R'                  U UUUS9nU(       d  GM  UR)                  W5      nGM     U$ )NrD   )rE   rb   rd   )_check_forward_argsr?   disable_adaptersrN   rT   r!   rn   active_adaptersr2   r3   rW   r   rY   r<   r`   rS   ra   rX   r9   forwardrR   )r    rb   rc   r$   rD   rd   rE   r2   rW   rj   rY   rk   rl   s                r%   rt   HqqLoraLinear.forward   s   $$Q888"JJ=M$$;;LLN<T<V<D MC *221ctc=c\bc@ M? <T<V<< M9 <T<V<&*&:&:N%[[-=-=-?? ![[8F![[8F"//?G"ll>:G.3.G.G.I*I'*)/ 221mm6I6IJ%->->>!'&
1C*Dw*N!N!%!2!2>!B!J!J +9#)	 "K " +*!'>!:1 ';4 Mr'   c                *   > [         TU ]  5       nSU-   $ )Nzlora.)r   __repr__)r    repr   s     r%   rw   HqqLoraLinear.__repr__   s    '"$CS= r'   )r   r!   r   )r   r
   g        TFFF)r!   torch.nn.Moduler"   strr#   intr   r|   r   floatr   boolr   r~   r   r~   r   r~   returnNone)r   r~   r   zOptional[LoraVariant])FN)rC   r~   rD   zOptional[list[str]]r   r   )r   r   )
rb   torch.Tensorrc   r   rD   z	list[str]r$   r   r   r   )rb   r   r   r   )r   r{   )r   
__module____qualname____firstlineno__r   r+   rK   rT   r:   rn   rt   rw   __static_attributes____classcell__)r   s   @r%   r   r       s     "%&*$"#	'	 	 		
 	  	  $	 	 	 	 	 	@	')	<V	08	$	!$	*-$	>G$	SV$	$	L)	V	! 	!r'   r   c                    S n[        U [        5      (       a  U R                  5       nOU n[        5       (       a!  [        U[        5      (       a  [        XA40 UD6nU$ )N)
isinstancer   r4   r   r   r   )targetr"   r$   
new_moduletarget_base_layers        r%   dispatch_hqqr      sU    J&.))"113"j):IFF"#4MfM
r'   )r   rz   r"   r{   )
__future__r   r5   rO   typingr   r   r<   peft.import_utilsr   peft.tuners.tuners_utilsr   r   peft.utils.otherr	   rF   r   r   hqq.core.quantizer   nnModuler   r    r'   r%   <module>r      sN    #      . L & ) +M! M!`r'   