
    oi                        S SK Jr  S SKrS SKrS SKJr  S SKrS SKJr  S SK	J
r
Jr  SSKJr  \" 5       (       a&  S SKJr   " S	 S
\R                   R"                  \5      rSS jrg)    )annotationsN)Optional)is_hqq_available)BaseTunerLayercheck_adapters_to_merge   )OFTLayer)	HQQLinearc                     ^  \ rS rSr         S	                       S
U 4S jjjrSSS jjrSS jrS rSS jrSU 4S jjr	Sr
U =r$ )HqqOFTLinear   c                   > [         TU ]  5         [        R                  " X5        SU l        X l        U R                  UUUUUUUU	U
US9
  g )NF)oft_block_sizemodule_dropoutinit_weightscoftepsblock_shareuse_cayley_neumannnum_cayley_neumann_terms)super__init__r	   fan_in_fan_out_active_adapterupdate_layer)self
base_layeradapter_namerr   r   r   r   r   r   r   r   kwargs	__class__s                M/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/oft/hqq.pyr   HqqOFTLinear.__init__!   s]     Gd/"'D#/ --)'#5)A      c                   [        X5      nU(       d  gU GH  nX0R                  R                  5       ;  a  M#  U R                  5       n0 [        R
                  " UR                  5      ESUR                  0EnUR                  5       nU R                  U5      n[        R                  " USS5      n[        R                  " XvR                  UR                  5      5      n[        R                  " USS5      nUR                  UR                  5      R                  UR                  5      nU(       a8  [        R                   " U5      R#                  5       (       d  [%        SU S35      e['        SXTR(                  UR                  S9n	UR+                  SS5        U	R,                  " U40 UD6  Xl        U R0                  R3                  U5        GM     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If True, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`list[str]`, *optional*):
        The list of adapter names that should be merged. If None, all active adapters will be merged.
        Defaults to `None`.
Noffload_metar   r   z1NaNs detected in the merged weights. The adapter z seems to be brokencompute_dtypedevice)r   lora_Akeysget_base_layercopydeepcopyquant_configr&   
dequantizeget_delta_weighttorch	transposemmtodtyper)   isfiniteall
ValueErrorr
   r(   popquantizer   merged_adaptersappend)
r   
safe_mergeadapter_namesactive_adapterlayerr/   outputoft_dataw_datanew_hqq_layers
             r"   mergeHqqOFTLinear.mergeB   su    4DHM "/!)9)9);;++-h$--0B0B"Ch^UZUgUgh))+00@A6(IIhnn,EFA68>>255hooFennV&<&@&@&B&B$KNK[[no  !*$L_L_hmhtht u  6&&v>>"/$$++N;1 #0r$   c                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  Ga  U R                  R                  5       nXR                  R                  5       ;  a  MS  U R                  5       n0 [        R                  " UR                  5      ESUR                  0EnUR                  5       nU R                  U5      n[        R                   " USS5      n[        R"                  " UR%                  5       UR'                  UR(                  5      5      n[        R                   " USS5      nUR'                  UR(                  5      R'                  UR*                  5      n[-        SX2R.                  UR*                  S9nUR                  SS5        UR0                  " U40 UD6  Xpl        [        U R                  5      S:  a  GM  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   r&   r   r'   )mergedwarningswarnlenr<   r:   oft_Rr+   r,   r-   r.   r/   r&   r0   r1   r2   r3   r4   tr5   r6   r)   r
   r(   r;   r   )r   r@   rA   r/   rB   rC   rD   rE   s           r"   unmergeHqqOFTLinear.unmergen   sl    ;;@Ad**+a/!%!5!5!9!9!;!)::++-h$--0B0B"Ch^UZUgUgh))+00@A6(**,		(..0IJA68>>255hooF )$L_L_hmhtht u  6&&v>>"/' d**+a//r$   c                <    U R                   U   R                  5       $ N)rM   
get_weight)r   adapters     r"   r1   HqqOFTLinear.get_delta_weight   s    ::g&1133r$   c                   U R                   " U/UQ70 UD6  UR                  SS 5      nU R                  (       a8  U R                  (       a  U R	                  5         U R
                  " U/UQ70 UD6nOU R                  (       a  U R
                  " U/UQ70 UD6nOU R                   H  nX`R                  R                  5       ;  a  M"  U R                  U   n[        R                  " 5       (       + nU(       a1  UR                  n	U R                  XR                  R                  5      nU" U5      nM     U R
                  " U/UQ70 UD6nW(       a  UR                  W	5      nU$ )Nr?   )_check_forward_argsr:   disable_adaptersrI   rO   r   active_adaptersrM   r+   r2   is_autocast_enabledr6   _cast_input_dtypeweightr5   )
r   xargsr    r?   resultr@   rM   requires_conversionexpected_dtypes
             r"   forwardHqqOFTLinear.forward   s   $$Q888"JJ=M$$;;LLN<T<V<<T<V<&*&:&:N%ZZ__->>  JJ~6E.3.G.G.I*I'*)* 221ll6H6HIaA '; __Q888F">2Mr$   c                *   > [         TU ]  5       nSU-   $ )Nzoft.)r   __repr__)r   repr!   s     r"   re   HqqOFTLinear.__repr__   s    '"$CC<r$   )r   r   r   )	   r   g        TFgiUMu?FF   )r   torch.nn.Moduler   strr   intr   rl   r   floatr   boolr   rn   r   rm   r   rn   r   rn   r   rl   returnNone)FN)r>   rn   r?   zOptional[list[str]]ro   rp   )ro   rp   )r]   torch.Tensorro   rq   )ro   rk   )__name__
__module____qualname____firstlineno__r   rF   rO   r1   rb   re   __static_attributes____classcell__)r!   s   @r"   r   r      s     "#$'!% %',,-	'	 	 		
  	 "	 	 	 	 	 !%	 '*	 	 	B*	<X	0:	4	8	  	 r$   r   c                    S n[        U [        5      (       a  U R                  5       nOU n[        5       (       a!  [        U[        5      (       a  [        XA40 UD6nU$ rR   )
isinstancer   r,   r   r
   r   )targetr   r    
new_moduletarget_base_layers        r"   dispatch_hqqr}      sU    J&.))"113"j):IFF!"3LVL
r$   )rz   rj   r   rk   )
__future__r   r-   rJ   typingr   r2   peft.import_utilsr   peft.tuners.tuners_utilsr   r   rA   r	   hqq.core.quantizer
   nnModuler   r}    r$   r"   <module>r      sK    #     . L  +M uxx M `r$   