
    +hM                     F	   S SK r S SKrS SKJr  S SKrS SKrS SKJr  SSKJ	r	J
r
  \	" 5       (       a  S SKrS SKJr  S SKJrJr  \R                   " SS5      R#                  5       S	;   =(       aA    \R$                  R'                  5       =(       a     \R$                  R)                  5       S    S
:  r\(       a  \
" 5       (       a  S SKJr  \" S5      rOSr\R2                  R4                  \R2                  R6                  \R2                  R8                  1r\R2                  R<                  \R2                  R>                  \R2                  R@                  \R2                  RB                  \R2                  RD                  \R2                  RF                  1r$\R2                  RJ                  \R2                  RL                  \R2                  RN                  \R2                  RP                  \R2                  RR                  1r*\R2                  RV                  \R2                  RX                  \R2                  RZ                  \R2                  R\                  \R2                  R^                  \R2                  R`                  \R2                  Rb                  \R2                  Rd                  \R2                  Rf                  1	r4\$\*-  \4-  r5\$\*-  \4-  r6\$\*-  r7S\Rp                  S\Rp                  S\9S\Rp                  4S jr:S r;S/ 4S jr</ 4S jr=Sr>Sr?S r@S rAS rBS,S jrCS,S jrDS,S jrES,S jrFS,S jrGS,S  jrHS,S! jrIS,S" jrJS,S# jrKS,S$ jrLS,S% jrM\R                  rN\R2                  R8                  \M\R2                  RD                  \C\R2                  RB                  \D\R2                  R@                  \E\R2                  R>                  \F\R2                  R<                  \G\R2                  RR                  \H\R2                  RP                  \I\R2                  RN                  \J\R2                  RL                  \K\R2                  RJ                  \L0rO\P" \OR                  5       5      rRS& rSS' rT " S( S)\R                  R                  5      rV " S* S+\R                  5      rXg)-    N)nullcontext   )is_accelerate_availableis_kernels_available)init_empty_weights)add_hook_to_moduleremove_hook_from_moduleDIFFUSERS_GGUF_CUDA_KERNELSfalse)1trueyes   )
get_kernelzIsotr0py/ggmlxqweightqweight_typereturnc                    U[         ;   a  XR                  -  $ U[        ;   a  [        R                  U   u  p4UR
                  S   UR
                  S   U-  U-  4n[        R                  " X/UQ76 nXR                  U R                  5      R                  -  nUR                  5       $ [        R                  " U5      n[        SU 35      e)Nr      z$Unsupported GGUF quantization type: )UNQUANTIZED_TYPESTDEQUANT_TYPESggufGGML_QUANT_SIZESshapeopsggml_dequantizetodtypeGGMLQuantizationTypeNotImplementedError	as_tensor)r   r   r   
block_size	type_sizer   weightys           Y/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/quantizers/gguf/utils.py_fused_mul_mat_ggufr)   O   s    ((99} }$ $ 5 5l C
q!7==#3y#@:#MN$$WCUC		!''"$$$ ;;= 00>!$H"WXX    c                 *   [        [        R                  U R                  R                  5      nU R
                  n0 n[        R                  " UR                  5      nUR                  5        H  nXTR                  ;   d  M  X%   X5'   M     U" S0 UD6nU$ )z
Creates a new hook based on the old hook. Use it only if you know what you are doing ! This method is a copy of:
https://github.com/huggingface/peft/blob/748f7968f3a31ec06a1c2b0328993319ad9a150a/src/peft/utils/other.py#L245 with
some changes
 )getattr
acceleratehooks	__class____name____dict__inspect	signature__init__keys
parameters)old_hookold_hook_clsold_hook_attrfiltered_old_hook_attrold_hook_init_signatureknew_hooks          r(   _create_accelerate_new_hookr?   m   s     :++X-?-?-H-HIL%%M%//0E0EF!222(5(8"% " 545HOr*    c           	      v   S n[        U R                  5       5      nU(       d  g U R                  5        H  u  pxX7-   S-   n	[        XX)U5        [	        U[
        R                  5      (       d  M:  U" X)5      (       d  MI  Xt;  d  MP  [        5       (       a  [        O[        n
U
" 5          [        UR                  UR                  UR                  S LUS9U R                  U'   S S S 5        [        U5      U R                  U   l        U R                  U   R#                  S5        M     U $ ! , (       d  f       NO= f)Nc                 F    US-   nX ;   =(       a    [        X   [        5      $ )Nr&   )
isinstanceGGUFParameter)
state_dictprefix
weight_keys      r(   _should_convert_to_gguf:_replace_with_gguf_linear.<locals>._should_convert_to_gguf   s$    h&
']Jz7M},]]r*   .)compute_dtypeF)listchildrennamed_children_replace_with_gguf_linearrC   nnLinearr   r   r   
GGUFLinearin_featuresout_featuresbias_modulestype
source_clsrequires_grad_)modelrK   rE   rF   modules_to_not_convertrH   has_childrennamemodulemodule_prefixctxs              r(   rO   rO   ~   s    ^ ()L,,.+!&Tjk vryy))'
BB2(?(A(A${C'1&&''KKt+"/	(t$  /36lENN4 +NN4 //6' /* L s   ':D**
D8	c           	      $   U R                  5        GHi  u  p#[        U[        5      (       Ga  X!;  Ga  UR                  R                  n[        USS 5      n[        5       (       a  [        O[        nU" 5          [        R                  " UR                  UR                  UR                  S LUS9nS S S 5        [        R                  " [        UR                  5      5      Wl        Ub  XWl        [!        US5      (       a-  UR"                  n[%        U5      n	['        U5        [)        Xy5        UR+                  U5        XpR,                  U'   [/        UR1                  5       5      n
U
(       d  GM^  [3        X15        GMl     U $ ! , (       d  f       N= f)NrU   )device_hf_hook)rN   rC   rR   r&   rb   r-   r   r   r   rP   rQ   rS   rT   rU   	Parameterdequantize_gguf_tensorhasattrrc   r?   r	   r   r   rV   rL   rM   #_dequantize_gguf_and_restore_linear)rZ   r[   r]   r^   rb   rU   r`   
new_moduler8   r>   r\   s              r(   rg   rg      s/   ,,.fj))d.P]]))F6640D(?(A(A${CYY&&''KKt+!	
  !#-CFMM-R SJ"& vz**!??6x@'/":8MM&!#-NN4 FOO-.</O= /@ L5 s   88F
F	      c                     U R                  [        R                  5      R                  [        R                  5      n U S S 2S4   U S S 2S4   S-  -  U S S 2S4   S-  -  U S S 2S4   S-  -  R                  S5      $ )Nr   r            r      )viewtorchuint8r   int32	unsqueeze)r   s    r(   	to_uint32ru      so    	u{{u{{+AadGa1gl"Qq!tW]2Qq!tW]BMMaPPr*   c                     U R                   S   n[        U5      U[        U5      -
  /-   n[        R                  " XSS9$ )Nr   dim)r   rL   sumrq   split)blocksargsn_maxdimss       r(   split_block_dimsr      s:    LLOE:T*++D;;v++r*   c                    U R                   S   nU R                  [        R                  5      n U R	                  USS45      n [        R
                  " X R                   S   S-  SS9u  p#n[        R                  " US-  US-  US-	  S	-  -  /S
S9n[        R                  " US-  US-	  US-	  S	-  -  /S
S9nUR	                  US45      UR	                  US45      4$ )Nr   r      rw   ?      rm   0   rl   )r   rp   rq   rr   reshaperz   cat)scalesn_blocksdmm_dscmins          r(   get_scale_minr      s    ||AH[[%F^^Xq!,-FFLL$4$9rBIA#	AHsTzqAvo>?R	HB
))QXqa1f_=>B
GCJJ!}%s{{Ha='ABBr*   c                     [        U S5      u  pEUR                  [        R                  5      R	                  U5      nUR                  [        R
                  5      nXE-  $ )Nrm   )r   rp   rq   float16r   int8)r{   r$   r%   r    r   r   s         r(   dequantize_blocks_Q8_0r      sF    FA&DA	u}}  'A	uzzA5Lr*   c                    U R                   S   n[        U SSS5      u  pVpxUR                  [        R                  5      R                  U5      nUR                  [        R                  5      R                  U5      n[        U5      nUR                  US45      [        R                  " SUR                  [        R                  S9R                  SS5      -	  nUR                  USSUS-  45      [        R                  " SS/UR                  [        R                  S9R                  SSSS5      -	  n	US-  R                  [        R                  5      nU	S-  R                  US45      n	XS-  -  nXX-  U-   $ )	Nr   rm   r   r       rb   r    r   r   )r   r   rp   rq   r   r   ru   r   arangerb   rs   tensorrr   )
r{   r$   r%   r    r   r   r   qhqsqls
             r(   dequantize_blocks_Q5_1r      s6   ||AH#FAq!4LA"	u}}  'A	u}}  'A	2B	XqM	"ell2ahhekk&Z&b&bcdfh&i	iB	Xr1jAo6	75<<	
Aqxxu{{<gaAq
B q&U[[	!B
t)		h^	,B	QwBFa<r*   c                    U R                   S   n[        U SS5      u  pVnUR                  [        R                  5      R                  U5      n[        U5      nUR                  US5      [        R                  " SUR                  [        R                  S9R                  SS5      -	  nUR                  USSUS-  5      [        R                  " SS/UR                  [        R                  S9R                  SSSS5      -	  nUS-  R                  [        R                  5      nUS-  R                  US5      nXS-  -  R                  [        R                  5      S	-
  nXW-  $ )
Nr   rm   r   r   r   r   r   r   rn   )r   r   rp   rq   r   r   ru   r   r   rb   rs   r   rr   r   )	r{   r$   r%   r    r   r   r   r   r   s	            r(   dequantize_blocks_Q5_0r      s   ||AH A.IA2	u}}  'A	2B	Ha	 ELLAHHEKK$X$`$`abdf$g	gB	Hb!Z1_	5	
Aqxxu{{:gaAq
B q&U[[	!B
t)		Xr	*B
a.		UZZ	(2	-B6Mr*   c                    U R                   S   n[        U SS5      u  pVnUR                  [        R                  5      R                  U5      nUR                  [        R                  5      R                  U5      nUR                  USSUS-  45      [        R                  " SS/UR                  [        R                  S9R                  SSSS5      -	  nUS-  R                  US5      nXW-  U-   $ )Nr   rm   r   r   r   r   r   )
r   r   rp   rq   r   r   r   r   rb   rr   )r{   r$   r%   r    r   r   r   r   s           r(   dequantize_blocks_Q4_1r     s    ||AH1-HA"	u}}  'A	u}}  'A	Xr1jAo6	75<<	
Aqxxu{{<gaAq
B t)		Xr	*BFa<r*   c                    U R                   S   n[        U S5      u  pVUR                  [        R                  5      R                  U5      nUR                  USSUS-  45      [        R                  " SS/UR                  [        R                  S9R                  S5      -	  nUS-  R                  US45      R                  [        R                  5      S	-
  nXV-  $ )
Nr   rm   r   r   r   r   r   r   rm   r   r   rl   )r   r   rp   rq   r   r   r   r   rb   rr   r   )r{   r$   r%   r    r   r   r   s          r(   dequantize_blocks_Q4_0r      s    ||AHVQ'EA	u}}  'A	Xr1jAo6	75<<	
Aqxxu{{<gl
B t)		h^	,	/	/

	;a	?B6Mr*   c                    U R                   S   n[        U [        S-  [        S-  [        S-  5      u  nnnnUR                  [        R
                  5      R                  U5      nUR                  [        R                  5      R                  U5      nX-  R                  U[        S-  S45      nUR                  USSS45      [        R                  " SS/UR                  [        R                  S9R                  S	5      -	  nUS
-  R                  USS45      nUR                  USSS45      [        R                  " / SQUR                  [        R                  S9R                  S5      -	  nUS-  R                  USS45      nXVS-  -  R                  [        R
                  5      S-
  n	U	R                  U[        S-  S45      n	X-  R                  U[        45      $ )Nr   rm   r   rn   r   r   @   r   r   r   r   r   rm   r      r   r   r   r   r   )r   r   QK_Krp   rq   r   r   r   r   r   rb   rr   )
r{   r$   r%   r    r   r   r   r   r   qs
             r(   dequantize_blocks_Q6_Kr   -  s   ||AH 	DAItrzB

	 [[$''.F	u}}  'A	
h
A67A	Xr1b)	*ellAq6!((Z_ZeZe.f.n.n/ 
B t)		hB/	0B	Xr1b)	*ell<PQPXPX`e`k`k.l.t.t/ 
B t)		hB/	0B	QwEJJ'",A			8TRZ,-AE??Hd+,,r*   c                 z   U R                   S   n[        U SS[        [        S-  5      u  pVpxn	UR	                  [
        R                  5      R                  U5      nUR	                  [
        R                  5      R                  U5      n[        U5      u  pXZ-  R                  USS45      nXk-  R                  USS45      nU	R                  USSS45      [
        R                  " SS/UR                  [
        R                  S9R                  S	5      -	  nUR                  USSS45      [
        R                  " SSUR                  [
        R                  S9R                  S
5      -	  nUS-  R                  USS45      nUS-  R                  USS45      nXS-  -  nX^-  U-
  R                  U[        45      $ )Nr   rm   rl   r   r   r   r   r   r   r   r   rl   r   r   )r   r   K_SCALE_SIZEr   rp   rq   r   r   r   r   r   rb   rr   r   )r{   r$   r%   r    r   r   dminr   r   r   r   r   dmr   r   s                  r(   dequantize_blocks_Q5_Kr   I  s   ||AH.vq!\4ST9UAV	u}}  'A99U]]#&&u-D&!EB	
(B*+A
(		Xr1-	.B	Xr1b)	*ellAq6!((Z_ZeZe.f.n.n/ 
B 
Xr1b)	*ell1aX]XcXc.d.l.l/ 
B t)		hB/	0B
t)		hB/	0B
AgAEBJ4 011r*   c                    U R                   S   n[        U SS[        5      u  pVpxUR                  [        R
                  5      R                  U5      nUR                  [        R
                  5      R                  U5      n[        U5      u  pXY-  R                  USS45      nXj-  R                  USS45      nUR                  USSS45      [        R                  " SS/UR                  [        R                  S9R                  S5      -	  nUS	-  R                  USS45      nXX-  U-
  R                  U[        45      $ )
Nr   rm   r   r   r   r   r   r   r   )r   r   r   rp   rq   r   r   r   r   r   rb   rr   r   )r{   r$   r%   r    r   r   r   r   r   r   r   r   s               r(   dequantize_blocks_Q4_Kr   c  s   ||AH*61aFAV	u}}  'A99U]]#&&u-D&!EB	
(B*+A
(		Xr1-	.B	Xr1b)	*ellAq6!((Z_ZeZe.f.n.n/ 
B t)		hB/	0BFRK  (D!122r*   c                    U R                   S   n[        U [        S-  [        S-  S5      u  pVpxUR                  [        R
                  5      R                  U5      nUS S 2S S24   US S 2SS 24   pU	R                  USS45      [        R                  " SS/UR                  [        R                  S9R                  S5      -	  n	U	R                  US45      n	U
R                  USS45      [        R                  " / S	QUR                  [        R                  S9R                  S
5      -	  n
U
R                  US45      n
U	S-  U
S-  S-  -  nUR                  [        R                  5      S-
  nX-  R                  USS45      nUR                  USSS45      [        R                  " / S	QUR                  [        R                  S9R                  S5      -	  nUR                  USSS5      [        R                  " SSUR                  [        R                  S9R                  S5      -	  nUR                  US[        S-  45      S-  nUR                  US[        S-  45      S-  S-  nUR                  [        R                  5      US-  R                  [        R                  5      -
  nX-  R                  U[        45      $ )Nr   rl   r   rj   r   r   )r   rm   r   rn   r   )r   r   r   r   r   r   r   r   r   rm   )r   r   r   rp   rq   r   r   r   r   rb   rr   r   r   )r{   r$   r%   r    r   hmaskr   r   r   lscaleshscalesdlr   r   r   s                  r(   dequantize_blocks_Q3_Kr   w  sh   ||AH+FDAItqy"MEv	u}}  'Aa!e}fQUmWooxA./5<<Aqxx_d_j_j3k3s3s4 G ooxn-GooxA./5<<QXXU[[4giG ooxn-Gn'D.Q!67FYYuzz"R'F
*		xQ/	0B	Xr1b)	*ell<PQPXPX`e`k`k.l.t.t/ 
B 
xQ	+u||AqY^YdYd/e/m/m0 
B 
Xr42:.	/!	3B
**hDBJ/
01
4	9B
ejjR1WLL44AFXt,--r*   c                    U R                   S   n[        U [        S-  [        S-  S5      u  pVpxUR                  [        R
                  5      R                  U5      nUR                  [        R
                  5      R                  U5      nXuS-  -  R                  U[        S-  S45      n	XS-	  -  R                  U[        S-  S45      n
[        R                  " / SQUR                  [        R                  S9R                  S	5      nUR                  US
SS45      U-	  S-  nUR                  U[        S-  S45      nX-  U
-
  nUR                  US
45      $ )Nr   rn   r   rm   r   r   r   r   r   r   r   r   )r   r   r   rp   rq   r   r   r   r   rb   rr   )r{   r$   r%   r    r   r   r   r   r   r   mlshifts               r(   dequantize_blocks_Q2_Kr     s*   ||AH*642:tqy!LF	u}}  'A99U]]#&&u-D |
	%	%xQ&?	@B
A+
	'	'42:q(A	BBLLahhekkJRRS_`E
**hAr*
+u
4	9B	Xtrz2.	/B	2B::xn%%r*   c                     U R                  [        R                  5      R                  [        R                  5      S-  R                  [        R
                  5      $ )Nrn   )rp   rq   int16r   rs   float32)r{   r$   r%   r    s       r(   dequantize_blocks_BF16r     s8    KK$''4:@@OOr*   c                 (    / U S S QU S   U-  U-  P7$ )Nr   r,   )r   r%   r$   s      r(   _quant_shape_from_byte_shaper     s&    =U3BZ=ri/*<==r*   c                 p   [        U S5      (       d  U $ U R                  n[        U   n[        U   u  p4U R	                  [
        R                  5      n [        U R                  XC5      nU R                  5       U-  nU R                  Xd45      nU" XsU5      nUR                  U5      nUR                  5       $ )N
quant_type)rf   r   dequantize_functionsr   rp   rq   rr   r   r   numelr   r#   )	r   r   
dequant_fnr$   r%   r   r   r{   dequants	            r(   re   re     s    6<((""J%j1J,Z8J[[%F(yME||~*H^^X12FY7Gooe$Gr*   c                   V   ^  \ rS rSrSS jrS r\S 5       r\SU 4S jj5       r	Sr
U =r$ )	rD   i  c                     Ub  UO[         R                  " S5      n[         R                  R                  XU5      nX4l        [
        U   u  pV[        UR                  Xe5      Ul        U$ Nr   )	rq   emptyTensor_make_subclassr   r   r   r   quant_shape)clsdatarequires_gradr   selfr$   r%   s          r(   __new__GGUFParameter.__new__  sV    'tU[[^||**3mD$ 0 <
7

IZr*   c                 r    [         R                  R                  [         R                  X R                  5      $ N)rq   r   r   r   )r   s    r(   r#   GGUFParameter.as_tensor  s#    ||**5<<?Q?QRRr*   c                     U  Hd  n[        U[        5      (       a)  [        US   [        5      (       a  US   R                  s  $ [        U[        5      (       d  MX  UR                  s  $    g r   )rC   rL   rD   r   )r|   args     r(   _extract_quant_type!GGUFParameter._extract_quant_type  sW    
 C#t$$CFM)J)J1v(((#}--~~%	 
 r*   c                   > Uc  0 n[         T	U ]  XX45      n[        U[        R                  5      (       a  U R                  U5      nU " XVS9$ [        U5      [        [        4;   aY  U R                  U5      nU Vs/ s H*  n[        U[        R                  5      (       a  U " XvS9OUPM,     nn[        U5      " U5      $ U$ s  snf )N)r   )	super__torch_function__rC   rq   r   r   rW   rL   tuple)
r   functypesr|   kwargsresultr   r   wrappedr0   s
            r(   r    GGUFParameter.__torch_function__  s    >F+DFfell++006Jv55&\dE]*006Jdjkdj_`
1ell8S8Ss14YZZdjGk<((M ls   ;1C)r   r   )FN)r,   N)r1   
__module____qualname____firstlineno__r   r#   staticmethodr   classmethodr   __static_attributes____classcell__r0   s   @r(   rD   rD     s6    S 	 	  r*   rD   c                      ^  \ rS rSr   S S	U 4S jjjrS\R                  4S jrS\R                  4S jrS\R                  4S jr	Sr
U =r$ )
rR   i  c                 >   > [         TU ]  XX55        X@l        XPl        g r   )r   r5   rK   rb   )r   rS   rT   rU   rK   rb   r0   s         r(   r5   GGUFLinear.__init__  s     	DA*r*   inputsc                     [         b=  U R                  R                  (       a"  UR                  (       a  U R                  U5      $ U R	                  U5      $ r   )r   r&   is_cudaforward_cudaforward_native)r   r   s     r(   forwardGGUFLinear.forward  s;    ?t{{22v~~$$V,,""6**r*   c                 "   [        U R                  5      nUR                  U R                  5      nU R                  b%  U R                  R                  U R                  5      OS n[
        R                  R                  R                  XU5      nU$ r   )	re   r&   r   rK   rU   rq   rP   
functionallinear)r   r   r&   rU   outputs        r(   r   GGUFLinear.forward_native  sh    '44--.37993Htyy||D../d$$++FDAr*   c                     U R                   R                  n[        UR                  U R                  5      U R                   U5      nU R
                  b'  X0R
                  R                  U R                  5      -  nU$ r   )r&   r   r)   r   rK   rU   )r   r   r   r   s       r(   r   GGUFLinear.forward_cuda!  s]    [[++
$VYYt/A/A%BDKKQ[\99 iill4#5#566Fr*   )rK   rb   )FNN)r   N)r1   r   r   r   r5   rq   r   r   r   r   r   r   r   s   @r(   rR   rR     sW    
 
 

 
+ell +
U\\ 5<<  r*   rR   r   )Yr3   os
contextlibr   r   rq   torch.nnrP   utilsr   r   r.   r   accelerate.hooksr   r	   getenvlowercudais_availableget_device_capabilitycan_use_cuda_kernelskernelsr   r   r!   F32F16BF16r   Q4_0Q4_1Q5_0Q5_1Q8_0Q8_1STANDARD_QUANT_TYPESQ2_KQ3_KQ4_KQ5_KQ6_KKQUANT_TYPESIQ1_MIQ1_SIQ2_XXSIQ2_XSIQ2_SIQ3_XXSIQ3_SIQ4_XSIQ4_NLIMATRIX_QUANT_TYPESr   MMVQ_QUANT_TYPESMMQ_QUANT_TYPESr   intr)   r?   rO   rg   r   r   ru   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rL   r6   SUPPORTED_GGUF_QUANT_TYPESr   re   rd   rD   rQ   rR   r,   r*   r(   <module>r,     s    	 "    B -L II+W5;;=AUU 3

!3

((*1-2 
 022"
_
%C
C..22D4M4M4Q4QSWSlSlSqSqr """"""""""""  	"""""""""" 	####%%$$##%%##$$$$
  %|36II',69LL &55<< %,, c V[VbVb <" HJbd B GI !R Q
,
C&&
-8243(.B&(P (( ""$:""$:""$:""$:""$:""$:""$:""$:""$:""$:""$:  ""6";";"=> >**EHH&& *Z r*   