hU &%SSKJrJrJrJr SSKrSSKJr SSKJ r SSK J r J r J r JrJrJrJrJrJrJrJr SSKJrJr SSKJrJrJrJrJrJrJ r /S Qr!S r"Sq#\\$\%S 'S r&S r'0r(\\\$\\\$\\\$\)444\%S'"SS\5r*"SS5r+S3Sjr,S\RZS\\)4Sjr.Sr/Sr0\Rb4Sjr2Sr3\Rb4Sjr4\Rb4Sjr5SS\Rl\ RnSS4S jr8\Rl4S!jr9S"r:SS\ Rn4S#jr;SS\ Rn4S$jrSS\Rb\ R~S4S'jr@SS\Rb\ R~4S(jrA\Rb\RbS4S)\RZS*\RS+\RS,\\CS\RZ4 S-jjrDS.rES/\\)S04S1\S\\)S044S2jrFg)5)DictListOptionalTupleN)TorchDispatchMode)int_scaled_matmul) MappingTypeZeroPointDomain)_choose_qparams_affine_dont_preserve_zero_choose_qparams_affine_tinygemm _dequantize_affine_no_zero_point_dequantize_affine_tinygemm_quantize_affine_no_zero_point_quantize_affine_tinygemmchoose_qparams_affinedequantize_affinequantize_affine)check_cpu_versioncheck_xpu_version) GranularityPerAxisPerBlockPerGroupPerRow PerTensorPerToken) compute_error%_quantize_activation_per_token_absmax$_quant_int8_dynamic_per_token_linear dynamically_quantize_per_channeldequantize_per_tensordequantize_per_channelget_groupwise_affine_qparamspack_tinygemm_scales_and_zeros unpack_tinygemm_scales_and_zeros-groupwise_affine_quantize_tensor_from_qparams/groupwise_affine_dequantize_tensor_from_qparams groupwise_affine_quantize_tensor"groupwise_affine_dequantize_tensorper_token_dynamic_quantget_group_qparams_symmetric"recommended_inductor_config_setterc[RRU5n[RRX- 5nS[R"X#- 5-$)N)torchlinalgnormlog10)xyPsPns T/home/james-whalen/.local/lib/python3.13/site-packages/torchao/quantization/utils.pyrr?sB   1 B   15 !B  BG$ $$_cur_fqnc^U4SjnU$)Nc >TqgN)r:)moduleinputfqns r8 forward_hook'_get_logging_hook..forward_hookKsr9)r@rAs` r8_get_logging_hookrDJs r9cjUR5HupUR[U55 M! gr=) named_modulesregister_forward_pre_hookrD)modelnamemods r8_apply_logging_hookrKRs+((*  %%&7&=>+r9_fqn_to_op_to_shape_to_countc\rSrSrSSjrSrg)LoggingTensorMode]rCNcUc0nU"U0UD6nURSUR3nSnUHHn[U[R5(dM$U[ [ UR55S-- nMJ US:waUSSn[[;a 0[['U[[;a0[[U'U[[U;aS[[UU'[[UU==S- ss'U$)N.z, rr) __module____name__ isinstancer0Tensorstrlistshaper:rL) selffunctypesargskwargsrsop_name shape_strargs r8__torch_dispatch__$LoggingTensorMode.__torch_dispatch__^s >F 4 "6 "//*!DMM?; C#u||,,Scii1D88  ?!#2I 7 757 ( 2 6x@ @>@ ( 27 ; 8B7K KIJ ( 27 ;I F$X.w7 BaGB r9)rCN)rUrT __qualname____firstlineno__rd__static_attributes__rCr9r8rNrN]sr9rNc2\rSrSrSrSrSrSrSrSr g) _MultiInputvc$[U5Ulgr=)rYvalues)r[inputss r8__init___MultiInput.__init__ws 6l r9c<URRU5 U$r=)rmappend)r[r?s r8 add_input_MultiInput.add_inputzs 5! r9c2[URU5$r=)rjrm)r[slices r8 __getitem___MultiInput.__getitem__~s4;;u-..r9cURVs/sH4n[U[R5(aUR 5OUPM6 snUlgs snfr=)rmrVr0rWcudar[vals r8rz_MultiInput.cudasALPKK LWS*S%,,77CHHJS @K   ;AcURVs/sH4n[U[R5(aUR 5OUPM6 snUlgs snfr=)rmrVr0rWxpur{s r8r_MultiInput.xpusAKO;; KVCC66CGGIC ?;   r~)rmN) rUrTrfrgrorsrwrzrrhrCr9r8rjrjvs#/  r9rjc Ub/URU:wa[SUSUSURS35eUb8UR5U:wa#[SUSUSUR5S35egg)NzExpected Tensor argument z to have dtype z , but got z instead.z to have size )dtype ValueErrorsize) tensor_argarg_namerrs r8_guard_dtype_sizers Z--6'zzR\RbRbQccl m   JOO-5'zvZPZP_P_PaObbk l  6r9r4returnc/n[[UR5S- 5HnURS5 M URURS5 U$)Nr)rangelenrZrr)r4 block_size_s r8_get_per_token_block_sizersMJ 3qwwi) scale_dtype) r SYMMETRICrYrZrrr0int8rfloat16float32rr) t mapping_typerireps quant_min quant_maxrscale zero_point quantizeds r8rrs((LaggJ 3z?Q& ' ( JJE CII#$77emm#;%--K-    E  u%II  r9cH[U5upV[XVXU5nUbXs-nU$)zY like F.linear, but with int8 dynamic quantization of activation, and a quantized weight )r_quant_int8_per_token_matmul)r4 w_vals_int8_tw_scalesbias out_dtype x_vals_int8x_scalesmm_outs r8r r s7B!DK )} F  Mr9cNUR[R:XdSURS35eUR[R:XdSURS35eUR[R[R4;dSUR35eUR SUR S5n[XRUR SS55nXc-R "/UR SSQUR SP76nURU5nU$)a Quantized matmul of int8 operands that accumulates to int32 and returns output_dtype. For now, this is written for approximate numerical Assumes that activation and weight quantization are symmetric, i.e. act_zp and w_zp is 0. Assumes that weight quantization is per-channel. see https://github.com/google/gemmlowp/blob/master/doc/quantization.md for an overview of quantized matmul compute in scalar form, assuming output_dtype is fp32 and zw == 0: Y_i_j_fp32 = sx * sw dot(X_i, W_j) zx dtype z not yet supportedzw dtype z?x_scales needs to be a torch.float32 or torch.bfloat16 but got rrN) rr0rfloatbfloat16reshaperZrto)rrrr output_dtypetmp y_dot_scaledr5s r8rrs6.    * ;$$%%78 *   %** , =&&''9: , >>    J(..IYZ  $   b+"3"3B"7 8C$S9I9I"a9PQL  ))    3B  !-!3!3B!7 A \A Hr9c >UR5S:XdS5e[R"[R5RnSUR S4n[R n[Rn[UUUUUUUUS9up[XXX1U5n XU 4$)zn assumes symmetric quantization assumes axis == 0 assumes dense memory format TODO(future): relax ^ as needed only support 2d Tensorsr) target_dtyperrrzero_point_dtype) dimr0finforrrZint64r rrr) r4rrrrrrrrrquants r8r!r!s 557a<222< ++emm $ ( (CQWWQZJ{{((L- ! ) E  u,9 E  ##r9c URnURnUR5S:XdSUR535e[XXXSS9nU$)Nrz scale size: r)rZrnumelr)int_reprrrrr input_dtype dequantizeds r8r"r"3sSJ..K ;;=A =ekkm_== #eK r9c UR5S:XdS5eUR5nURSS4nURn[ XXXSS9nUR5nU$)Nrrrrr)rrrZrr)rscales zero_pointsrrrrs r8r#r#>si <<>Q 9 99  zz|H..#Q'J..K#f;K--/K r9Fc X RS:aURSnUS:deURSU-S:XdeUR5S:XdeUS::d SU35e[Rn[R nSU4n Sn SU-S- n UcSnUn U[ R:waUO[R n U[ R:XaU(d[UUU UU U UU U S9 upOBU[ R:XaU(d[UUU UU U UU U S9 upO[UUU UU U UU U S9 upURUS 9RURSS5URU S 9RURSS54$) Nrrrrz-only n_bit smaller than 8 is supported, got: gư>)rrr)rZrr ASYMMETRICr0int32r INTFLOATr r rrr)wn_bit groupsizerzero_point_domain preserve_zerorrrrrrrrrrs r8r$r$Os772;GGBK q== 772; "a '' ' 557a<< A:NFugNN:))L;;LYJI51 I {K"o&9&99u{{O111-;      #-  z o11 1-E      #-  z2      #-   88%8 ( (R 8*--;H; gaggaj" r9c[USX!R5S9 [USUS9 UR5n[R"UR S5UR S5/U5R SS5R5$)Nr)rrzerosrrrS)rrrr0cat unsqueeze transpose contiguous)rrrrs r8r%r%sufhe**,GeWE2 **,C   $#     2r   r9c|URSS:Xde[R"URSS5SS5$)NrrrrSr)rZr0splitr)scales_and_zeross r8r&r&s<  ! !" % ** * ;;'11"b91b AAr9c US:deX@RS:a"URSS:XaURSnURSU-S:XdeUR5S:XdeSU4n[RnSnSU-S- n U[R :Xa[ n OHU[R:Xa[n O-[[R:Xa[n O[SU35eU "UUUUUUU 5n URSS:a[U R5(dS[U R5(d9U SS2SSS24S-U SS2SSS24-R[R 5n [U R5(a9U SS2SSS24S-U SS2SSS24-R[R 5n U $)Nrrrrz Unrecognized zero point domain: r)rZrr0rr rrrrNONErrrdevicerruint8) rrrrrrrrrr_quantize_affineint_datas r8r'r's q==772;6<<#3q#8GGBK 772; "a '' ' 557a<<YJ;;LI51 IO///* o33 34 O00 09;Q  %++%);a)?(//**{{5;;'AI $;++ ^^A q 1A 5 6++??  "(//22(B!G  (B1H 'B!G  )B1H ==$$b)9Q)>MM"% == y (A -- -YJ++KI51 IO///. o33 38=  \\  r9c V[UUUUUUS9upg[XXqX$S9n[XgU5n X4$)N)rr)r)r$r'r%) rrrrrrrrrrs r8r)r)sM1   +# MF= 5H6fUK  %%r9c6[U5upE[XXRU5$r=)r&r()rrrrrrs r8r*r*'s% 55EFMF :%  r9c \X RS:aURSnUS:deURSU-S:XdeUR5S:XdeUS::d SU35eSU4nUc*[R"UR5R n0nSUS'[ SS5HnSUS- -*SUS- -S- 4Xx'M Xqup[UUU[RU U UUUS 9 upU RURSS5U RURSS54$) Nrrrrrzunsupported n_bit: )rr )rrrrrr) rZrr0rrrrrrr) rrr precisionrrrrangesrrrrrs r8r,r,4sC772;GGBK q== 772; "a '' ' 557a<< A:4,UG44:YJ {kk!''"&& FF1I 1a[QU|_aAElQ&67 !=I- ZZ " E ==R (**<*JM1Q6 0 =Z[eZfg 6( K&(!3 4 4s;'!+, B/AAA K * *2!7!771< &{2&77WXcXnXnWo p <s;'!+, 0F0F/HHH 0 > ??r9)NN)rr)Gtypingrrrrr0torch.utils._python_dispatchrtorchao.kernelr%torchao.quantization.quant_primitivesr r r r r rrrrrr torchao.utilsrrrrrrrrrr__all__rr:rX__annotations__rDrKrLintrNrjrrWrrr rrr!r"r#rrr$r%r&r'r(r)r*rr,rrrr+r-r rCr9r8rs} /. :      *%(3-?d SM4 tHSM3,>'??@@ )2  , $s)B2 9 x$@BGEJMM&  ..%++ CL9>"B @U@U)` %++ 6v  ..%++ &4   mm&& $MR mm&& !8 %}}$)MM + <<++kk+ % +  \\ +\/$@sCx@/:@ 38_@r9