hSSKrSSKJr SSKJr SSKJrJrJrJ r SSK r SSK J r SSK Jr SSKJr SSKJr SS KJrJr SS KJr SS KJrJrJrJr SS KJr SS KJ r SSK!J"r" SSK#J$r$ \ RLROSS5r(\""\(5r)S/r*Sr+Sr,Sr-Sr.Sr/\)S\ R`S\1S\ R`S\ R`S\ R`S\ \ R`S\\2S\ R`4S j5r3S!r4S\54S"jr6S#\4S$jr7S%r8S&r9S'r:S(r;S)rS,r?S-r@S.rAS/rBS0rCS1rDS2rES3rFS4rGS5\ RR 4S6jrI\"S7S855rJS9\S\4S:jrKg! \RJr$GN=f);N) dataclass)partial)AnyCallableListOptional) out_dtype)quantized_decomposed_lib) GraphModule) InternalMatch)ReplacedPatternsreplace_pattern_with_filters) WrapperModule)"_get_aten_graph_module_for_pattern,_replace_literals_with_existing_placeholders'_replace_literals_with_new_placeholders"remove_tensor_overload_for_qdq_ops) MappingType)_get_per_token_block_size)_register_custom_op)$_disable_aten_to_metadata_assertionstorchaoFRAGMENT reference_representation_rewritec[RRRXX#U[R5n[RRRUUUUU [R5n[RR R RUUU 5n[RRRUXX[R5nU$N) torchopsquantized_decomposeddequantize_per_tensorint8atenlineardefaultquantize_per_tensor)x_i8x_scale x_zero_point x_quant_min x_quant_max weight_i8 weight_scaleweight_zero_pointweight_quant_minweight_quant_max bias_fp32 out_scaleout_zero_point out_quant_min out_quant_maxx_fp32 weight_fp32out_fp32out_i8s t/home/james-whalen/.local/lib/python3.13/site-packages/torchao/quantization/pt2e/reference_representation_rewrite.py_qdq_quantized_linearr:,s"YY + + A A |+uzzF))00FF Kyy~~$$,,V[)LH YY + + ? ?)]5::F Mc[RRRXU5n[RRRXXU 5nUR [R 5nUR [R 5n[ [RRRR[RX- UU- S5nX-n[ [RRRR[RU U5nUU-n[ [RRRR[RUX-U - 5U -n[RRRUX5R [R5nU$r)rrr"clamptoint16r r#r$int32divTensormulr!)r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4x_i16 weight_i16acc_i32 bias_scalebias_i32r8s r9_reference_quantized_linearrIOsJ( 99>>  ; ?D $$YBRSI GGEKK Eekk*J %%  && G'J++22EKKJWH G  IINN   % % KK   "Y .     YY^^ ! !'= H K KEJJ WF Mr;c D[RRRXX#[R5up[RRR X XU[R5n [RRR XXU[R5n[RRR UUUUU[R5n [RRRRX U 5nU$r) rrrchoose_qparamsr!r%r r"r#r$)r5r)r*x_epsr+r,r-r.r/r0r'r(r&r6r7s r9_qdq_dynamic_quantized_linearrMs"II::II[G 99 ) ) = =K DYY + + A A |+uzzF))00FF Kyy~~$$,,V)LH Or;c t[RRRXX#[R5upX - n[R "U5nUR [RS9n X-n [R"XU5n U R [RS9n [RRRXGU5nU R [R5nUR [R5n[[RRRR[RX- X- S5nX-n[[RRRR[RU U5nUU-nUX--nU$Ndtype)rrrrKr!roundr>r@r=r"r?r r#r$rArB)r5r)r*rLr+r,r-r.r/r0r'r(x_i32r&rDrErFrGrHr7s r9#_reference_dynamic_quantized_linearrTsL"II::II[G  F[[ F IIEKKI (E  E KKK 8E 88%**8 %D $$YBRSI GGEKK Eekk*J %%  & G'J++22EKKJWH G'01H Or;c [Rn[U5nSn Sn [RR R UURU[RU U U[R[R5 up[RR RUUU U [RU U 5n [RR RU UU U [RU U [R5nUS:dS5eURSU-S:XdS5eUR5S:XdS5eSU4n[RR RUUUU[RS S 5n[RRR R#XU5nU$) NrzGroup size must be positivez&Weight must be divisible by group_sizezWeight must be 2D tensor)r ASYMMETRICrrrrchoose_qparams_affinenamer!float32r@quantize_affinedequantize_affineshapedimr"r#r$)r5rL weight_i4r,r-r0 group_sizex_mapping_typeper_token_block_sizer)r*r'r(r& block_sizer6r7s r9,_qdq_dynamic_quantized_linear_4bit_groupwiseris!++N4V<KK!II--CC     G 99   , ,  DYY   0 0    F >888> ??1  *a /0 / ==?a ;!;; ZJ))##55   Kyy~~$$,,V)LH Or;r5rLrdr,r-r0rereturnc USn[Rn[U5nSn Sn [RR R UURU[RU U U[R[R5 up[RR RUUU U [RU U 5n URupX-nUR[R5R[R5nU RnU RSnU RSU5n U RSnU RUUU5nURUUU5nUR[R5nUR[R5n[R "UU[RUR"S9n[%U5nUUS'UR&S:XaUR)S5n[+U5GH nUSS2USS24nUSS2USS24nUR-SS9nUSS2U4n[/[RR0R2R4[RUUS5nUUR[R5URSS5--nUR[R5RSS5U RSS5-URSS5-n UU - nGM U RSS5n!UU!-n"UbU"U-n"U"RU5$) av Reference implementation for dynamically quantized linear 4-bit groupwise operation. This implementation emulates actual numerics of on-device integer compute. Args: x_fp32: Input activation tensor in fp32 x_eps: Epsilon for quantization parameter computation weight_i4: 4-bit quantized weight (stored as int8 with values in [-8, 7]) weight_scale: Groupwise scales for weight dequantization weight_zero_point: Groupwise zero points for weight (unused for symmetric) bias_fp32: Optional bias tensor in fp32 group_size: Size of each group for groupwise quantization Returns: Output tensor in fp32 rXrVrWr)rQdeviceN)rc)rr\rrrrr]r^r!r_r@r`rbr>bfloat16viewzerosrmlistndim unsqueezerangesumr r"r#r$)#r5rLrdr,r-r0rerfrgr)r*r'r(r& out_features in_features num_groups x_orig_shapek_dim batch_size x_i8_groupedweight_i4_grouped x_i32_groupedweight_i32_groupedacc_fp32 out_shape group_idxx_group weight_groupweight_group_col_sumweight_scale_group group_accweights_col_sum_adjustedx_scale_multiplierr7s# r9_reference_dqlinear_int4rs6AJ ++N4V<KK!II--CC     G 99   , ,  D!*L*J ??5>>255emmDL::L JJrNE 99R DAJ99ZZ@L"|ZL!OOEKK0M*--ekk:{{L fmmH\"I IbMA#--a0 :& 9a0)!Y/: +//B/7*!Y,7 IINN ! ! ) ) KK     ill5==9A& ' %%a, - ! 66C'D!b!,,,Hi' == ##r;c d[RRRUUUUUUSU45$)z Reference implementation for dynamically quantized linear 4-bit groupwise operation. This function now delegates to the custom op implementation. rX)rrrreference_dqlinear_int4)r5rLrdr,r-r0res r92_reference_dynamic_quantized_linear_4bit_groupwisers: 99   4 4  J r;cSnSnURR5GH[n[U[RR 5(a~UR S:XanUR[RRRR:Xa2URn[U5S:aUSS:H=(a USS:Hn[U[RR 5(dMUR S:XdMUR[RRRR:XdGM'URn[U5S:dGMEUS[R:HnGM^ U=(a U$)NF call_functionr[rZ) nodes_mapvalues isinstancerfxNodeoptargetrrrar$argslenr`r!)matchoriginal_graph pattern_graphweight_is_int4act_quant_is_int8noders r96_filter_fn_for_dynamic_quantized_linear_4bit_groupwisers N&&( tUXX]] + +?* uyy00BBJJJ99D4yA~!%aB!?47a< tUXX]] + +?* uyy00@@HHH99D4yA~$(Guzz$9!!)"  ///r;replacement_patterncSSKJnJn SnSnURR 5HEupVUR [ RRRR:XdMCUn O [UR5S:aURSnUbFUbBURR5Ul U"USUR5/URS'ggg)z Port metadata for dynamically quantized linear 4-bit groupwise operation. It custom_op node's metadata with corresponding linear node's metadata. r) NodeSourceNodeSourceActionNrl#ReplaceInt4DynamicQuantWithCustomOp from_node)torch.fx.tracebackrrritemsrrrr"r#r$r replacementsmetacopyREPLACE)rrr linear_nodeint4_custom_op_node_g_ns r9:_port_metadata_for_dynamic_quantized_linear_4bit_groupwisers@K%//557 ::..66 6K 8  + +,q01>>rB#6#B#.#3#3#8#8#: 5 (( 1   -$Cr;c SS/nSS/nSS/nSnSS/nSn[RRRXX#U[R5n[RRRUUUUU [R5n[RR R RUUU UUUUUU5 n[RRRUXX[R5nU$NrXrF) rrrr r!r" convolutionr$r%)r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4stridepaddingdilation transposedoutput_paddinggroupsr5r6r7r8s r9_qdq_quantized_conv2drs"VF!fG1vHJVN F YY + + A A |+uzzF))00FF Kyy~~))11 HYY + + ? ?)]5::F Mr;c SS/nSS/nSS/nSnSS/nSn[RRRXU5n[RRRXXU 5nUR [R 5nUR [R 5n[ [RRRR[RUU- UU- SUUUUUU5 nX-n[ [RRRR[RU U5nURS5nURS5nUU-n[ [RRRR[RUX-U - 5U -n[RRRUX5R [R5nU$)NrXrFrl)rrr"r=r>r?r rr$r@rArBrsrCr!)r&r'r(r)r*r+r,r-r.r/r0r1r2r3r4rrrrrrrDrErFrGrHr8s r9_reference_quantized_conv2drs"VF!fG1vHJVN F 99>>  ; ?D $$YBRSI GGEKK Eekk*J ""**  &&  G'J++22EKKJWH!!"%H!!"%H G  IINN   % % KK   "Y .     YY^^ ! !'= H K KEJJ WF Mr;c [RRRXX(U [R5n [RRRX4XXU [R5n X-n [RR R U 5n [RRRXXxU [R5n U $r)rrrr r!r"relur%r&r'r(y_i8y_scale y_zero_pointr1r2 quant_min quant_maxr5y_fp32r7r8s r9_qdq_quantized_add_relurksYY + + A A | 5::FYY + + A A | 5::FHyy~~""8,H YY + + ? ?^ 5::F Mr;c NUR[R5n UR[R5n [[RR R R[RX- X- 5n [[RR R R[RX- XF- 5n X-U-n [RR RXU 5R[R5n U $)z See comments for `_reference_quantized_add` for more information on how to derive the formula for out_i8 based on x_i8 and y_i8 ) r>rr@r rr"rCrBr=r!r&r'r(rrrr1r2rrrSy_i32out_i32r8s r9_reference_quantized_add_relurs GGEKK E GGEKK E  !!      E  !!      E mn,G YY^^ ! !'9 E H H TF Mr;c j[RRRXX(U [R5n [RRRX4XXU [R5n X-n [RRR XXxU [R5n U $r)rrrr r!r%rs r9_qdq_quantized_addrsYY + + A A | 5::FYY + + A A | 5::FH YY + + ? ?^ 5::F Mr;c UR[R5n UR[R5n [R"X- X- -5R[R5n [R"XF- X- -5R[R5n X-U-n SnSn [RR R XU 5R[R5n U $)a # How to Derive the formula for out_i8 based on x_i8 and y_i8 # (since quantized add takes x_i8, y_i8 and their quantization parameters, and produce an out_i8) # out_i8 is quantized output, we can write down the formula for it first: out_i8 = out_f32 / out_scale + out_zero_point (1) # then out_fp32 is computed from x_f32 + y_f32, and the x_fp32 and y_fp32 are the dequantized x_i8 and y_i8 out_f32 = x_f32 + y_f32 (2) x_fp32 = (x_i8 - x_zero_point) * x_scale (3) y_fp32 = (y_i8 - y_zero_point) * y_scale (4) # applying the above fomula to the out_i8 equation we can get the following: out_i8 = out_fp32 / out_scale + out_zero_point # (1) = (x_f32 + y_f32) / out_scale + out_zero_point # applying (2) to substitute out_fp32 with x_fp32 + y_fp32 = ((x_i8 - x_zero_point) * x_scale + (y_i8 - y_zero_point) * y_scale) / out_scale + out_zero_point # apply (3) and (4) rVrW)r>rr@rRrr"r=r!rs r9_reference_quantized_addrs: GGEKK E GGEKK E KK,1EF G J J5;; WE KK,1EF G J J5;; WEmn,GII YY^^ ! !'i @ C CEJJ OF Mr;c rSn Sn Sn Sn Sn [RRRXX#U[R5n[RR R RXXX5unn[RRRXXgU[R5nU$r) rrrr r!r"max_pool2d_with_indicesr$r%)r&r'r(r)r*r1r2r3r4 kernel_sizerrr ceil_moder5r7rr8s r9_qdq_quantized_max_pool2drsK FGHI YY + + A A |+uzzF))..88@@VhKHaYY + + ? ?^M5::F Mr;c |Sn Sn Sn Sn Sn [R"XU5nUR[R5n[RR R RX- XXU 5unnXU- -U-n[R"UXx5nUR[R5nU$r) rr=r>r@rr"rr$r!)r&r'r(r)r*r1r2r3r4rrrrrrSrrr7r8s r9_reference_quantized_max_pool2drsK FGHI ;;t+ 6D GGEKK E77?? k7iJGQI-.?H{{8]BH [[ $F Mr;cz[RRRXX#U[R5nU$r)rrrr%r!r5scale zero_pointrrxs r9_quantize_per_tensor_int8r$s0 &&::zi A Hr;cX- n[R"U5nUR[RS9nXR-n[R"XSU5nUR[R S9nU$rO)rrRr>r@r=r!rs r9#_reference_quantize_per_tensor_int8r+s^ A  AA 5;;A A A),A 5::A Hr;cz[RRRXX#U[R5nU$r)rrrr r!)r&rrrrr5s r9_dequantize_per_tensor_int8r;s0 YY + + A A ZIuzzF Mr;c[RRRXU5nUR [R 5U- U-R [R S9$rO)rrr"r=r>r_)r&rrrrs r9%_reference_dequantize_per_tensor_int8rBsO 99>>   ;DWWU]] #j 0E 9 = =EMM = RRr;c z[RRRXX#XE[R5nU$r)rrrquantize_per_channelr!)r5scales zero_pointsch_axisrrr8s r9_quantize_per_channel_int8rOs2YY + + @ @ iEJJF Mr;c\[R"XS5n[RRR [R "X- 5R [R5U-XE5n[R"XcS5nUR [R5$Nrl) r transposerr"r=rRr>r@r!)r5rrrrrrs r9$_reference_quantize_per_channel_int8rXsu__Vb 1Fiinn"" FO$'' 4{BIGoog3G ::ejj !!r;c z[RRRXX#XE[R5nU$r)rrrdequantize_per_channelr!)r&rrrrrr7s r9_dequantize_per_channel_int8rcs2yy--DD kI%**H Or;c<[RRRXU5n[R"XS5nUR [R 5nXb- R [R5U-n[R"XsS5nU$r)rrr"r=rr>r@float)r&rrrrrrSr7s r9&_reference_dequantize_per_channel_int8rmsq 99>>   ;D ??4" -D GGEKK E#'' 4v=Hx"5H Or;gmc"[US/SSSS.S9$)Nrlrr)rXrVrW)exclude_literalsliteral_to_ph_idx)r)rs r9'_replace_ph_qdq_per_channel_replacementr{s 7 bT!!5L r;c*\rSrSr%Sr\\S4\S'\\S'\\S'Sr \ \\ /\ 4\S'Sr \ \\ /\ 4\S 'Sr \ \\S \R R"\R R"/\4\S 'S r\\S 'Sr\ \S/S4\S'Srg) _RewriteInfoizData needed for rewrite, this includes example inputs, pattern and replacement functions and post transformation functions for the exported pattern and replacement GraphModule .example_inputspattern replacementNpattern_post_transreplacement_post_transr filter_fnFignore_literalsr port_metadata_fn)__name__ __module__ __qualname____firstlineno____doc__tupler__annotations__rrrr rrrqrrGraphboolrr__static_attributes__rr;r9rrs #s(O# IM;-*D!EFMMQHX{m[.H%IJQ x XGM NO "OT!GKhx);(5[.[.5[U[[@5[[B55[U[[D5[[F55[U[[H5[[J5[L[L5[U [[N5[[P5[L[L5/ n [SU5 [U5 U Hn U RVnU RXnU RZnU R\nU R^n[aX5n[SU5 [aUU5n[SU5 U(aU"U5nU(aU"U5nURc5 URc5 [eUUUU RfU RhS9nU Rj(dMUHnU RkU5 M M SSS5 U$!,(df  U$=f)NrVrW)rYrrPrX)rri)rXrrrr)rX rZr[)rrr)rXrXrrY)r)rXrrT)rrrrl)r) match_filtersr)6rrandintr!randnrrpinttensorfinfor_epsrrrMrTrrrirrrr:rIrrrrrrrrrrrrrrrrrrrrrrrrrr recompilerrrr)r _QUANTIZED_LINEAR_EXAMPLE_INPUTS(_DYNAMIC_QUANTIZED_LINEAR_EXAMPLE_INPUTS _QUANTIZED_CONV2d_EXAMPLE_INPUTS)_QUANTIZED_ADD_OR_ADD_RELU_EXAMPLE_INPUTS$_QUANTIZED_MAX_POOL2D_EXAMPLE_INPUTS(_QUANTIZE_PER_TENSOR_INT8_EXAMPLE_INPUTS*_DEQUANTIZE_PER_TENSOR_INT8_EXAMPLE_INPUTS)_QUANTIZE_PER_CHANNEL_INT8_EXAMPLE_INPUTS+_DEQUANTIZE_PER_CHANNEL_INT8_EXAMPLE_INPUTS9_DYNAMIC_QUANTIZED_LINEAR_4BIT_GROUPWISE_EXAMPLE_INPUTS_19_DYNAMIC_QUANTIZED_LINEAR_4BIT_GROUPWISE_EXAMPLE_INPUTS_2_REWRITE_INFO_LIST rewrite_inforrrrrmatchesms r9rrs  dCuzz: AU[[) AUYY' dV599- cU%)), dCuzz: AU[[) AUYY' dV599- cU%)), AU[[) AU[[) AUYY' dV599- cU%)),($&  F%++.   EMM"&& dCuzz: AU[[) AUYY' dV599- cU%)), AU[[) 0,  dCUZZ@ AU[[) AUYY' dV599- cU%)), dCUZZ@ AU[[) AUYY' dV599- cU%)), AU[[) AU[[) AUYY' dV599- cU%)),($&  dCUZZ@ AU[[) AUYY' dCUZZ@ AU[[) AUYY' AU[[) AUYY' dV599- cU%)), 1-  dCUZZ@ AU[[) AUYY' dV599- cU%)), AU[[) AUYY' dV599- cU%)), ,(  Aq!Qekk2 AU[[) AUYY' dV599- cU%)), 0,  dCUZZ@ AU[[) AUYY' dV599- cU%)), 2.  Aq!Qekk2 AU[[) AUYY'   1-  dCUZZ@ AU[[) AUYY'   3/  G5;;/ EMM"&& b!WEJJ7 Aq , q    AU[[)  A=  Jekk2 EMM"&& b!WEJJ7 Aq , q    AU[[)  A=  4 7 8 = > <#'CEKK 4N4R4RTU"V  <#'CEKK 4N4R4RTU"V    E F G L M <KK .22AA#  <KK .22AA# NN W) ,  E F G L M <KK .22AA#  <KK .22AA# NN W) ,  , / 0 5 6 3 3    , / 0 5 6 ;rd S ;rd S    5 1 2 7 8  5 , - 2 3  0 3 4 9 : 3 3    4 3 4 = >  6 5 6 ? @  5 4 5 > ? 3 3    7 6 7 @ A 3 3  Uqf'u- - /.L)88N"**G&22K!-!@!@ %1%H%H "8QG .w 7<^K /{ ;!,W5%4[A      ! ! #2*44 , < < G,,, A 11!4!5/ 0< L= 0 /< Ls/CAA8AAA8A8 AB)L contextlib dataclassesr functoolsrtypingrrrrr!torch._higher_order_ops.out_dtyper $torch.ao.quantization.fx._decomposedr torch.fxr 6torch.fx.passes.utils.matcher_with_name_node_map_utilsr torch.fx.subgraph_rewriterr r&torchao.quantization.pt2e.export_utilsrtorchao.quantization.pt2e.utilsrrrr%torchao.quantization.quant_primitivesrtorchao.quantization.utilsr torchao.utilsrtorch._export.utilsr nullcontextlibraryLibrary quant_libregister_custom_op__all__r:rIrMrTrirBrrrrrrrrrrrrrrrrrrrrrrrrrrrrr;r9rAs!00 7I PU@ >@-BH MM ! !)Z 8 (3'  F2jB,^>B~$ LL~$ ~$||~$,, ~$ || ~$  % ~$S ~$ \\~$~$B00 06 ) 80fOd4"J2&R88    S" 0D0D  LL L&YKYKYiB+5+A+A(s &E33F