h& SSKrSSKJr SSKrSSKJr \R R SS5r\RS5 \RS5 \RS5 \RS 5 \RS 5 \RS 5 \RS 5 \RS 5 \RS5 \RS5 \RS5 \RS5 \RS5 \RS\RRR/S9 \RS\RRR/S9 \RS5 \RS5 \RS5 \RS5 Sr Sr \RS5rSS\S\S\S \S!\S"\S#\4S$jjr\ "S%5SS\S\S\S \S!\S"\S#\4S&jj5rSS'\S(\S)\S*\\S+\S,\S-\\S.\S/\S0\S1\S2\S3\S4\S5\S6\S7\S#\4$S8jjr\ "S95SS'\S(\S)\S*\\S+\S,\S-\\S.\S/\S0\S1\S2\S3\S4\S5\S6\S7\S#\4$S:jj5rS;\S<\S#\4S=jr\ "S>5S;\S<\S#\4S?j5rS;\S@\SA\S<\S#\4 SBjr\ "SC5S;\S@\SA\S<\S#\4 SDj5rSE\SF\SG\SH\SI\SJ\SK\SL\SM\S#\4SNjr\ "SO5SE\SF\SG\SH\SI\SJ\SK\SL\SM\S#\4SPj5rSE\SF\SQ\SR\SS\SI\SK\SL\SM\S#\4STjr\ "SU5SE\SF\SQ\SR\SS\SI\SK\SL\SM\S#\4SVj5rSSW\SX\SY\SZ\S[\\S\\\R4S#\4S]jjr\ "S^5SSW\SX\SY\SZ\S[\\S\\\R4S#\4S_jj5rSSW\SX\SY\SZ\S[\\S\\\R4S#\4S`jjr\ "Sa5SSW\SX\SY\SZ\S[\\S\\\R4S#\4Sbjj5rSSW\SX\SY\Sc\SZ\S[\\S\\\R4S#\4Sdjjr\ "Se5SSW\SX\SY\Sc\SZ\S[\\S\\\R4S#\4Sfjj5rSY\S#\\44Sgjr\ "Sh5SY\S#\\44Sij5rSSj\Sk\Sl\Sm\S#\\44 Snjjr \ "So5SSj\Sk\Sl\Sm\4Spjj5rSSq\SG\Sr\S4\\Ss\\St\Su\Sv\S#\4Swjjr!\ "Sx5SSq\SG\Sr\S4\\Ss\\St\Su\Sv\4Syjj5rSz\S{\S|\S}\S#\4 S~jr"\ "S5Sz\S{\S|\S}\S#\4 Sj5rSz\S{\S|\S}\S\S\S[\\S\\S\\\R4S#\4Sjr#\ "S5Sz\S{\S|\S}\S\S\S[\\S\\S\\\R4S#\4Sj5r\R"5S5r$Sr%\ "S5S\S\S\S\4Sj5r&S\S\S\S\4Sjr'\ "S5S\S\S\S\4Sj5r(SY\S\S\S#\4Sjr)\ "S5SY\S\S\S#\4Sj5rSW\S\S\SY\S\S\S\S[\\S\\R44Sjr*\ "S5SW\S\S\SY\S\S\S\S[\\S\\R4S#\4Sj5r\ "S5S\S\S\S\S6\S\S\S#\4Sj5rg)N)Optional)TensortorchaoFRAGMENTzuquant_llm_linear(int EXPONENT, int MANTISSA, Tensor _in_feats, Tensor _weights, Tensor _scales, int splitK) -> TensorzMunpack_tensor_core_tiled_layout(Tensor packed_w, int inner_k_tiles) -> Tensorzzdequantize_tensor_core_tiled_layout(Tensor packed_w, Tensor scales_and_zeros, int group_size, int inner_k_tiles) -> Tensorzmarlin_24_gemm(Tensor x, Tensor weight_marlin, Tensor meta, Tensor s, Tensor workspace, int bits, int size_m, int size_n, int size_k) -> Tensorzmarlin_qqq_gemm(Tensor x, Tensor weight_marlin, Tensor s_tok, Tensor s_ch, Tensor s_group, Tensor workspace, int size_m, int size_n, int size_k) -> Tensorzrowwise_scaled_linear_cutlass_s8s4(Tensor input, Tensor input_scale, Tensor weight, Tensor weight_scale, Tensor? bias=None, ScalarType? out_dtype=None) -> Tensorzrowwise_scaled_linear_cutlass_s4s4(Tensor input, Tensor input_scale, Tensor weight, Tensor weight_scale, Tensor? bias=None, ScalarType? out_dtype=None) -> Tensorzrowwise_scaled_linear_sparse_cutlass_f8f8(Tensor input, Tensor input_scale, Tensor weight, Tensor weight_meta, Tensor weight_scale, Tensor? bias=None, ScalarType? out_dtype=None) -> TensorzLto_sparse_semi_structured_cutlass_sm9x_f8(Tensor weight) -> (Tensor, Tensor)zsparse24_sm90_sparsify(Tensor input, str metadata_fmt, str activation, str sp_selection_algo, *, ScalarType? dtype = None, Tensor? scale=None) -> (Tensor, Tensor)zsparse24_fp8_sm90_cutlass_gemm(Tensor a, Tensor a_mdata, Tensor b, *, Tensor? a_scale = None, Tensor? b_scale = None, int swizzle_size=8, str swizzle_axis='n', int sm_count=128) -> Tensorz\swizzle_mm(Tensor mat1, Tensor mat2, bool mat1_is_swizzled, bool mat2_is_swizzled) -> Tensorzswizzle_scaled_mm(Tensor mat1, Tensor mat2, bool mat1_is_swizzled, bool mat2_is_swizzled, Tensor scale_a, Tensor scale_b, Tensor? bias=None, Tensor? scale_result=None, ScalarType? out_dtype=None) -> TensorzImx_fp8_bf16(Tensor a, Tensor b, Tensor a_scale, Tensor b_scale) -> Tensor)tagszImx_fp4_bf16(Tensor a, Tensor b, Tensor a_scale, Tensor b_scale) -> Tensora6qscaled_dot_product(Tensor query, Tensor key, Tensor value, Tensor? attn_mask=None, float dropout_p=0.0, bool is_causal=False, float? scale=None, float q_scale=1.0, int q_zp=0, float k_scale=1.0, int k_zp=0, float v_scale=1.0, int v_zp=0, float a_scale=1.0, int a_zp=0, float o_scale=1.0, int o_zp=0) -> Tensorzida8w4_linear_prepack_cpu(Tensor weight, Tensor scales, Tensor qzeros) -> (Tensor, Tensor, Tensor, Tensor)zda8w4_linear_cpu(Tensor input, Tensor input_scales, Tensor input_qzeros, Tensor weight, Tensor weight_scales, Tensor weight_qzeros, Tensor compensation, Tensor? bias, ScalarType output_dtype) -> Tensorz_scaled_embedding_bag(Tensor qweight, Tensor indices, Tensor offsets, Tensor weight_scale, float o_scale, int mode, bool include_last_offset) -> Tensorc^U4SjnU$)NcP>[RRT5"U5$N)torchlibrary register_fakefuncnames E/home/james-whalen/.local/lib/python3.13/site-packages/torchao/ops.py decorator%register_custom_op..decoratorMs}}**dV5d;;rrs` rregister_custom_oprLs< rc^U4SjnU$)NcN>[RRTSS9"U5$)Nr) mutates_args)r r custom_oprs rr*register_custom_op_impl..decoratorTs$}}&&$r&B4HHrrrs` rregister_custom_op_implrSsI rc[RR[RR55nURS-UR -nU$)N )r cudaget_device_propertiescurrent_devicemajorminor) device_propscompute_capabilitys rcached_compute_capabilityr'ZsE::33EJJ4M4M4OPL%++b0<3E3EE rEXPONENTMANTISSA _in_feats_weights_scalessplitKreturnc^[5m[R"TS:U4Sj5 [RRR R XX#XE5$)aG Quant-LLM linear layer A @ W.T. See https://arxiv.org/abs/2401.14112 for more details. Arguments EXPONENT: number of exponent bits MANTISSA: number of mantissa bits _in_feats: input activations in FP16 _weights: packed Floatx weights _scales: scale splitK: split K Returns output of linear layer Kc>ST3$)NzLquant_llm_linear requires sm7.5+ GPU architecture, but current device has smr)r&sr"quant_llm_linear..{s^_q^rsr)r'r _checkopsrquant_llm_lineardefault)r(r)r*r+r,r-r&s @rr6r6asQ.34 LLb s 99   - - 5 5I rztorchao::quant_llm_linearc^^^[R"TR5S:HU4Sj5 [R"TR[R[R 4;U4Sj5 [R"TR5S:HU4Sj5 [R"TR[R LU4Sj5 [R"TR5S:HU4Sj5 [R"TR[R[R 4;U4Sj5 TRupgTRupSU-U-n [R"US -U -TRS:HS 5 [R"UTRS :HS 5 TRXh45$) Nc,>STR5S3$)Nz!input should be a 2d tensor, got Ddimr*srr2_..s3IMMO3DAFrc">STR3$)Nz!weight must be FP16 or BF16, got dtyper>srr2r?s3IOO3DErc,>STR5S3$)Nz"weight should be a 2d tensor, got r;r<r+srr2r?s4X\\^4DAFrc">STR3$)Nzweight must be UINT8, got rArDsrr2r?,X^^,<=rc,>STR5S3$)Nz!scale should be a 2d tensor, got r;r<r,srr2r?s&G VW$Xrc">STR3$)Nz scale must be FP16 or BF16, got rArIsrr2r?s27==/BrcgNzDimensions mismatchedrrrrr2r?s@WrrcgrMrrrrr2r?s1Hr) r r4r=rBfloat16bfloat16uint8shape new_empty) r(r)r*r+r,r-BSICOC_N_BITSs ``` rrWrWsQ LL 1F LLEMM5>>::E LL !F LL%++%= LL X LL %--88B __FB NNEB \H $F LLq6!X^^A%668WX LLw}}Q'')HI   x ((rquerykeyvalue attn_mask dropout_p is_causalscaleq_scaleq_zpk_scalek_zpv_scalev_zpa_scalea_zpo_scaleo_zpc[RRRR UUUUUUUUUU U U U U UUU5$)a Quantized SDPA with quantized inputs and outputs. Arguments query: input query tensor, key: input key tensor, value: input value tensor, attn_mask: attention mask tensor, dropout_p: dropout probability, is_causal: causal flag, scale: scaling factor applied prior to softmax, q_scale: scale for query from linear quantization, q_zp: zero point for query from linear quantization, k_scale: scale for key from linear quantization, k_zp: zero point of key from linear quantization, v_scale: zero point for value from linear quantization, v_zp: zero point of value from linear quantization, a_scale: scale for attention from softmax quantization, a_zp: zero point for attention from softmax quantization, o_scale: scale for output from linear quantization, o_zp: zero point for output from linear quantization, Returns output of quantized SDPA )r r5rqscaled_dot_productr7rYrZr[r\r]r^r_r`rarbrcrdrerfrgrhris rrkrks[T 99   0 0 8 8         # rztorchao::qscaled_dot_productcU$r rrls rrWrWs ( Lrpacked_w inner_k_tilescd[RRRR XS9$)a Unpacks weights that were packed with `torch.ops.aten._convert_weight_to_int4pack` to original tensor of shape `N x K`. Assumes that the packed weights were generated with `torch.ops.aten._convert_weight_to_int4pack` with `inner_k_tiles = 2 | 4 | 8`" Args: packed_w: torch.tensor: 4D tensor with shape (N / 8) x (K / (inner_k_tiles * 16)) x 32 x inner_k_tiles, dtype is torch.int32 inner_k_tiles: int Returns: torch.tensor of shape is N x K, dtype is torch.int32 rnro)r r5runpack_tensor_core_tiled_layoutr7rqs rrrrrs0 99   < < D D E rz(torchao::unpack_tensor_core_tiled_layoutc^[R"TR5S:HU4Sj5 [R"TR[RLU4Sj5 [R"US:H=(d US:H=(d US:HS5 [R"TR S5S:HS5 [R"TR S 5US- :HS 5 TR S 5S-nTR S 5U-S -n[R "X#4[RTRS9$)Nc,>STR5S3$)Nz*packed weight should be a 42d tensor, got r;r<rnsrr2r?sSTR3$Nzweight must be INT32, got rArvsrr2r?rFrr9rKcgNz inner_k_tiles must be 2, 4, or 8rrrrr2r?!2r cgNz#packed weight must have 32 at dim 2rrrrr2r?#1VrcgNz0packed weight must have inner_k_tiles/2 at dim 3rrrrr2r?&BrrrGrBdevice)r r4r=rBint32sizeemptyr)rnroNKs` rrWrWs LL !N LL%++%= LLFmq0FMQ4F2 LLq!R')VW LL aMA--B  a1A a=(2-A ;;vU[[ IIrscales_and_zeros group_sizecj[RRRR XX#5$)a Dequantizes by: - Unpacking weights that were packed with `torch.ops.aten._convert_weight_to_int4pack` to original tensor of shape `N x K` - Upcasting to bfloat16 - Dequantizing with the scales_and_zeros that were packed with `torchao.quantization.utils.pack_tinygemm_scales_and_zeros` Assumes: - packed weights were generated with `torch.ops.aten._convert_weight_to_int4pack` with `inner_k_tiles = 2 | 4 | 8`" - packed scales_and_zeros were generated with `torchao.quantization.utils.pack_tinygemm_scales_and_zeros` - qGroupSize is 32 | 64 | 128 | 256 Args: packed_w: torch.tensor: 4D tensor with shape `(N / 8) x (K / (inner_k_tiles * 16)) x 32 x inner_k_tiles / 2`, dtype is torch.int32 scales_and_zeros: torch.tensor: 3D tensor with shape `numQGroups x N x 2`, dtype is torch.bfloat16 where numQGroups is K / qGroupSize group_size: int inner_k_tiles: int Returns: torch.tensor of shape is N x K, dtype is torch.bfloat16 )r r5r#dequantize_tensor_core_tiled_layoutr7)rnrrros rrr.s,0 99   @ @ H HJ rz,torchao::dequantize_tensor_core_tiled_layoutc^[R"TR5S:HU4Sj5 [R"TR[RLU4Sj5 [R"US:H=(d US:H=(d US:HS5 [R"TR S5S:HS5 [R"TR S 5US- :HS 5 TR S 5S-nTR S 5U-S -n[R"UR[R LS5 [R"UR5S :HS5 [R"US:H=(d US:H=(d US:H=(d US:HS5 [R"UR S 5XR-:HS5 [R"UR S 5U:HS5 [R"UR S5S:HS5 [R"XE4[R TRS9$)Nrtc,>STR5S3$)Nz)packed weight should be a 4d tensor, got r;r<rvsrr2r?Rs;HLLN;K1Mrc">STR3$rxrArvsrr2r?VrFrr9rKcgrzrrrrr2r?Zr{rr|cgr~rrrrr2r?\rrrcgrrrrrr2r?_rrrrGrcg)Nz!scales_and_zeros must be bfloat16rrrrr2r?gs3rcg)Nz9scales_and_zeros must be 3D, got {scales_and_zeros.dim()}rrrrr2r?ksKr@cg)Nz&qGroupSize must be 32, 64, 128, or 256rrrrr2r?os8rcg)Nz3scales_and_zeros must have K // qGroupSize at dim 0rrrrr2r?ssErcg)Nz%scales_and_zeros must have N at dim 1rrrrr2r?v/Vrcg)Nz%scales_and_zeros must have 2 at dim 2rrrrr2r?yrrr) r r4r=rBrrrPrr)rnrrrorrs` rrWrWKs  LL !M LL%++%= LLFmq0FMQ4F2 LLq!R')VW LL aMA--B  a1A a=(2-A LL%..03 LL!#K LLbVJ",V c0AVZSVEV8 LLa AO3E LLa A%'V LLa A%'V ;;vU^^HOO LLrx weight_marlinmetas workspacebitssize_msize_nsize_kc p[RRRR XX#XEXgU5 $)a Sparse Marlin 2:4 matrix multiplication. Reference: https://github.com/IST-DASLab/Sparse-Marlin/tree/main Args: x: input matrix of shape `(n, k/2)` in column-major layout. weight_marlin: weight matrix of original shape `(m, k)` in Marlin format; see `Layer.pack()`. meta: metadata information for 2:4 sparsity. s: scales of shape `(n / groupsize / 2, m)`. workspace: tensor with at least `m / 128 * max_par` entries that are all zero. bits: number of bits for quantization. size_m: number of rows in input matrix. size_n: number of columns in weight matrix. size_k: number of columns in input matrix. Returns: output matrix of shape `(n, m)` in column-major layout. )r r5rmarlin_24_gemmr7) rrrrrrrrrs rrrs24 99   + + 3 3 $9FF rztorchao::marlin_24_gemmc ^^^^^^^^^^ ^ ^ ^^Sm Sm Sn [R"TS:H=(d TS:HU4Sj5 ST-n [R"TTRS5:HUU4S j5 [R"TTRS 5:HUU4S j5 [R"TT -S:HU U4S j5 [R"TT -S -TRS5:HU UU4Sj5 [R"TRS 5T:HUU4Sj5 [R"TRS 5T -S:HU U4Sj5 TRS 5T -U -m [R"TT :HU U4Sj5 [R"TRS5TS-S -S -:HUU4Sj5 [R"TRS 5TS -:HUU4Sj5 [R"TRS5 [R"TR 5S5 [R"TRS5 [R"TR 5S5 [R"TRS5 [R"TR 5S5 [R"TRS5 [R"TR 5S5 SmTRS5S :aJ[R"TTRS5-S:HUU4Sj5 TTRS5-mTS -m[R"TS:H=(d TS:HU4Sj5 [R"TT -S:HU U4Sj5 TT -U -m[R"TR 5T:UU4S j5 [R "TRS5TRS 54TRTRS!9$)"NrrrrtrKc>ST3$)Nznum_bits must be 4 or 8. Got = r)rsrr2r?s *I$(Prr|rc2>STRS5ST3$NzShape mismatch: x.size(0) = r , size_m = rrrsrr2r?.qvvaykVHMrrGc2>STRS5ST3$NzShape mismatch: x.size(1) = rG , size_k = rrrsrr2r?rrc>STST3$N size_k = ! is not divisible by tile_size = r TILE_SIZErsrr2r?)F8#DYKPrr9c8>STRS5STST3$Nz(Shape mismatch: weight_marlin.size(0) = rrz, tile_size = rrrrsrr2r?+:=;M;Ma;P:QQ\]c\ddrs|r}~rc2>STRS5ST3$)Nz s.size(1) = rG , size_n = r)rrsrr2r?s|AFF1I;k&%Rrc2>STRS5ST3$Nzweight_marlin.size(1) = rGrrrrsrr2r?"*=+=+=a+@*AAbclbmnrc>STST3$)N size_n = , actual_size_n = r actual_size_nrsrr2r?s)F8#5m_ErcD>STRS5STS-S-S-3$)Nzmeta.size(0) = rz is not size_k / 8 / 2 / 2 = rKr9r)rrsrr2r?s-/$))A,/LVWX[\]M]abMbLcdrc8>STRS5STS-3$)Nzmeta.size(1) = rGz is not size_n * 2 = r9r)rrsrr2r?s/$))A,/DVaZLQrcgNzx is not on GPUrrrrr2r?$5rcgNzx is not contiguousrrrrr2r?,ArcgNzweight_marlin is not on GPUrrrrr2r?0MrcgNzweight_marlin is not contiguousrrrrr2r?/Prcg)Nzmeta is not on GPUrrrrr2r?';rcg)Nzmeta is not contiguousrrrrr2r?/Grcg)Nzs is not on GPUrrrrr2r?rrcg)Nzs is not contiguousrrrrr2r?rrc2>STSTRS53$)Nrz! is not divisible by s.size(0) = rr)rrsrr2r?six'HPQ Trc>ST3$NzUnexpected groupsize = r groupsizesrr2r?s ))5rc>STST3$Nrz$ is not divisible by min_thread_n = r MIN_THREAD_Nrsrr2r?)F8#G ~Vrc0>STR5ST3$)Nzworkspace.numel =  is below min_workspace_size = numelmin_workspace_sizersrr2r? s$Y__%6$77VWiVjkrr) r r4ris_cuda is_contiguousrrrBr)rrrrrrrrrMAX_PARALLELISM pack_factorrrrrrs````````` @@@@@rrWrWsLILO LL  TQY P*K LL!&&)M  LL!&&)M LLaP LL 9  !m&8&8&;;~  LL q VR LL1 )Q.n #''*i7;FM LL-E  LL ! ! q(A--d LL !  "Q  LL56 LL"$AB LL&&(MN LL##%'P  LL;< LL##%'GH LL56 LL"$ABIvvay1} QVVAY ! # T affQi' a  LLR*9?5  LL"V!L0OC LL//k ;;q 166!9-QWWQXX NNrs_toks_chs_groupc p[RRRR XX#XEXgU5 $)a Marlin for W4A8 mixed precision matrix multiplication. See https://arxiv.org/pdf/2406.09904 for more details. Reference: https://github.com/HandH1998/QQQ/tree/main Args: x: `torch.int8` input matrix of shape `(m, k)` in standard row-major layout. weight_marlin: `torch.int32` weight matrix of original shape `(k, n)` in the specified format. s_tok: `torch.float32` activation per-token quantization scales of shape `(m, 1)`. s_ch: `torch.float32` weight per-channel quantization scales of shape `(1, n)`. s_group: `torch.half` weight per-group quantization scales of shape `(m / groupsize, n)`, it should be empty when group_size != -1. workspace: `torch.int32` tensor with at least `n / 128 * max_par` entries that are all zero. size_m: number of rows in input matrix. size_n: number of columns in weight matrix. size_k: number of columns in input matrix. Returns: `torch.half` out matrix of shape `(m, n)` in standard row-major layout. )r r5rmarlin_qqq_gemmr7) rrrrrrrrrs rrrs28 99   , , 4 4 %w66 rztorchao::marlin_qqq_gemmc x ^^^^^^^^^^ ^ ^ ^^Sm Sm Sn Sn [R"TTRS5:HUU4Sj5 [R"TTR5:HUU4Sj5 [R"TTRS5:HUU4Sj5 [R"TT -S:HU U4S j5 [R"TT -TRS5:HU UU4S j5 TR5S:XaS OTTRS5-m[R"TS ;U4S j5 [R"TR5T:HUU4Sj5 [R"TRS5T -S:HU U4Sj5 TS :wa_[R"TRS5T:HUU4Sj5 [R"TTRS5-S:HUU4Sj5 TRS5T -U -m [R"TT :HU U4Sj5 [R"TRS5 [R"TR 5S5 [R"TRS5 [R"TR 5S5 [R"TRS5 [R"TR 5S5 [R"TR [R:HS5 [R"TRS5 [R"TR 5S5 [R"TR [R:HS5 [R"TRS5 [R"TR 5S5 [R"TR [R:HS5 [R"TT -S:HU U4S j5 TT -U -m[R"TR5T:UU4S!j5 [R"TT4[RTRS"9$)#NrrrKrc2>STRS5ST3$rrrsrr2r?Drrc0>STR5ST3$)Nz Shape mismatch: s_tok.numel() = rr)rrsrr2r?Hs25;;=/VHUrrGc2>STRS5ST3$rrrsrr2r?Nrrc>STST3$rrrsrr2r?Rrrc8>STRS5STST3$rrrsrr2r?Vrrr)rrc>ST3$rrrsrr2r?[s 3J9+1Vrc0>STR5ST3$)NzShape mismatch: s_ch.numel() = rr)rrsrr2r?`s1$**,{6(Src2>STRS5ST3$rrrsrr2r?drrc2>STRS5ST3$)Nz"Shape mismatch: s_group.size(1) = rGrr)rrsrr2r?is8a8IU[T\]rc2>STSTRS53$)Nrz' is not divisible by s_group.size(0) = rr)rrsrr2r?msix'Nw||\]N_`rc>STST3$)NzShape mismatch: size_n = rrrsrr2r?ss+F83Em_Urcgrrrrrr2r?wrrcgrrrrrr2r?xrrcgrrrrrr2r?{rrcgrrrrrr2r?}rrcg)Nzs_tok is not on GPUrrrrr2r?s(=rcg)Nzs_tok is not contiguousrrrrr2r?s0Ircg)Nzs_tok's dtype is not float32rrrrr2r?s7Urcg)Nzs_ch is not on GPUrrrrr2r?rrcg)Nzs_ch is not contiguousrrrrr2r?rrcg)Nzs_ch's dtype is not float32rrrrr2r?s6Srcg)Nzs_group is not on GPUrrrrr2r?s*Arcg)Nzs_group is not contiguousrrrrr2r?s2Mrzs_group's dtype is not float16c>STST3$rrrsrr2r?rrc0>STR5ST3$)Nzworkspace.numel() = rrrsrr2r?s&y'8&99XYkXlmrr) r r4rrrrrBfloat32rOrr)rrrrrrrrrr PACK_FACTORrrrrrs````````` @@@@@rrWrW0skILOK LL!&&)M LL%++-U  LL!&&)M LLaP LL 9 !3!3A!66~ mmo*',,q/0II LLi')VW LL S LL1 )Q.nB LLOv % ]   W\\!_ $ ) ` #''*i7;FM LL-U  LL56 LL"$AB LL&&(MN LL##%'P  LL => LL$$&(IJ LL -/UV LL;< LL##%'GH LLu}},.ST LL"AB LL&&(*MN LL%--/1QR LL"V!L0OC LL//m ;;'u}}QXX NNrinput input_scaleweight weight_scalebias out_dtypecr[RRRR UUUUUU5$)a CUTLASS-based row-wise scaled W4A8 linear operator. Args: input: quantized input tensor, in row-major layout. input_scale: scale factors for input tensor, has to be tensor of the same shape as the input tensor, minus the last dimension. weight: quantized weight matrix, in row-major layout. weight_scale: scale factors for weight tensor, one value per row of weight matrix (thus also tensor of the same shape as the weight tensor, minus the last dimension). bias: an optional vector of size equal to number of rows of weight tensor, or None. out_dtype: optional data type for output tensor. Returns: output: result tensor, in row-major layout. )r r5r"rowwise_scaled_linear_cutlass_s8s4r7rrrrrrs rrrs9* 99   ? ? G G    rz+torchao::rowwise_scaled_linear_cutlass_s8s4cUbUO URnURn[R"/URSSQURSP7XgS9$NrrrrBrr rrRrrrrrrrBrs rrWrWN#.IK4E4EE \\F ;;;Sb);6<<?;5 XXrcl[RRRR XX#XE5$)a CUTLASS-based row-wise scaled W4A4 linear operator. Args: input: quantized input tensor, in row-major layout. input_scale: scale factors for input tensor, has to be tensor of the same shape as the input tensor, minus the last dimension. weight: quantized weight matrix, in row-major layout. weight_scale: scale factors for weight tensor, one value per row of weight matrix (thus also tensor of the same shape as the weight tensor, minus the last dimension). bias: an optional vector of size equal to number of rows of weight tensor, or None. out_dtype: optional data type for output tensor. Returns: output: result tensor, in row-major layout. )r r5r"rowwise_scaled_linear_cutlass_s4s4r7rs rr&r&s.* 99   ? ? G G F$ rz+torchao::rowwise_scaled_linear_cutlass_s4s4cUbUO URnURn[R"/URSSQURSP7XgS9$r!r"r#s rrWrWr$r weight_metac n[RRRR XX#XEU5$)a CUTLASS-based row-wise scaled F8F8 linear operator, for sparsified weight case. Args: input: quantized input tensor, in row-major layout. input_scale: scale factors for input tensor, has to be tensor of the same shape as the input tensor, minus the last dimension. weight: sparsified quantized weight matrix, in row-major layout. weight_meta: sparsify metadata for weight tensor. weight_scale: scale factors for weight tensor, one value per row of weight matrix (thus also tensor of the same shape as the weight tensor, minus the last dimension). bias: an optional vector of size equal to number of rows of weight tensor, or None. out_dtype: optional data type for output tensor. Returns: output: result tensor, in row-major layout. )r r5r)rowwise_scaled_linear_sparse_cutlass_f8f8r7)rrrr(rrrs rr*r*s0. 99   F F N N FY rz2torchao::rowwise_scaled_linear_sparse_cutlass_f8f8cUbUO URnURn[R"/URSSQURSP7XxS9$r!r") rrrr(rrrrBrs rrWrWsN#.IK4E4EE \\F ;;;Sb);6<<?;5 XXrch[RRRR U5$)a CUTLASS-based conversion from sparsified input tensor to corresponding compressed tensor, along with corresponding metadata tensor. Args: weight: input tensor, in row-major layout. Returns: weight_compressed: compressed weight tensor, with sparsity eliminated, in row-major layout. weight_meta: metadata tensor, describing the sparsity structure of the input tensor, also in row-major layout. )r r5r)to_sparse_semi_structured_cutlass_sm9x_f8r7rs rr-r-&s% 99   F F N Nv VVrz2torchao::to_sparse_semi_structured_cutlass_sm9x_f8cURUSUSS-5URUS[USS-S5[RS94$)NrrGr9rKrrA)rSmaxr charr.s rrWrW5sV F1IN3Cq Q$;5::N r input_tensormetadata_format activation algorithmc T[RRRXX#XES9$)N)rBr_)r r5rsparse24_sm90_sparsify)r2r3r4r5rBr_s rr7r7Bs- 99   3 3zE 4 rztorchao::sparse24_sm90_sparsifyc<UbUO URn[R"URSURSS-4UURS9[R"URSURSS-4[R URS94$)NrrGr9rrK)rBr rrRrrQ)r2r3r4r5rBr_rs rrWrWOs* 0B0BI    "L$6$6q$9Q$> ?&&     "L$6$6q$9Q$> ?++&&  rabb_scale swizzle_size swizzle_axissm_countc ^[RRRUUUUUUUUS9$)N)rfr;r<r=r>)r r5rsparse24_fp8_sm90_cutlass_gemmr9rr:rfr;r<r=r>s rr@r@gs> 99   ; ;   !! <  rz'torchao::sparse24_fp8_sm90_cutlass_gemmc[R"URSURS4[RURS9$)NrrGr)r rrRrPrrAs rrWrW}s4 ;; AGGAJ/u~~ahh WWrmat1mat2mat1_is_swizzledmat2_is_swizzledcj[RRRR XX#5$zH Similar to torch.mm but Tensor inputs can be SwizzleTensor instances. )r r5r swizzle_mmr7rCrDrErFs rrIrIs- 99   ' ' / / $ rztorchao::swizzle_mmcZURURSURS5$NrrGrSrRrJs rrWrWs% >>$**Q-A 77rscale_ascale_b scale_resultc x[RRRR UUUUUUUUU5 $rH)r r5rswizzle_scaled_mmr7 rCrDrErFrNrOrrPrs rrRrRsB 99   . . 6 6     rztorchao::swizzle_scaled_mmc ZURURSURS5$rLrMrSs rrWrWs% >>$**Q-A 77rc[[S5(a [R[R4$[R4$)zGTODO: when e8m0 is hardened and major release lets remove uint8 supportfloat8_e8m0fnu)hasattrr rQrVrrr _get_dtypesrXs3u&'' U1122 KK>rc^^[5n[R"TRU;U4Sj5 [R"TRU;U4Sj5 g)Nc">STR3$)Nz4A_scale tensor must be uint8 or float8_e8m0fnu, got rA)A_scalesrr2%_check_scale_dtypes..Fw}}oVrc">STR3$)Nz4B_scale tensor must be uint8 or float8_e8m0fnu, got rA)B_scalesrr2r\r]r)rXr r4rB)r[r_allowed_dtypess`` r_check_scale_dtypesrasB ]N LL 'V LL 'Vrztorchao::mx_fp8_bf16ABr[r_c[R"URS5URS54[RURS9$)zMeta impl for mx_fp8_bf16rrGrr rrrPrrbrcr[r_s rmeta_mx_fp8_bf16rgs4 ;;q 166!9-U^^AHH UUrc[X#5 [RRRR XX#5$)a Defines a matmul between two fp4 tensors w/ MX scales in E8MO and returns a bf16 tensor. The expected format is fp4_e2m1 specified: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final.pdf (Section 5.3.3) Note: The mx scales are E8MO tensors stored in uint8 tensors (for now). The layout of the scales is very particular, see: https://docs.nvidia.com/cuda/cublas/index.html#d-block-scaling-factors-layout Args: A: fp4 tensor (2 fp4 elements are packed into 1 byte -> elem0|elem1) B: fp4 tensor (2 fp4 elements are packed into 1 byte -> elem0|elem1) A_scale: E8M0 scale tensor for A with groupsize=32 in swizzled layout B_scale: E8M0 scale tensor for B with groupsize=32 in swizzled layout Returns: MXN bf16 Tensor )rar r5r mx_fp4_bf16r7rfs rriris/*) 99   ( ( 0 0w HHrztorchao::mx_fp4_bf16c[R"URS5URS54[RURS9$)zMeta impl for mx_fp4_bf16rrGrrerfs rmeta_mx_fp4_bf16rks4 ;;q 166!9-U^^AHH UUrscalesqzeroscj[RRRR XU5$)z Prepack weights for DA8W4 linear operator on CPU. Args: weight: weight tensor. scales: scales for weight tensor. qzeros: zero points for weight tensor. Returns: packed weight, scales, and zero points. )r r5rda8w4_linear_prepack_cpur7rrlrms rroros' 99   5 5 = =ff UUrz!torchao::da8w4_linear_prepack_cpuc2XU[R"54$r )r rrps rrWrWs 65<<> 11r input_scales input_qzeros weight_scales weight_qzeros compensationc x[RRRR UUUUUUUUU5 $)a DA8W4 linear operator on CPU. Args: input: input tensor. input_scales: scales for input tensor. input_qzeros: zero points for input tensor. weight: weight tensor. weight_scales: scales for weight tensor. weight_qzeros: zero points for weight tensor. compensation: compensation tensor for weight. bias: optional bias tensor. out_dtype: output data type. Returns: output tensor in out_dtype. )r r5rda8w4_linear_cpur7) rrrrsrrtrurvrrs rrxrxsB4 99   - - 5 5    rztorchao::da8w4_linear_cpuc UR5S:XdeURS5URS5-S-n UR"/URSSQU P7SU06$)Nrtrrr9rrB)r=rrSrR) rrrrsrrtrurvrrrs rrWrWAs^ ::<1   AQ'!+A ?? AEKK, Aa Ay AArztorchao::_scaled_embedding_bagqweightindicesoffsetsw_scalesmodeinclude_last_offsetcUS:XdeURSS- nURXpRSURS9$)NTrrGrA)rRrSrB)rzr{r|r}rhr~r batch_sizes rrWrWRsH $ && &q!A%J   Zq)9  OOr)rG)NgFN?rrrrrrrrr)NN)NNrKnr)+ functoolstypingrr rr Librarylibdefine_CTagneeds_fixed_stride_orderrr lru_cacher'intr6rWfloatboolrkrrrrrrBrr&r*r-strr7r@rIrRrXrargrirkrorxrrrrs   mmIz2 { S A V a h h C R i B b T  O ((,, / / 0  O ((,, / / 0  } o P ^         B/0 &)&)&)&) &)  &)  &) &)1&)Z#'!#< < < < <  <  < E?<< << << << < !<" #<$ %<~23 #'!#         E?     !" #$ %4,fSV&>?JJsJvJ@J0(.-.mO mOmO mO mO  mO  mO mO mO mO mO/mO`            B./iO iOiO iO  iO  iO  iO iO iO iO iO0iOb"'+     6    $  >AB "'+ Y  Y Y  Y Y 6  Y  $ Y  YC Y*"'+     6    $  4AB "'+ Y  Y Y  Y Y 6  Y  $ Y  YC Y,"'+      6   $ 8HI"'+Y YY Y Y  Y 6  Y $Y YJY" W  W f WHI    f J "        f 56    76!% $  f   f     ,=> !% $ X  X  X Xf  X f  X  X X X? X    26 JN   )*8 88268JN8 8+8         6 6" $ 801 8  8  8 8 8  8  8 6  86" 8 $ 8  82 8  *+VV6VFVVV,V I6IfIvII2*+VV6VFVVV,V V V V V V"782f2f2f2292$ $$$  $  $  $$ 6 ${{$N/0 B  B B B  B  B  B B 6  B{{ B  B1 B 45 P  P  P P P  P  P P  P6 Pr