
    hv!                        S SK r S SKrS SKJr  S SKJs  Jr   " S S\R                  R                  5      r " S S\R                  5      r	 " S S\R                  5      r
 " S S	\R                  5      r " S
 S\R                  R                  5      r\R                  SS4S\S\S\S\S\R                   S\S\4S jjr " S S\R                  5      rg)    Nc                   L   ^  \ rS rSrSS\R
                  4U 4S jjrS rSrU =r	$ )ToyLinearModel   @       c                    > [         TU ]  5         [        R                  R	                  XSS9R                  U5      U l        g )NFbias)super__init__torchnnLineartolinear1)selfkndtype	__class__s       ]/home/james-whalen/.local/lib/python3.13/site-packages/torchao/testing/model_architectures.pyr   ToyLinearModel.__init__   s1    xxq%8;;EB    c                 (    U R                  U5      nU$ Nr   r   xs     r   forwardToyLinearModel.forward   s    LLOr   r   
__name__
__module____qualname____firstlineno__r   bfloat16r   r   __static_attributes____classcell__r   s   @r   r   r      s    r C r   r   c                   V   ^  \ rS rSr S SU 4S jjjrS\R                  4S jrSrU =r	$ )#ConvWithSharedWeightInExportedModel   returnc                    > [         TU ]  5         Xl        [        R                  " X#XEU5      U l        [        R                  " U5      U l        [        R                  " SS9U l	        g )NT)inplace)
r   r   n_chunksr   Conv2dconvBatchNorm2dbnReLUrelu)r   r0   in_channelsout_channelskernel_sizestridepaddingr   s          r   r   ,ConvWithSharedWeightInExportedModel.__init__   sI     	 IIkgV	...GGD)	r   c                    [         R                  " XR                  SS9n/ nU HG  nU R                  U5      nU R	                  U5      nU R                  U5      nUR                  U5        MI     [         R                  " USS9$ )N   dim)r   chunkr0   r2   r4   r6   appendcat)r   r   chunksoutputsrA   outs         r   r   +ConvWithSharedWeightInExportedModel.forward#   sl    Q15E))E"C''#,C))C.CNN3	 
 yya((r   )r4   r2   r0   r6   )   r>   r>   )r-   N)
r"   r#   r$   r%   r   r   Tensorr   r'   r(   r)   s   @r   r+   r+      s-    TU*	* *)ELL ) )r   r+   c                   J   ^  \ rS rSr\R
                  S4U 4S jjrS rSrU =r	$ )LNLinearActivationModel.   sigmoidc                   > [         TU ]  5         UR                  5       n[        R                  " 5       [        R
                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       [        R                  " 5       S.nXE;  a  [        SU 35      e[        R                  " USS9U l        [        R                  " XSS9R                  US9U l        XT   U l        g )N)r6   rM   	leakyrelurelu6gelusilu	hardswishzUnsupported activation: F)elementwise_affiner	   )r   )r   r   lowerr   r5   Sigmoid	LeakyReLUReLU6GELUSiLU	Hardswish
ValueError	LayerNormlnr   r   fc
activation)r   fc_dim1fc_dim2r   r`   activation_mapr   s         r   r    LNLinearActivationModel.__init__/   s    %%'
GGIzz|XXZGGIGGI
 +7
|DEE,,w5A))G59<<5<I(4r   c                 h    U R                  U5      nU R                  U5      nU R                  U5      $ r   )r^   r_   r`   r   s     r   r   LNLinearActivationModel.forwardD   s+    GGAJGGAJq!!r   )r`   r_   r^   r!   r)   s   @r   rK   rK   .   s    /4~~) 5*" "r   rK   c                   x   ^  \ rS rSrS
S\S\4U 4S jjjrS rS\R                  S\R                  4S jr
S	rU =r$ )RMSNormJ   r@   epsc                    > [         TU ]  5         X l        [        R                  " [
        R                  " U5      5      U l        g r   )r   r   rj   r   	Parameterr   onesweight)r   r@   rj   r   s      r   r   RMSNorm.__init__K   s+    ll5::c?3r   c           	      z    U[         R                  " [         R                  " X-  SSS9U R                  -   5      -  $ )NT)r@   keepdim)r   rsqrtmeanrj   r   s     r   _normRMSNorm._normP   s.    5;;uzz!%RFQRRRr   r   r-   c                 z    U R                  UR                  5       5      R                  U5      nX R                  -  $ r   )ru   floattype_asrn   )r   r   outputs      r   r   RMSNorm.forwardS   s.    AGGI&..q1##r   )rj   rn   )gh㈵>)r"   r#   r$   r%   intrx   r   ru   r   rI   r   r'   r(   r)   s   @r   rh   rh   J   sA    4C 4e 4 4
S$ $%,, $ $r   rh   c                   L   ^  \ rS rSrSS\R
                  4U 4S jjrS rSrU =r	$ )TransformerBlockX         c                    > [         TU ]  5         Xl        X l        X-  U l        [
        R                  R                  USU-  SS9R                  U5      U l	        [
        R                  R                  XSS9R                  U5      U l
        X0l        [        X-  5      U l        [
        R                  R                  XR                  SS9R                  U5      U l        [
        R                  R                  U R                  USS9R                  U5      U l        [!        U5      R                  U5      U l        [!        U5      R                  U5      U l        [
        R                  R'                  5       U l        g )NrH   Fr	   )r   r   
hidden_dim	num_headshead_dimr   r   r   r   qkvproj	mlp_ratior|   mlp_hidden_dimmlp_fc1mlp_fc2rh   norm1norm2rY   r`   )r   r   r   r   r   r   s        r   r   TransformerBlock.__init__Y   s&   $""/ 88??:q:~E?JMMeTHHOOJOGJJ5Q	 #!*"89xxz3F3FUSVV
 xxt':':JUSVV

 Z(++E2
Z(++E2
  ((--/r   c                    UR                   u  p#nUnU R                  U5      nU R                  U5      nUR                  X#SU R                  U R
                  5      nUR                  SSSSS5      nUu  pxn	UR                  X R                  -  X0R
                  5      nUR                  X R                  -  X0R
                  5      nU	R                  X R                  -  X0R
                  5      n	XxR                  SS5      -  SU R
                  S	-  -  -  n
[        R                  " U
SS
9n
X-  nUR                  X R                  X0R
                  5      nUR                  SS5      R                  X#U R                  5      nU R                  U5      nXQ-   nUnU R                  U5      nU R                  U5      nU R                  U5      nU R                  U5      nXQ-   nU$ )NrH      r   r>   r   rq   g      ?g      ?r?   )shaper   r   reshaper   r   permute	transposer   softmaxr   r   r   r   r`   r   )r   r   
batch_sizeseq_len_residualr   qr   vattns              r   r   TransformerBlock.forwardt   s   !"
Q JJqM hhqkkk*q$..$--Pkkq!Q
 a IIj>>17MMJIIj>>17MMJIIj>>17MMJ KKB''C4==#3E,FG}}Tr* H IIj..'==IKK1%%j4??K IIaLL JJqMLLOOOALLOLr   )r`   r   r   r   r   r   r   r   r   r   r   r   r!   r)   s   @r   r~   r~   X   s    -.!5>> *6, ,r   r~   cudar6   
model_typemr   r   high_precision_dtypedevicer`   c                    U S:X  a3  [        X#U5      R                  U5      n[        R                  " XXTS9nXx4$ SU ;   ax  [        R
                  " SU 5      n	U	(       a'  U	R                  S5      (       a  U	R                  S5      OSn[        X#XFS9R                  U5      n[        R                  " XXTS9nXx4$ U S:X  a5  [        US	S
US9R                  U5      n[        R                  " USX%US9nXx4$ [        SU  35      e)aF  Create a model and input data for benchmarking.

Args:
    model_type (str): type of the model to be created
    batch_size (int): batch size of the input data
    device (str): device to run the model on
    high_precision_dtype (torch.dtype): data type of the model
    m, k, n (int): dimensions of the model and input data
linear)r   r   	ln_linearzln_linear_?(\w+)?r>   r6   )r`   transformer_blockr   r   )r   r   r      zUnknown model type: )
r   r   r   randnresearchgrouprK   r~   r\   )
r   r   r   r   r   r   r`   model
input_datamatchs
             r   create_model_and_input_datar      s   $ Xq%9:==fE[[fQ
$ # 

	"		.
;',QU[[^V
'&

"V* 	 [[fQ
  
*	* a/C

"V* 	 [[B@TU
  /
|<==r   c                      ^  \ rS rSrS\S\S\S\R                  S\R                  SS4U 4S	 jjrS
\R                  S\R                  4S jr
SrU =r$ )LlamaModelsLlama4Experts   num_local_expertsr@   r   r   r   r-   Nc           
      `  > [         TU ]  5         Xl        X l        [        R
                  " [        R                  " UUUUUS95      U l        [        R
                  " [        R                  " UUUUUS95      U l	        [        R
                  " [        R                  " UUUUUS95      U l
        g )N)r   r   )r   r   r   r@   r   rl   r   r   w1w2w3)r   r   r@   r   r   r   r   s         r   r   !LlamaModelsLlama4Experts.__init__   s     	!2 "KK!!
 !#KK!!
 !#KK!!
r   routed_in_egDc                 j   U R                   nU R                  nUR                  USU5      n[        R                  " [
        R                  " X@R                  5      5      [
        R                  " X@R                  5      -  n[
        R                  " XPR                  5      nUR                  SU5      nU$ )Nrq   )
r   r@   viewFrR   r   bmmr   r   r   )r   r   eDx_egDmiddle_out_egFout_egDs          r   r    LlamaModelsLlama4Experts.forward   s     ""HH""1b!,		% 9:UYYugg=VV))NGG4,,r1%r   )r@   r   r   r   r   )r"   r#   r$   r%   r|   r   r   r   r   rI   r   r'   r(   r)   s   @r   r   r      so    )
)
 )
 	)

 {{)
 )
 
)
V|| 
 r   r   )r   r   torch.nnr   torch.nn.functional
functionalr   Moduler   r+   rK   rh   r~   r&   strr|   r   r   r    r   r   <module>r      s    
    UXX__ )")) )*"bii "8$bii $Huxx H` ).&&
& & 	&
  ++& & &T9ryy 9r   