
    h<                        S SK JrJr  S SKrS SKrS SKJr  S SKJr  S SK	J
r
Jr  S SKJr  S SKJr   " S S	\R                   5      r " S
 S\5      r " S S\5      r\R(                  SSSS4S\S\S\\   S\\R                      S\S\S\\\\R                            S\\\\R                            S\\R                      4S jjr " S S\R                   5      r " S S\5      rS\\\   \\\\   4   4   S\\\   \\   4   4S jrg)    )OptionalUnionN)spaces)nn)get_flattened_obs_dimis_image_space)
TensorDict)
get_devicec                   l   ^  \ rS rSrSrS
S\R                  S\SS4U 4S jjjr\	S\4S j5       r
S	rU =r$ )BaseFeaturesExtractor   z
Base class that represents a features extractor.

:param observation_space: The observation space of the environment
:param features_dim: Number of features extracted.
observation_spacefeatures_dimreturnNc                 J   > [         TU ]  5         US:  d   eXl        X l        g )Nr   )super__init___observation_space_features_dim)selfr   r   	__class__s      _/home/james-whalen/.local/lib/python3.13/site-packages/stable_baselines3/common/torch_layers.pyr   BaseFeaturesExtractor.__init__   s(    a"3)    c                     U R                   $ )z2The number of features that the extractor outputs.)r   )r   s    r   r   "BaseFeaturesExtractor.features_dim   s     !!!r   )r   r   )r   )__name__
__module____qualname____firstlineno____doc__gymSpaceintr   propertyr   __static_attributes____classcell__r   s   @r   r   r      sF    *#)) *3 *t * * "c " "r   r   c                      ^  \ rS rSrSrS\R                  SS4U 4S jjrS\R                  S\R                  4S jr
S	rU =r$ )
FlattenExtractor!   z
Feature extract that flatten the input.
Used as a placeholder when feature extraction is not needed.

:param observation_space: The observation space of the environment
r   r   Nc                 l   > [         TU ]  U[        U5      5        [        R                  " 5       U l        g N)r   r   r   r   Flattenflatten)r   r   r   s     r   r   FlattenExtractor.__init__)   s'    *,ABS,TUzz|r   observationsc                 $    U R                  U5      $ r-   r/   r   r1   s     r   forwardFlattenExtractor.forward-   s    ||L))r   r3   )r   r   r   r    r!   r"   r#   r   thTensorr5   r&   r'   r(   s   @r   r*   r*   !   s?    $#)) $ $*BII *")) * *r   r*   c            	          ^  \ rS rSrSr  SS\R                  S\S\SS4U 4S jjjr	S	\
R                  S\
R                  4S
 jrSrU =r$ )	NatureCNN1   a  
CNN from DQN Nature paper:
    Mnih, Volodymyr, et al.
    "Human-level control through deep reinforcement learning."
    Nature 518.7540 (2015): 529-533.

:param observation_space: The observation space of the environment
:param features_dim: Number of features extracted.
    This corresponds to the number of unit for the last layer.
:param normalized_image: Whether to assume that the image is already normalized
    or not (this disables dtype and bounds checks): when True, it only checks that
    the space is a Box and has 3 dimensions.
    Otherwise, it checks that it has expected dtype (uint8) and bounds (values in [0, 255]).
r   r   normalized_imager   Nc                   > [        U[        R                  5      (       d   SSU 345       e[        TU ]  X5        [        USUS9(       d   SU S35       eUR                  S   n[        R                  " [        R                  " USS	S
SS9[        R                  " 5       [        R                  " SSS
SSS9[        R                  " 5       [        R                  " SSSSSS9[        R                  " 5       [        R                  " 5       5      U l        [        R                  " 5          U R                  [        R                  " UR!                  5       S    5      R#                  5       5      R                  S   nS S S 5        [        R                  " [        R$                  " WU5      [        R                  " 5       5      U l        g ! , (       d  f       NS= f)Nz-NatureCNN must be used with a gym.spaces.Box zobservation space, not F)check_channelsr<   z3You should use NatureCNN only with images not with a  
(you are probably using `CnnPolicy` instead of `MlpPolicy` or `MultiInputPolicy`)
If you are using a custom environment,
please check it using our env checker:
https://stable-baselines3.readthedocs.io/en/master/common/env_checker.html.
If you are using `VecNormalize` or already normalized channel-first images you should pass `normalize_images=False`: 
https://stable-baselines3.readthedocs.io/en/master/guide/custom_env.htmlr             )kernel_sizestridepadding@            )
isinstancer   Boxr   r   r   shaper   
SequentialConv2dReLUr.   cnnr7   no_grad	as_tensorsamplefloatLinearlinear)r   r   r   r<   n_input_channels	n_flattenr   s         r   r   NatureCNN.__init__A   sx    +VZZ88 	
;%&7%89;
 	
8 	*9 /Xhi 
	
)):(; <WW
	
i -2215==II&!QOGGIIIb"!AqAGGIIIb"!AqAGGIJJL
 ZZ\.?.F.F.H.N!O!U!U!WX^^_`aI  mmBIIi$F	R \s   *AG		
Gr1   c                 B    U R                  U R                  U5      5      $ r-   )rU   rO   r4   s     r   r5   NatureCNN.forwardj   s    {{488L122r   )rO   rU   )i   F)r   r   r   r    r!   r"   r#   r$   boolr   r7   r8   r5   r&   r'   r(   s   @r   r:   r:   1   sk    $  !&	'S99'S 'S 	'S
 
'S 'SR3BII 3")) 3 3r   r:   FT	input_dim
output_dimnet_archactivation_fnsquash_output	with_biaspre_linear_modulespost_linear_modulesr   c           	         U=(       d    / nU=(       d    / n/ n[        U5      S:  a  U H  n	UR                  U	" U 5      5        M     UR                  [        R                  " XS   US95        U H  n	UR                  U	" US   5      5        M     UR                  U" 5       5        [	        [        U5      S-
  5       H  n
U H  n	UR                  U	" X*   5      5        M     UR                  [        R                  " X*   X*S-      US95        U H  n	UR                  U	" X*S-      5      5        M!     UR                  U" 5       5        M     US:  aZ  [        U5      S:  a  US   OU nU H  n	UR                  U	" U5      5        M     UR                  [        R                  " XUS95        U(       a$  UR                  [        R
                  " 5       5        U$ )a`  
Create a multi layer perceptron (MLP), which is
a collection of fully-connected layers each followed by an activation function.

:param input_dim: Dimension of the input vector
:param output_dim: Dimension of the output (last layer, for instance, the number of actions)
:param net_arch: Architecture of the neural net
    It represents the number of units per layer.
    The length of this list is the number of layers.
:param activation_fn: The activation function
    to use after each layer.
:param squash_output: Whether to squash the output using a Tanh
    activation function
:param with_bias: If set to False, the layers will not learn an additive bias
:param pre_linear_modules: List of nn.Module to add before the linear layers.
    These modules should maintain the input tensor dimension (e.g. BatchNorm).
    The number of input features is passed to the module's constructor.
    Compared to post_linear_modules, they are used before the output layer (output_dim > 0).
:param post_linear_modules: List of nn.Module to add after the linear layers
    (and before the activation function). These modules should maintain the input
    tensor dimension (e.g. Dropout, LayerNorm). They are not used after the
    output layer (output_dim > 0). The number of input features is passed to
    the module's constructor.
:return: The list of layers of the neural network
r   )biasrH   )lenappendr   rT   rangeTanh)r\   r]   r^   r_   r`   ra   rb   rc   modulesmoduleidxlast_layer_dims               r   
create_mlpro   n   s   H ,1r-3G
8}q(FNN6),- ) 	ryyQKiHI *FNN6(1+./ * 	}'S]Q&'(FNN6(-01 ) 	ryyq0A	RS)FNN6(7"345 * 	}' ( A~),X):"	(FNN6.12 ) 	ryy)LMrwwy!Nr   c                   n  ^  \ rS rSrSr SS\S\\\   \\	\\   4   4   S\
\R                     S\\R                  \	4   SS4
U 4S	 jjjrS
\R                   S\\R                   \R                   4   4S jrS
\R                   S\R                   4S jrS
\R                   S\R                   4S jrSrU =r$ )MlpExtractor   a  
Constructs an MLP that receives the output from a previous features extractor (i.e. a CNN) or directly
the observations (if no features extractor is applied) as an input and outputs a latent representation
for the policy and a value network.

The ``net_arch`` parameter allows to specify the amount and size of the hidden layers.
It can be in either of the following forms:
1. ``dict(vf=[<list of layer sizes>], pi=[<list of layer sizes>])``: to specify the amount and size of the layers in the
    policy and value nets individually. If it is missing any of the keys (pi or vf),
    zero layers will be considered for that key.
2. ``[<list of layer sizes>]``: "shortcut" in case the amount and size of the layers
    in the policy and value nets are the same. Same as ``dict(vf=int_list, pi=int_list)``
    where int_list is the same for the actor and critic.

.. note::
    If a key is not specified or an empty list is passed ``[]``, a linear network will be used.

:param feature_dim: Dimension of the feature vector (can be the output of a CNN)
:param net_arch: The specification of the policy and value networks.
    See above for details on its formatting.
:param activation_fn: The activation function to use for the networks.
:param device: PyTorch device.
feature_dimr^   r_   devicer   Nc                   > [         TU ]  5         [        U5      n/ n/ nUnUn[        U[        5      (       a%  UR                  S/ 5      n	UR                  S/ 5      n
OU=pU	 H@  nUR                  [        R                  " X{5      5        UR                  U" 5       5        UnMB     U
 H@  nUR                  [        R                  " X5      5        UR                  U" 5       5        UnMB     Xpl	        Xl
        [        R                  " U6 R                  U5      U l        [        R                  " U6 R                  U5      U l        g )Npivf)r   r   r
   rI   dictgetrh   r   rT   latent_dim_pilatent_dim_vfrL   to
policy_net	value_net)r   rs   r^   r_   rt   r}   r~   last_layer_dim_pilast_layer_dim_vfpi_layers_dimsvf_layers_dimscurr_layer_dimr   s               r   r   MlpExtractor.__init__   s    	F#&(
%'	'' h%%%\\$3N%\\$3N.66N,Nbii(9JKmo. . -
 -NRYY'8IJ]_- . - /. --477?	255f=r   featuresc                 F    U R                  U5      U R                  U5      4$ )z
:return: latent_policy, latent_value of the specified network.
    If all layers are shared, then ``latent_policy == latent_value``
)forward_actorforward_criticr   r   s     r   r5   MlpExtractor.forward   s%    
 !!(+T-@-@-JJJr   c                 $    U R                  U5      $ r-   )r}   r   s     r   r   MlpExtractor.forward_actor  s    x((r   c                 $    U R                  U5      $ r-   )r~   r   s     r   r   MlpExtractor.forward_critic  s    ~~h''r   )rz   r{   r}   r~   )auto)r   r   r   r    r!   r$   r   listrx   strtyper   Moduler7   rt   r   r8   tupler5   r   r   r&   r'   r(   s   @r   rq   rq      s    : )/'>'> S	4T#Y#778'> BII	'>
 biin%'> 
'> '>RK		 KeBIIryy4H.I K)bii )BII )(ryy (RYY ( (r   rq   c            	          ^  \ rS rSrSr  SS\R                  S\S\SS4U 4S jjjr	S	\
S\R                  4S
 jrSrU =r$ )CombinedExtractori
  a  
Combined features extractor for Dict observation spaces.
Builds a features extractor for each key of the space. Input from each space
is fed through a separate submodule (CNN or MLP, depending on input shape),
the output features are concatenated and fed through additional MLP network ("combined").

:param observation_space:
:param cnn_output_dim: Number of features to output from each CNN submodule(s). Defaults to
    256 to avoid exploding network sizes.
:param normalized_image: Whether to assume that the image is already normalized
    or not (this disables dtype and bounds checks): when True, it only checks that
    the space is a Box and has 3 dimensions.
    Otherwise, it checks that it has expected dtype (uint8) and bounds (values in [0, 255]).
r   cnn_output_dimr<   r   Nc                 <  > [         TU ]  USS9  0 nSnUR                  R                  5        HJ  u  pg[	        XsS9(       a  [        XrUS9XF'   XR-  nM%  [        R                  " 5       XF'   U[        U5      -  nML     [        R                  " U5      U l
        XPl        g )NrH   )r   r   )r<   )r   r<   )r   r   r   itemsr   r:   r   r.   r   
ModuleDict
extractorsr   )	r   r   r   r<   r   total_concat_sizekeysubspacer   s	           r   r   CombinedExtractor.__init__  s     	*;+-
.55;;=MChJ"+Hdt"u
!3! #%**,
!%:8%DD! > --
3 /r   r1   c                     / nU R                   R                  5        H  u  p4UR                  U" X   5      5        M      [        R                  " USS9$ )NrH   )dim)r   r   rh   r7   cat)r   r1   encoded_tensor_listr   	extractors        r   r5   CombinedExtractor.forward4  sI     "oo335NC&&y1B'CD 6vv)q11r   )r   r   )   F)r   r   r   r    r!   r   Dictr$   r[   r   r	   r7   r8   r5   r&   r'   r(   s   @r   r   r   
  s`    $ "!&	/!;;/ / 	/
 
/ /42J 2299 2 2r   r   c                     [        U [        5      (       a  X p!X4$ [        U [        5      (       d   S5       eSU ;   d   S5       eSU ;   d   S5       eU S   U S   p!X4$ )a  
Get the actor and critic network architectures for off-policy actor-critic algorithms (SAC, TD3, DDPG).

The ``net_arch`` parameter allows to specify the amount and size of the hidden layers,
which can be different for the actor and the critic.
It is assumed to be a list of ints or a dict.

1. If it is a list, actor and critic networks will have the same architecture.
    The architecture is represented by a list of integers (of arbitrary length (zero allowed))
    each specifying the number of units per layer.
   If the number of ints is zero, the network will be linear.
2. If it is a dict,  it should have the following structure:
   ``dict(qf=[<critic network architecture>], pi=[<actor network architecture>])``.
   where the network architecture is a list as described in 1.

For example, to have actor and critic that share the same network architecture,
you only need to specify ``net_arch=[256, 256]`` (here, two hidden layers of 256 units each).

If you want a different architecture for the actor and the critic,
then you can specify ``net_arch=dict(qf=[400, 300], pi=[64, 64])``.

.. note::
    Compared to their on-policy counterparts, no shared layers (other than the features extractor)
    between the actor and the critic are allowed (to prevent issues with target networks).

:param net_arch: The specification of the actor and critic networks.
    See above for details on its formatting.
:return: The network architectures for the actor and the critic
z@Error: the net_arch can only contain be a list of ints or a dictrv   zAError: no key 'pi' was provided in net_arch for the actor networkqfzBError: no key 'qf' was provided in net_arch for the critic network)rI   r   rx   )r^   
actor_archcritic_archs      r   get_actor_critic_archr   <  s|    < (D!!"*K ""	 (D))m+mm)xd!ddxe!ee"*4.(4.K""r   ) typingr   r   	gymnasiumr"   torchr7   r   r   &stable_baselines3.common.preprocessingr   r   %stable_baselines3.common.type_aliasesr	   stable_baselines3.common.utilsr
   r   r   r*   r:   rN   r$   r   r   r[   ro   rq   r   rx   r   r   r    r   r   <module>r      sb   "     X < 5"BII "(*, * :3% :3B &(WW:>;?III 3iI 		?	I
 I I !d299o!67I "$tBII"78I 
"))_IXM(299 M(`/2- /2d%#E$s)T#tCy.5I*I$J %#uUYZ]U^`deh`iUiOj %#r   