
    6bi                       S r SSKrSSKrSSKrSSKrSSKJr  SSKJs  J	r
  SSKJr  SSKJr  SSKJr  SSKJr  SSKJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJr  \" \
R:                  \
R<                  \
R>                  \
R@                  \
RB                  \
RD                  /5      r#S r$ " S S5      r%S r&\" SSS/S9 " S S\
RN                  RP                  RR                  5      5       r*S r+S r, " S S\*5      r- \
RN                  R\                  R^                  Ra                  SS \
RN                  R\                  R^                  Rc                  S SSS\-Rd                  S 9/S!9  g! \3 an    \
RN                  R\                  R^                  Ra                  S"S# \
RN                  R\                  R^                  Rc                  S$ SSS\-Rd                  S 9/S!9   gf = f)%zVersion 2 of class Optimizer.    N)deepcopy)backend)initializers)base_layer_utils)utils)learning_rate_schedule)generic_utils)layer_utils)
tf_inspect)tf_utils)keras_exportc                     [         R                  " U5      u  p#[         R                  R                  X[         R                  " U5      S   5      nXB4$ )a  Sums `values` associated with any non-unique `indices`.

Args:
  values: A `Tensor` with rank >= 1.
  indices: A one-dimensional integer `Tensor`, indexing into the first
    dimension of `values` (as in an IndexedSlices object).

Returns:
  A tuple of (`summed_values`, `unique_indices`) where `unique_indices` is a
  de-duplicated version of `indices` and `summed_values` contains the sum of
  `values` slices associated with each unique index.
r   )tfuniquemathunsorted_segment_sumshape)valuesindicesunique_indicesnew_index_positionssummed_valuess        e/home/james-whalen/.local/lib/python3.13/site-packages/tf_keras/src/optimizers/legacy/optimizer_v2.py_deduplicate_indexed_slicesr   3   sG     +-))G*<'NGG00RXXn%=a%@M **    c                   &    \ rS rSrS rS rS rSrg)NullContextmanagerG   c                     g N )selfargskwargss      r   __init__NullContextmanager.__init__H       r   c                     g r    r!   r"   s    r   	__enter__NullContextmanager.__enter__K   r'   r   c                     g)NFr!   )r"   type_arg	value_argtraceback_args       r   __exit__NullContextmanager.__exit__N   s    r   r!   N)__name__
__module____qualname____firstlineno__r%   r*   r0   __static_attributes__r!   r   r   r   r   G   s    r   r   c                 v    [         R                  " 5       (       d  [         R                  " U 5      $ [        5       $ )zInternal-only entry point for `name_scope*`.

Enters a compat.v1.name_scope only when in a function or graph,
not when running fully eagerly.

Args:
  name: The name argument that is passed to the op function.

Returns:
  `name_scope*` context manager.
)r   executing_eagerly
name_scoper   names    r   $name_scope_only_in_function_or_graphr<   R   s+     !!}}T""!##r   z!keras.optimizers.legacy.Optimizerzkeras.optimizers.Optimizer)v1c                   `  ^  \ rS rSrSrSr  SBS jrS r\S 5       r	\S 5       r
\	R                  S	 5       r	\
R                  S
 5       r
\S 5       r\R                  S 5       rS rSCS jrS rS rS rSDS jrSBS jr SES jrS rS rS rS rSCS jrS rS rS rU 4S jrU 4S jrU 4S jrS  r SFS" jr!S# r"S$ r#S% r$S& r%S' r&\S( 5       r'\'R                  S) 5       r'S* r(\)RT                  S+ 5       r+\,SCS, j5       r-S- r.S. r/\S/ 5       r0S0 r1S1 r2SS!S\3Rh                  Rj                  \3Rl                  Rn                  4S2 jr8SGS3 jr9S4 r:S5 r;S6 r<S7 r=S8 r>S9 r?S: r@S; rA\\BR                  S< 5       5       rD\\BR                  S= 5       5       rES> rFS? rG\HR                  S@ 5       rJSArKU =rL$ )HOptimizerV2d   a  Base class for legacy TF-Keras optimizers.

You should not use this class directly, but instead instantiate one of its
subclasses such as `tf.keras.optimizers.legacy.SGD`,
`tf.keras.optimizers.legacy.Adam`, etc.

This is the default TF-Keras optimizer base class until v2.10 (included).
In v2.11 and later, `tf.keras.optimizers.Optimizer`
points to a new base class implementation. The legacy class won't be
deleted in the future and will continue to be available at
`tf.keras.optimizers.legacy.Optimizer`.

### Usage

```python
# Create an optimizer with the desired parameters.
opt = tf.keras.optimizers.legacy.SGD(learning_rate=0.1)
# `loss` is a callable that takes no argument and returns the value
# to minimize.
var1 = tf.Variable(2.0)
var2 = tf.Variable(5.0)
loss = lambda: 3 * var1 * var1 + 2 * var2 * var2
# In graph mode, returns op that minimizes the loss by updating the listed
# variables.
opt_op = opt.minimize(loss, var_list=[var1, var2])
opt_op.run()
# In eager mode, simply call minimize to update the list of variables.
opt.minimize(loss, var_list=[var1, var2])
```

### Usage in custom training loops

In TF-Keras models, sometimes variables are created when the model is first
called, instead of construction time. Examples include 1) sequential models
without input shape pre-defined, or 2) subclassed models. Pass var_list as
callable in these cases.

Example:

```python
opt = tf.keras.optimizers.legacy.SGD(learning_rate=0.1)
model = tf.keras.Sequential()
model.add(tf.keras.layers.Dense(num_hidden, activation='relu'))
model.add(tf.keras.layers.Dense(num_classes, activation='sigmoid'))
loss_fn = lambda: tf.keras.losses.mse(model(input), output)
var_list_fn = lambda: model.trainable_weights
for input, output in data:
  opt.minimize(loss_fn, var_list_fn)
```

### Processing gradients before applying them

Calling `minimize()` takes care of both computing the gradients and
applying them to the variables.  If you want to process the gradients
before applying them you can instead use the optimizer in three steps:

1.  Compute the gradients with `tf.GradientTape`.
2.  Process the gradients as you wish.
3.  Apply the processed gradients with `apply_gradients()`.

Example:

```python
# Create an optimizer.
opt = tf.keras.optimizers.legacy.SGD(learning_rate=0.1)

# Compute the gradients for a list of variables.
with tf.GradientTape() as tape:
  loss = <call_loss_function>
vars = <list_of_variables>
grads = tape.gradient(loss, vars)

# Process the gradients, for example cap them, etc.
# capped_grads = [MyCapper(g) for g in grads]
processed_grads = [process_gradient(g) for g in grads]

# Ask the optimizer to apply the processed gradients.
opt.apply_gradients(zip(processed_grads, var_list))
```

### Use with `tf.distribute.Strategy`

This optimizer class is `tf.distribute.Strategy` aware, which means it
automatically sums gradients across all replicas. To average gradients,
you divide your loss by the global batch size, which is done
automatically if you use `tf.keras` built-in training or evaluation loops.
See the `reduction` argument of your loss which should be set to
`tf.keras.losses.Reduction.SUM_OVER_BATCH_SIZE` for averaging or
`tf.keras.losses.Reduction.SUM` for not.

To aggregate gradients yourself, call `apply_gradients` with
`experimental_aggregate_gradients` set to False. This is useful if you need
to process aggregated gradients.

If you are not using these and you want to average gradients, you should use
`tf.math.reduce_sum` to add up your per-example losses and then divide by
the global batch size. Note that when using `tf.distribute.Strategy`, the
first component of a tensor's shape is the *replica-local* batch size, which
is off by a factor equal to the number of replicas being used to compute a
single step. As a result, using `tf.math.reduce_mean` will give the wrong
answer, resulting in gradients that can be many times too big.

### Variable Constraints

All TF-Keras optimizers respect variable constraints. If constraint function
is passed to any variable, the constraint will be applied to the variable
after the gradient has been applied to the variable.
Important: If gradient is sparse tensor, variable constraint is not
supported.

### Thread Compatibility

The entire optimizer is currently thread compatible, not thread-safe. The
user needs to perform synchronization if necessary.

### Slots

Many optimizer subclasses, such as `Adam` and `Adagrad` allocate and manage
additional variables associated with the variables to train.  These are
called <i>Slots</i>.  Slots have names and you can ask the optimizer for the
names of the slots that it uses.  Once you have a slot name you can ask the
optimizer for the variable it created to hold the slot value.

This can be useful if you want to log debug a training algorithm, report
stats about the slots, etc.

### Hyperparameters

These are arguments passed to the optimizer subclass constructor
(the `__init__` method), and then passed to `self._set_hyper()`.
They can be either regular Python values (like 1.0), tensors, or
callables. If they are callable, the callable will be called during
`apply_gradients()` to get the value for the hyper parameter.

Hyperparameters can be overwritten through user code:

Example:

```python
# Create an optimizer with the desired parameters.
opt = tf.keras.optimizers.legacy.SGD(learning_rate=0.1)
# `loss` is a callable that takes no argument and returns the value
# to minimize.
loss = lambda: 3 * var1 + 2 * var2
# In eager mode, simply call minimize to update the list of variables.
opt.minimize(loss, var_list=[var1, var2])
# update learning rate
opt.learning_rate = 0.05
opt.minimize(loss, var_list=[var1, var2])
```

### Callable learning rate

Optimizer accepts a callable learning rate in two ways. The first way is
through built-in or customized
`tf.keras.optimizers.schedules.LearningRateSchedule`. The schedule will be
called on each iteration with `schedule(iteration)`, a `tf.Variable`
owned by the optimizer.

Example:

>>> var = tf.Variable(np.random.random(size=(1,)))
>>> learning_rate = tf.keras.optimizers.schedules.ExponentialDecay(
... initial_learning_rate=.01, decay_steps=20, decay_rate=.1)
>>> opt = tf.keras.optimizers.legacy.SGD(learning_rate=learning_rate)
>>> loss = lambda: 3 * var
>>> opt.minimize(loss, var_list=[var])
<tf.Variable...

The second way is through a callable function that
does not accept any arguments.

Example:

>>> var = tf.Variable(np.random.random(size=(1,)))
>>> def lr_callable():
...   return .1
>>> opt = tf.keras.optimizers.legacy.SGD(learning_rate=lr_callable)
>>> loss = lambda: 3 * var
>>> opt.minimize(loss, var_list=[var])
<tf.Variable...

### Creating a custom optimizer

If you intend to create your own optimization algorithm, simply inherit from
this class and override the following methods:

  - `_resource_apply_dense` (update variable given gradient tensor is a
    dense `tf.Tensor`)
  - `_resource_apply_sparse` (update variable given gradient tensor is a
    sparse `tf.IndexedSlices`. The most common way for this to happen
    is if you are taking the gradient through a `tf.gather`.)
  - `_create_slots`
    (if your optimizer algorithm requires additional variables)
  - `get_config`
    (serialization of the optimizer, include all hyper parameters)
FNc                    1 SknU H`  nXe;  a  [        S[        U5       SU S35      eXF   b  XF   S:  a  [        SU SXF    35      eUS	:X  d  MK  [        R                  " S
SS9  Mb     SU l        U R                  U5        0 U l        0 U l        / U l	        / U l
        SU l        0 U l        UR                  SS5      nUS:  a  [        SU S35      eXpl        SU l        [         R"                  R%                  5       (       a$  [         R"                  R'                  5       U l        OSU l        Uc  [*        R,                  nX l        Uc  / nX0l        UR                  SS5      U l        UR                  SS5      U l        U R2                  b<  U R4                  b/  [        SR7                  U R2                  U R4                  5      5      eUR                  SS5      U l        g)a  Create a new Optimizer.

This must be called by the constructors of subclasses.
Note that Optimizer instances should not bind to a single graph,
and so shouldn't keep Tensors as member variables. Generally
you should be able to use the _set_hyper()/state.get_hyper()
facility instead.

This class is stateful and thread-compatible.

Example of custom gradient transformations:

```python
def my_gradient_transformer(grads_and_vars):
  # Simple example, double the gradients.
  return [(2. * g, v) for g, v in grads_and_vars]

optimizer = tf.keras.optimizers.legacy.SGD(
    1e-3, gradient_transformers=[my_gradient_transformer])
```

Args:
  name: String. The name to use for momentum accumulator weights created
    by the optimizer.
  gradient_aggregator: The function to use to aggregate gradients across
    devices (when using `tf.distribute.Strategy`). If `None`, defaults
    to summing the gradients across devices. The function should accept
    and return a list of `(gradient, variable)` tuples.
  gradient_transformers: Optional. List of functions to use to transform
    gradients before applying updates to Variables. The functions are
    applied after `gradient_aggregator`. The functions should accept and
    return a list of `(gradient, variable)` tuples.
  **kwargs: keyword arguments. Allowed arguments are `clipvalue`,
    `clipnorm`, `global_clipnorm`.
    If `clipvalue` (float) is set, the gradient of each weight
    is clipped to be no higher than this value.
    If `clipnorm` (float) is set, the gradient of each weight
    is individually clipped so that its norm is no higher than this
    value. If `global_clipnorm` (float) is set the gradient of all
    weights is clipped so that their global norm is no higher than this
    value.

Raises:
  ValueError: in case of any invalid argument.
>   lrdecayclipnorm	clipvalueglobal_clipnormz1Unexpected keyword argument passed to optimizer: z. Allowed kwargs are .Nr   z	Expected z >= 0, received: rB   z=The `lr` argument is deprecated, use `learning_rate` instead.   )
stacklevelTrC           z-decay cannot be less than 0. Received: decay=FrD   rF   zcCannot accept both `clipnorm` and `global_clipnorm`. Received: `clipnorm`={}, `global_clipnorm`={}.rE   )	TypeErrorstr
ValueErrorwarningswarn_use_locking_init_set_name_hyper_slots_slot_names_weights_iterations_deferred_slot_restorationspop_initial_decay_hypers_createdr   
distributehas_strategyget_strategy_distribution_strategyoptimizer_utilsall_reduce_sum_gradientsgradient_aggregatorgradient_transformersrD   rF   formatrE   )r"   r;   ra   rb   r$   allowed_kwargskrC   s           r   r%   OptimizerV2.__init__8  s   h
 A&,,/F83H%&a)  y$Q 9QC/@!LMMDy3  " !D! ,.(

7C(3;?waH  $$ ==%%''*,--*D*D*FD'*.D' &"1"J"J#6  ($&!%:"

:t4%zz*;TB==$)=)=)IAAGMM4#7#7B   K6r   c           	          U R                   nUR                  U5      nX1[        U 5      '   U R                  R	                  5        H"  u  pEUS:X  a  M  [        X4[        XQ5      5        M$     U R                  Ul        U$ )Nr^   )	__class____new__id__dict__itemssetattrr   r^   )r"   memoclsresultre   vs         r   __deepcopy__OptimizerV2.__deepcopy__  sm    nnS!RXMM'')DA,,Fx01	 *
 )-(C(C%r   c                     U R                   $ )z=`float` or `None`. If set, clips gradients to a maximum norm.)	_clipnormr)   s    r   rD   OptimizerV2.clipnorm  s     ~~r   c                     U R                   $ )zq`float` or `None`.

If set, clips gradients to a maximum norm.

Check `tf.clip_by_global_norm` for more details.
)_global_clipnormr)   s    r   rF   OptimizerV2.global_clipnorm  s     $$$r   c                     Ub-  U R                   (       a  [        SU SU R                    S35      eXl        [        R                  " U R                  5      U l        g )Nz`clipnorm` cannot be set when `gradient_transformers` is set. Instead, use the `gradient_transformers` to specify clipping and other transformations. Received: val=, gradient_transformers=rG   )rb   rM   ru   r_   make_gradient_clipnorm_fn_clipnorm_fnr"   vals     r   rD   rv     sb    ?t99 e ))-)C)C(DA	G  +EENN
r   c                     Ub-  U R                   (       a  [        SU SU R                    S35      eXl        [        R                  " U R                  5      U l        g )Nz`global_clipnorm` cannot be set when `gradient_transformers` is set. Instead, use the `gradient_transformers` to specify clipping and other transformations. Received: val=r{   rG   )rb   rM   rx   r_    make_global_gradient_clipnorm_fn_global_clipnorm_fnr~   s     r   rF   ry     sh    ?t99 e ))-)C)C(DAG  !$<<%% 	 r   c                     U R                   $ )z>`float` or `None`. If set, clips gradients to a maximum value.)
_clipvaluer)   s    r   rE   OptimizerV2.clipvalue  s     r   c                     Ub-  U R                   (       a  [        SU SU R                    S35      eXl        [        R                  " U R                  5      U l        g )Nz`clipvalue` cannot be set when `gradient_transformers` is set. Instead, use the `gradient_transformers` to specify clipping and other transformations. Received: val=r{   rG   )rb   rM   r   r_   make_gradient_clipvalue_fn_clipvalue_fnr~   s     r   rE   r     sb    ?t99 e ))-)C)C(DA	G  ,GGOO
r   c                     U$ )zCCalled in `.minimize` to transform loss before computing
gradients.r!   )r"   losss     r   _transform_lossOptimizerV2._transform_loss  s	     r   c                 N    UR                  X#U5      n[        [        XS5      5      $ )z4Called in `minimize` to compute gradients from loss.)gradientlistzip)r"   taper   var_list	grad_lossgradss         r   _get_gradientsOptimizerV2._get_gradients  s"    di8C())r   c                     U$ )z8Called in `apply_gradients` before gradient aggregation.r!   r"   grads_and_varss     r   !_transform_unaggregated_gradients-OptimizerV2._transform_unaggregated_gradients  s    r   c                 $    U R                  U5      $ )aI  Called in `apply_gradients` to aggregate gradients across devices.

Note that user subclasses may override this, so the interface should not
be changed.

Args:
  grads_and_vars: List of (gradient, variable) pairs.

Returns:
  A list of (aggregrated_gradient, variable) pairs. By default, this
  calls `self.gradient_aggregator`.
)ra   r   s     r   _aggregate_gradients OptimizerV2._aggregate_gradients  s     ''77r   c                     U R                   b  U R                  U5      nU R                  b  U R                  U5      nU R                  b  U R                  U5      nU R                   H  nU" U5      nM     U$ )z.Called in `apply_gradients` after aggregation.)r   r   ru   r}   rx   r   rb   )r"   r   fns      r   _transform_gradients OptimizerV2._transform_gradients#  sr    ??&!//?N>>%!..~>N  ,!55nEN,,B/N -r   c                 @    U R                  XX5S9nU R                  XdS9$ )a  Minimize `loss` by updating `var_list`.

This method simply computes gradient using `tf.GradientTape` and calls
`apply_gradients()`. If you want to process the gradient before applying
then call `tf.GradientTape` and `apply_gradients()` explicitly instead
of using this function.

Args:
  loss: `Tensor` or callable. If a callable, `loss` should take no
    arguments and return the value to minimize. If a `Tensor`, the
    `tape` argument must be passed.
  var_list: list or tuple of `Variable` objects to update to minimize
    `loss`, or a callable returning the list or tuple of `Variable`
    objects.  Use callable when the variable list would otherwise be
    incomplete before `minimize` since the variables are created at the
    first time `loss` is called.
  grad_loss: (Optional). A `Tensor` holding the gradient computed for
    `loss`.
  name: (Optional) str. Name for the returned operation.
  tape: (Optional) `tf.GradientTape`. If `loss` is provided as a
    `Tensor`, the tape that computed the `loss` must be provided.

Returns:
  An `Operation` that updates the variables in `var_list`. The
  `iterations` will be automatically increased by 1.

Raises:
  ValueError: If some of the variables are not `Variable` objects.

)r   r   r   r:   )_compute_gradientsapply_gradients)r"   r   r   r   r;   r   r   s          r   minimizeOptimizerV2.minimize0  s4    > 00y 1 
 ##N#>>r   c                 2   [        U5      (       d  Uc  [        SU SU S35      eUb  UO[        R                  " 5       n[        U5      (       aJ  U   [        U5      (       d  UR	                  U5        U" 5       n[        U5      (       a  U" 5       nSSS5        U   U R                  U5      nSSS5        [        R                  R                  U5      n[        R                  " U R                  S-   5         U R                  XAX#5      nSSS5        U R                  W VVs/ s H,  u  pgUc  M
  UR                  [        R                  :w  d  M*  UPM.     snn5        U$ ! , (       d  f       N= f! , (       d  f       N= f! , (       d  f       N}= fs  snnf )a  Compute gradients of `loss` for the variables in `var_list`.

This is the first part of `minimize()`.  It returns a list
of (gradient, variable) pairs where "gradient" is the gradient
for "variable".  Note that "gradient" can be a `Tensor`, an
`IndexedSlices`, or `None` if there is no gradient for the
given variable.

Args:
  loss: `Tensor` or callable. If a callable, `loss` should take no
    arguments and return the value to minimize. If a `Tensor`, the
    `tape` argument must be passed.
  var_list: list or tuple of `Variable` objects to update to minimize
    `loss`, or a callable returning the list or tuple of `Variable`
    objects.  Use callable when the variable list would otherwise be
    incomplete before `minimize` and the variables are created at the
    first time when `loss` is called.
  grad_loss: Optional. A `Tensor` holding the gradient computed for
    `loss`.
  tape: (Optional) `tf.GradientTape`. If `loss` is provided as a
    `Tensor`, the tape that computed the `loss` must be provided.

Returns:
  A list of (gradient, variable) pairs. Variable is always present, but
  gradient can be `None`.

Raises:
  TypeError: If `var_list` contains anything else than `Variable`
    objects.
  ValueError: If some arguments are invalid, or var_list is None.
NzB`tape` is required when a `Tensor` loss is passed. Received: loss=z, tape=rG   
/gradients)callablerM   r   GradientTapewatchr   nestflattenr9   _namer   _assert_valid_dtypesdtyperesource)r"   r   r   r   r   r   grq   s           r   r   OptimizerV2._compute_gradientsT  sU   B ~~$,""&wtfA7  'tR__->D>>))JJx(vH%%'zH  ''-D  77??8,]]4::45!00HN 6
 	!! +*DA %&WW%; *	
 1  T 65s7   A E E18F#	F
0F
F
 
E.1
E?
Fc                 <   [         R                  " U5      nU VVs/ s H  u  pEUPM	     nnn[        R                  " U R                  5         [        R
                  " 5          U R                  U5        SSS5        U(       d  [        R                  " 5       sSSS5        $ [        R                  R                  5       (       a  [        S5      e[        R                  R                  5       nU(       d  U(       a  [        U[        R                  R                  R                  R                  R                   [        R                  R                  R                   [        R                  R                  R"                  [        R                  R                  R                  R                  R"                  45      (       a  [%        SU S35      eU R'                  U5      nU(       a"  U R)                  U5      nU R+                  U5      nU R-                  U5      n[        R.                  R                  R0                  R3                  [4        R6                  " U R8                  US9UUUS9sSSS5        $ s  snnf ! , (       d  f       GN= f! , (       d  f       g= f)a  Apply gradients to variables.

This is the second part of `minimize()`. It returns an `Operation` that
applies gradients.

The method sums gradients from all replicas in the presence of
`tf.distribute.Strategy` by default. You can aggregate gradients
yourself by passing `experimental_aggregate_gradients=False`.

Example:

```python
grads = tape.gradient(loss, vars)
grads = tf.distribute.get_replica_context().all_reduce('sum', grads)
# Processing aggregated gradients.
optimizer.apply_gradients(zip(grads, vars),
    experimental_aggregate_gradients=False)

```

Args:
  grads_and_vars: List of (gradient, variable) pairs.
  name: Optional name for the returned operation. When `None`, uses the
    name passed to the `Optimizer` constructor. Defaults to `None`.
  experimental_aggregate_gradients: Whether to sum gradients from
    different replicas in the presence of `tf.distribute.Strategy`. If
    False, it's user responsibility to aggregate the gradients. Default
    to `True`.

Returns:
  An `Operation` that applies the specified gradients. The `iterations`
  will be automatically increased by 1.

Raises:
  TypeError: If `grads_and_vars` is malformed.
  ValueError: If none of the variables have gradients.
  RuntimeError: If called in a cross-replica context.
Nz`apply_gradients() cannot be called in cross-replica context. Use `tf.distribute.Strategy.run` to enter replica context. For more information, please see the docstring of `tf.distribute.get_replica_context`.z`experimental_aggregate_gradients=False is not supported for ParameterServerStrategy and CentralStorageStrategy. Used: strategy=rG   )apply_stater:   )r_   filter_empty_gradientsr   r9   r   
init_scope_create_all_weightsno_opr[   in_cross_replica_contextRuntimeErrorr]   
isinstancecompatr=   experimentalParameterServerStrategyCentralStorageStrategyNotImplementedError_preparer   r   r   __internal__interimmaybe_merge_call	functoolspartial_distributed_apply)	r"   r   r;    experimental_aggregate_gradients_rq   r   strategyr   s	            r   r   OptimizerV2.apply_gradients  s   R )??O$23N&1AN3]]4::&((2 ! " xxz '& }}5577"H  }}113H4		//<<TT22JJ22II		//<<SS	  *&&.Zq2  --1K/!%!G!G"" "&!:!:>!J!66~FN??--55FF!!++  G ] '& 4 ! '&s0   I5J#I;5#J"G	J;
J
	J
Jc                   ^ ^ UU 4S jn[         R                  R                  R                  5       n/ n[	        U=(       d    T R
                  5         U H  u  pUR                  R                  U	5         [	        U(       a  SOSU	R                  R                  -   5         UR                  R                  U	UU4SS9n
[         R                  R                  5       (       a  UR                  U
5        OUR                  U
5        SSS5        SSS5        M     [        S U 5       5      n[         R                   " 5       (       a  U(       a  ["        R$                  " U5      R'                  5          [         R(                  " [         R*                  " U5      /5         T R,                  R/                  SSS	9sSSS5        sSSS5        sSSS5        $ T R,                  R/                  S5      sSSS5        $ ! , (       d  f       N= f! , (       d  f       GM  = f! , (       d  f       O= f SSS5        Md  ! , (       d  f       Nr= f! , (       d  f       g= f)
z1`apply_gradients` using a `DistributionStrategy`.c                   > [        U [        R                  5      (       a  [        SU  S35      e0 n[        U[        R                  5      (       af  U R
                  b  [        SU SU R
                   S35      eSTR                  ;   a  TUS'   TR                  " UR                  XR                  40 UD6$ STR                  ;   a  TUS'   TR                  " X40 UD6nU R
                  bA  [        R                  " U/5         U R                  U R                  U 5      5      sSSS5        $ U$ ! , (       d  f       g= f)zApply gradient to variable.z6Updating a `Tensor` is not implemented. Received: var=rG   NzFCannot use a constraint function on a sparse variable. Received: grad=z, var.constraint=r   )r   r   Tensorr   IndexedSlices
constraintr   _sparse_apply_args(_resource_apply_sparse_duplicate_indicesr   r   _dense_apply_args_resource_apply_densecontrol_dependenciesassign)vargradapply_kwargs	update_opr   r"   s       r   apply_grad_to_update_var@OptimizerV2._distributed_apply.<locals>.apply_grad_to_update_var  s<   #ryy)))%%(E, 
 L$ 0 011>>-&4486 :**-..)9< 
 !D$;$;;2=L/DDKKll6B   6 66.9]+224MMI~~),,i[9::cnnS&9: :9 !  :9s    D//
D=updateupdate_F)r#   groupNc              3      #    U  H<  n[        U[        R                  5      =(       d    [        R                  " U5      v   M>     g 7fr    )r   r   	Operationr   is_symbolic_tensor).0is     r   	<genexpr>1OptimizerV2._distributed_apply.<locals>.<genexpr><  s5      #A 1bll+Mx/J/J1/MM#s   AA   )
read_value)r   r   r=   #executing_eagerly_outside_functionsr<   r   extendedcolocate_vars_withopr;   r   r[   r   extendappendanyr8   r   _current_graph
as_defaultr   r   
iterations
assign_add)r"   distributionr   r   r;   r   eagerly_outside_functions
update_opsr   r   r   any_symbolics   `  `        r   r   OptimizerV2._distributed_apply  s   
	!@ IILL<<> 	" 
1$2D$**E+	 "**==cB=4 !&4
 %1$9$9$@$@4"&"'	 %A %	 ==AACC '--i8 '--i8' CB ,2  # L ''))\ ++J7BBD00"((:2F1GH#99!9N IH EDE FEL ??--a0M FE
  CB> IHH EDDE FEsz   #I5,H!A$G6	HA#I0,H6H6	H6?	II6
H H
H	I
H($H6,
I6
I	 I
Ic                    [         R                  R                  U5      n[        R                  " 5       R                  5          [        R                  " U R                  S-   5         [         R                  R                  R                  X5      n[        X25       H"  u  pEUb  M
  [        SR                  U5      5      e   SSS5        SSS5        W$ ! , (       d  f       N= f! , (       d  f       W$ = f)a3  Returns gradients of `loss` with respect to `params`.

Should be used only in legacy v1 graph mode.

Args:
  loss: Loss tensor.
  params: List of variables.

Returns:
  List of gradient tensors.

Raises:
  ValueError: In case any gradient cannot be computed (e.g. if gradient
    function not implemented).
r   NzVariable {} has `None` for gradient. Please make sure that all of your ops have a gradient defined (i.e. are differentiable). Common ops without gradient: K.argmax, K.round, K.eval.)r   r   r   r   	get_graphr   r9   r   r   r=   	gradientsr   rM   rc   )r"   r   paramsr   r   params         r   get_gradientsOptimizerV2.get_gradientsJ  s      ( ++-w/A/AJJ%0
 IILL**48E"51<$5 6<VE]   2	0
- 0
 0
-- s*   $C%';C&CC%
C"	C%%
C4c                    U R                  X5      n[        [        X25      5      nU R                  U VVs/ s H,  u  pVUc  M
  UR                  [
        R                  :w  d  M*  UPM.     snn5        U R                  U5      /$ s  snnf r    )r   r   r   r   r   r   r   r   )r"   r   r   r   r   r   rq   s          r   get_updatesOptimizerV2.get_updatesj  s    ""40c%01!! +*DA %&WW%; *	
 $$^455s   	B
B
#B
c                    [        U[        R                  R                  R                  5      (       a  U R                  X!SS9  XR                  ;  a  X R                  U'   gU R                  U   n[        U5      (       dX  [        U[        R                  [        [        [        R                  45      (       d  [        U[        R                  5      (       a  X R                  U'   g[        R                  " U R                  U   U5        g)zBset hyper `name` to value. value can be callable, tensor, numeric.T)	overwriteN)r   r   r   tracking	Trackable_track_trackablerR   r   r   intfloatr   LearningRateScheduler   	set_value)r"   r;   value
prev_values       r   
_set_hyperOptimizerV2._set_hyperv  s    eR__55??@@!!%!>{{" %KKT*J$$		.CC	  1FF  %*D!!!$++d"3U;r   c                    U R                   (       d  U R                  5         U R                  U   n[        U[        R
                  5      (       a  U$ [        U5      (       a  U" 5       nU(       a  [        R                  " X25      $ U$ r    )	rZ   _create_hypersrR   r   r   r  r   r   cast)r"   r;   r   r  s       r   
_get_hyperOptimizerV2._get_hyper  se    ##!D!e3HHIILE??GE775((Lr   c                     g r    r!   )r"   r   s     r   _create_slotsOptimizerV2._create_slots  r'   r   c                    [        5       nU H6  n[        USS5      (       d  M  UR                  UR                  5       5        M8     U H9  n[	        U5      n0 nU R                  5        H  nXFU'   M	     X`R                  U'   M;     g)a  Add ShardedVariables to slots to later reconstruct for checkpointing.

ShardedVariables don't have slot variables created for them; their
shards do. This function allows users to call get_slot with a
ShardedVariable input and receive a ShardedVariable output containing
the appropriate slot vars.

Iterate over the variables to find shards, and aggregate the sharded
containers in a set. Add these ShardedVariables to _slots so that
get_slot can retrieve the proper slot variables for their component
shards, and reconstruct those into a ShardedVariable.

Args:
  var_list: list or tuple of `Variable` objects that will be minimized
    using this optimizer.
_sharded_containerFN)setgetattraddr  _var_keyget_slot_namesrS   )r"   r   sharded_varsr   sharded_varsharded_key	slot_dictslots           r   #_create_slots_for_sharded_variables/OptimizerV2._create_slots_for_sharded_variables  s}    " uCs0%88  !7!7!9:  (K";/KI++-"-$ .'0KK$ (r   c                     U R                   nU R                  5         U R                  U5        U R                  U5        g)au  Creates all weights, including iterations, hyperparameters and slot
vars.

This will add newly created variables to `optimizer.weights`.

New variables are only created when this method is called the first
time, or when called with different variables in the var_list.

Args:
  var_list: list or tuple of `Variable` objects that will be minimized
    using this optimizer.
N)r   r  r  r&  )r"   r   r   s      r   r   OptimizerV2._create_all_weights  s5     OO8$00:r   c                    >  [         TU ]  U5      $ ! [         a<  nUS:X  a  UeUS:X  a  SnXR                  ;   a  U R	                  U5      s SnA$ UeSnAff = f)z,Overridden to support hyperparameter access.rR   rB   learning_rateN)super__getattribute__AttributeErrorrR   r  )r"   r;   erh   s      r   r-  OptimizerV2.__getattribute__  s_    	7+D11 		xt|&{{"t,,G		s     
A/AAAAc                    > [        [        TU ]	  5       5      nSU;   aK  XR                  R	                  5       -  nSU R                  R	                  5       ;   a  UR                  S5        [        U5      $ )NrR   r+  rB   )r  r,  __dir__rR   keysr  r   )r"   rp   rh   s     r   r2  OptimizerV2.__dir__  s\    UW_&'vkk&&((F$++"2"2"44

4 F|r   c                    > US:X  a  Sn[        U S5      (       a!  XR                  ;   a  U R                  X5        g[        TU ]  X5        g)z;Override setattr to support dynamic hyperparameter setting.rB   r+  rR   N)hasattrrR   r  r,  __setattr__)r"   r;   r  rh   s      r   r7  OptimizerV2.__setattr__  s@     4<"D4""t{{':OOD(G,r   c                     U R                   $ )z+A list of names for this optimizer's slots.)rT   r)   s    r   r   OptimizerV2.get_slot_names  s    r   zerosc           	      p   X R                   ;  a  U R                   R                  U5        [        U5      nU R                  R	                  U0 5      nUR                  US5      nUGc  [        U[        5      (       d  [        U5      (       a{  [        R
                  " U5      n[        U[        R                  R                  R                  5      (       d  Ub  UnOUR                  n[        R                   " X8UR"                  S9n	OUn	U R%                  5          [        R&                  R)                  5       n
U
R*                  R-                  U5      (       d  [/        SR1                  X5      5      eU
R*                  R3                  U5         [        R4                  " UR6                   SU 3UR"                  SU	S9nSSS5        SSS5        [8        R:                  " U5        XvU'   U R=                  X!US9  U R>                  R                  U5        U$ ! , (       d  f       N]= f! , (       d  f       Nf= f)a  Add a new slot variable for `var`.

A slot variable is an additional variable associated with `var` to
train.  It is allocated and managed by optimizers, e.g. `Adam`.

Args:
  var: a `Variable` object.
  slot_name: name of the slot variable.
  initializer: initializer of the slot variable
  shape: (Optional) shape of the slot variable. If not set, it will
    default to the shape of `var`.

Returns:
  A slot variable.
N)r   r   a1  Trying to create optimizer slot variable under the scope for tf.distribute.Strategy ({}), which is different from the scope used for the original variable ({}). Make sure the slot variables are created under the same strategy scope. This may happen if you're restoring from a checkpoint outside the scope./F)r;   r   	trainableinitial_value)	slot_namevariableslot_variable) rT   r   r  rS   
setdefaultgetr   rL   r   r   r   r   r  CheckpointInitialValueCallabler   r   r   r   _distribution_strategy_scoper[   r]   r   variable_created_in_scoperM   rc   r   Variable_shared_namer   track_variable_restore_slot_variablerU   )r"   r   r@  initializerr   var_keyr$  weight
slot_shaper?  r   s              r   add_slotOptimizerV2.add_slot  s     ,,,##I.3-KK**7B7	y$/>+s++x/D/D*..{;OO,,KK  '!&J!$J ) 1 1! !,224==557((BB3GG$- .4VH-B  &&99#>[[ # 0 019+>!ii"'&3	F ? 5( ""6*#)i ''# (  MM  ( ?> 54s%   A4H'1H?H'
H$	 H''
H5c                 x   [        U5      nU R                  U   nXB   n[        U[        R                  R
                  R                  5      (       ah  / nUR                   H%  nU R                  Xr5      nUR                  U5        M'     [        R                  R
                  R                  XeR                  S9nU$ )Nr:   )r  rS   r   r   r   r[   ShardedVariable	variablesget_slotr   r;   )	r"   r   r@  rM  r$  rB  
shard_varsshard
slot_shards	            r   rU  OptimizerV2.get_slot7  s    3-KK(	!,2??55EE
 

 J&00!]]5<
!!*- 1 OO66FF!3!3 G M r   c                    [        5       nU Hw  n[        U[        R                  R                  5      (       a  UR
                  nOUR                  /nUR                  R                  nU H  nUR                  Xe45        M     My     0 nU H<  u  pe0 XvU4'   [        R                  " U5         U R                  XeU5        S S S 5        M>     U$ ! , (       d  f       MQ  = fr    )r  r   r   r[   DistributedValues_devicesdevicer   
base_dtyper  _prepare_local)r"   r   r3  r   var_devices	var_dtype
var_devicer   s           r   r   OptimizerV2._prepareI  s    uC#r}}>>??!ll"zzl		,,I)
*01 *  %)!J35KY/0:&##J;G '& &*
  '&s   -C
C	c                     SU R                   ;   a.  [        R                  " U R                  U5      5      nXCX4   S'   g g )Nr+  lr_t)rR   r   identity_decayed_lr)r"   rb  ra  r   re  s        r   r_  OptimizerV2._prepare_local\  s<    dkk);;t//	:;D;?/08 *r   c                 <    X40 0nU R                  XU5        X1U4   $ )zACompatibility for subclasses that don't pass apply_state through.)r_  )r"   rb  ra  r   s       r   _fallback_apply_state!OptimizerV2._fallback_apply_statea  s-    ".3J;?	233r   c           
         U R                   (       a  g U R                  5          [        U R                  R	                  5       5       H~  u  p[        U[        R                  [        R                  45      (       d  [        U5      (       a  MF  U R                  U/ SU[        R                  R                  S9U R                  U'   M     S S S 5        SU l         g ! , (       d  f       N= f)NF)r   r>  rL  aggregationT)rZ   rF  sortedrR   rl   r   r   r   rH  r   
add_weightVariableAggregationONLY_FIRST_REPLICA)r"   r;   r  s      r   r  OptimizerV2._create_hypersg  s    ..0%dkk&7&7&9:ebii%=>>(C C (, "'$)$&$:$:$M$M )8 )DKK%  ; 1$  $% 10s   B&C
C'c           	      N   U R                   c|  U R                  5          U R                  S/ [        R                  S[        R
                  R                  S9U l         SSS5        U R                  R                  U R                   5        U R                   $ ! , (       d  f       N?= f)z>Variable. The number of training steps this Optimizer has run.NiterF)r   r   r>  rm  )	rV   rF  ro  r   int64rp  rq  rU   r   r)   s    r   r   OptimizerV2.iterations~  s     #224#'??((# " 6 6 I I $3 $  5 MM  !1!12 54s   ?B
B$c                     U R                   b  [        SU S35      eXl         U R                  R                  U R                   5        g )NzCannot set `iterations` to a new Variable after the Optimizer weights have been created. Here it is attempting to set `iterations` to rG   )rV   r   rU   r   )r"   rA  s     r   r   rv    sM    '55=JaA 
 $T--.r   c                    U R                  SU5      n[        U[        R                  5      (       a>  [        R
                  " U R                  U5      n[        R
                  " U" U5      U5      nU R                  S:  aL  [        R
                  " U R                  U5      n[        R
                  " U R                  U5      nUSXC-  -   -  nU$ )z;Get decayed learning rate as a Tensor with dtype=var_dtype.r+  rJ   g      ?)r  r   r   r  r   r  r   rY   )r"   ra  re  
local_stepdecay_ts        r   rg  OptimizerV2._decayed_lr  s    	:d2GGHH)<J774
+Y7D$)<Jggd119=G3!556Dr   c                     SU R                   0nU R                  b  U R                  US'   U R                  b  U R                  US'   U R                  b  U R                  US'   U$ )a  Returns the config of the optimizer.

An optimizer config is a Python dictionary (serializable)
containing the configuration of an optimizer.
The same optimizer can be reinstantiated later
(without any saved state) from this configuration.

Returns:
    Python dictionary.
r;   rD   rE   rF   )r   rD   rE   rF   )r"   configs     r   
get_configOptimizerV2.get_config  sd     $**%==$!%F:>>%"&..F;+(,(<(<F$%r   c                     SU;   a  UR                  S5      US'   SU;   a3  [        US   [        5      (       a  [        R                  " US   US9US'   U " S0 UD6$ )a  Creates an optimizer from its config.

This method is the reverse of `get_config`,
capable of instantiating the same optimizer from the config
dictionary.

Args:
    config: A Python dictionary, typically the output of get_config.
    custom_objects: A Python dictionary mapping names to additional
      Python objects used to create this optimizer, such as a function
      used for a hyperparameter.

Returns:
    An optimizer instance.
rB   r+  )custom_objectsr!   )rX   r   dictr   deserialize)ro   r}  r  s      r   from_configOptimizerV2.from_config  se    " 6>&,jj&6F?#f$&1488*@*L*L?+N+' }V}r   c                    U R                   U   n[        U[        R                  5      (       a  [        R                  " U5      $ [        U5      (       a  U" 5       $ [        R                  " U5      (       a  [        R                  " U5      $ U$ )zDSerialize a hyperparameter that can be a float, callable, or
Tensor.)
rR   r   r   r  	serializer   r   	is_tensorr   	get_value)r"   hyperparameter_namer  s      r   _serialize_hyperparameter%OptimizerV2._serialize_hyperparameter  sm     /0e3HHII)33E::E??7N<<$$U++r   c                     U R                   $ z?Returns variables of this Optimizer based on the order created.rU   r)   s    r   rT  OptimizerV2.variables  s    }}r   c                     U R                   $ r  r  r)   s    r   weightsOptimizerV2.weights  s     }}r   c                 F    U R                   n[        R                  " U5      $ )a  Returns the current weights of the optimizer.

The weights of an optimizer are its state (ie, variables).
This function returns the weight values associated with this
optimizer as a list of Numpy arrays. The first value is always the
iterations count of the optimizer, followed by the optimizer's state
variables in the order they were created. The returned list can in turn
be used to load state into similarly parameterized optimizers.

For example, the RMSprop optimizer for this simple model returns a list
of three values-- the iteration count, followed by the root-mean-square
value of the kernel and bias of the single Dense layer:

>>> opt = tf.keras.optimizers.legacy.RMSprop()
>>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)])
>>> m.compile(opt, loss='mse')
>>> data = np.arange(100).reshape(5, 20)
>>> labels = np.zeros(5)
>>> results = m.fit(data, labels)  # Training.
>>> len(opt.get_weights())
3

Returns:
    Weights values as a list of numpy arrays.
)r  r   batch_get_value)r"   r   s     r   get_weightsOptimizerV2.get_weights  s    4 &&v..r   c                 P   U R                   n[        U5      [        U5      :w  aR  [        SU R                   S[	        [        U5      5       S[	        [        U5      5       S[	        U5      SS  S3	5      eU(       d  g/ n[
        R                  " U5      n[        XBU5       Hj  u  pVnUR                  UR                  :w  a8  [        S[	        UR                  5       S	[	        UR                  5       S
35      eUR                  Xg45        Ml     [
        R                  " U5        g)aF  Set the weights of the optimizer.

The weights of an optimizer are its state (ie, variables).
This function takes the weight values associated with this
optimizer as a list of Numpy arrays. The first value is always the
iterations count of the optimizer, followed by the optimizer's state
variables in the order they are created. The passed values are used to
set the new state of the optimizer.

For example, the RMSprop optimizer for this simple model takes a list of
three values-- the iteration count, followed by the root-mean-square
value of the kernel and bias of the single Dense layer:

>>> opt = tf.keras.optimizers.legacy.RMSprop()
>>> m = tf.keras.models.Sequential([tf.keras.layers.Dense(10)])
>>> m.compile(opt, loss='mse')
>>> data = np.arange(100).reshape(5, 20)
>>> labels = np.zeros(5)
>>> results = m.fit(data, labels)  # Training.
>>> new_weights = [np.array(10), np.ones([20, 10]), np.zeros([10])]
>>> opt.set_weights(new_weights)
>>> opt.iterations
<tf.Variable 'RMSprop/iter:0' shape=() dtype=int64, numpy=10>

Args:
    weights: weight values as a list of numpy arrays.
z/You called `set_weights(weights)` on optimizer z with a  weight list of length z", but the optimizer was expecting z weights. Provided weights: N2   z...zOptimizer weight shape z+ not compatible with provided weight shape rG   )r  lenrM   r   rL   r   r  r   r   r   batch_set_value)r"   r  r   weight_value_tuplesparam_valuespvpws           r   set_weightsOptimizerV2.set_weights  s   8 v;#g,&A$** N114S\1B0C D336s6{3C2D E..1'l3B.?-@E   ..v6L':HB1xx177" -c"((m_ =--0\N!= 
  &&v. ; 	 34r   c                    Uc  [         R                  n[        U[        5      (       d  [	        U5      (       a  [
        R                  " U5      nU[         R                  R                  :X  a  U(       a  [        S5      eSnOUc  SnU R                  UU[        R                  SUUUSUUS9
n[        R                  " U5        U$ )NzSynchronization value can be set to VariableSynchronization.ON_READ only for non-trainable variables. You have specified trainable=True and synchronization=VariableSynchronization.ON_READ.FT)
r;   r   getterr  rL  r   r>  use_resourcesynchronizationrm  )r   float32r   rL   r   r   rD  VariableSynchronizationON_READrM    _add_variable_with_custom_getterr   make_variabler   rJ  )	r"   r;   r   r   rL  r>  r  rm  rA  s	            r   ro  OptimizerV2.add_weight8  s     =JJEk3''8K+@+@&**;7Kb88@@@ G  "	I88#11#+# 9 
 	x(r   c                     U(       dC  [         R                  " [        R                  " U R                  R
                  5      US9U l        g Xl        g )N)
zero_based)r   unique_object_namer	   to_snake_caserh   r2   r   )r"   r;   r  s      r   rQ   OptimizerV2._init_set_namef  s9     33++DNN,C,CD%DJ
 Jr   c                     U R                  5       nU HS  nUR                  R                  nXB;  d  M   [        SR	                  XCR
                  U Vs/ s H  oUPM     sn5      5      e   gs  snf )zAsserts tensors are all valid types (see `_valid_dtypes`).

Args:
  tensors: Tensors to check.

Raises:
  ValueError: If any tensor is not a valid type.
z%Invalid type {} for {}, expected: {}.N)_valid_dtypesr   r^  rM   rc   r;   )r"   tensorsvalid_dtypestr   rq   s         r   r    OptimizerV2._assert_valid_dtypeso  sj     ))+AGG&&E( ;BBvv<'@<a<'@  
 (As   A+c                     [         $ )zValid types for loss, variables and gradients.

Subclasses should override to allow other float types.

Returns:
  Valid types for loss, variables and gradients.
)_DEFAULT_VALID_DTYPESr)   s    r   r  OptimizerV2._valid_dtypes  s
     %$r   c                 4    [        U5      (       a  U" 5       $ U$ )z'Call the function if param is callable.)r   )r"   r   s     r   _call_if_callableOptimizerV2._call_if_callable  s    "5//uw4u4r   c                     [        S5      e)aN  Add ops to apply dense gradients to the variable `handle`.

Args:
  grad: a `Tensor` representing the gradient.
  handle: a `Tensor` of dtype `resource` which points to the variable to
    be updated.
  apply_state: A dict which is used across multiple apply calls.

Returns:
  An `Operation` which updates the value of the variable.
z:`_resource_apply_dense` must be implemented in subclasses.r   )r"   r   handler   s       r   r   !OptimizerV2._resource_apply_dense  s     "H
 	
r   c                 @    [        XS9u  pVU R                  " XRU40 UD6$ )a  Add ops to apply sparse gradients to `handle`, with repeated indices.

Optimizers which override this method must deal with repeated indices.
See the docstring of `_apply_sparse_duplicate_indices` for details. By
default the correct behavior, to sum non-unique indices and their
associated gradients, is enforced by first pre-processing `grad` and
`indices` and passing them on to `_resource_apply_sparse`. Optimizers
which deal correctly with duplicate indices may instead override this
method to avoid the overhead of summing.

Args:
  grad: a `Tensor` representing the gradient for the affected indices.
  handle: a `Tensor` of dtype `resource` which points to the variable to
    be updated.
  indices: a `Tensor` of integral type representing the indices for
    which the gradient is nonzero. Indices may be repeated.
  **kwargs: May optionally contain `apply_state`

Returns:
  An `Operation` which updates the value of the variable.
)r   r   )r   _resource_apply_sparse)r"   r   r  r   r$   summed_gradr   s          r   r   4OptimizerV2._resource_apply_sparse_duplicate_indices  s7    0 'B'
# **
39
 	
r   c                     [        S5      e)a  Add ops to apply sparse gradients to the variable `handle`.

Similar to `_apply_sparse`, the `indices` argument to this method has
been de-duplicated. Optimizers which deal correctly with non-unique
indices may instead override `_resource_apply_sparse_duplicate_indices`
to avoid this overhead.

Args:
  grad: a `Tensor` representing the gradient for the affected indices.
  handle: a `Tensor` of dtype `resource` which points to the variable to
    be updated.
  indices: a `Tensor` of integral type representing the indices for
    which the gradient is nonzero. Indices are unique.
  apply_state: A dict which is used across multiple apply calls.

Returns:
  An `Operation` which updates the value of the variable.
z;`_resource_apply_sparse` Must be implemented in subclasses.r  )r"   r   r  r   r   s        r   r  "OptimizerV2._resource_apply_sparse  s    & "I
 	
r   c                     [         R                  " [         R                  R                  UR                  X#S9/5         UR                  5       sS S S 5        $ ! , (       d  f       g = fN)r   r   updates)r   r   raw_opsResourceScatterAddr  r  r"   xr   rq   s       r   _resource_scatter_add!OptimizerV2._resource_scatter_add  sP    $$

--XXq . 
 779
 
 
   A
A&c                     [         R                  " [         R                  R                  UR                  X#S9/5         UR                  5       sS S S 5        $ ! , (       d  f       g = fr  )r   r   r  ResourceScatterUpdater  r  r  s       r   _resource_scatter_update$OptimizerV2._resource_scatter_update  sP    $$

00XXq 1 
 779
 
 
r  c                 V    [         R                  " U R                  5      R                  $ r    )r   getfullargspecr   r#   r)   s    r   r   OptimizerV2._dense_apply_args  s!     (()C)CDIIIr   c                 V    [         R                  " U R                  5      R                  $ r    )r   r  r  r#   r)   s    r   r   OptimizerV2._sparse_apply_args  s!     (()D)DEJJJr   c                     [        U5      nU R                  R                  U0 5      R                  U/ 5      nUR	                  S SS9  U H  nUR                  U5        M     g)z.Restore a newly created slot variable's value.c                     U R                   $ r    )restore_uid)positions    r   <lambda>4OptimizerV2._restore_slot_variable.<locals>.<lambda>  s
    !5!5r   T)keyreverseN)r  rW   rD  rX   sortrestore)r"   r@  rA  rB  variable_keydeferred_restorationscheckpoint_positions          r   rK  "OptimizerV2._restore_slot_variable  sk    ) $ @ @ D Dr!

#lB
 	
 	""5t 	# 	
 $9''6 $9r   c                    [        U5      nU R                  R                  U0 5      nUR                  US5      nUc  [        R                  " 5       (       a  UR                  5       (       a  [        R                  R                  R                  5       R                  (       a  U R                  (       aG  [        R                  R                  R                  US9nU R                  UUUUR                  5       S9nUbJ  XcL aD  [!        U[        R                  R"                  R$                  5      (       a  U R'                  X25      $ U$ U R(                  R+                  U0 5      R+                  U/ 5      R-                  U5        g)a|  Returns the slot variable that should have a value restored into it.

It is up to the caller to restore the value into the slot variable if a
valid slot variable is returned.

Called when a variable which has an associated slot variable is created
or restored. When executing eagerly, we create the slot variable with a
restoring initializer.

No new variables are created when graph building. Instead,
_restore_slot_variable catches these after normal creation and adds
restore ops to the graph. This method is nonetheless important when
graph building for the case when a slot variable has already been
created but `variable` has just been added to a dependency graph
(causing us to realize that the slot variable needs to be restored).

Args:
  slot_variable_position: A `trackable._CheckpointPosition` object
    indicating the slot variable `Trackable` object to be restored.
  slot_name: The name of this `Optimizer`'s slot to restore into.
  variable: The variable object this slot is being created for.

Returns:
  A slot variable that should have a value restored into it, or None if
  a slot variable should not be restored at this time.
N)r  )r   rL  r@  r   )r  rS   rD  r   r8   is_simple_variabler   r=   get_default_graph_variable_creator_stackr^   r   r  rE  rP  value_shaper   r[   rS  rU  rW   rC  r   )r"   slot_variable_positionr@  rA  r  r$  rB  rL  s           r    _create_or_restore_slot_variable,OptimizerV2._create_or_restore_slot_variable  sE   :  )KKOOL"5	!i6!$$&&&99;; IILL224LL.. ((GG(> H  
 !MM'#,88:	 * M $ )8R__%?%?%O%OPP}}X99 !  ,,772jr*662H+Ir   c              #     #    U R                   (       ac  [        R                  R                  5       (       d@  U R                   R	                  5          U R                   R	                  5       v   SSS5        gSv   g! , (       d  f       g= f7f)zFReturns the `tf.distribute.Strategy` this optimizer was created
under.N)r^   r   r[   r\   scoper)   s    r   rF  (OptimizerV2._distribution_strategy_scopeb  s^      &&r}}/I/I/K/K,,224117799 54  54s   ABA<.B<
B
B)ru   r}   r   r   rW   r^   rx   r   rR   rZ   rY   rV   r   rT   rS   rP   rU   rD   rE   rF   ra   rb   )NNr    )NNN)NT)r;  N)T)Mr2   r3   r4   r5   __doc___HAS_AGGREGATE_GRADr%   rr   propertyrD   rF   setterrE   r   r   r   r   r   r   r   r   r   r   r  r  r  r  r&  r   r-  r2  r7  r   rP  rU  r   r_  rj  r  r   rg  abcabstractmethodr~  classmethodr  r  rT  r  r  r  r   r  AUTOrp  NONEro  rQ   r   r  r  r   r   r  r  r  r
   cached_per_instancer   r   rK  r  
contextlibcontextmanagerrF  r6   __classcell__rh   s   @r   r?   r?   d   s   
DZ  
 !"	|7|
   % % __
 
 
 
"   
 

*
8"?HAH KOaFN1`@
6<61:;&- ?B$&@
4$.     / /
 	 (  2
  /<05l 2277**//,\&%5
 
>
. $$J % J $$K % K7Zx  r   r?   c                 \   [        U S5      (       a  U R                  5       n Oa[        R                  " U 5      (       aF  [        U S5      (       a5  [        U R                  S5      (       a  U R                  R                  5       n [        U SS5      (       a  U R                  $ U R                  $ )a@  Key for representing a primary variable, for looking up slots.

In graph mode the name is derived from the var shared name.
In eager mode the name is derived from the var unique id.
If distribution strategy exists, get the primary variable first.

Args:
  var: the variable.

Returns:
  the unique name of the variable.
_distributed_containerr  _in_graph_modeF)r6  r  r   is_extension_typer  r  rI  
_unique_id)r   s    r   r  r  m  s     s,--((*""3''C""CJJ 899 jj//1s$e,,>>r   c                 (    [        U 5      nUS-   U-   $ )z6Get the slot key for the variable: var_name/slot_name.r=  )r  )r   r@  r;   s      r   _get_slot_key_from_varr    s     C=D#:	!!r   c                   2   ^  \ rS rSrSrU 4S jrS rSrU =r$ )RestoredOptimizeri  ah  A non-functional Optimizer implementation for checkpoint compatibility.

Holds slot variables and hyperparameters when an optimizer is restored from
a SavedModel. These variables may be referenced in functions along with ops
created by the original optimizer, but currently we do not support using the
optimizer object itself (e.g. through `apply_gradients`).
c                 2   > [         TU ]  S5        SU l        g )Nr
  T)r,  r%   rZ   )r"   rh   s    r   r%   RestoredOptimizer.__init__  s    ,-#r   c                     [        S5      e)NzRestoring functional Optimizers from SavedModels is not currently supported. Please file a feature request if this limitation bothers you.r  r)   s    r   r~  RestoredOptimizer.get_config  s    !
 	
r   )rZ   )	r2   r3   r4   r5   r  r%   r~  r6   r   r  s   @r   r
  r
    s    $
 
r   r
  	optimizerc                 "    [        U [        5      $ r    r   r?   objs    r   r  r        JsK0r   c                     [        5       $ r    r
  protos    r   r  r        ->-@r   rH   r   )object_factoryversionmin_producer_versionmin_consumer_versionr  )versionstf_keras_optimizerc                 "    [        U [        5      $ r    r  r  s    r   r  r    r  r   c                     [        5       $ r    r  r  s    r   r  r    r  r   )4r  r  r  r   rN   copyr   tensorflow.compat.v2r   v2r   tf_keras.srcr   r   tf_keras.src.enginer   tf_keras.src.optimizersr   r_   !tf_keras.src.optimizers.schedulesr   tf_keras.src.utilsr	   r
   r   r    tensorflow.python.util.tf_exportr   	frozensetfloat16bfloat16r  float64	complex64
complex128r  r   r   r<   r   r  r  r?   r  r  r
  saved_modelloadregister_revived_typeVersionedTypeRegistrationr  AssertionErrorr!   r   r   <module>r6     s   $      ! !   % 0 < D , * ) ' :!











	 +( $$ '$&IJB"//**44 B	BJ0<"
 
6OO$$::0OO'',,FF@%&%&(33 G 
 ;   OO$$::0OO'',,FF@%&%&(33 G 
 ; s   ?A+E+ +A1GG