
    6bi                         S r SSKJs  Jr  SSKJr  SSKJr  SSK	J
r
  \" 5       \
" S/ S9 " S S	\R                  5      5       5       r\R                   R                  S
\R                  5      \l         g)zLion optimizer implementation.    N)	optimizer)register_keras_serializable)keras_exportzkeras.optimizers.Lion)v1c                   f   ^  \ rS rSrSr            SU 4S jjrU 4S jrS rU 4S jrSr	U =r
$ )	Lion   aT  Optimizer that implements the Lion algorithm.

The Lion optimizer is a stochastic-gradient-descent method that uses the
sign operator to control the magnitude of the update, unlike other adaptive
optimizers such as Adam that rely on second-order moments. This make
Lion more memory-efficient as it only keeps track of the momentum. According
to the authors (see reference), its performance gain over Adam grows with
the batch size. Because the update of Lion is produced through the sign
operation, resulting in a larger norm, a suitable learning rate for Lion is
typically 3-10x smaller than that for AdamW. The weight decay for Lion
should be in turn 3-10x larger than that for AdamW to maintain a
similar strength (lr * wd).

Args:
    learning_rate: A `tf.Tensor`, floating point value, a schedule that is a
        `tf.keras.optimizers.schedules.LearningRateSchedule`, or a callable
        that takes no arguments and returns the actual value to use. The
        learning rate. Defaults to 0.0001.
    beta_1: A float value or a constant float tensor, or a callable
        that takes no arguments and returns the actual value to use. The
        rate to combine the current gradient and the 1st moment estimate.
    beta_2: A float value or a constant float tensor, or a callable
        that takes no arguments and returns the actual value to use. The
        exponential decay rate for the 1st moment estimate.
    {{base_optimizer_keyword_args}}

References:
    - [Chen et al., 2023](http://arxiv.org/abs/2302.06675)
    - [Authors' implementation](
        http://github.com/google/automl/tree/master/lion)

c                    > [         TU ]  " SUUUUUUU	U
US.	UD6  U R                  U5      U l        X l        X0l        US::  d  US:  a  [        SU S35      eg )N)	nameweight_decayclipnorm	clipvalueglobal_clipnormuse_emaema_momentumema_overwrite_frequencyjit_compiler      z	`beta_1`=zI must be between ]0, 1]. Otherwise, the optimizer degenerates to SignSGD. )super__init___build_learning_rate_learning_ratebeta_1beta_2
ValueError)selflearning_rater   r   r   r   r   r   r   r   r   r   r   kwargs	__class__s                 V/home/james-whalen/.local/lib/python3.13/site-packages/tf_keras/src/optimizers/lion.pyr   Lion.__init__>   s      	 	
%+%$;#	
 	
 #77FQ;&1*F8 $8 8  %    c                    > [         TU ]  U5        [        U S5      (       a  U R                  (       a  g/ U l        U H,  nU R                  R                  U R                  USS95        M.     SU l        g)zInitialize optimizer variables.

Lion optimizer has one variable `momentums`.

Args:
    var_list: list of model variables to build Lion variables on.
_builtNm)model_variablevariable_nameT)r   buildhasattrr%   	momentumsappendadd_variable_from_reference)r   var_listvarr    s      r!   r)   
Lion.buildc   sk     	h4""t{{CNN!!00#&c 1   r#   c           	         [         R                  " U R                  UR                  5      n[         R                  " U R                  UR                  5      n[         R                  " U R
                  UR                  5      nU R                  U5      nU R                  U R                  U      n[        U[         R                  5      (       a  UR                  Xt-  5        UR                  [         R                  " UR                  SU-
  -  UR                  5      5        UR                  U[         R                   R#                  U5      -  5        UR                  Xu-  U-  5        UR                  [         R                  " UR                  SXT-  -
  -  UR                  5      5        gUR                  U[         R                   R#                  Xt-  USU-
  -  -   5      -  5        UR                  Xu-  USU-
  -  -   5        g)z=Update step given gradient and the associated model variable.g      ?N)tfcastr   dtyper   r   _var_keyr+   _index_dict
isinstanceIndexedSlicesassignscatter_addvaluesindices
assign_submathsign)r   gradientvariablelrr   r   var_keyr&   s           r!   update_stepLion.update_stepw   s   WWT''8hnn5hnn5--)NN4++G45h 0 011HHQZ MM  OOsV|4h6F6F
 RWW\\!_ 45HHQZ&()MM  OOsV_'<=x?O?O RWW\\!*x3</H"HII HHQZ(cFl";;<r#   c                    > [         TU ]  5       nUR                  U R                  U R                  5      U R
                  U R                  S.5        U$ )N)r   r   r   )r   
get_configupdate_serialize_hyperparameterr   r   r   )r   configr    s     r!   rG   Lion.get_config   sO    #%!%!?!?''" ++++	
 r#   )r%   r   r   r   r+   )g-C6?g?Gz?NNNNFrL   NTr   )__name__
__module____qualname____firstlineno____doc__r   r)   rD   rG   __static_attributes____classcell__)r    s   @r!   r   r      sI    F  $#J(=> r#   r   z{{base_optimizer_keyword_args}})rQ   tensorflow.compat.v2compatv2r2   tf_keras.src.optimizersr   'tf_keras.src.saving.object_registrationr    tensorflow.python.util.tf_exportr   	Optimizerr   replacebase_optimizer_keyword_argsr   r#   r!   <module>r]      sv    % ! ! - O : %"-F9 F . FR ||##%y'L'Lr#   