
    6bi                     J    S r SSKJs  Jr  SSKJr  S rS r	S r
S rS rS	 rg)
zOptimizer utilities.    N)
tf_loggingc                    [        U 5      n [        U 5      nU(       a  [        R                  R                  R                  5       (       ae  U Vs/ s H  o"S   PM	     nn[        R                  R                  5       R                  [        R                  R                  R                  U5      nO4[        R                  R                  5       R                  [        U4S9nO/ n/ nSnU  H7  u  pxUc  UR                  SU45        M  UR                  XF   U45        US-  nM9     U[        U5      :X  d   S5       eU$ s  snf )zReturns all-reduced gradients aggregated via summation.

Args:
  grads_and_vars: List of (gradient, variable) pairs.

Returns:
  List of (gradient, variable) pairs where gradients have been all-reduced.
r   )argsN   zFailed to add all gradients)listfilter_empty_gradientstf__internal__
distributestrategy_supports_no_merge_callget_replica_context
all_reduceReduceOpSUM
merge_call_all_reduce_sum_fnappendlen)	grads_and_varsfiltered_grads_and_varspairgradsreducedreduced_with_nonesreduced_posgvs	            W/home/james-whalen/.local/lib/python3.13/site-packages/tf_keras/src/optimizers/utils.pyall_reduce_sum_gradientsr      s(    .)N4^D??%%EEGG)@A)@!W)@EAmm779DD&&**EG
 mm779DD"*A)C E G K9%%tQi0%%w';Q&?@1K  #g,&E(EE&+ Bs   D=c                    [        U 5      n U (       d  U $ / n/ nU  H-  u  p4Uc  UR                  U5        M  UR                  X445        M/     [        U5      nU(       d4  U  VVs/ s H  u  pVUR                  PM     snn4n[        SU SU  S35      eU(       a1  [        R
                  " SU Vs/ s H  ofR                  PM     sn5        U$ s  snnf s  snf )zDFilter out `(grad, var)` pairs that have a gradient equal to `None`.z(No gradients provided for any variable: z. Provided `grads_and_vars` is .zGradients do not exist for variables %s when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss` argument?)tupler   name
ValueErrorloggingwarning)r   filteredvars_with_empty_gradsgradvar_r   variables           r   r   r   <   s    >*NH#	<!((-OOTK(	 $
 XH(67QVV796xj A,,:+;1>
 	
 ) 443ff34		
 O 8 5s   #C.C
c                     ^  T c  S $ U 4S jnU$ )@Creates a gradient transformation function for clipping by norm.c                     U $ N r   s    r   <lambda>+make_gradient_clipnorm_fn.<locals>.<lambda>^       n    c                   > [        [        R                  R                  5       [        R                  R                  R
                  [        R                  R                  R                  R                  R
                  45      (       a+  [        S[        R                  R                  5        S35      eU  VVs/ s H  u  p[        R                  " UT5      U4PM      nnnU$ s  snnf )NzQ`clipnorm` is not supported with `CenteralStorageStrategy`. The strategy used is r!   )

isinstancer	   r   get_strategyexperimentalCentralStorageStrategycompatv1r$   clip_by_norm)r   r   r   clipped_grads_and_varsclipnorms       r   gradient_clipnorm_fn7make_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fn`   s    MM&&(**AA		''44KK
 
 ((*(B(B(D'EQH  ;I"
:H$!R__Q)1-. 	 "
 &%"
s   9%C#r1   r@   rA   s   ` r   make_gradient_clipnorm_fnrD   [   s    44&$  r6   c                     ^  T c  S $ U 4S jnU$ )r.   c                     U $ r0   r1   r2   s    r   r3   2make_global_gradient_clipnorm_fn.<locals>.<lambda>x   r5   r6   c                   > [        [        R                  R                  5       [        R                  R                  R
                  [        R                  R                  R                  R                  R
                  45      (       a+  [        S[        R                  R                  5        S35      e[        U 6 u  p[        R                  " UT5      u  p4[        [        X25      5      nU$ )NzX`global_clipnorm` is not supported with `CenteralStorageStrategy`. The strategy used is r!   )r8   r	   r   r9   r:   r;   r<   r=   r$   zipclip_by_global_normr   )r   r   	variablesclipped_gradsr+   r?   r@   s         r   rA   >make_global_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnz   s    MM&&(**AA		''44KK
 
 ((*(B(B(D'EQH  /11%B!%c-&C!D%%r6   r1   rC   s   ` r    make_global_gradient_clipnorm_fnrN   u   s    44&&  r6   c                     ^  T c  S $ U 4S jnU$ )zACreates a gradient transformation function for clipping by value.c                     U $ r0   r1   r2   s    r   r3   ,make_gradient_clipvalue_fn.<locals>.<lambda>   r5   r6   c           	        > [        [        R                  R                  5       [        R                  R                  R
                  [        R                  R                  R                  R                  R
                  45      (       a+  [        S[        R                  R                  5        S35      eU  VVs/ s H   u  p[        R                  " UT* T5      U4PM"     nnnU$ s  snnf )NzR`clipvalue` is not supported with `CenteralStorageStrategy`. The strategy used is r!   )
r8   r	   r   r9   r:   r;   r<   r=   r$   clip_by_value)r   r   r   r?   	clipvalues       r   gradient_clipvalue_fn9make_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fn   s    MM&&(**AA		''44KK
 
 ((*(B(B(D'EQH  '"
& a)Y7;& 	 "
 &%	"
s   9'C%r1   )rT   rU   s   ` r   make_gradient_clipvalue_fnrW      s    44&& ! r6   c                 ~    U R                   R                  [        R                  R                  R
                  U5      $ r0   )extendedbatch_reduce_tor	   r   r   r   )distributionr   s     r   r   r      s0      00
""N r6   )__doc__tensorflow.compat.v2r<   v2r	   tensorflow.python.platformr   r%   r   r   rD   rN   rW   r   r1   r6   r   <module>r`      s4     ! ! ="J> 4 6!6r6   