
    +h.              	          S SK r S SKJrJr  S SKJrJrJr  S SKrSSK	J
r
  SSKJr  SSKJrJrJrJr  SS	KJrJr  SS
KJrJr  \
" \5      rSr\ " S S5      5       r " S S\R8                  R:                  5      r " S S\5      r " S S\5      r  " S S\5      r!S\RD                  RF                  S\SS4S jr$SS\RD                  RF                  S\S\\%   SS4S jjr&g)    N)asdict	dataclass)CallableListOptional   )
get_logger)unwrap_module   )"_ALL_TRANSFORMER_BLOCK_IDENTIFIERS_ATTENTION_CLASSES_FEEDFORWARD_CLASSES_get_submodule_from_fqn)AttentionProcessorRegistryTransformerBlockRegistry)HookRegistry	ModelHooklayer_skip_hookc                       \ rS rSr% Sr\\   \S'   Sr\	\S'   Sr
\\S'   Sr\\S	'   Sr\\S
'   Sr\\S'   S rS r\S\SS 4S j5       rSrg)LayerSkipConfig(   ay  
Configuration for skipping internal transformer blocks when executing a transformer model.

Args:
    indices (`List[int]`):
        The indices of the layer to skip. This is typically the first layer in the transformer block.
    fqn (`str`, defaults to `"auto"`):
        The fully qualified name identifying the stack of transformer blocks. Typically, this is
        `transformer_blocks`, `single_transformer_blocks`, `blocks`, `layers`, or `temporal_transformer_blocks`.
        For automatic detection, set this to `"auto"`. "auto" only works on DiT models. For UNet models, you must
        provide the correct fqn.
    skip_attention (`bool`, defaults to `True`):
        Whether to skip attention blocks.
    skip_ff (`bool`, defaults to `True`):
        Whether to skip feed-forward blocks.
    skip_attention_scores (`bool`, defaults to `False`):
        Whether to skip attention score computation in the attention blocks. This is equivalent to using `value`
        projections as the output of scaled dot product attention.
    dropout (`float`, defaults to `1.0`):
        The dropout probability for dropping the outputs of the skipped layers. By default, this is set to `1.0`,
        meaning that the outputs of the skipped layers are completely ignored. If set to `0.0`, the outputs of the
        skipped layers are fully retained, which is equivalent to not skipping any layers.
indicesautofqnTskip_attentionFskip_attention_scoresskip_ff      ?dropoutc                     SU R                   s=::  a  S::  d  O  [        SU R                    S35      e[        R                  " U R                   S5      (       d  U R                  (       a  [        S5      eg g )Nr   r   z6Expected `dropout` to be between 0.0 and 1.0, but got .r   bCannot set `skip_attention_scores` to True when `dropout` is not 1.0. Please set `dropout` to 1.0.)r   
ValueErrormathiscloser   selfs    T/home/james-whalen/.local/lib/python3.13/site-packages/diffusers/hooks/layer_skip.py__post_init__LayerSkipConfig.__post_init__I   sg    T\\&Q&UVZVbVbUccdeff||DLL#..43M3Mt  4N.    c                     [        U 5      $ N)r   r&   s    r(   to_dictLayerSkipConfig.to_dictQ   s    d|r+   datareturnc                     [        S0 U D6$ )N )r   )r0   s    r(   	from_dictLayerSkipConfig.from_dictT   s    &&&r+   r3   N)__name__
__module____qualname____firstlineno____doc__r   int__annotations__r   strr   boolr   r   r   floatr)   r.   staticmethoddictr4   __static_attributes__r3   r+   r(   r   r   (   ss    0 #YCND"'4'GTGU ' '!2 ' 'r+   r   c                       \ rS rSrSS jrSrg)AttentionScoreSkipFunctionModeY   r3   Nc                 V   Uc  0 nU[         R                  R                  R                  L av  UR	                  SS 5      nUR	                  SS 5      nUR	                  SS 5      nUb  UOUS   nUb  UOUS   nUb  UOUS   nUR
                  S   UR
                  S   :X  a  U$ U" U0 UD6$ )Nquerykeyvaluer   r   r   )torchnn
functionalscaled_dot_product_attentiongetshape)r'   functypesargskwargsrG   rH   rI   s           r(   __torch_function__1AttentionScoreSkipFunctionMode.__torch_function__Z   s    >F588&&CCCJJw-E**UD)CJJw-E".EDGE#d1gC".EDGE {{1~Q/T$V$$r+   )r3   N)r6   r7   r8   r9   rT   rB   r3   r+   r(   rD   rD   Y   s    %r+   rD   c                   d    \ rS rSrS
S\S\S\4S jjrS\R                  R                  4S jrSrg	)AttentionProcessorSkipHookn   skip_processor_output_fnr   r   c                 (    Xl         X l        X0l        g r-   )rY   r   r   )r'   rY   r   r   s       r(   __init__#AttentionProcessorSkipHook.__init__o   s    (@%%:"r+   modulec                 &   U R                   (       ab  [        R                  " U R                  S5      (       d  [	        S5      e[        5          U R                  R                  " U0 UD6nS S S 5        U$ [        R                  " U R                  S5      (       a  U R                  " U/UQ70 UD6nU$ U R                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ ! , (       d  f       W$ = f)Nr   r"   p)r   r$   r%   r   r#   rD   fn_reforiginal_forwardrY   rJ   rK   rL   r'   r]   rR   rS   outputs        r(   new_forward&AttentionProcessorSkipHook.new_forwardt   s    %%<<c22 x  0155tFvF 2  ||DLL#..66vOOO  55tFvF,,44V||4L 21 s   D
D)r   r   rY   N)Fr   )r6   r7   r8   r9   r   r>   r?   r[   rJ   rK   Modulere   rB   r3   r+   r(   rW   rW   n   s2     RV in 
%((// r+   rW   c                   f   ^  \ rS rSrS\4U 4S jjrS\R                  R                  4S jr	Sr
U =r$ )FeedForwardSkipHook   r   c                 .   > [         TU ]  5         Xl        g r-   superr[   r   r'   r   	__class__s     r(   r[   FeedForwardSkipHook.__init__       r+   r]   c                 l   [         R                  " U R                  S5      (       a@  UR                  SS 5      nUc  UR                  SS 5      nUc  [	        U5      S:  a  US   nU$ U R
                  R                  " U0 UD6n[        R                  R                  R                  X@R                  S9nU$ )Nr   hidden_statesxr   r_   )
r$   r%   r   rN   lenra   rb   rJ   rK   rL   rc   s        r(   re   FeedForwardSkipHook.new_forward   s    <<c**ZZ6F~C.~#d)a-a  [[114B6BFXX((00<<0HFr+   )r   )r6   r7   r8   r9   r?   r[   rJ   rK   rg   re   rB   __classcell__ro   s   @r(   ri   ri      s(     
%((// 
 
r+   ri   c                   l   ^  \ rS rSrS\4U 4S jjrS rS\R                  R                  4S jr
SrU =r$ )TransformerBlockSkipHook   r   c                 .   > [         TU ]  5         Xl        g r-   rl   rn   s     r(   r[   !TransformerBlockSkipHook.__init__   rq   r+   c                 b    [         R                  " [        U5      R                  5      U l        U$ r-   )r   rN   r
   ro   	_metadata)r'   r]   s     r(   initialize_hook(TransformerBlockSkipHook.initialize_hook   s$    155mF6K6U6UVr+   r]   c                    [         R                  " U R                  S5      (       aY  U R                  R	                  SX#5      nU R                  R
                  c  UnU$ U R                  R	                  SX#5      nXF4n U$ U R                  R                  " U0 UD6n[        R                  R                  R                  XPR                  S9nU$ )Nr   rs   encoder_hidden_statesr_   )r$   r%   r   r   _get_parameter_from_args_kwargs"return_encoder_hidden_states_indexra   rb   rJ   rK   rL   )r'   r]   rR   rS   original_hidden_statesrd   original_encoder_hidden_statess          r(   re   $TransformerBlockSkipHook.new_forward   s    <<c**%)^^%S%STcei%r"~~@@H/  261_1_+T2. 1Q  [[114B6BFXX((00<<0HFr+   )r   r   )r6   r7   r8   r9   r?   r[   r   rJ   rK   rg   re   rB   rw   rx   s   @r(   rz   rz      s-     %((//  r+   rz   r]   configr1   c                     [        X5        g)ag  
Apply layer skipping to internal layers of a transformer.

Args:
    module (`torch.nn.Module`):
        The transformer model to which the layer skip hook should be applied.
    config (`LayerSkipConfig`):
        The configuration for the layer skip hook.

Example:

```python
>>> from diffusers import apply_layer_skip_hook, CogVideoXTransformer3DModel, LayerSkipConfig

>>> transformer = CogVideoXTransformer3DModel.from_pretrained("THUDM/CogVideoX-5b", torch_dtype=torch.bfloat16)
>>> config = LayerSkipConfig(layer_index=[10, 20], fqn="transformer_blocks")
>>> apply_layer_skip_hook(transformer, config)
```
N)_apply_layer_skip_hook)r]   r   s     r(   apply_layer_skipr      s    ( 6*r+   namec                    U=(       d    [         nUR                  (       a  UR                  (       a  [        S5      e[        R
                  " UR                  S5      (       d  UR                  (       a  [        S5      eUR                  S:X  a0  [         H  n[        X5      (       d  M  X1l          O   [        S5      e[        XR                  5      nUb)  [        U[        R                  R                  5      (       d  [        SUR                   S35      e[        UR                   5      S:X  a  [        S	5      eS
n[#        U5       GHQ  u  pgXaR                   ;  a  M  SnUR                  (       au  UR$                  (       ad  [&        R)                  SUR                   SU S35        [*        R,                  " U5      n[/        UR                  5      n	UR1                  X5        GO	UR                  (       d  UR                  (       a  UR3                  5        H  u  p[        U[4        5      (       d  M  UR6                  (       a  M/  [&        R)                  SUR                   SU SU
 S35        [8        R:                  " UR<                  R>                  5      R@                  n[*        R,                  " U5      n[C        XR                  UR                  5      n	UR1                  X5        M     UR$                  (       d  GM  UR3                  5        H  u  p[        U[D        5      (       d  M  [&        R)                  SUR                   SU SU
 S35        [*        R,                  " U5      n[G        UR                  5      n	UR1                  X5        M     GMT     U(       d&  [        SUR                    SUR                   S35      eg )NzXCannot set both `skip_attention` and `skip_attention_scores` to True. Please choose one.r   r"   r   zCould not find a suitable identifier for the transformer blocks automatically. Please provide a valid `fqn` (fully qualified name) that identifies a stack of transformer blocks.zCould not find z in the provided module, or configured `fqn` (fully qualified name) does not identify a `torch.nn.ModuleList`. Please provide a valid `fqn` that identifies a stack of transformer blocks.r   zTLayer index list is empty. Please provide a non-empty list of layer indices to skip.FTz&Applying TransformerBlockSkipHook to 'r!   'z(Applying AttentionProcessorSkipHook to 'z!Applying FeedForwardSkipHook to 'zDCould not find any transformer blocks matching the provided indices z and fully qualified name 'z4'. Please check the indices and fqn for correctness.)$_LAYER_SKIP_HOOKr   r   r#   r$   r%   r   r   r   hasattrr   
isinstancerJ   rK   
ModuleListru   r   	enumerater   loggerdebugr   check_if_exists_or_initializerz   register_hooknamed_modulesr   is_cross_attentionr   rN   	processorro   rY   rW   r   ri   )r]   r   r   
identifiertransformer_blocksblocks_foundiblockregistryhooksubmodule_name	submodule	output_fns                r(   r   r      s   ##D!=!=stt<<,,1M1Mp
 	
 zzV<Jv**'
 =
 ^ 
 1D!4FH[H[)\)\fjj\ *s t
 	
 6>>aoppL01NN"  V^^LLA&**QqcQRST#AA%HH+FNN;D""4.""f&B&B-2-@-@-B)i);<<YEaEaEaLL#KFJJ<WXYZX[[\]k\llm!no : > >y?R?R?\?\ ] v vI+II)TH5iA]A]_e_m_mnD**46 .C >>>-2-@-@-B)i)=>>LL#DVZZLPQRSQTTUVdUeef!gh+II)TH.v~~>D**46 .C- 2: RSYSaSaRb c%%+ZZL0df
 	
 r+   r-   )'r$   dataclassesr   r   typingr   r   r   rJ   utilsr	   utils.torch_utilsr
   _commonr   r   r   r   _helpersr   r   hooksr   r   r6   r   r   r   	overridesTorchFunctionModerD   rW   ri   rz   rK   rg   r   r=   r   r3   r+   r(   <module>r      s     ) + +   -  K * 
H	$ 
 -' -' -'`%U__%F%F %* .) $y 2+UXX__ +o +$ +.@
588?? @
O @
S[\_S` @
lp @
r+   