
    oip'                        S SK Jr  S SKrS SKJr  S SKJrJr  S SKJ	r	J
r
Jr  S SKrS SKJr  S SKJr  SSKJrJrJrJr  \ " S	 S
\5      5       r " S S\5      r " S S\5      rg)    )annotationsN)abstractmethod)	dataclassfield)AnyOptionalUnion)
PeftConfig   )	BaseTunerBaseTunerLayer_get_in_out_featurescheck_adapters_to_mergec                  R    \ rS rSr% Sr\" \SS0S9rS\S'   \" \SS0S9r	S\S	'   S
r
g)LycorisConfig"   z)
A base config for LyCORIS like adapters
helpzThe mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. For example, `{'^model.decoder.layers.0.encoder_attn.k_proj': 16}`.)default_factorymetadatazOptional[dict]rank_patternzThe mapping from layer names or regexp expression to alphas which are different from the default alpha specified by `alpha`. For example, `{'^model.decoder.layers.0.encoder_attn.k_proj': 16}`.alpha_pattern N)__name__
__module____qualname____firstlineno____doc__r   dictr   __annotations__r   __static_attributes__r       S/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/lycoris_utils.pyr   r   "   sO     $)V
$L.  %*V
%M> r!   r   c                      \ rS rSrSrSrSS jr\\SS j5       5       r	SS jr
\SS j5       r\SS j5       r\SS	 j5       rSSS jjr\SS j5       rS rSS jrSS jrSSS jjr\SS j5       rSrg
)LycorisLayer<   z(
A base layer for LyCORIS like adapters
)ralphascalingrank_dropoutmodule_dropoutc                    Xl         0 U l        0 U l        0 U l        0 U l        0 U l        0 U l        SU l        / U l        SU l	        [        U R                  5       5      u  p#X l        X0l        g )NFT)
base_layerr&   r'   r(   r)   rank_dropout_scaler*   _disable_adaptersmerged_adapterscast_input_dtype_enabledr   get_base_layerin_featuresout_features)selfr,   r2   r3   s       r"   __init__LycorisLayer.__init__D   sp    $
"$  "'!(,%$89L9L9N$O!&(r!   c                    g Nr   )r4   s    r"   _available_adapters LycorisLayer._available_adaptersW   s    /2r!   c                    UR                  5       nUR                  SS5      nUR                  " U /UQ7SS0UD6  U R                  US9  g )Ndevicecpumeta)r<   )copypopr5   to_empty)r4   clsargskwargsfinal_devices        r"   _init_empty_weights LycorisLayer._init_empty_weights[   sH     zz(E2T:D::6:\*r!   c                    g r8   r   )r4   adapter_namer&   rD   s       r"   create_adapter_parameters&LycorisLayer.create_adapter_parametersg   s    NQr!   c                    g)zZActivations added on top of the base layer output (i.e. after the base layer forward pass)Nr   )r4   rI   xrC   rD   s        r"   _get_delta_activations#LycorisLayer._get_delta_activationsk   s    r!   c                    g r8   r   r4   rI   s     r"   get_delta_weightLycorisLayer.get_delta_weighto   s    CFr!   Nc                   [        X5      nU(       d  gU H  nX0R                  ;   d  M  U R                  5       nU(       a  UR                  R                  R                  5       nXPR                  U5      -  n[        R                  " U5      R                  5       (       d  [        SU S35      eXTR                  l        O.UR                  =R                  U R                  U5      -  sl        U R                  R                  U5        M     g)a  
Merge the active adapter weights into the base weights

Args:
    safe_merge (`bool`, *optional*):
        If `True`, the merge operation will be performed in a copy of the original weights and check for NaNs
        before merging the weights. This is useful if you want to check if the merge operation will produce
        NaNs. Defaults to `False`.
    adapter_names (`List[str]`, *optional*):
        The list of adapter names that should be merged. If `None`, all active adapters will be merged.
        Defaults to `None`.
Nz1NaNs detected in the merged weights. The adapter z seems to be broken)r   r9   r1   weightdataclonerR   torchisfiniteall
ValueErrorr/   append)r4   
safe_mergeadapter_namesactive_adapterr,   orig_weightss         r"   mergeLycorisLayer.merger   s     0D+N!9!99!002
#-#4#4#9#9#?#?#AL $9$9.$IIL >>,7;;==(OP^O__rs  .:%%*%%**d.C.CN.SS*$$++N; ,r!   c                    g r8   r   rQ   s     r"   reset_adapter_parameters%LycorisLayer.reset_adapter_parameters   s    ;>r!   c                    XR                   ;  a  g X R                  U   -  U R                  U   -  U R                  U'   g r8   )r9   r'   r&   r(   )r4   adapterscales      r"   	set_scaleLycorisLayer.set_scale   s8    222 %

7(; ;dffWo MWr!   c                    US:X  a  g U R                    H+  nX R                  ;  a  M  U R                  U==   U-  ss'   M-     g )Nr   )active_adaptersr9   r(   r4   rh   r_   s      r"   scale_layerLycorisLayer.scale_layer   s?    A:"22N%=%==LL(E1(	 3r!   c                   U R                   (       d  [        R                  " S5        g[        U R                  5      S:  a  U R                  R                  5       nXR                  ;   a<  U R                  5       R                  =R                  U R                  U5      -  sl	        [        U R                  5      S:  a  M  gg)zG
This method unmerges all merged adapter layers from the base weights.
z Already unmerged. Nothing to do.Nr   )mergedwarningswarnlenr/   r@   r9   r1   rU   rV   rR   )r4   r_   s     r"   unmergeLycorisLayer.unmerge   s     {{MM<=$&&'!+!11557N!9!99##%,,11T5J5J>5ZZ1 $&&'!+r!   c                    U R                    H\  nX R                  ;  a  M  Uc.  U R                  U   U R                  U   -  U R                  U'   ME  U R                  U==   U-  ss'   M^     g r8   )rl   r9   r'   r&   r(   rm   s      r"   unscale_layerLycorisLayer.unscale_layer   s`    "22N%=%==}/3zz./IDFFSaLb/b^,^,5, 3r!   c                    g r8   r   )r4   rI   r&   r'   rD   s        r"   update_layerLycorisLayer.update_layer   s    ORr!   )r.   r'   r,   r0   r2   r/   r*   r3   r&   r)   r-   r(   )r,   	nn.ModulereturnNone)r~   zset[str])r~   r   )rI   strr&   int)
rI   r   rM   torch.TensorrC   r   rD   r   r~   r   )rI   r   r~   r   )FN)r]   boolr^   zOptional[list[str]]r~   r   )rI   r   )rh   floatr~   r   r8   )rI   r   r&   r   r'   r   )r   r   r   r   r   other_param_namesr5   propertyr   r9   rF   rJ   rN   rR   ra   rd   ri   rn   ru   rx   r{   r    r   r!   r"   r$   r$   <   s    
 T)& 2  2
+ Q Q i i F F!<F > >N2
[6 R Rr!   r$   c                  f    \ rS rSr% SrS\S'   \rS\S'   \      SS j5       r	\
SS j5       rS	rg
)LycorisTuner   a  
A base tuner for LyCORIS like adapters

Args:
    model ([`torch.nn.Module`]): The model to be adapted.
    config ([`LoraConfig`]): The configuration of the Lora model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

r   prefixz/dict[type[torch.nn.Module], type[LycorisLayer]]layers_mappingc                    g r8   r   )r4   configrI   targettarget_nameparentcurrent_keys          r"   _create_and_replace LycorisTuner._create_and_replace   s     r!   c                   S nU R                   R                  5        Hb  u  pg[        US5      (       a8  [        UR	                  5       U5      (       a  [        U[
        5      (       a  Un  O[        X65      (       d  M`  Un  O   UcK  SR                  S U R                   R                  5        5       5      n[        S[        U5       SU S35      e[        U[
        5      (       a  UR	                  5       n	OUn	[        U	[        R                  R                  [        R                  R                  45      (       a  U" U4SU0UD6n
U
$ [        U	[        R                  R                  5      (       a  U" U4SU0UD6n
U
$ SR                  S U R                   R                  5        5       5      n[        S[        U5       SU S35      e)	Nr,   z, c              3  8   #    U  H  oR                   v   M     g 7fr8   r   .0layers     r"   	<genexpr>2LycorisTuner._create_new_module.<locals>.<genexpr>        )`F_U..F_   zTarget module of type z, not supported, currently only adapters for z are supportedrI   c              3  8   #    U  H  oR                   v   M     g 7fr8   r   r   s     r"   r   r     r   r   )r   itemshasattr
isinstancer1   r   joinkeysr[   typerX   nnConv2dConv1dLinear)rB   r   rI   r   rD   new_module_clssubtype
target_clssupported_modulestarget_base_layer
new_modules              r"   _create_new_moduleLycorisTuner._create_new_module   s    #&#5#5#;#;#=G--v446@@v~66 ",F,,!+ $> ! $		)`cFXFXF]F]F_)` `(f 7//@.AQ 
 fn-- & 5 5 7 &'%((//588??)KLL'T\TVTJ  )588??;;'T\TVTJ  !%		)`cFXFXF]F]F_)` `(f 7//@.AQ r!   r   N)r   r   rI   r   r   zUnion[LycorisLayer, nn.Module])r   r   rI   r   r   r}   r~   r$   )r   r   r   r   r   r   r$   tuner_layer_clsr   r   classmethodr   r    r   r!   r"   r   r      s[    
 K"OCC  /	  ( (r!   r   )
__future__r   rr   abcr   dataclassesr   r   typingr   r   r	   rX   torch.nnr   peft.configr
   tuners_utilsr   r   r   r   r   r$   r   r   r!   r"   <module>r      sf    #   ( ' '   "  J  2CS> CSLE9 Er!   