
    oiR;                        S SK Jr  S SKrS SKrS SKJr  S SKrS SKJr  S SK	J
r
  S SKJr  S SKJrJr  S SKJrJr  S SKJr  S	S
KJr  S	SKJr  SSKJr  SSKJrJr        SS jr " S S\5      rg)    )annotationsN)Union)is_bf16_available)Conv1D)is_bnb_4bit_availableis_bnb_available)	BaseTunerBaseTunerLayer)6TRANSFORMERS_MODELS_TO_RANDLORA_TARGET_MODULES_MAPPING   )
BufferDict) _maybe_include_all_linear_layers   )RandLoraConfig)LinearRandLoraLayerc                   [        U [        5      (       aC  [        R                  " U [	        5       (       a  [        R
                  O[        R                  S9nOU n[        R                  " 5          [        R                  R                  R                  U[        R                  " S5      US9nUsSSS5        $ ! , (       d  f       g= f)a  
Kaiming Uniform Initialisation adapted to accept a `torch.Generator` object for PRNG.

Args:
    tensor_or_shape (`Union[torch.Tensor, tuple[int, ...]]`):
        Tensor to initialise, or shape of new tensor to create and then initialise.
    generator: (`torch.Generator`):
        Generator object that manages the state of the PRNG algorithm in use.

Returns:
    `torch.Tensor`: The initialised tensor.
)dtype   )a	generatorN)
isinstancetupletorchemptyr   bfloat16float16no_gradnninitkaiming_uniform_mathsqrt)tensor_or_shaper   tensorbasiss       T/home/james-whalen/.local/lib/python3.13/site-packages/peft/tuners/randlora/model.py_kaiming_initr(   &   s|      /5))$5$7$7%..U]]

 !	..v1QZ.[ 
s   0?B99
Cc                     ^  \ rS rSr% SrSrS\S'   \r\	r
SS jrSSS jjrSS jrSS	 jrSU 4S
 jjrS r\S 5       rSrU =r$ )RandLoraModelC   a  
Creates a RandLoRA model from a pretrained transformers model.

Args:
    model ([`~transformers.PreTrainedModel`]): The model to be adapted.
    config ([`RandLoraConfig`]): The configuration of the RandLora model.
    adapter_name (`str`): The name of the adapter, defaults to `"default"`.
    low_cpu_mem_usage (`bool`, `optional`, defaults to `False`):
        Create empty adapter weights on meta device. Useful to speed up the loading process.

Returns:
    `torch.nn.Module`: The RandLora model.

Example:

    ```py
    >>> from transformers import AutoModelForCausalLM
    >>> from peft import RandLoraConfig, get_peft_model

    >>> base_model = AutoModelForCausalLM.from_pretrained("facebook/opt-125m")
    >>> config = RandLoraConfig(r=32)
    >>> model = get_peft_model(base_model, config)
    ```

**Attributes**:
    - **model** ([`~transformers.PreTrainedModel`]) -- The model to be adapted.
    - **peft_config** ([`RandLoraConfig`]): The configuration of the RandLora model.
	randlora_strprefixc                   U R                  U R                  5      nU R                  X5      n[        X0R                  5      nSnU R                  R	                  5        H  u  pVU R                  X55      (       d  M  [        U[        R                  5      (       a  UR                  UR                  4nOg[        U[        5      (       aP  [        UR                  S5      (       a  UR                  R                  OUR                  R                  nUSSS2   nOM  Uc  UnM  Xt:w  d  M  [!        S [#        XG5       5       5      nM     Uc  Sn[%        U5      eU$ )z
Finds the largest input and output dimensions across linear layers that have been wrapped with RandLora.

This will be used for determining the size of the shared randlora_A and randlora_B matrices.
Nds_shapec              3  <   #    U  H  u  p[        X5      v   M     g 7f)N)max).0r   bs      r'   	<genexpr>*RandLoraModel._find_dim.<locals>.<genexpr>   s     %]<\DAc!ii<\s   z_No layers types compatible with RandLora were found. Please check `peft_config.target_modules`.)get_model_configmodel_prepare_adapter_configr   named_modules_check_target_module_existsr   r   r   out_featuresin_featuresr   hasattrweightr0   shaper   zip
ValueError)	selfconfigmodel_configpeft_configlargest_shapekeymodulemodule_shapemsgs	            r'   	_find_dimRandLoraModel._find_dime   s"    ,,TZZ8226H6{JJO::335KC33KEE&")),,%22F4F4FFFF++9@PZ9[9[v}}55agananatat+DbD1$ ,, %%]C<\%] ]# 6&  sCS/!    c                   U R                  U5      u  pE[        XE5      [        XE5      pv[        0 UR                  S9U l        [        0 UR                  S9U l        [        R                  " SS9R                  UR                  5      n[        R                  " UR                  SU4US9n	XqR                  -  n
U
R                  5       (       a  [        U
5      O[        U
5      S-   n
[        R                  " XjUR                  4US9n[        R                  " UR                   5      n[        R                  " U	R                   5      nSXSSU-  -  :  '   SXSSSU-  -  -
  :  '   SXSSU-  -  :  '   SXSSSU-  -  -
  :  '   XR#                  5       -  XR#                  5       -  pXR
                  U'   XR                  U'   g)	z^
Sparse random projections as described in https://cs-people.bu.edu/evimaria/cs565/kdd-rp.pdf

persistentcpudevicer   r   r1   r   N)rM   r3   minr   save_projection
randlora_A
randlora_Br   	Generatormanual_seedprojection_prng_keyrandr
is_integerintzerosrA   std)rD   rE   adapter_namesparsitylinear_out_dimlinear_in_dimmax_dimmin_dimr   rY   	num_basesrZ   randlora_B_sparserandlora_A_sparses                 r'   "_init_randlora_A_randlora_B_sparse0RandLoraModel._init_randlora_A_randlora_B_sparse   s   
 )-v(>%~=s>?a %RF4J4JK$RF4J4JK OO51==f>X>XY	 ZZ1g 6)L
 hh&	&/&:&:&<&<C	N#i.STBT	ZZVXX >)T
 "KK
(8(89!KK
(8(89=?qAL'99:ABq1H+='==>=?qAL'99:ABq1H+='==>  5 5 77 5 5 77  )3%(2%rO   c           
        U R                  U5      u  p4[        X45      [        X45      pe[        0 UR                  S9U l        [        0 UR                  S9U l        [        R                  " SS9R                  UR                  5      n[        UR                  SU4US9n[        X45      UR                  -  n	U	R                  5       (       a  [        U	5      O[        U	5      S-   n	[        R                  " [!        U	5       V
s/ s H  n
[        USUR                  4US9PM     sn
SS9nXR#                  5       -  XR#                  5       -  pXR
                  U'   XR                  U'   g s  sn
f )NrQ   rS   rT   r   rV   )dim)rM   r3   rW   r   rX   rY   rZ   r   r[   r\   r]   r(   r_   r`   ra   catrangerc   )rD   rE   rd   rf   rg   rh   ri   r   rY   rj   _rZ   s               r'   _init_randlora_A_randlora_B)RandLoraModel._init_randlora_A_randlora_B   s8   (,v(>%~=s>?a %RF4J4JK$RF4J4JK OO51==f>X>XY	 #FHHa#9YO
 6A	&/&:&:&<&<C	N#i.STBT	YYQVW`QabQaA]GQ1YGQabhi


 ",nn.>!>
^^M]@]J(2%(2% cs   !E)c           
        UR                   (       aA  U R                  U5      u  pEU R                  X#[        R                  " [        XE5      5      S9  g UR                  (       a  U R                  X#SS9  g U R                  X#5        g )N)re      )very_sparserM   rm   r"   r#   rW   sparsert   )rD   r9   rE   rd   rf   rg   s         r'   _pre_injection_hook!RandLoraModel._pre_injection_hook   sm    ,0NN6,B)N33tyy^9[/\ 4  ]]33FST3U,,VBrO   c                  > [         TU ]  U5        U R                  R                  5        HJ  nX!L a  M	  UR                  UR                  :w  d  M%  [        SUR                  < SUR                   S35      e   [        U R                  R                  5        Vs1 s H  oR                  iM     sn5      n[        U5      S:  a  [        SU 35      egs  snf )z
A helper method to check the config when a new adapter is being added.

Raise a ValueError if there is something wrong with the config or if it conflicts with existing adapters.

zcRandLora PRNG initialisation key must be the same for all adapters. Got config.projection_prng_key=z but previous config had .r   zgRandLora projection weights must be saved for all adapters or none, but got multiple different values: N)	super_check_new_adapter_configrG   valuesr]   rC   sortedrX   len)rD   rE   existing_configsave_project_unique_values	__class__s       r'   r   'RandLoraModel._check_new_adapter_config   s     	)&1#//668O(22f6P6PP z_e_y_y^{ |++:+N+N*OqR   9 &,RVRbRbRiRiRk,lRk-C-CRk,l%m")*Q.y-.0  / -ms   Cc                   Uc  [        S5      eUR                  n[        US5      =(       a    UR                  S Ln	UUR                  UR
                  UR                  UR                  [        U R                  SS5      [        U R                  SS5      S.n
XS'   [        U[        5      (       aJ  UR                  UU R                  U R                  UUR                  UR
                  UR                  5        g U R                  " XR                  U R                  X#40 U
D6nX R                   ;  a  UR#                  S5        U R%                  XTX5        g )NzCurrent Key shouldn't be `None`biasis_loaded_in_8bitFis_loaded_in_4bit)r_   randlora_alpharandlora_dropoutfan_in_fan_outinit_weightsloaded_in_8bitloaded_in_4bit)rC   r_   r?   r   r   r   r   r   getattrr9   r   r   update_layerrY   rZ   _create_new_moduleactive_adapterrequires_grad__replace_module)rD   randlora_configrd   targettarget_nameparentcurrent_keyoptional_kwargsr_   r   kwargs
new_modules               r'   _create_and_replace!RandLoraModel._create_and_replace   s3    >??vv&B6;;d+B-<< / @ @-<<+88%djj2EuM%djj2EuM
 vff%%..00,, 00$//<[aJ #6#66))%0  jIrO   c                   [        5       (       a
  SS KnSSKJn  [	        5       (       a  SSKJn  UR                  SS5      n	UR                  SS5      n
UR                  SS5      n[        U[        5      (       a  UR                  5       nOUnU
(       a  [        UWR                  R                  5      (       aa  UR                  5       nUR                  UR                  R                  UR                  R                   UR"                  S	.5        W" XCX40 UD6$ U(       a  [        UWR                  R
                  5      (       aa  UR                  5       nUR                  UR$                  UR&                  R(                  UR&                  R*                  S
.5        W" XCX40 UD6$ [        U[,        R                  R.                  5      (       a-  US   (       a"  [0        R2                  " S5        S=US'   U l        OV[        U[6        5      (       a2  SUS'   US   (       d"  [0        R2                  " S5        S=US'   U l        O[9        SU S35      e[/        UUUU4SU	0UD6nU$ )Nr   r   )Linear8bitLt)
Linear4bitr   Fr   r   )has_fp16_weights	thresholdindex)compute_dtypecompress_statistics
quant_typer   zjfan_in_fan_out is set to True but the target module is `torch.nn.Linear`. Setting fan_in_fan_out to False.Tis_target_conv_1d_layerzafan_in_fan_out is set to False but the target module is `Conv1D`. Setting fan_in_fan_out to True.zTarget module z is not supported. Currently, only the following modules are supported: `torch.nn.Linear`, `transformers.pytorch_utils.Conv1D`.)r   bitsandbytesbnbr   r   r   popgetr   r
   get_base_layerr   copyupdatestater   r   r   r   r@   r   r   r   r   warningswarnr   r   rC   )r   rY   rZ   rd   r   r   r   r   r   r   r   r   target_base_layereightbit_kwargsfourbit_kwargsr   s                   r'   r    RandLoraModel._create_new_module   s?    &) ""'zz&%($4e<$4e<fn-- & 5 5 7 &j):CFF<O<OPP$kkmO""(9(?(?(P(P!2!8!8!B!B.44  j`P_``
+<cff>O>O P P#[[]N!!%6%D%D+<+C+C+W+W"3":":"E"E fJ]n]])588??;;&'7 MRQ'(?+I)62204F,-*+w MQP'(?+I  )J J  	

 
 

 rO   )rY   rZ   )returnztuple[int, int])rw   )rE   r   rd   r-   re   ra   r   None)rE   r   rd   r-   r   r   )r9   z	nn.ModulerE   r   rd   r-   r   r   )rE   r   r   r   )__name__
__module____qualname____firstlineno____doc__r.   __annotations__r   tuner_layer_clsr   target_module_mappingrM   rm   rt   rz   r   r   staticmethodr   __static_attributes____classcell__)r   s   @r'   r*   r*   C   sX    : FC#OR#J'3R36	C6*JX C CrO   r*   )r$   z$Union[torch.Tensor, tuple[int, ...]]r   ztorch.Generatorr   ztorch.Tensor) 
__future__r   r"   r   typingr   r   torch.nnr   accelerate.utils.importsr   transformers.pytorch_utilsr   peft.import_utilsr   r   peft.tuners.tuners_utilsr	   r
   
peft.utilsr   _buffer_dictr   tuners_utilsr   rE   r   layerr   r   r(   r*    rO   r'   <module>r      se    #      6 - E > & ; " (9 :aI arO   