
    -jiI6                       S SK Jr  S SKrS SKJr  S SKrS SKJr  S SK	J
r
  \(       a  S SKJr  S SKrS SKJr  S SKJr  OS SKJr  \" S	5      r\" S
5      r\" S5      r\
" \5      r              SS jr            SS jr              SS jr              SS jr              SS jrSSS.         SS jjrSSSSSS.             SS jjrg)     )annotationsN)TYPE_CHECKING)*single_blas_thread_if_scipy_v1_15_or_newer)
get_logger)batched_lbfgsb)BaseAcquisitionFunc)_LazyImportzscipy.optimizetorchzoptuna._gp.batched_lbfgsbc                  ^ ^^ UR                   S:X  d   e[        T5      S:X  a$  X[        R                  " [        U5      [        S94$       S	U UU4S jjn[        5          [        R                  " UUSS2T4   T-  UR                  5        Vs/ s H  owPM     sn4T Vs/ s H
  nSSU-  4PM     sn[        R                  " U5      SS9u  pnSSS5        UR                  5       nW	T-  USS2T4'   W
* nX:  WS:  -  n[        R                  " USS2S4   X5      [        R                  " XU5      U4$ s  snf s  snf ! , (       d  f       Nw= f)
a  
This function optimizes the acquisition function using preconditioning.
Preconditioning equalizes the variances caused by each parameter and
speeds up the convergence.

In Optuna, acquisition functions use Matern 5/2 kernel, which is a function of `x / l`
where `x` is `normalized_params` and `l` is the corresponding lengthscales.
Then acquisition functions are a function of `x / l`, i.e. `f(x / l)`.
As `l` has different values for each param, it makes the function ill-conditioned.
By transforming `x / l` to `zl / l = z`, the function becomes `f(z)` and has
equal variances w.r.t. `z`.
So optimization w.r.t. `z` instead of `x` is the preconditioning here and
speeds up the convergence.
As the domain of `x` is [0, 1], that of `z` becomes [0, 1/l].
   r   dtypec                  > [         R                  " U5      nU R                  S:X  a  UR                  S:X  d   eU T	-  US S 2T4'   [        R                  " U5      R                  S5      nTR                  U5      * nUR                  5       R                  5         UR                  R                  5       R                  5       n[         R                  " UR                  5       R                  5       5      nXeS S 2T4   T	-  4$ )Nr   T)nparrayndimr
   
from_numpyrequires_grad_	eval_acqfsumbackwardgraddetachnumpy
atleast_1d)
scaled_xfixed_paramsnext_paramsx_tensor	neg_fvalsgrads
neg_fvals_acqfcontinuous_indiceslengthscaless
          P/home/james-whalen/.local/lib/python3.13/site-packages/optuna/_gp/optim_mixed.pynegative_acqf_with_grad9_gradient_ascent_batched.<locals>.negative_acqf_with_grad8   s     hh|,}}!k&6&6!&;;;-5-DA))*##K0??E^^H--	  "$$&,,.]]9#3#3#5#;#;#=>
 $6!67,FFF    N      )func_and_grad
x0_batchedbatched_argsboundspgtol	max_iters)r   
np.ndarrayr   zlist[np.ndarray]returntuple[np.ndarray, np.ndarray])r   lenr   zerosboolr   r   copymathsqrtwhere)r#   initial_params_batchedinitial_fvalsr$   r%   tolr'   paramsscaled_cont_xs_optneg_fvals_optn_iterationsxs_opt	fvals_optis_updated_batchs   `  ``          r&   _gradient_ascent_batchedrG      st   . "&&!+++
!#%bhhs=?QY]6^^^GG,<G	&G G" 
4	5:H:W:W1-a1C.CD|S.D.I.I.KL.KU5.KLN(451QAJ5))C.;
7< 
6 $((*F$6$EF1  ! I!1lQ6FG 	!!T'*FK
!m<  M5 
6	5s*   #/ED:E$D?5E:
E
Ec                   [        U5      S:X  a  XS4$ XDX   :g     n[        R                  " US S S 24   [        U5      SS9nXVS S 2U4'   U R                  U5      n[        R                  " U5      nXx   U:  a  XhS S 24   Xx   S4$ XS4$ )Nr*   Fr   )axisT)r5   r   repeateval_acqf_no_gradargmax)	r#   initial_paramsinitial_fval	param_idxchoiceschoices_except_current
all_paramsfvalsbest_idxs	            r&   _exhaustive_searchrU   a   s     7|qU22$0I%IJ>$'2C8N4OVWXJ5q)|"":.EyyH%A+&==..r)   c                  ^ ^^^^^ [        T5      S:X  a  XS4$ SU4S jjnU" UT   5      n[        R                  " UT   TU   5      (       d   eXr* 0mUR                  5       mSU UUUU4S jjmSUU4S jjnSn	[        R
                  " UTS   U	-
  TU   TS   U	-   4S	US
9n
U" U
R                  5      nT" U5      * nX:w  a  X:  a  TU   TT'   TUS4$ XS4$ )Nr*   Fc           	        > [        [        R                  " [        R                  " TU 5      S[	        T5      S-
  5      5      n[        U TUS-
     -
  5      [        U TU   -
  5      :  a  US-
  $ U$ )Nr*   )intr   clipsearchsortedr5   abs)xigridss     r&   find_nearest_index1_discrete_line_search.<locals>.find_nearest_index   sb    q11c%j1nEFAa!e,-AaL0AAq1uHqHr)   c                   > TR                  U 5      nUb  U$ TU    TT'   [        TR                  T5      5      * nUTU '   U$ )N)getfloatrK   )r]   	cache_valnegvalr#   r^   negative_fval_cachenormalized_paramsrO   s      r&   negative_acqf_with_cache7_discrete_line_search.<locals>.negative_acqf_with_cache   sY    '++A.	 ',Qx)$ ../@ABB!'Ar)   c           	     8  > U TS   :  d	  U TS   :  a  [         R                  $ [        [         R                  " [         R                  " TU 5      S[        T5      S-
  5      5      nUS-
  nT" U5      T" U5      pCTU   U -
  TU   TU   -
  -  nSU-
  nXS-  Xd-  -   $ )Nr   r*   g      ?)r   infrX   rY   rZ   r5   )	r\   rightleftneg_acqf_leftneg_acqf_rightw_leftw_rightr^   rh   s	          r&   interpolated_negative_acqf9_discrete_line_search.<locals>.interpolated_negative_acqf   s    uQx<1uRy=66MBGGBOOE15q#e*q.IJqy$T*$U+ & ,"uU|eDk'AB,%(@@@r)   g-q=r   rk   brent)bracketmethodr>   T)r\   rc   r3   rX   )r]   rX   r3   rc   )r\   rc   r3   rc   )r5   r   iscloser8   sominimize_scalarr\   )r#   rM   rN   rO   r^   xtolr_   current_choice_irs   EPSresopt_idxfval_optrh   rf   rg   s   `  ``        @@@r&   _discrete_line_searchr   y   s    5zQU22I *.*CD::nY/7G1HIIII+];&++-
 
A A C


" qC'7!8%)c/JC !'G(11H "x'>',W~)$ (D00..r)   c                    SnU R                   R                  U   nU(       d  [        U5      U::  a  [        XX#U5      $ [	        XX#XE5      $ )N   )search_spaceis_categoricalr5   rU   r   )r#   rM   rN   rO   rP   r{    MAX_INT_EXHAUSTIVE_SEARCH_PARAMSr   s           r&   _local_search_discreter      sO     (*$&&55i@NW)II!$QXYY$T<T[bbr)   c           	         UR                  5       nUR                  5       n[        R                  " [        U5      [        S9n[        U5       H$  u  p[        X
Xy   X4U5      u  pnXU	'   XU	'   XU	'   M&     XgU4$ )Nr   )r8   r   r6   r5   r7   	enumerater   )r#   r<   r=   rO   rP   r{   best_normalized_params_batched
best_fvalsrF   batchrg   best_normalized_params	best_fvalupdateds                 r&   _local_search_discrete_batchedr      s     &<%@%@%B"##%JxxM 2$?$-.D$E 5KZ%6	D6
27 1Gu-%5") %F *7GGGr)   g-C6?d   )r>   max_iterc          
        U R                   U R                  R                  =n   nU R                  R                  nU R                  R	                  5       nU Vs/ s H=  n[
        R                  " [
        R                  " U5      [
        R                  S9S-  PM?     n	nU R                  UR                  5       =n
5      nSn[
        R                  " [        U
5      U[        S9n[
        R                  " [        U
5      5      n[        U5       H  n[!        X
U   X   XEU5      u  X'   X'   n[
        R"                  " UX5      n[%        XgU	5       Hb  u  nnnXU:H  =n)    nUU)    nUR&                  S:X  a  X4s  s  $ [)        X
U   X   UUU5      u  X'   X'   n[
        R"                  " UUU5      nMd     XU:H  =n)    nUU)    nUR&                  S:X  d  M  X4s  $    [*        R-                  S5        X4$ s  snf )N)initial   rk   r   r   z2local_search_mixed: Local search did not converge.)length_scalesr   r$   discrete_indicesget_choices_of_discrete_paramsr   mindiffrl   rK   r8   fullr5   rX   arangerangerG   r;   zipsizer   _loggerwarning)r#   xs0r>   r   	cont_indsr%   r   choices_of_discrete_paramsrP   discrete_xtolsbest_xsr   
CONTINUOUSlast_changed_dimsremaining_inds_r   r]   r{   is_convergeds                       r&   local_search_mixed_batchedr      s    %%D4E4E4X4X'XyZL((99!%!2!2!Q!Q!S
 2	 2G 	rwww0141	   ''CHHJ)>@JJGjDYYs7|,N8_G_.):+Ey`cH
D!;W HHWjL #$4R` aAw 1YZDZ4ZL2[ \+\M:N""a'**..1:3MqRY[_ IG#Z%?
 !#!5F G !b (z:Y*Y,(Z[-|m<!#&&+ . 	LMEs   AG8i   
   )!warmstart_normalized_params_arrayn_preliminary_samplesn_local_searchr>   rngc                   U=(       d    [         R                  R                  5       nUc,  [         R                  " SU R                  R
                  45      n[        U5      US-
  ::  d   S5       eU R                  R                  X%S9nU R                  U5      n[        U[         R                  5      (       d   e[         R                  " U5      n[         R                  " XwU   -
  5      n	SX'   XR                  5       -  n	[        [         R                  " U	S:  5      5      n
[!        U[        U5      -
  S-
  U
5      nX:X  a  ["        R%                  S5        [         R&                  " U/5      nUS:  a1  UR)                  [        U5      USU	S9n[         R*                  " X5      n[         R,                  " XlS S 24   U/5      n[/        XUS	9u  nn[         R                  " U5      R1                  5       nUU   UU   4$ )
Nr   r*   zPWe must choose at least 1 best sampled point + given_initial_xs as start points.)r   g        zBStudy already converged, so the number of local search is reduced.F)r   replacep)r>   )r   randomRandomStateemptyr   dimr5   sample_normalized_paramsrK   
isinstancendarrayrL   expr   rX   count_nonzeror   r   r   r   choiceappendvstackr   item)r#   r   r   r   r>   r   
sampled_xsf_valsmax_iprobsn_non_zero_probs_improvementn_additional_warmstartchosen_idxsadditional_idxsx_warmstartsr   r   rT   s                     r&   optimize_acqf_mixedr     s    
(&&(C(0,.HHa9J9J9N9N5O,P)01^a5GG ZG "";;<Q;[J ##J/Ffbjj))))IIfE
 FF65M)*EEL	YY[E#&r'7'7'D#E  >??!CEa =\]((E7#K!**
O"8%5 % 
 ii=99ja8:[\]L4TSQGZyy$))+H8j222r)   )r#   r   r<   r2   r=   r2   r$   r2   r%   r2   r>   rc   r3   )tuple[np.ndarray, np.ndarray, np.ndarray])r#   r   rM   r2   rN   rc   rO   rX   rP   r2   r3   tuple[np.ndarray, float, bool])r#   r   rM   r2   rN   rc   rO   rX   r^   r2   r{   rc   r3   r   )r#   r   rM   r2   rN   rc   rO   rX   rP   r2   r{   rc   r3   r   )r#   r   r<   r2   r=   r2   rO   rX   rP   r2   r{   rc   r3   r   )
r#   r   r   r2   r>   rc   r   rX   r3   r4   )r#   r   r   znp.ndarray | Noner   rX   r   rX   r>   rc   r   znp.random.RandomState | Noner3   ztuple[np.ndarray, float])
__future__r   r9   typingr   r   r   "optuna._gp.scipy_blas_thread_patchr   optuna.loggingr   scipy.optimizeoptimizery   r
   
optuna._gpr   optuna._gp.acqfr   optunar	   __name__r   rG   rU   r   r   r   r   r    r)   r&   <module>r      se   "     Y % )3"	%	&B E !<=N X
A
A&A A #	A
 A 
A /AH/
// / 	/
 / $/0A/
A/A/ A/ 	A/
 A/ A/ $A/Hc
cc c 	c
 c c $c&H
H&H H 	H
 H H /H2 AEVY-
-$.-8=-PS-"-f <@!%(,13
13 (913 	13
 13 
13 
&13 13r)   