
    Lh)                     j   S r SSKrSSKrSSKrSSKJrJrJrJrJ	r	  SSK
Jr  SSKJr  SSKr " S S\5      r " S S	\5      r " S
 S5      r\R"                  \R$                  S4S\S\S\4S jjr\S:X  aK  \" 5       rS r\R1                  S\5        \R3                  SSSS9  \R5                  SS/SS.SS9   \" S5      eg! \ a>  r\R=                  \SSS.\R>                  \R@                  S9r!\"" S \! 35         SrCOSrCff = f\RG                  5       r$\"" S!\RJ                  " \$S"S#9 35        g)$zy
Eden Core - Robust Error Handling & Recovery System
Addresses: Exception handling, graceful degradation, state recovery
    N)AnyCallableDictOptionalList)Enum)datetimec                   (    \ rS rSrSrSrSrSrSrSr	g)	ErrorSeverity   z)Error severity levels for proper handlingcriticalhighmediumlow N)
__name__
__module____qualname____firstlineno____doc__CRITICALHIGHMEDIUMLOW__static_attributes__r       @/home/james-whalen/eden-agi-project/eden_robust_error_handler.pyr   r      s    3HDF
Cr   r   c                   ,    \ rS rSrSrSrSrSrSrSr	Sr
g	)
RecoveryStrategy   z-Recovery strategies for different error typesretryfallbackdegradeabortcheckpoint_restorer   N)r   r   r   r   r   RETRYFALLBACKDEGRADEABORTCHECKPOINT_RESTOREr   r   r   r   r   r      s    7EHGE-r   r   c                      \ rS rSrSrSS\4S jjrS\S\4S jr  SS\S\	S	\
4S
 jjrSS\\\4   S\4S jjrSS\\   S\\   4S jjr\R$                  \R(                  4S\S\\\4   S\S\S\\\4   4
S jjrS\\\4   S\\\4   4S jrS\\\4   S\\\4   4S jrS\\\4   S\\\4   4S jrS\\\4   S\\\4   4S jrS\\\4   4S jrSrg) RobustErrorHandler    z<Comprehensive error handling system with recovery strategieslog_filec                     Xl         / U l        / U l        0 U l        0 U l        [
        R                  " U[
        R                  SS9  [
        R                  " S5      U l	        g )Nz4%(asctime)s - %(name)s - %(levelname)s - %(message)s)filenamelevelformatEdenRobustHandler)
r.   error_historycheckpoint_stackfallback_handlersretry_policiesloggingbasicConfigINFO	getLoggerlogger)selfr.   s     r   __init__RobustErrorHandler.__init__#   sY     )+,.68/1 	,,I	

 ''(;<r   operation_namehandlerc                 \    X R                   U'   U R                  R                  SU 35        g)z3Register a fallback handler for specific operationszRegistered fallback for: N)r6   r<   info)r=   r@   rA   s      r   register_fallback$RobustErrorHandler.register_fallback2   s+    18~.4^4DEFr   max_retriesbackoff_factorc                 (    UUS.U R                   U'   g)z(Set retry policy for specific operationsrF   rG   N)r7   )r=   r@   rF   rG   s       r   set_retry_policy#RobustErrorHandler.set_retry_policy7   s     ',/
N+r   statelabelc                 .   [         R                  " 5       R                  5       UUS.nU R                  R	                  U5        U R
                  R                  SU 35        [        U R                  5      S:  a  U R                  R                  S5        gg)z+Create a checkpoint of current system state)	timestamprM   rL   zCheckpoint created: 
   r   N)	r	   now	isoformatr5   appendr<   rC   lenpop)r=   rL   rM   
checkpoints       r   create_checkpoint$RobustErrorHandler.create_checkpoint?   s     "113


 	$$Z0/w78 t$$%*!!%%a( +r   Nreturnc                    U R                   (       d  U R                  R                  S5        gU(       aj  [        U R                   5       H1  nUS   U:X  d  M  U R                  R	                  SU 35        US   s  $    U R                  R                  SU S35        gU R                   S   nU R                  R	                  S	5        US   $ )
z/Restore from most recent or specific checkpointz(No checkpoints available for restorationNrM   zRestored checkpoint: rL   zCheckpoint z
 not foundzRestored most recent checkpoint)r5   r<   warningreversedrC   )r=   rM   rV   s      r   restore_checkpoint%RobustErrorHandler.restore_checkpointM   s    $$KK JK&t'<'<=
g&%/KK$$'<UG%DE%g.. > KK+eWJ ?@..r2JKK>?g&&r   errorcontextseveritystrategyc                    [         R                  " 5       R                  5       [        U5      R                  [        U5      UUR                  UR                  [        R                  " 5       S.nU R                  R                  U5        U R                  R                  SU 35        U[        R                  :X  a  U R                  U5      $ U[        R                   :X  a  U R#                  U5      $ U[        R$                  :X  a  U R'                  U5      $ U[        R(                  :X  a  U R+                  U5      $ U[        R,                  :X  a  S[        U5      SS.$ S[        U5      SS.$ )zr
Comprehensive error handling with recovery strategies

Returns: Dict with keys: success, result, recovery_action
)rO   
error_typeerror_messagera   rb   rc   	tracebackzError occurred: Fabortedsuccessr`   recovery_actionnone)r	   rQ   rR   typer   strvaluerg   
format_excr4   rS   r<   r`   r   r&   _retry_operationr'   _fallback_operationr(   _degrade_gracefullyr*   _restore_from_checkpointr)   )r=   r`   ra   rb   rc   error_records         r   handle_errorRobustErrorHandler.handle_error_   s1    "113u+.. Z  "--/
 	!!,/,\N;< '---((11)222++G44)111++G44)<<<0099)///$s5ziXX 3u:&QQr   c                    UR                  SS5      nU R                  R                  USSS.5      n[        US   5       H1  n U R                  R	                  SUS-    S	U 35        S
SUS-   S.s  $    SSS.$ ! [
         aG  nXCS   S-
  :X  a  S[        U5      SS.s SnAs  $ SSKnUR                  US   U-  5         SnAM  SnAff = f)z(Retry operation with exponential backoffr@   unknown          @rI   rF   zRetry attempt    z for Tretry_successful)rj   rk   attemptsFretry_failedri   Nr   rG   retry_exhaustedrj   rk   )	getr7   ranger<   rC   	Exceptionrn   timesleep)r=   ra   r@   policyattempter   s          r   rq   #RobustErrorHandler._retry_operation   s     %5yA$$((^a9bcVM23G@  >'A+eNCS!TU#'<N\cfg\ghh	 4 !5FGG  @]3a77',s1vR`aa

6"23w>??	@s$   ,A::
CCC%CCc           	      b   UR                  SS5      nX R                  ;   a:   U R                  U   " U5      nU R                  R                  SU 35        SUSS.$ S	SS.$ ! [         aC  nU R                  R                  SU S[        U5       35        S	[        U5      S
S.s SnA$ SnAff = f)z%Execute fallback handler if availabler@   ry   zFallback successful for Tr"   )rj   resultrk   zFallback failed for z: Ffallback_failedri   Nno_fallback_availabler   )r   r6   r<   rC   r   r`   rn   )r=   ra   r@   r   r   s        r   rr   &RobustErrorHandler._fallback_operation   s     %5yA333a//?H  #;N;K!LM#'6jYY
 !5LMM	  a!!$88H3q6("ST#(3q6N_``as   8A! !
B.+8B)#B.)B.c                 F    U R                   R                  S5        SSSSS.$ )z Degrade to simpler functionalityz'Degrading to reduced functionality modeTdegradedgraceful_degradationz$Operating with reduced functionality)rj   moderk   message)r<   rC   )r=   ra   s     r   rs   &RobustErrorHandler._degrade_gracefully   s+    BC5=	
 	
r   c                 F    U R                  5       nU(       a  SUSS.$ SSS.$ )z#Restore from most recent checkpointTcheckpoint_restored)rj   rL   rk   Fno_checkpoint_availabler   )r^   )r=   ra   rL   s      r   rt   +RobustErrorHandler._restore_from_checkpoint   s-    '')#eH]^^ 5NOOr   c                 x   U R                   (       d  SS0$ [        U R                   5      0 0 0 U R                   SS S.nU R                    Hl  nUS   nUS   R                  US5      S-   US   U'   US	   nUS
   R                  US5      S-   US
   U'   US   nUS   R                  US5      S-   US   U'   Mn     U$ )z*Get statistics about errors and recoveriestotal_errorsr   N)r   by_severityby_typeby_strategyrecent_errorsrb   r   r|   re   r   rc   r   )r4   rT   r   )r=   statsr`   rb   re   rc   s         r   get_error_statistics'RobustErrorHandler.get_error_statistics   s    !!"A&&   2 23!//4
 ''EZ(H-2=-A-E-EhPQ-RUV-VE- * |,J+0+;+?+?
A+NQR+RE)Z( Z(H-2=-A-E-EhPQ-RUV-VE- * ( r   )r5   r4   r6   r.   r<   r7   )z./eden_errors.log)rz   r{   )auto)N)r   r   r   r   r   rn   r>   r   rD   intfloatrJ   r   r   rW   r   r^   r   r   r   r&   r   rv   rq   rr   rs   rt   r   r   r   r   r   r,   r,       s   F= =G Gh G
 HI03
s 
 
(-
)tCH~ )c )' '$ '& /<.B.B1A1G1G!R) !Rd38n !R+!R.!RLPQTVYQYN!RFHS#X H4S> H$N4S> Nd38n N
4S> 
d38n 
PS#X P4S> Pd38n r   r,   rb   rc   r@   c                 8   ^ ^^ S[         S[         4UU U4S jjnU$ )zDDecorator for making operations robust with automatic error handlingfuncrY   c                 N   >^  [         R                  " T 5      U UUU4S j5       nU$ )Nc                  &  > [        5       n T" U 0 UD6$ ! [         aq  nT=(       d    TR                  [        U 5      [        U5      S.nUR	                  X4TT	5      nUS   (       d  [        SU 35      eUR                  S5      s S nA$ S nAff = f)N)r@   argskwargsrj   z)Operation failed after recovery attempt: r   )r,   r   r   rn   rv   r   )
r   r   rA   r   ra   r   r   r@   rb   rc   s
         r   wrapper4robust_operation.<locals>.decorator.<locals>.wrapper   s    (*G,T,V,, ,&4&EI!&k
 !--a(HMi(#&OPVx$XYYzz(++,s    
BA&BBB)	functoolswraps)r   r   r@   rb   rc   s   ` r   	decorator#robust_operation.<locals>.decorator   s%    			, 
	,$ r   )r   )rb   rc   r@   r   s   ``` r   robust_operationr      s#     X  * r   __main__c                     g)Nz*Using cached knowledge instead of learningr   )ra   s    r   learning_fallbackr      s    ;r   meta_learning   g      ?rI   	reasoninglearningactive)capabilitiesrL   pre_operation)rM   zSimulated learning errortraining_batch_1)r@   data)rb   rc   zRecovery result: z
Error Statistics:    )indent)&r   r8   rg   jsontypingr   r   r   r   r   enumr   r	   r   r   r   r,   r   r&   rn   r   r   rA   r   rD   rJ   rW   
ValueErrorr   r   rv   r   r'   r   printr   r   dumpsr   r   r   <module>r      s|  
    6 6   D .t .n nb 0=/C/C2B2H2H+/} /%(: z "G< o/@A_!CP %z2XF  	,344% &  ,%%.8JK"''%..	 & 
 	!&*++, ((*E	 E!!< =
>?s   :C D	4DD