
    ^h%+                    9   % S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKrS SKrS SK	r	S SK
r
S SKrS SKrS SKrS SKJrJr  S SKJrJrJrJrJrJrJrJrJr  \(       a	  S SKJr  \rO\rGSS jr\\l        \R<                  " S\S9  / r \!\"S'   \	RF                  RI                  S \RF                  RK                  S	5      5         S SK&r&S SK'r'S SK(r(S SK)r)S SK*r*S S
K+J,r,  / SQr/S r0S SK1J2r2  S SK3r3S SK3J4r4J5r5J6r6J7r7J8r8  S SK9J:r:J;r;  S SK<J=r=J>r>  S SK?J@r@  S SKAJBrB  S SKCJDrDJErE  S SKFJGrGJHrH  S SKI7  S SKJJKrL  S SKMJNrN  S SKOJPrP  S SKQJRrR  S SKSJTrT  S SKUJVrVJWrWJXrX  S SKYJZrZJ[r[  S SK\JKr]  S SK^J_r_  S S K`JaraJbrbJcrc  S S!KdJere  S SKdJKrf  S S"KgJhrhJiri  S S#KjJkrkJlrl  S S$KmJnrnJoro  S S%KpJqrq  S S&KrJsrs  S S'KtJuru  S SKvJKrw  S S(KvJxrx  S S)KyJzrzJ{r{  S S*K|J}r}  S SK~JKr  S S+KJr  S S,KJr  S S-KJr  S S.KJr  S S/KJr  S SKJKr  S SKJKr  S S0KJr  S S1KJrJrJr  S SKJKr  S SKJKr  S SKJKr  S SKJKr  S S2KJr  S SKJKr  S S3KJr  S S4KJr  S SKJKr  S S5KJr  S SKJKr  S S6KJr  S SKJKr  S SKJKr  S S7KJr  S SKJKr  S S8KJr  S SKJKr  S S9KJrJrJrJrJrJrJrJrJrJrJrJrJr  S SKJKr  S SKJKr  S S:KJr  S S;KJrJrJrJr  S S<KJrJrJr  S S=KJr  S S>KJr  S S?KJrJrJrJr  S S@KJr  S SAKJrJrJrJr  S SBKJr  S SCKJr  S SDKJrJr  S SEKJr  S SFKJr   S SGKJr  SI\3l        S SKrS SJKJr  S SKK(JrJrJrJrJrJrJrJrJrJrJr  S SLKJr  S SMKJr  S SNKJr  S SOGK GJGrGJGrGJGrGJGrGJGr  S SPGKGJGr  S SQGKGJ	Gr	  S SRGK
GJGr  S SSGKGJGr   S SGKGJs  GJGr  \GR"                  " STSU5      Gr\T" 5       GqG[&        GR)                  5       GqG\\"SV'   \GR"                  " SW5      Gr\G\   \"SX'   \s" 5       GrG\c
  SYGrG\\"SZ'   OG\" G\5      Gr\GR"                  " S[5      Gr\G\   \"S\'   G\c
  S]GrG\\"S^'   OG\" G\5      GrG\ S_3GrS`G\ Sa3Gr G\ Sb-  Gr ScGr!G[*        (       a  \GR"                  " SdSe5      OSeGr"G[*        (       a  \GR"                  " SfSgG\! ShG\  35      O	SiG\! ShG\  3Gr#\" \" 5       \" 5       G\"G\#\G\Sj9Gr$Sk Gr%\GR"                  " SlSm5      Sn:X  a  G[*        (       a
  G\%G\$Gl&         " So SpG\'GRP                  5      Gr)G\$GRU                  G\+5      Sq\SrG\+4Ss j5       Gr,G\" 5       rKSt/Gr- \RF                  GR]                  \RF                  RK                  G\/5      5      Gr0\RF                  GRc                  G\0SuSv5      Gr2G\$GRg                  SwG\" G\2SISx9SySz9  \GRh                  " G\25       H  Gr5G\5GRm                  S{5      (       d  M  G\5S|:w  d  M'  \RF                  GRo                  G\55      S    Gr8\RF                  GRc                  G\2G\85      Gr9\GRt                  " G\9SIS}9  \RF                  GRc                  G\2G\55      Gr;\RF                  GRc                  G\9S|5      Gr<\GRz                  " G\;G\<5        M     G\SU:w  aQ  G\>" S~G\ Sw35        \GR"                  " S5      c  G\\GR~                  S'   G\$GR                  S5      Sq\4S j5       GrAG\$GR                  \G\-SISt/St/S9  S SKGJCGrC  SGqDSGqESGqFSGqGSGqHSGqISIGqJSGrKSGqLSGqM\G\   \"S'   SIGrNSGqOSGqP\\8   \"S'   SGqQ\\!   \"S'   0 GqRG\S\"S'   0 GqTG\S\"S'   SGrUSGrVSGqW\G\   \"S'   SGqXSGqY\\   \"S'   \=" G\)GR                  GR                  S9Gr\\" G\\S9Gr]\3GR                  GR                  G\]5        SGq`\\>   \"S'   SGqaSGqbSGqcSGqdSGqeSGqfSGqg0 Gqh/ Gri\\"S'   SGrjSGqkSGqlG\mS   \"S'   SGqnSGqoSGqpSGqqSGqr\R" 5       GrsSGqt\\   \"S'   SGquG\\"S'   SGqv\\   \"S'   \" G\\G[*        S9GqwSGrxSGrySGrzSGq{\\G   \"S'   SSSSSSSS0 S.	SG\|S\G\   S\G\   S\G\   S\G\   S\G\   S\G\   S\\G\G\4      S\G\   S\G\S   SG\S4S jjGr}Sq\4S jGr~S GrS\4S jGrGSSG\4S jjGrS GrS Gr GSS\G\SG\4   S\S\G\   4S jjGr        GSS jGrS\G\   S\G\   S\G\   S\G\   S\G\   S\\   4S jGrS GrS Gr " S S5      GrG\" 5       GqS Gr                 GSS jGrS GrSG\|SG\S4S jGrSG\|SG\S4S jGrSG\|SG\S4S jGrSG\|SG\S4S jGr0 4SG\S4S jjGrS GrS Gr " S S5      Gr\KGR/                  S5      S 5       Gr\KGR3                  S\" \Z5      /S/S9\KGR3                  S\" \Z5      /S/S9\" \Z5      4SG\|4S jj5       5       Gr\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/SSS00\uES9\'GR8                  " \'GR:                  \G\G\G\G\\:S9S\" \Z5      4Sq\S\S\G\   SG\|4S jj5       5       5       5       5       Gr\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9S\" \Z5      4Sq\S\S\G\   SG\|4S jj5       5       5       5       Gr\KGR7                  S\" \Z5      /G\S/S9\KGR7                  S\" \Z5      /G\S/S9\KGR7                  S\" \Z5      /G\S/S9\KGR7                  S\" \Z5      /G\S/S9S\" \Z5      4Sq\S\S\G\   SG\|4S jj5       5       5       5       Gr\KGR7                  S\" \Z5      /G\S/S9\KGR7                  S\" \Z5      /G\S/S9\" \Z5      4Sq\S\SG\|4S jj5       5       Gr\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\" \Z5      4Sq\S\SG\|4S jj5       5       Gr\KGR7                  S\" \Z5      /S/S9\KGR7                  S\" \Z5      /S/S9\" S5      \" \Z5      4Sq\S\S\SG\|4S jj5       5       GrS SK(JrGJGrGJGr  S SK3GJGr  G\$GRO                  S5      \" \[5      4SG\SG\4S jj5       Gr\KGR3                  S\" \Z5      /GS /S9\KGR3                  GS\" \Z5      /GS /S9\" \Z5      4Sq\S\SG\|4GS jj5       5       Gr\KGR7                  S\" \Z5      /GS /S9\KGR7                  GS\" \Z5      /GS /S9\" \Z5      4Sq\S\SG\|4GS jj5       5       Gr\KGRW                  GS\" \Z5      /GS /S9\KGRW                  GS\" \Z5      /GS /S9\" \Z5      4Sq\GSG\S\SG\|4GS jj5       5       Gr\KGR7                  GS\" \Z5      /GS /S9\KGR7                  GS	\" \Z5      /GS /S9\" \Z5      4Sq\S\SG\|4GS
 jj5       5       Gr\KGR3                  GS\" \Z5      /GS /S9\KGR3                  GS\" \Z5      /GS /S9\" \Z5      4Sq\GSG\S\SG\|4GS jj5       5       Gr\KGR7                  GS\" \Z5      /GS /S9\KGR7                  GS\" \Z5      /GS /S9\" \Z5      4Sq\GSG\S\SG\|4GS jj5       5       Gr\KGR3                  GS\" \Z5      /GS /S9\KGR3                  GS\" \Z5      /GS /S9\" \Z5      4Sq\GSG\S\SG\|4GS jj5       5       Gr\KGR7                  GS\" \Z5      /GS /S9\KGR7                  GS\" \Z5      /GS /S9\" \Z5      4Sq\GSG\S\SG\|4GS jj5       5       Gr\KGR7                  GS\" \Z5      /GS/S9\KGR7                  GS\" \Z5      /GS/S9\KGR7                  GS\" \Z5      /GS/S9S\" \Z5      4Sq\S\GS\G\   SG\|4GS jj5       5       5       Gr\KGR3                  GS\" \Z5      /GS/S9\KGR3                  GS\" \Z5      /GS/S9\KGR3                  GS\" \Z5      /GS/S9\" \Z5      S\" GSGS GS!94Sq\S\SG\|GS\G\   GS"G\4
GS# jj5       5       5       Gr\KGR3                  GS\" \Z5      /GS/S9\KGR3                  GS\" \Z5      /GS/S9\KGR3                  GS\" \Z5      /GS/S9SSS\" \Z5      4S\GS\G\   GS$\G\   GS%\G\   SG\|4
GS& jj5       5       5       Gr\KGR7                  GS'\" \Z5      /G\GS(/S9\KGR7                  GS)\" \Z5      /G\GS(/S9\" \Z5      4Sq\S\SG\|4GS* jj5       5       Gr\KGR7                  GS+GS,/\" \Z5      /\SGS-9\" \Z5      4GS.\S\Sq\SG\|4GS/ jj5       Gr\KGR7                  GS0GS1/\" \Z5      /G\GS29SqG\4GS3 j5       Gr\KGR3                  GS4GS1/\" \Z5      /GS59SG\4GS6 j5       Gr\KGR7                  GS7GS8/\" \Z5      /GS59\" \Z5      4GS9G\SG\|4GS: jj5       Gr\KGR7                  GS;GS8/\" \Z5      /GS59\" \Z5      4GS9G\SG\|4GS< jj5       Gr\KGR7                  GS=GS8/\" \Z5      /GS59GS>G\4GS? j5       Gr\KGR3                  GS@GS8/\" \Z5      /GS59\" \Z5      4GSAG\SG\|4GSB jj5       Gr\KGR3                  GSCGS8/\" \Z5      /GS59\" \Z5      4SG\|4GSD jj5       Gr\KGR7                  GSEGS8/\" \Z5      /GS59\" \Z5      4GS>G\SG\|4GSF jj5       GrGSG\SG\|S\4GSH jGrGSG\SG\|S\4GSI jGrS\G\   SG\|SG\4GSJ jGr\KGR7                  GSKGSLS/\" \Z5      /GSM9\" \Z5      4GSG\SG\|4GSN jj5       Gr\KGR7                  GSOGSPS/\" \Z5      /GSM9\" \Z5      4GSG\SG\|4GSQ jj5       Gr\KGR3                  GSRGSSS/\" \Z5      /SGST9\" \Z5      \(GR                  " SGSUGSV9S4SG\|S\G\   GSW\G\   4GSX jj5       Gr\KGR3                  GSYGSZS/S\" \Z5      /GS[9\" \Z5      SSS4SG\|GS\\G\   GS]\\   GS^\\   4GS_ jj5       Gr\KGR3                  GS`GSaS/S\" \Z5      /GS[9\" \Z5      GSbSSSS4SG\|GS\\G\   GS]\\   GS^\\   GSc\G\   GSd\G\   4GSe jj5       Gr\KGR3                  GSfGSgS/S\" \Z5      /GS[9\" \Z5      GSbSSSS4SG\|GS\\G\   GS]\\   GS^\\   GSc\G\   GSd\G\   4GSh jj5       Gr\KGR3                  GSiGSjS/S\" \Z5      /GS[9\" \Z5      SSSSS4SG\|GS\\G\   GS]\\   GS^\\   GSc\G\   GSd\G\   4GSk jj5       Gr\KGR3                  GSlS/\" \Z5      /GS59\KGR3                  GSmS/\" \Z5      /GS59\" \Z5      S4SG\|GSn\G\   4GSo jj5       5       Gr\KGR3                  GSpS/\" \Z5      /GS59\" \Z5      4SG\|4GSq jj5       Gr\KGR7                  GSrGSsS/\" \Z5      /GSM9GStG\4GSu j5       Gr\KGR3                  GSvGSwS/\" \Z5      /SGST9GSx 5       Gr\KGR3                  GSyGSzGS{/\" \Z5      /SGST9\" \Z5      4SG\|4GS| jj5       Gr\KGR7                  GS}GS~/\" \Z5      /SGS9S\" \Z5      4Sq\S\S\G\   SG\|4GS jj5       GrG\$GR3                  GSGS~/SGS9Sq\4GS j5       Gr\KGR7                  GSSGS9Sq\4GS j5       GrG\$GR3                  GSSGS9GSG\4GS j5       GrG\$GR7                  GSSGS9GS>G\4GS j5       GrG\$GR3                  GSSGS9GS 5       Gr\KGR7                  GSGS/\" \Z5      /G\SGS-9\" \Z5      4GS>G\SG\|4GS jj5       Gr\KGR3                  GSGS/\" \Z5      /G\SGS-9\" \Z5      4GSG\SG\|4GS jj5       Gr\KGR7                  GSGS/\" \Z5      /G\SGS-9\" \Z5      4GS>G\SG\|4GS jj5       Gr\KGR7                  GSGS/\" \Z5      /G\SGS-9\" \Z5      4GS>G\SG\|4GS jj5       Gr\KGR7                  GSGS/\" \Z5      /SGS9GSG\4GS j5       Gr \KGR7                  GSGS/\" \Z5      /SGS9\" \Z5      4GS>G\SG\|4GS jj5       Gr\KGR3                  GSGS/\" \Z5      /G\SGS-9\" \Z5      4GSG\SG\|4GS jj5       Gr\KGR3                  GSGS/\" \Z5      /SGS9\" \Z5      4GSG\mS   SG\|S\G\   4GS jj5       Gr\KGR7                  GSGS/\" \Z5      /SGS9\" \Z5      4GS>G\SG\|4GS jj5       Gr\KGR3                  GSGS/S\" \Z5      /GS9GS 5       Gr\KGR3                  GSGS/\" \Z5      /SGS9GSG\4GS j5       Gr\KGR3                  GSS\" \Z5      /GS9GS 5       Gr\KGR3                  GS\" \Z5      /GS9Sq\4GS j5       Gr\KGR3                  GS\" \Z5      /GS9GS 5       Gr\KGR/                  GS5      GS 5       GrGS GrG\$GR                  \K5        G\$GR                  \5        G\$GR                  \w5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \]5        G\$GR                  \L5        G\$GR                  \f5        G\$GR                  \5        G\$GR                  \5        G\$GR                  \5        g! \- a  r.\-" S\. S35      eSr.C.ff = f! \ a    SHr GNf = f! \ a     S SGKGr GN! \ a      GNf = ff = f! \ a     GNf = f(      N)datetime	timedelta)	TYPE_CHECKINGAnyListOptionalTuplecastget_args
get_originget_type_hints)Spanc                 ^    U SU SUR                    SU  S3nUb  UR                  U5        g g )N:: 
)__name__write)messagecategoryfilenamelinenofilelinetraceback_infos          T/home/james-whalen/.local/lib/python3.13/site-packages/litellm/proxy/proxy_server.pyshowwarningr   $   s>     z6("X->->,?r'"MN

>"     default)r   messagesz../..)AsyncIOSchedulerzMissing dependency z$. Run `pip install 'litellm[proxy]'`)z%'The thing I wish you improved is...'z'A feature I really want is...'z*'The worst thing about this product is...'z$'This product would be better if...'z 'I don't like how this works...'z&'It would help me if you could add...'z/'This feature doesn't meet my needs because...'z&'I get frustrated when the product...'c                     Sn [         R                  " [        5      n[        5         [        SSU -  -   S-   5        [        SSU -  -   S-   5        [        SSR	                  U5      -   5        [        SSR	                  S5      -   5        [        SSU -  -   S-   5        [        SSU -  -   S-   5        [        5         [        S	5        [        5         [        5         [        5         [        S
5        [        5         [        5         g )N<   z[1;37m#-z#[0m z[1;37mz# {:^59} #[0mz-https://github.com/BerriAI/litellm/issues/newz/ Thank you for using LiteLLM! - Krrish & IshaanzR[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new[0m)randomchoicelist_of_messagesprintformat)	box_widthr   s     r   generate_feedback_boxr,   L   s    I mm,-G	G	
sY
.
;<	
sY
.
;<	,,33G<
<=	

$
$%T
U	V 

sY
.
;<	
sY
.
;<	G	
;<	G	G	G	b 
G	Gr   )defaultdict)CancelBatchRequestCreateBatchRequestListBatchRequestRetrieveBatchRequestRouterverbose_proxy_loggerverbose_router_logger)	DualCache
RedisCache)RejectedRequestError)SlackAlerting)!_get_parent_otel_span_from_kwargs get_litellm_metadata_from_kwargs)AsyncHTTPHandlerHTTPHandler)*router)log_db_metrics)check_response_size_is_safe)
JWTHandler)LicenseCheck)get_complete_model_listget_key_modelsget_team_models)user_api_key_authuser_api_key_auth_websocket)	html_form)get_logging_caching_headers3get_remaining_tokens_and_requests_from_request_datainitialize_callbacks_on_proxy)init_verbose_loggers)decrypt_value_helperencrypt_value_helper)_read_request_bodycheck_file_size_under_limit)!get_config_file_contents_from_gcsget_file_contents_from_s3)%remove_sensitive_info_from_deployment)
ProxyState)ERROR_RESPONSES)set_fine_tuning_config)init_guardrails_v2initialize_guardrails)perform_health_check)&_PROXY_VirtualKeyModelMaxBudgetLimiter)"_OPTIONAL_PromptInjectionDetection)_PROXY_failure_handler)_PROXY_track_cost_callback)add_litellm_data_to_request)user_update)delete_verification_tokenduration_in_secondsgenerate_key_helper_fn)update_team)create_audit_log_for_update)is_known_model)set_files_config)!initialize_pass_through_endpoints)route_request)get_logging_payload)PrismaClientProxyLogging_cache_user_row_get_docs_url_get_projected_spend_over_limit_get_redoc_url_is_projected_spend_over_limit_is_valid_team_configsget_error_message_strget_instance_fn
hash_tokenreset_budgetupdate_spend)set_default_vertex_config)AssistantsTypedDict
DeploymentLiteLLM_ParamsModelGroupInfo)DefaultPrioritiesFlowItem	Scheduler)load_aws_kms)load_google_kms)
get_secretget_secret_boolget_secret_strstr_to_bool)SlackAlertingArgs)AnthropicMessagesRequestAnthropicResponse!AnthropicResponseContentBlockTextAnthropicResponseUsageBlock)HttpxBinaryResponseContent)	ModelInfo)RouterGeneralSettingsupdateDeployment)StandardLoggingPayload)!get_end_user_id_for_cost_tracking)versionz0.0.0T)Union)DependsFastAPIFileFormHeaderHTTPExceptionPathRequestResponse
UploadFilestatus)jsonable_encoder)CORSMiddleware)get_openapi)FileResponseJSONResponseORJSONResponseRedirectResponseStreamingResponse)	APIRouter)OAuth2PasswordBearer)APIKeyHeader)StaticFilesSERVER_ROOT_PATH premium_user+LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES'global_max_parallel_request_retries_env   #global_max_parallel_request_retries1LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT-global_max_parallel_request_retry_timeout_envg      N@)global_max_parallel_request_retry_timeout/ui/u(   👉 [```LiteLLM Admin Panel on /ui```](z). Create, Edit Keys with SSOuB   

💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai/).zr[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)
DOCS_TITLEzLiteLLM APIDOCS_DESCRIPTIONzJEnterprise Edition 

Proxy Server to call 100+ LLMs in the OpenAI format. 

z5Proxy Server to call 100+ LLMs in the OpenAI format. )docs_url	redoc_urltitledescriptionr   	root_pathc                  p   [         R                  (       a  [         R                  $ [        [         R                  [         R                  [         R
                  [         R                  S9n [        R                  R                  n0 nU H  nU S   U   X#'   M     X S'   U [         l        [         R                  $ )N)r   r   r   routespaths)
appopenapi_schemar   r   r   r   r   LiteLLMRoutesopenai_routesvalue)r   r   paths_to_includeroutes       r   custom_openapir   }  s    
!!! iiOOzz	N "//55M"0"9%"@ .7'Cr   DOCS_FILTEREDFalseTruec                       \ rS rSrSrSrg)UserAPIKeyCacheTTLEnum  r#    N)r   
__module____qualname____firstlineno__in_memory_cache_ttl__static_attributes__r   r   r   r   r     s    r   r   requestexcc           	         #    UR                   n[        UR                  (       a  [        UR                  5      O[        R
                  SUR                  UR                  UR                  UR                  S.0US9$ 7f)Nerrorr   typeparamcode)status_codecontentheaders)	r   r   r   intr   HTTP_500_INTERNAL_SERVER_ERRORr   r   r   )r   r   r   s      r   openai_exception_handlerr     sf      kkG XXCM6+P+P ;;	
  s   A;A=r>   _experimentalout/ui)	directoryhtmlui)namez.htmlz
index.html)exist_okz8server_root_path is set, forwarding any /ui requests to PROXY_BASE_URLhttpc                    #    U R                   R                  R                  S5      (       a9  U R                   R                  R                  S[         S3S5      n[        U5      $ U" U 5      I S h  vN $  N7f)Nr      )urlpath
startswithreplaceserver_root_pathr   )r   	call_nextnew_paths      r   redirect_ui_middlewarer     sd     {{**511";;++33E>N=Os;SUVW'11"7++++s   A/A81A62A8)allow_originsallow_credentialsallow_methodsallow_headers)DictFuser_config_file_path
llm_routerllm_model_listgeneral_settingscallback_settingszapi_log.json
master_keyprisma_client)default_in_memory_ttl)
dual_cacheredis_usage_cachequeuezlitellm-proxy-budgetdefault_user_idall)adminr  ui_access_modeiU  i]  
   prompt_injection_detection_objstore_model_in_dbopen_telemetry_logger)user_api_key_cacher   db_writer_client)	call_idmodel_id	cache_keyapi_baser   model_regionresponse_cost!fastest_response_batch_completionrequest_datauser_api_key_dictr  r  r  r  r   r  r  r  r  returnc        
         v   SS 1nUUUUUU[        U5      [        U R                  5      [        U R                  5      Ub  [        U5      OS S.
U
R                  5        VVs0 s H  u  pU[        U5      _M     snnEnU	(       a?  [	        U	5      nUR                  U5        [        U	5      nU(       a  UR                  U5         UR                  5        VVs0 s H  u  nnUU;  d  M  U[        U5      _M     snn$ s  snnf s  snnf ! [         a%  n[        R                  " SU 35        0 s S nA$ S nAff = f)Nr   )
x-litellm-call-idzx-litellm-model-idzx-litellm-cache-keyzx-litellm-model-api-basezx-litellm-versionzx-litellm-model-regionzx-litellm-response-costzx-litellm-key-tpm-limitzx-litellm-key-rpm-limitz+x-litellm-fastest_response_batch_completionzError setting custom headers: )
str	tpm_limit	rpm_limititemsrL   updaterK   	Exceptionr4   r   )r  r  r  r  r  r   r  r  r  r  kwargsexclude_valueskvr   remaining_tokens_headerlogging_caching_headerskeyr   es                       r   get_custom_headersr+    sB    $ZN$&($,$".#&}#5#&'8'B'B#C#&'8'B'B#C 1< 12 "(
01c!f9
0G" "U#
 	./"=l"K"NN23 &mmo
-
UN* CUO-
 	
 1

  ""%CA3#GH	s<   !C=D	 D)D9D	 D	 	
D8D3-D83D8c                 ^  #    [         R                   " 5       n[         R                   " 5       U-
  S:  at  [        R                  " S5      I Sh  vN   U R                  5       I Sh  vN (       a  UR	                  5         [        SSS9e[         R                   " 5       U-
  S:  a  Ms  gg N[ NE7f)aI  
Asynchronously checks if the request is disconnected at regular intervals.
If the request is disconnected
- cancel the litellm.router task
- raises an HTTPException with status code 499 and detail "Client disconnected the request".

Parameters:
- request: Request: The request object to check for disconnection.
Returns:
- None
X  r   Ni  zClient disconnected the requestr   detail)timeasynciosleepis_disconnectedcancelr   )r   llm_api_call_task
start_times      r   check_request_disconnectionr7  S  s      J
))+

"S
(mmA((*** $$&8  ))+

"S
(*s*   AB-B)B-%B+&?B-'B-+B-c                     SSK Jn  [        U 5      nU[        L a'  [	        U 5       H  n[        X15      (       d  M  Us  $    g[        U [        5      (       a  [        U [        5      (       a  U $ g)CResolve the actual TypedDict class from a potentially wrapped type.r   )_TypedDictMetaN)typing_extensionsr:  r   r   r   
isinstancer   dict)typr:  originargs       r   _resolve_typed_dict_typerA  p  s[    0_FC=C#..
 !
  
C		:c4#8#8
r   c                 :   [        U 5      n/ nU[        L aW  [        U 5       HF  nUc  M  [        U[	        S5      5      (       a  M$  S[        U5      ;  d  M5  UR                  U5        MH     U$ [        U [        5      (       a  [        U [        5      (       a  U /$ U$ )r9  NNoneType)r   r   r   r<  r   r  append	BaseModel)r>  r?  typsr@  s       r   _resolve_pydantic_typerG  ~  s    _FDC=C"3T
33c#h.C  ! K 
C		:c9#=#=uKr   use_azure_key_vaultc                 H   U SL a  g  SSK Jn  SSKJn  [        R
                  " SS 5      nUc  [        S5      eU" 5       nU" X4S9nU[        l        [        R                  [        l        g ! [         a,  n[        U5      n[        R                  " SU5         S nAg S nAff = f)	NFr   )DefaultAzureCredential)SecretClientAZURE_KEY_VAULT_URIzMError when loading keys from Azure Key Vault: AZURE_KEY_VAULT_URI is not set.)	vault_url
credentialztError when loading keys from Azure Key Vault: %s .Ensure you run `pip install azure-identity azure-keyvault-secrets`)azure.identityrJ  azure.keyvault.secretsrK  osgetenvr"  litellmsecret_manager_clientKeyManagementSystemAZURE_KEY_VAULT_key_management_systemr  r4   	exception)rH  rJ  rK  KVUrirN  clientr*  
_error_strs           r   load_from_azure_key_vaultr\    s    e#
97 		/6=_  ,-
 E(.%)<)L)L& 
V
&& C	
 	

s   A"A+ +
B!5"BB!c                      [         bw  [        [        R                  [        5      (       aS  [
        R                  " S5        [        [        R                  ;  a$  [        R                  R                  [        5        g g g g )Nz.setting litellm success callback to track cost)	r  r<  rS  _async_success_callbacklistr4   debugr_   rD  r   r   r   cost_trackingra    s]     g55t<< &&'WX*73R3RR//667QR S = !r   c                      [         bw  [        [        R                  [        5      (       aS  [
        R                  " S5        [        [        R                  ;  a$  [        R                  R                  [        5        g g g g )Nz.setting litellm failure callback to track cost)	r  r<  rS  failure_callbackr_  r4   r`  r^   rD  r   r   r   error_trackingrd    s]     g..55 &&'WX&w/G/GG((//0FG H 6 !r   payloadspend_logs_urlc                 @   Ub|  Uby  [        U S   [        5      (       a  U S   R                  5       U S'   [        U S   [        5      (       a  U S   R                  5       U S'   UR                  R	                  U 5        U$ Ub  UR                  R	                  U 5        U$ )N	startTimeendTime)r<  r   	isoformatspend_log_transactionsrD  )re  r  rf  s      r   _set_spend_logs_payloadrl    s    
  ^%?gk*H55#*;#7#A#A#CGK gi((33!(!3!=!=!?GI,,33G<  
	",,33G<r   c
           
      
  ^^^^^^^^^	^#     [         R                  " ST SU  ST ST 35        U b5  [        U [        5      (       a   U R	                  S5      (       a
  [        U S9mOU mUUU4S jn
UU4S jnUUUUUU4S	 jnUUU4S
 jnU	U4S jn[        R                  " U
" 5       5        [        R                  " U" 5       5        [        R                  " U" 5       5        [        R                  " U" 5       5        [        SL a  U" 5       I S h  vN   O[         R                  " S5        [         R                  " S5        g  N2! [         a/    [         R                  " S[        R                  " 5        35         g f = f7f)Nz&Enters prisma db call, response_cost: z	, token: z; user_id: z; team_id: sk-tokenc            	        >#    [         R                  TS9I Sh  vN n U b   [        U [        5      (       a  [	        S0 U D6n  [
        b  T/n[        R                  S:  a  UR                  [        5        U H<  nUc  M  T[
        R                  R                  US5      -   [
        R                  U'   M>     Tb5  T[
        R                  R                  TS5      -   [
        R                  T'   ggg N! [         aE  n[        R                  " SS[!        U5       S["        R$                  " 5        3-   5         SnAgSnAff = f7f)zQ
- Update that user's row
- Update litellm-proxy-budget row (global proxy spend)
r)  Nr   z[91mz&Update User DB call failed to execute r   r   )r  async_get_cacher<  r=  LiteLLM_UserTabler  rS  
max_budgetrD  litellm_proxy_budget_nameuser_list_transactonsgetend_user_list_transactonsr"  r4   infor  	traceback
format_exc)existing_user_objuser_ids_idr*  end_user_idr  user_ids       r   _update_user_db(update_database.<locals>._update_user_db  s@     '9&H&HW&H&U U ,<Mt1T1T$5$J8I$J! , 'yH**Q. (AB'? -"/"E"E"I"I#q"Q!R *??D  ( #.)+EEII +Q &??L / -	 !V0  $))>s1vhbI]I]I_H`ab s@   EC2'E9C4  A/C4 /E4
E>;D>9E>EEc                  6  >#     [         R                  " ST ST S35        Tc  g [        b5  T[        R                  R	                  TS5      -   [        R                  T'   g g ! [
         a)  n [         R                  " S[        U 5       35        U eS n A ff = f7f)Nz'adding spend to key db. Response cost: z	. Token: .r   z'Update Key DB Call failed to execute - )r4   r`  r  key_list_transactonsrx  r"  rX  r  )r*  hashed_tokenr  s    r   _update_key_db'update_database.<locals>._update_key_db  s     $**=m_IVbUccde  ' ,%'<<@@qQR "66|D -
  $..=c!fXF 	s2   B A# B;A# !B#
B-$BBBc                  "  >#     [         b6  [        TTTTTS9n TU S'   [        U [        R                  " S5      [         S9q g g ! [
         a?  n[        R                  " S[        U5       S[        R                  " 5        35        UeS nAff = f7f)N)r#  response_objr6  end_timer  spendSPEND_LOGS_URL)re  rf  r  z)Update Spend Logs DB failed to execute - r   )r  rk   rl  rQ  rR  r"  r4   r`  r  r{  r|  )re  r*  completion_responser  r  r#  r  r6  s     r   _insert_spend_log_to_db0update_database.<locals>._insert_spend_log_to_db  s      ,1%%8#-!)$/G (5GG$$; '')yy1A'B&3%M -   $**?Axr)J^J^J`Iab 	s(   B<A B
B:BBBc                  (  >#     [         R                  " ST ST S35        Tc  [         R                  " S5        g [        br  T[        R                  R	                  TS5      -   [        R                  T'    ST ST 3n T[        R
                  R	                  U S5      -   [        R
                  U '   g g ! [         a     g f = f! [         a?  n[         R                  " S[        U5       S	[        R                  " 5        35        UeS nAff = f7f)
Nz(adding spend to team db. Response cost: z. team_id: r  zAtrack_cost_callback: team_id is None. Not tracking spend for teamr   z	team_id::z::user_id::z#Update Team DB failed to execute - r   )r4   r`  r  team_list_transactonsrx  team_member_list_transactonsr"  rz  r  r{  r|  )team_member_keyr*  r  team_idr  s     r   _update_team_db(update_database.<locals>._update_team_db9  s$    $**>}o[Y`Xaabc ?(..[  ,%'==AA'1MN "77@
 -6gYk'*S)+HHLL / &BB?S -  %  $))9#a&IDXDXDZC[\ 	sV   D6C D;C 8<B6 4D6
C C DCC 
D:D

DDc                    >#     [         R                  " SR                  TT5      5        Tc  [         R                  " S5        g [        b5  T[        R                  R                  TS5      -   [        R                  T'   g g ! [         a?  n [         R                  " S[        U 5       S[        R                  " 5        35        U eS n A ff = f7f)Nz6adding spend to org db. Response cost: {}. org_id: {}.z?track_cost_callback: org_id is None. Not tracking spend for orgr   z"Update Org DB failed to execute - r   )r4   r`  r*   r  org_list_transactonsrx  r"  rz  r  r{  r|  )r*  org_idr  s    r   _update_org_db'update_database.<locals>._update_org_db\  s     $**LSS%v
 >(..Y  ,%'<<@@KL "66v> -
  $))8Q9CWCWCYBZ[ 	s4   C?B C;B  C
C:CCCFzwdisable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur.zRuns spend update on all tablesz Error updating Prisma database: )r4   r`  r<  r  r   rv   r1  create_taskdisable_spend_logsrz  r"  r{  r|  )rp  r  r  r  r  r#  r  r6  r  r  r  r  r  r  r  r  s    `````````     @r   update_databaser    sF    g
""4]O9UGS^_f^ggrszr{|	
 E3!7!7E<L<LU<S<S%E2L L"	J	&	 	4 	F	. 	O-.N,-O-.N,-&)+++ %% J 	""#DE ,  
"".y/C/C/E.FG	

s;   FDE E1E FE 6F =F?F  Frp  r  r  r  parent_otel_spanc                 j  ^^^^^
#    / m
S[         S[        4U
4S jjnUUU
4S jnUUU
4S jnUUU
4S jn	U b  Tb  U" U TS9I Sh  vN   Tb  U" 5       I Sh  vN   Tb  U" 5       I Sh  vN   Tb  U	" 5       I Sh  vN   [        R                  " [        R                  T
S	US
95        g Nd NT ND N47f)zT
Use this to update the cache with new user spend.

Put any alerting logic in here.
rp  r  c           
        >#    [        U [        5      (       a   U R                  S5      (       a
  [        U S9nOU n[        R
                  " SU5        [        R                  US9I S h  vN n[        R
                  " SU 35        [        R
                  " SU 35        Uc  g UR                  nXA-   nUR                  SL a  UR                  b  [        UUR                  S   S	9S
L a  [        UUR                  R                  SS 5      S	9u  pgUR                  R                  S[        S5      5      n[        UR                   =(       d    SUUR"                  UUR$                  UUS9n	[&        R(                  " [*        R-                  SU	S95        Ub,  [/        USS 5      b  UR0                  =(       d    Sn
X-   Ul        Ub,  [/        USS 5      b  UR2                  =(       d    SnX-   Ul        XSl        TR5                  X#45        g  GN7f)Nrn  ro  z"_update_key_cache: hashed_token=%srr  z&_update_key_cache: existing_spend_obj=z#_update_key_cache: existing spend: Fsoft_budget)current_spendsoft_budget_limitTinfr   )rp  r  	key_aliasru  r  projected_spendprojected_exceeded_dateprojected_limit_exceeded)r   	user_info
team_spendr   team_member_spend)r<  r  r   rv   r4   r`  r  rs  r  soft_budget_cooldownlitellm_budget_tablerr   rp   rx  floatCallInforp  r  r  r1  r  proxy_logging_objbudget_alertsgetattrr  r  rD  )rp  r  r  existing_spend_objexisting_spend	new_spendr  r  
soft_limit	call_infoexisting_team_spendexisting_team_member_spendvalues_to_update_in_caches               r   _update_key_cache'update_cache.<locals>._update_key_cache  s/    eS!!e&6&6u&=&=%E2L L""#GVBTBdBdiuBdBv<v""45G4HI	
 	""12D1EF	
 %/55N"2	 33u<"77C."+&8&M&M%'  8W'"4"I"I"M"M!4#84O ,@@DDuU|J !(..4",66%*22 /(?I !//3' 0  **L$?K"4"?"?"D1,?,O) **,?FR);)M)M)RQR& +: 0
 $- !((,)KLW =ws   A%H(H	)F!Hc                    >#    T	/n  U  H  nUc  M  [         R                  US9I S h  vN nUc    g [        R                  " SU ST 35        [	        U[
        5      (       a  US   nOUR                  nUT-   n[	        U[
        5      (       a  XBS'   T
R                  X45        M  XBl        T
R                  XR                  5       45        M     [         R                  SR                  [        5      S9I S h  vN nUc  g Tb0  Ub,  UT-   nT
R                  SR                  [        5      U45        g g g  GN N?! [         aB  n[        R                  " S[        U5       S[        R                  " 5        35         S nAg S nAff = f7f)Nrr  z!_update_user_db: existing spend: ; response_cost: r  z{}:spendz'An error occurred updating user cache: r   )r  rs  r4   r`  r<  r=  r  rD  jsonr*   litellm_proxy_admin_namer"  r  r{  r|  )r~  r  r  r  r  global_proxy_spend	incrementr*  r  r  r  s           r   _update_user_cache(update_cache.<locals>._update_user_cache  s    9*	;+=+M+MRU+M+V%V"%-$**78J7KK\]j\kl 0$77%7%@N%7%=%=N*]:	 0$772;w/-44c5NO/8,-44c;R;R;T5UV3  6 (:'I'I%%&>? (J ( " ")*/A/M.>	)00&&'?@)L 0N*; &W."  	 &&9#a&iFZFZF\E]^ 	sg   F D< D7D< FCD< :D:;D< F1D< 4F7D< :D< <
F8F>FFFc                  4  >#    Tb  Tc  g SR                  T5      n  [        R                  U S9I S h  vN nUc  g [        R                  " SU ST 35        Uc  SnO'[        U[        5      (       a  US   nOUR                  nUT-   n[        U[        5      (       a  X1S'   TR                  X45        g X1l        TR                  XR                  5       45        g  N! [         a,  n[        R                  " S[        U5       35         S nAg S nAff = f7f)Nzend_user_id:{}rr  z%_update_end_user_db: existing spend: r  r   r  +An error occurred updating end user cache: )r*   r  rs  r4   r`  r<  r=  r  rD  r  r"  rX  r  )r  r  r  r  r*  r  r  r  s        r   _update_end_user_cache,update_cache.<locals>._update_end_user_cache  s)    -"7%%k2	'9'I'Ic'I'R!R!)  &&78J7KK\]j\kl ")!"0$77%7%@N%7%=%=N&6I ,d33.77+)00#1JK+4()00#7N7N7P1QR1 "S2  	 **=c!fXF 	sU   DC CC DA9C 5D6&C DC 
D)"DDDDc                  "  >#    Tb  Tc  g SR                  T5      n  [        R                  U S9I S h  vN nUc  g [        R                  " SU ST 35        Uc  SnO'[        U[        5      (       a  US   nOUR                  nUc  SnUT-   n[        U[        5      (       a  X1S'   TR                  X45        g X1l        TR                  X45        g  N! [         a,  n[        R                  " S[        U5       35         S nAg S nAff = f7f)Nz
team_id:{}rr  z!_update_team_db: existing spend: r  g        r  r  )r*   r  rs  r4   r`  r<  r=  r  rD  r"  rX  r  )r  r  r  r  r*  r  r  r  s        r   _update_team_cache(update_cache.<locals>._update_team_cacheB  s.    ?m3!!'*"	 )88S8AA  ") &&34F3GGXYfXgh ")250$77%7%@N%7%=%=N%!$&6I ,d33.77+)00#1JK+4()00#1JK7 B8  	 **=c!fXF 	sU   DC CC DA>C :D;C DC 
D "DDDDN)rp  r  r#   )
cache_listttllitellm_parent_otel_span)r  r  r1  r  r  async_set_cache_pipeline)rp  r  r  r  r  r  r  r  r  r  r  s    ````     @r   update_cacher    s      8:RMs RM5 RMj-`#L'R ]6e=III """$&&& """330%5 	4 	
 	J 	# 	' 	#sG   ?B3B+B3B-B3*B/+B3<B1=/B3-B3/B31B3c                      SS/n [        [        R                  S5       n[        R                  " XUS9  S S S 5        g ! , (       d  f       g = f! [
         a$  n[        R                  " SU S35         S nAg S nAff = f)Nollamaservew)stdoutstderrzd
            LiteLLM Warning: proxy started with `ollama` model
`ollama serve` failed with Exceptionz). 
Ensure you run `ollama serve`
        )openrQ  devnull
subprocessPopenr"  r4   r`  )commandr  r*  s      r   run_ollama_server    sv    

W%"**c"gWWE #"" 
""eefdg h		
 	

s1   A ?A 
A	A A 
A>A99A>c                  t  #    [         R                  " [        5      n U c  g [        U [        S9I Sh  vN u  pU[
        S'   U[
        S'   [        U5      [
        S'   [        U5      [
        S'   [        b;  [        [        [        5      (       a"  [        R                  " [        5      I Sh  vN   M   N N7f)zq
Periodically run health checks in the background on the endpoints.

Update health_check_results, based on this.
N)
model_listdetailshealthy_endpointsunhealthy_endpointshealthy_countunhealthy_count)copydeepcopyr   r[   health_check_detailshealth_check_resultslenhealth_check_intervalr<  r  r1  r2  )_llm_model_listr  r  s      r   _run_background_health_checkr    s      mmN3O
7K&0D8
 2
.
 5F016I23034E0F_-256I2J./ ,!52
 2
 -- 5666 2
 7s"   1B8B4A9B8-B6.B86B8c            	          \ rS rSrSrS0S jrS\S\4S jrS\S\	4S	 jr
 S1S\\   S\	4S
 jjrS\	S\S\	4S jrS\	4S jr S2S\	S\S\S\	4S jjrS\S\\   S\4S jrS\4S jrS\	4S jrS1S\\   S\	4S jjrS\	4S jrS\\R2                     S\4S jrS3S\4S jjrS\S\4S jrS\S\4S  jrS!\S"\ 4S# jr!S$\\"   4S% jr#S&\	S'\S(\$S\	4S) jr%S*\&S\	S+\\   4S, jr'S*\&S\4S- jr(S*\&S"\ 4S. jr)S/r*g)4ProxyConfigi  zu
Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
r  Nc                     0 U l         g Nconfig)selfs    r   __init__ProxyConfig.__init__  s	    &(r   config_file_pathc                     [         R                  R                  U5      (       d  g[         R                  R                  U5      u  p#UR	                  5       S:H  =(       d    UR	                  5       S:H  $ )NFz.yamlz.yml)rQ  r   isfilesplitextlower)r  r  _file_extensions       r   is_yamlProxyConfig.is_yaml  sX    ww~~.//GG,,-=>##%0TN4H4H4Jf4TTr   	file_pathc           	           [        US5       n[        R                  " U5      =(       d    0 sSSS5        $ ! , (       d  f       g= f! [         a  n[        SU S[	        U5       35      eSnAff = f)z
Load and parse a YAML file
rNzError loading yaml file r   )r  yaml	safe_loadr"  r  )r  r  r   r*  s       r   _load_yaml_fileProxyConfig._load_yaml_file  s_    	Ni%~~d+1r &%% 	N6ykCF8LMM	Ns1   A 7	A 
AA A 
A1A,,A1c           	      *  #    U=(       d    [         nUb  Uq [        R                  R                  U 5      (       a-  [	        U S5       n[
        R                  " U5      nSSS5        OUb  [        SU 35      e/ 0 0 0 S.nU R                  W[        R                  R                  [        R                  R                  U=(       d    S5      5      S9n[        R                  " S[        R                  " USS	9 35        U$ ! , (       d  f       N= f7f)
z
Given a config file path, load the config from the file.
Args:
    config_file_path (str): path to the config file
Returns:
    dict: config
Nr  zConfig file not found: )r  r   router_settingslitellm_settingsr   )r  base_dirzloaded config=   indent)r   rQ  r   existsr  r  r  r"  _process_includesdirnameabspathr4   r`  r  dumps)r  r  r  config_filer  s        r   _get_config_from_file!ProxyConfig._get_config_from_file  s      %=(=	'$4! 77>>YK))c*k4 +*"5i[ABB !$&#%$&	F ''BGGOOBGGOOIOQS4T$U ( 
 	""^DJJva4P3Q#RS% +*s   AD	D B"D
DDr  r  c                    SU;  a  U$ [        US   [        5      (       d  [        S5      eUS    H  n[        R                  R                  X#5      n[        R                  R                  U5      (       d  [        SU 35      eU R                  U5      nUR                  5        H8  u  pg[        U[        5      (       a  Xa;   a  X   R                  U5        M4  XqU'   M:     M     US	 U$ )a%  
Process includes by appending their contents to the main config

Handles nested config.yamls with `include` section

Example config: This will get the contents from files in `include` and append it
```yaml
include:
    - model_config.yaml

litellm_settings:
    callbacks: ["prometheus"]
```
includez&'include' must be a list of file pathszIncluded file not found: )r<  r_  
ValueErrorrQ  r   joinr  FileNotFoundErrorr  r   extend)r  r  r  include_filer  included_configr)  r   s           r   r  ProxyConfig._process_includes  s     F"M&+T22EFF #9-LX<I77>>),,'*CI;(OPP"229=O-335
eT**s}K&&u-"'3K	 6 . 9r   
new_configc                 F  #     [         bS  [        R                  SS5      SL d  [        (       a/  UR	                  SS 5        [         R                  USS9I S h  vN   g [        [         S5       n[        R                  " XSS9  S S S 5        g  N5! , (       d  f       g = f7f)	Nr  FTr  r  )data
table_namer  )default_flow_style)
r  r   rx  r  popinsert_datar  r   r  dump)r  r   r  s      r   save_configProxyConfig.save_config  s     	 $  !4e<D   NN<.+++QQQ ./#6+		*eL 76	 R 76s*   AB!BB!/B
B!
BB!depth	max_depthc                    X#:  a  [         R                  " SU S35        U$ UR                  5        H  u  pE[        U[        5      (       a  U R                  XRS-   US9X'   M1  [        U[        5      (       a5  U H-  n[        U[        5      (       d  M  U R                  XbS-   US9nM/     M{  [        U[        5      (       d  M  UR                  S5      (       d  M  [        U5      X'   M     U$ )ai  
Check for os.environ/ variables in the config and replace them with the actual values.
Includes a depth limit to prevent infinite recursion.

Args:
    config (dict): The configuration dictionary to process.
    depth (int): Current recursion depth.
    max_depth (int): Maximum allowed recursion depth.

Returns:
    dict: Processed configuration dictionary.
zMaximum recursion depth (z") reached while processing config.r   )r  r*  r+  os.environ/)
r4   warningr   r<  r=  _check_for_os_environ_varsr_  r  r   r   )r  r  r*  r+  r)  r   items          r   r/  &ProxyConfig._check_for_os_environ_vars(  s      ((+I;6XY M ,,.JC%&&"== 	Y >  E4((!D!$--#>>#'qyI  ?   " E3''E,<,<],K,K(/ ) r   r  all_teams_configc                    0 nU H#  nSU;  a  [        SU 35      eXS   :X  d  M!  Un  O   UR                  5        HA  u  pV[        U[        5      (       d  M  UR	                  S5      (       d  M4  [        U5      X5'   MC     U$ )Nr  zteam_id missing from team: r-  )r"  r   r<  r  r   r   )r  r  r2  team_configteamr%  r&  s          r   _get_team_configProxyConfig._get_team_configM  s    $D$"=dV DEEy/)" %  %%'DA!S!!all=&A&A!+A ( r   c                    #    U R                   nUR                  S0 5      nUR                  SS5      nUc  0 $ U R                  XS9nU$ 7f)zF
- for a given team id
- return the relevant completion() call params
r
  default_team_settingsN)r  r2  )r  rx  r6  )r  r  r  r
  r2  r4  s         r   load_team_configProxyConfig.load_team_configZ  sc       "::&8"=+//0GN#I++ , 
    AA	cache_paramsc                 6   SSK Jn  SU;   a  US   [         l        SU;   a  US   [         l        U" S0 UD6[         l        [         R                  bI  [        [         R                  R                  [        5      (       a  [         R                  R                  qg g g )Nr   Cacher  default_redis_ttldefault_in_redis_ttlr   )rS  r@  r  rA  cacher<  r7   r  )r  r=  r@  s      r   _init_cacheProxyConfig._init_cachem  s{    
 	""l2,89P,QG),.(45K(LG%--==$GMM4G4G)T)T ' 3 3 *U$r   c                   #    [         R                  R                  S5      b  [         R                  R                  S5      n[         R                  R                  S5      n[         R                  R                  S5      n[        R                  " SX#5        US:X  a  [        X#S9I Sh  vN nO	[        X#S9nUc  [        S5      eOU R                  US	9I Sh  vN n[        b!  U R                  U[        [        S
9I Sh  vN n[        R                  " U5      nUR                  SS5        U R                  US9nU R!                  US9  U$  N Nv NP7f)z
Load config file
Supports reading from:
- .yaml file paths
- LiteLLM connected DB
- GCS
- S3

Args:
    config_file_path (str): path to the config file
Returns:
    dict: config

LITELLM_CONFIG_BUCKET_NAMEN LITELLM_CONFIG_BUCKET_OBJECT_KEYLITELLM_CONFIG_BUCKET_TYPEzbucket_name: %s, object_key: %sgcs)bucket_name
object_keyz(Unable to load config from given source.r  )r  r  r  environment_variablesr  )rQ  environrx  r4   r`  rS   rT   r"  r  r  _update_config_from_dbr  r  r  r%  r/  update_config_state)r  r  rK  rL  bucket_typer  printed_yamls          r   
get_configProxyConfig.get_config  sF    $ ::>>67C**..)EFK(JKJ**..)EFK &&1; e#@ +   3 + ~ JKK   55GW5XXF$66+"3 7  F }}V,0$7000?   /9 Ys7   B'E)E*/EE'EEAEEEc                     Xl         g r  r  )r  r  s     r   rQ  ProxyConfig.update_config_state  s    r   r@   c           
        #    U R                  US9I Sh  vN nUR                  SS5      nU(       av  UR                  5        H(  u  pV[        [	        XVS95      [
        R                  U'   M*     SU;   a4  [
        R                  " SS5      [        l	        [        R                  5       qUR                  SS5      qUR                  SS5      nUc  0 nU(       Ga  SnS	n	UR                  5        GH  u  pVUS
:X  Ga  USL Ga  [        U S35        SSKJn
  0 nSU;   a  US   nUR!                  U5        UR                  SS5      n["        R$                  " SU5        US:X  d  US:X  a  ['        UR)                  5       5      S:X  a  [	        SS5      n[	        SS5      nSnUR!                  UUUS.5        [	        SS5      b  [	        SS5      nUR!                  SU05        ["        R$                  " SUU	U5        ["        R$                  " SUU	U5        ["        R$                  " SUU	U5        ["        R$                  " SUU	U5        US:X  a  UR!                  SS05        UR                  5        H>  u  pV[+        U5      [        L d  M  UR-                  S5      (       d  M1  [	        U5      X'   M@     U R/                  US9  [0        R2                  b  ["        R$                  " U S U	 35        GM  GM  US
:X  a  US!L a  GM  US":X  a  [5        U[        UUS#9nU[0        l        GM4  US$:X  a  [9        U[        UUS%9  GMM  US&:X  a=  [;        XbS'9/[0        l        ["        R$                  " S([0        R<                   35        GM  US):X  a  [?        U5      [0        l         GM  US*:X  aD  [?        U5      [0        l!        [0        R@                  c  [0        RB                  [0        l         GM  GM  US+:X  a=  SS,K"J#n  U Vs/ s H  nUS-   [;        US.   US'9S/.PM     sn[0        l$        U" 5         GM:  US0:X  a  / [0        l%        U H  nS1U;   a(  [0        RJ                  RM                  [;        US295        M1  [0        RJ                  RM                  U5        S3U;   d  MX  [        (       d#  [O        [P        RR                  RT                  5      e["        R$                  " S45        SS5K+J,n  U" 5       n[Z        R]                  S6U5        M     [        U S7[0        RJ                   S8U	 35        GM4  US9:X  a  / [0        l/        U HP  nS1U;   a(  [0        R^                  RM                  [;        US295        M1  [0        R^                  RM                  U5        MR     [        U S:[0        R^                   S8U	 35        GM  US:X  a  GM  US;:X  aT  [a        U5       H  u  nn [c        S0 UD6  M     ["        R$                  " U S>U S?U U	 35        [e        [0        XV5        GM"  US@:X  a  Ub  [g        U[h        5      (       an  UR                  5        HC  u  nn[g        U[        5      (       d  M  UR-                  S5      (       d  M5  [	        U5      UU'   ME     [k        S0 UD6[0        l6        GM  [O        SAU 35      e["        R$                  " U S>U S?U U	 35        [e        [0        XV5        GM     UR                  SB0 5      nUc  0 nU(       Ga  UR                  SCS5      nUb  U[n        Rp                  RT                  :X  a
  [s        SSD9  OU[n        Rt                  RT                  :X  a
  [w        SSE9  OU[n        Rx                  RT                  :X  a  SSFK=J>n  UR                  SSG9  O_U[n        R                  RT                  :X  a
  [        SSH9  O7U[n        R                  RT                  :X  a  SSIKCJDn  U" 5         O[        SJ5      eUR                  SKS5      nUb  [        S0 UD6[0        lG        UR                  SLS!5      n [w        U SE9  UR                  SMS!5      n![s        U!SD9  [        R                  UR                  SNS5      UR                  SOSP5      UR                  SQS5      UR                  SRS5      UR                  SSS5      [        ST9  UR                  SUS5      n"U"(       aN  U"R-                  S5      (       a8  ["        R$                  " SV5        [	        U"5      n"["        R$                  " SWU"5        UR                  SX[	        SYS5      5      qK[        (       ai  [        R-                  S5      (       aO  [	        [        5      qK[g        [        [        5      (       d'  [O        SZR                  [+        [        5      5      5      e[        b(  [g        [        [        5      (       a  [        [        5      qNUR                  S[S5      n#U#b  [        R                  [?        U#5      SS\9  UR                  S]S!5      qQ[        c  S!qQUR                  S^S5      n$U$b
  [;        U$US'9qRUR                  S_S5      n%U%b
  [;        U%US'9qSUR                  S`S5      n&U&b
  [;        U&US'9qTUR                  SaS5      b  [        USa   Sb9I Sh  vN   UR                  ScSd5      qVUR                  SeS5      n'U'b  [        S!L a  [        Sf5      eUR                  Sg[        5      qWUR                  Sh[        5      qXUR                  Si[        5      qYUR                  Sj[        5      qZUR                  SkS!5      q[UR                  SlSm5      q\UR                  SnS5      q]UR                  So5      b/  [        SLa&  [        Sp[P        RR                  RT                  -   5      eSqU;   a"  USq   [        l	        [        R                  5       qSr[0        R2                  SL0n(UR                  SsS5      n)U)(       a  U)U(Ss'   [        St5        U) H  n*U*Su   R                  5        HF  u  n+n,[g        U,[        5      (       d  M  U,R-                  S5      (       d  M5  [	        U,5      U*Su   U+'   MH     [        SvU*R                  SwSx5       S	35        U*Su   Sy   n-U*Su   R                  SzS5      n.S{U-;   d  M  U.b  M  [        5         M     Sn/UR                  S|S5      n0U0(       a  U0Su   R                  5        He  u  n+n,[g        U,[        5      (       d  M  U,R-                  S5      (       d  M5  U,R                  SSx5      n[
        R                  " U5      n,U,U0Su   U+'   Mg     [        S0 U0D6n/UR                  S}S5      n1[        U1S~9  UR                  SS5      n2[        U2S~9  UR                  SS5      n3[        U3S~9  UR                  SS5      n4U4(       a  [g        U4[h        5      (       as  [        R                  " [0        R                  5      n5SSs1n6U5R                   V7s/ s H  n7U7U6;  d  M  U7PM     n8n7U4R                  5        H  u  n+n,U+U8;   d  M  U,U(U+'   M     [0        R                  " S0 U(DU/[        SS9S.D6nSn9Ub  UR                  S"S5      n9U9(       a
  [        U9US9  XR                  5       U4$  GNs  snf ! [N         a    [O        S<U S=U 35      ef = f GNDs  sn7f 7f)z,
Load config values into proxy global state
rM  NrN  )secret_namedefault_valueLITELLM_LICENSEr   r
  z[94mz[0mrC  Tz
Setting Cache on Proxyr   r?  r=  r   rediszpassed cache type=%szredis-semantic
REDIS_HOST
REDIS_PORT)r   hostportREDIS_PASSWORDpasswordz%sCache Type:%s %sz%sCache Host:%s %sz%sCache Port:%s %sz%sCache Password:%s %sredis_semantic_cache_use_asyncr-  )r=  zSet Cache on LiteLLM ProxyF
guardrails)guardrails_configr   r  r
  	callbacks)r   r   r  r
  post_call_rules)r   r  zlitellm.post_call_rules: max_internal_user_budget default_max_internal_user_budgetcustom_provider_map)custom_llm_setupprovidercustom_handler)rl  rm  success_callbackr  r   
prometheusz'Starting Prometheus Metrics on /metrics)make_asgi_appz/metricsz! Initialized Success Callbacks - r%   rc  z! Initialized Failure Callbacks - r9  z4team_id missing from default_team_settings at index=z
passed in value=z setting litellm.=upperbound_key_generate_paramsz=Invalid value set for upperbound_key_generate_params - value=r   key_management_system)rH  )use_google_kms)AWSSecretsManagerV2)use_aws_secret_manager)use_aws_kms)GoogleSecretManagerz&Invalid Key Management System selectedkey_management_settingsru  rH  alertingalerting_thresholdr-  alert_typesalert_to_webhook_urlalerting_args)r{  r|  r}  r~  r  redis_cachedatabase_urlzGOING INTO LITELLM.GET_SECRET!zRETRIEVED DB URL: %sr  LITELLM_MASTER_KEYz.Master key must be a string. Current type - {}user_api_key_cache_ttl)r  rA  r  custom_authcustom_key_generate
custom_ssopass_through_endpointsr  r
  r  allowed_ipsz\allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment.!proxy_budget_rescheduler_min_time!proxy_budget_rescheduler_max_timeproxy_batch_write_atr  background_health_checksr  ,  r  enforced_paramszTrying to use `enforced_params`litellm_licensecache_responsesr  z<[32mLiteLLM: Proxy initialized with Config, Set models:[0mlitellm_paramsz	[32m    
model_namer   modelr  r  assistant_settingsfinetune_settingsr  files_settingsdefault_vertex_configr	  r  async_only_mode)assistants_configrouter_general_settings)all_guardrailsr  r   )krT  rx  r   r  r   rQ  rO  rR  _license_checklicense_str
is_premiumr   r   r)   litellm.caching.cachingr@  r!  r4   r`  r  keysr   r   rD  rS  rC  rZ   guardrail_name_config_maprM   ru   rg  r  rh  ri  litellm.utilsrk  rj  rn  rD  r"  CommonProxyErrorsnot_premium_userr   prometheus_clientrq  r   mountrc  	enumerateTeamDefaultSettingssetattrr<  r=  #LiteLLM_UpperboundKeyGenerateParamsrs  rU  rV  r\  
GOOGLE_KMSr   AWS_SECRET_MANAGER-litellm.secret_managers.aws_secret_manager_v2rv  load_aws_secret_managerAWS_KMSr   GOOGLE_SECRET_MANAGER-litellm.secret_managers.google_secret_managerry  r  KeyManagementSettings_key_management_settingsr  update_valuesr  r  r*   rv   litellm_master_key_hashr  update_cache_ttlr  user_custom_authuser_custom_key_generateuser_custom_ssori   r
  r  r  r  r  use_background_health_checksr  r  r  r   rz   rX   rh   ry   inspectgetfullargspecr2   argsr   rY   get_model_list):r  r@   r  r  rN  r)  r   r
  blue_color_codereset_color_coder@  r=  cache_params_in_config
cache_type
cache_host
cache_portcache_passwordr  rk  r0  callbackrq  metrics_appidxteam_setting_k_vr   rt  rv  ry  rz  ru  rH  r  r  r  r  r  r  router_paramsr  r  r%  r&  litellm_model_namelitellm_model_api_baser  r  finetuning_configfiles_configr  r	  arg_specexclude_argsxavailable_argsguardrails_v2s:                                                             r   load_configProxyConfig.load_config  s     "__>N_OO !'

+BD I 399;
"%jS&V"W

3 < !$99-/YY7H$-O*-88: #JJ':DA "::&8$?#!(O(.446
'>etm_--EFG=#%L%)991A.1Q.$++,BC!-!1!1&'!BJ(../EzR #g-?O1Ol//12a7%/d%C
%/d%C
)-$++(2(2(2 &&6=I-78H$-ON(//$.!" -220+,&	 -220+,&	 -220+,&	 -224+,*	 "%55$++-Mt,TU '3&8&8&:
;#-%2B2B=2Q2Q0:50AL- ';
 $$,$?}}0,22.//IJZI[\ 1 G^L(0E*/%1)9)9	1- 9RG5K'1#%1)9)9	 --'eW/G+ )..3G4K4K3LM 667<U|G4>>?DU|G<77?#DD  8 @ 11> %*	3 %*D )-Z(8.=&*+;&<1A/ %*	3G/ %&../1G, %*(?#44;; /h ?
 $44;;HE+x7'3|*3(9(J(J(P(P+& %& !5 : :$M!" !L /<o #		*k B+ %*, *++LWMeMeLffghxgyz ../1G, %*(?#44;; /h ?
 $44;;HE %* *++LWMeMeLffghxgyz N* 33-6.)\/?,?	. )..*++<SE5'JZI[\ GS0<<(Zt-D-D&+kkmFB)"c22r}}]7S7S,6rNb	 '4 @H%H  > ([\a[bc  )..*++<SE5'JZI[\ GS0] 7b "::&8"=#!$4$8$89PRV$W!$0(,?,O,O,U,UU-$G*.A.L.L.R.RR#48)*==CCD (??/3 @  +.A.I.I.O.OO T2)*@@FFG ()$%MNN&6&:&:)4'# '23H 4-40 .112BEJN>:"2"6"67Le"T%:MN ++)--j$?#3#7#78Lc#R,00E%5%9%9:PRV%W.22?DI- ,  ,//EL 7 7 F F$**+KL),7$**+A<P)--j)=tDJ zj33MBB'
3
!*c22#HOO ,  %*Z*E*E*4Z*@'%5%9%9($&" &1"33*/0F*G&* 4 
 !1 4 45H% P ($)! +..}dCK&#2%8H$  #3"6"67Ld"S".+:-@P,( *--lDAJ%"1$7G#
  ##$<dCO7+;<T+U   .11 %N +..}dCK&<5+@ r  1A0D0D35V1- 1A0D0D35V1- $4#7#7&(<$  "2!5!5$&8"
 ,<+?+?*E,( %5$8$89PRU$V!#3#7#78NPT#U  !$$%67C , 5'88>>?  !$44-=>O-P*-88: w}} 

 ZZd3
*4M,'T $!"2399;DAq!!S))all=.I.I5?]./2 < UYY|R%@$AIJ%*+;%<W%E")./?)@)D)DZQU)V&116L6T$& $ <@#ZZ(<dC*+;<BBD1a%%!,,}*E*E="5B		"A>?&'78;	 E
 !4 I6H I #JJ':DA&78 zz"2D9- !'

+BD I!)>? !**%6=z/4@@--gnn=H L
 *2PA!<:OaNP'--/1&'(M!$ 0  

/$9 $%
 /3"JJ|T:M,?O ,,.0@@@} PV	3F  ) "+"VWZV[[mnzm{ |# `T Qs   {?{I,{?{?E{?5{A={?D:{?{A>{?{?3Q {?3{74F${?{?4A{?{?A{?+{?C4{?7
{:{:{?+A&{?{?{44{?:{?c                    [        USS5      nUb  X1R                  S'   SUR                  S'   [        SL ah  [        USS5      UR                  S'   [        USS5      UR                  S'   [        USS5      UR                  S'   [        US	S5      UR                  S	'   UR                  b  [        UR                  [        5      (       ap  SUR                  ;  a  UR
                  UR                  S'   SUR                  ;   a   UR                  S   S
L a  X!R                  S'   [        S0 UR                  D6nU$ [        UR
                  US9nU$ )z
Common logic across add + delete router models
Parameters:
- deployment
- db_model -> flag for differentiating model stored in db vs. config -> used on UI

Return model info w/ id
r  NidTdb_model
created_at
updated_at
created_by
updated_byF)r  r  r   )r  
model_infor   r<  r=  r  RouterModelInfo)r  r  r  r  _model_infos        r   get_model_info_with_id"ProxyConfig.get_model_info_with_id  sC    %UJ=?%(T"+/EZ(4-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*'Ju7G7G,N,N5+++).  &U---%2B2B:2NRW2W/7  ,)=E,<,<=K  *U^^hOKr   	db_modelsc                 "  #    / n[         b  [        U5      S:X  a  gU H<  nU R                  US9nUR                  c  M!  UR	                  UR                  5        M>     U R                  [        S9I Sh  vN nUR                  SS5      nU(       a  U H  nUS   R                  5        HE  u  p[        U	[        5      (       d  M  U	R                  S5      (       d  M4  [        U	5      US   U'   MG     UR                  S0 5      R                  S	S5      n
U
c  [         R                  US
   US   S9n
UR	                  U
5        M     [         R                  5       nSnU H'  n
X;  d  M
  [         R                  U
S9nUc  M"  US-  nM)     U$  GN7f)z
(Helper function of add deployment) -> combined to reduce prisma db calls

- Create all up list of model id's (db + config)
- Compare all up list to router model id's
- Remove any that are missing

Return:
- int - returns number of deleted deployments
Nr   r  rM  r  r  r-  r  r  r  )model_groupr  r  r   )r   r  r  r  rD  rT  r   rx  r   r<  r  r   r   _generate_model_idget_model_idsdelete_deployment)r  r  combined_id_listmr  r  r  r  r%  r&  r  router_model_idsdeleted_deployments
is_deleteds                 r   _delete_deploymentProxyConfig._delete_deployment  s      Y1!4 A4414=J}}( ''
6  8MNNZZd3
#!"2399;DA!!S))all=.I.I5?]./2 <
 !99\26::4F#)<<$),$7',-='>  =  H !''1 $  &335  (H/'99X9F
)'1,'	 )
 #"9 Os0   9F5F4F5AFFBF)FFc           	      ~   SSK n[        b  [        [        [        5      (       d  [	        S[         35      e[
        c  gSnU H  nUR                  n[        U[        5      (       aw  UR                  5        HW  u  pg[        U[        5      (       d  M  [        US9nUc  [	        SR                  U5      5      e[        U5      S:  d  MS  XU'   MY     [        S0 UD6nO[        R                  " SU 35        M  U R                  USS9n	[
        R!                  [#        UR$                  UU	S	9S
9n
U
c  M  US-  nM     U$ )zd
Iterate through db models

for any not in router - add them.

Return - number of deployments added
r   N7Master key is not initialized or formatted. master_key=ro  zUnable to decrypt value={}HInvalid model added to proxy db. Invalid litellm params. litellm_params=T)r  r  r  r  r  )
deploymentr   r   )base64r  r<  r  r"  r   r  r=  r   rO   r*   r  r|   r4   r   r  upsert_deploymentr{   r  )r  r  r  added_modelsr  _litellm_paramsr%  r&  _valuer  addeds              r   _add_deploymentProxyConfig._add_deployment  sP    	Z
C%@%@I*V  A..O/400+113DA!!S))!5A!>!>"+,H,O,OPQ,R"SSv;?17A. 4 #1"C?"C %**^_n^op 55$ 6 K 00% ||#2* 1 E  !C D r   
new_modelsr  c           	        #    SS K n [        GcE  [        Gb=  [        R                  " S[        U5       35        / nU H  nUR                  n[        U[        5      (       a2  UR                  5        H  u  px[        US9n	XU'   M     [        S0 UD6nO[        R                  " SU 35        Mq  U R                  US9n
UR                  [        UR                   UU
S9R#                  SS95        M     [        U5      S:  aR  [        R                  " S	U 35        [$        R&                  " U[)        SS
9S9q[        R                  " S[         35        OI[        R                  " S[        U5       35        U R+                  US9I S h  vN   U R-                  US9   [        b  [        R5                  5       q[8        R;                  5       I S h  vN nUR=                  S0 5      =(       d    0 nUR=                  SS 5      nUR=                  SS 5      nUbS  [        U[>        5      (       a>  U H8  nU[$        R@                  ;  d  M  [$        R@                  R                  U5        M:     UbS  [        U[>        5      (       a>  U H8  nU[$        RB                  ;  d  M  [$        RB                  R                  U5        M:     UR=                  S0 5      nUR                  5        H'  u  px [        US9n	U	b  U	[D        RF                  U'   M'  M)     [        bj  [H        bc  [H        RJ                  RL                  RO                  SS0S9I S h  vN nUb/  URP                  b"  URP                  n[        RR                  " S0 UD6  UR=                  S0 5      nSU;   Ga5  [T        b  [T        R=                  SS 5      b  [        [T        S   [>        5      (       at  UR=                  SS 5      ba  [        US   [>        5      (       aI  [        R                  " S5        US   [T        S'   [T        S   Ul+        [T        S   URX                  l+        O[T        c5  0 q*US   [T        S'   [T        S   Ul+        [T        S   URX                  l+        OK[        [T        [        5      (       a2  US   [T        S'   [T        S   Ul+        [T        S   URX                  l+        SU;   a?  US   [T        S'   [T        S   Ul-        URX                  R]                  [T        S   [        S9  SU;   a2  US   [T        S'   URX                  R]                  [T        S   [        S9  g g  GN! [.         a-  n[        R0                  " S[3        U5       35         S nAGNS nAff = f GN! [.         a-  n[        R                  " SU[3        U5      5         S nAGM  S nAff = f GN}7f)Nr   zlen new_models: ro  r  r  r  Texclude_nonez_model_list: r  )r  r  zupdated llm_router: )r  z+Error adding/deleting model to llm_router: r
  rn  rc  rN  z#Error setting env variable: %s - %s
param_namer	  wherer   r{  z?Overriding Default 'alerting' values with db 'alerting' values.r}  )r}  r   r~  )r~  r   r   )/r  r   r  r4   r`  r  r  r<  r=  r   rO   r|   r   r  rD  r{   r  to_jsonrS  r2   r   r  r   r"  rX  r  r  r   proxy_configrT  rx  r_  rn  rc  rQ  rO  r  dblitellm_config
find_firstparam_valueupdate_settingsr   r{  slack_alerting_instancer}  r  )r  r  r  r  _model_listr  r  r%  r&  decrypted_valuer  r*  config_datar
  success_callbacksfailure_callbacksrn  rc  rN  db_router_settings_router_settings_general_settingss                         r   _update_llm_routerProxyConfig._update_llm_router3  s     	/	!j&<$**-=c*o=N+OP$&#A&'&6&6O!/488$3$9$9$;DA.B.KO1@A. %< +9*K?*K,22fgvfwx !"&"="=A"="FK&&"'(||+:'2 "'t'4 $, {#a'(..{m/LM!(#.0E,01"J )..1Ej\/RS$**-=c*o=N+OP--
-CCC $$z$: !'668N )3355&??+=rBHb,001CTJ,001CTJ(Z8I4-P-P$5 #7+C+CC,,334DE %6
 (Z8I4-P-P$5 #7+C+CC,,334DE %6 !,0G L)//1DA"6Q"?".$3BJJqM / 2 !m&?'4'7'7'F'F'Q'Q#%67 (R ( " #.&22>#5#A#A **>-=> (OO,>C** ,$((T:F/
;TBB%))*d;G0<dCC$**U 0A0 , .>j-I!*EUF!99B ")#% /@/L ,-=j-I!*EUF!99B ,d33/@/L ,-=j-I!*EUF!99B --.?.N]+,<],K)55CC,];
 D  "%667H&834 55CC%56L%M% D 	 7} D
  	 **=c!fXF 	 6,  $**91c!f "s   WEU W4U 	U

U 3WVA2WAWA
W(V
A WWHW
U 
V"U?9W?VW

W!V<5W<WWdb_general_settingsc                 8  #    Uc  g[        U5      nSU;   a  US   [        S'   SU;   a  US   [        S'   SU;   a0  US   [        S'   [        R                  R	                  [        S   S9  SU;   a%  US   [        S'   [        [        S   S9I Sh  vN   gg N7f)z+
Pull from DB, read general settings value
Nmax_parallel_requestsglobal_max_parallel_requestsr  )r  r  r  )r=  r   r  r  r  ri   )r  r  r  s      r   _update_general_settings$ProxyConfig._update_general_settings  s     
 & !45"&778I'945 *->>?P.@;<
 //0A/0R_-55CC.? D 
 $'889J(:56 4'78P'Q  	 9s   BBBBcurrent_configr  db_param_valuec                     [        X   [        5      (       a+  UR                  5        H  u  pEU(       d  M  XQU   U'   M     U$ X1U'   U$ r  )r<  r=  r   )r  r!  r  r"  r)  r   s         r   _update_config_fields!ProxyConfig._update_config_fields  sU     n0$77 -224
56;:.s3 5  *8:&r   r  r  c                   #    USLa  [         R                  " S5        U$ / n/ SQnU H%  nUR                  SUSS9nUR                  U5        M'     [        R
                  " U6 I S h  vN nU H  nUc  M  [        USS 5      n	[         R                  " SU	 S35        U	S	:X  a#  [         R                  " S
UR                   35        [        USS 5      n
U	c  Mj  U
c  Mo  X;   a  U R                  UU	U
S9nM  XU	'   M     U$  N7f)NTz4'store_model_in_db' is not True, skipping db updates)r   r	  r
  rN  r  r  )r)  r   r#  zloading z settings from dbr
  zlitellm_settings: r  )r!  r  r"  )	r4   rz  get_generic_datarD  r1  gatherr  r  r$  )r  r  r  r  _tasksr  r%  response	responsesr  r  s              r   rP  "ProxyConfig._update_config_from_db  s/     D( %%F M
 A$55 h 6 H MM(#	  "..&11	!H#$X|TB
$))HZL@Q*RS!33(--,X-A-A,BC &htD)k.E!+!%!;!;+1'1+6 "< " .9z*' "* - 2s%   A%D'C?(
D6ADD#Dc                    #     UR                   R                  R                  5       I S h  vN nU$  N! [         a;  n[        R
                  " SR                  [        U5      5      5        / n S nAU$ S nAff = f7f)NzQlitellm.proxy_server.py::add_deployment() - Error getting new models from DB - {})r  litellm_proxymodeltable	find_manyr"  r4   rX  r*   r  )r  r  r  r*  s       r   _get_models_from_dbProxyConfig._get_models_from_db0	  st     	,//GGQQSSJ  T 	 **cjjF
 J	s6   A<'4 24 A<4 
A90A4.A<4A99A<c                   #     [         b  [        [         [        5      (       d  [        S[          35      eU R	                  US9I Sh  vN nU R                  X2S9I Sh  vN   UR                  R                  R                  SS0S9I Sh  vN nUb"  U R                  UR                  S9I Sh  vN   gg Nn NY N. N! [         a8  n[        R                  " S	R                  [        U5      5      5         SnAgSnAff = f7f)
z[
- Check db for new models
- Check if model id's in router already
- If not, add to router
Nr  )r  )r  r  r  r   r  )r  z>litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {})r  r<  r  r  r0  r  r  r  r  r  r  r"  r4   rX  r*   )r  r  r  r  r  r*  s         r   add_deploymentProxyConfig.add_deployment=	  s!    	!J)D)D Mj\Z   $77m7TTJ ))% *    )6(8(8(G(G(R(R#%78 )S ) #
 #.33(;(G(G 4    / U#  	 **PWWF 	sw   DAB? B7	B? B9 ,B? B;#B? 0B=1B? 5D7B? 9B? ;B? =B? ?
D	.C<7D<DDr  )r  Nr  )r   r  F)+r   r   r   r   __doc__r  r  boolr  r=  r  r   r  r  r(  r   r/  r   r   r6  r:  rD  rT  rQ  rS  r2   r  r  r  r_  r  r   rm   r  Jsonr  r   r$  rl   rP  r0  r3  r   r   r   r   r  r    s   )U U UN N N 15$ ($	$L% % % %NMD M, >@###&#7:#	#J tDz d c &44&6# 6$ 6p$ fAw~~.fABEfAP @6#$ 6#3 6#p6 6# 6pTT (Tl!(4. !F  	
 
$0#0 0 $D>	0d|  (#( ((r   r  c                  R    SS K nUR                  " U 5      [        R                  S'   g )Nr   WORKER_CONFIG)r  r  rQ  rO  )r"  r  s     r   save_worker_configr;  k	  s    "&**T"2BJJr   c                   #    [         R                  " SS5      R                  5       S:w  a
  [        5         U qUqUSL a[  SS KnSSKJnJ	nJ
n  UR                  UR                  S9  UR                  " UR                  S9  UR                  " UR                  S9  USL a\  SS KnSSKJnJ	nJ
n  UR                  UR                  S9  UR                  " UR                  S9  UR                  " UR                  S9  OUSL a  USL a  [         R                  R                  S	S5      nUb  UR!                  5       S
:X  aA  SS KnSSKJ	nJ
n  UR                  " UR                  S9  UR                  " UR                  S9  OTUR!                  5       S:X  a@  SS KnSSKJ	nJ
n  UR                  " UR                  S9  UR                  " UR                  S9  S0 [        0 0nU(       a$  ["        R%                  [&        US9I S h  vN u  qqqU(       a  UqUU[           S'   U(       a  UqUU[           S'   U(       a  U[         R                  S'   U(       a  UU[           S'   U(       a  UqUU[           S'   U(       a  UqUU[           S'   U(       a  UU[           S'   USL a  S[4        l        SUS   S'   USL a  S[4        l        SUS   S'   U	(       a  U	[4        l        U	US   S'   [<        (       a   U
qg  N7f)NLITELLM_DONT_SHOW_FEEDBACK_BOXr   trueTr   )verbose_loggerr4   r5   )levelFLITELLM_LOGINFOr3   DEBUGgeneralr@   r  r   r  AZURE_API_VERSION
max_tokenstemperaturerequest_timeoutaliasdrop_paramsadd_function_to_promptru  ) rQ  rR  r  r,   
user_model
user_debuglogginglitellm._loggingr?  r4   r5   setLevelrB  rC  rO  rx  upperr
  r  r   r   r   user_headersuser_api_baseuser_temperatureuser_request_timeoutrS  rK  rL  ru  experimentaluser_telemetry)r  rJ  r  api_versionr`  detailed_debugrH  rG  rI  ru  	telemetryrK  rL  r   save	use_queuer  rO  r?  r4   r5   litellm_log_settingdynamic_configs                          r   
initializer`  q	  s    ( 
yy126<<>&HJJ}	
 	
 	gll3&&W\\:%%GLL9	
 	
 	gmm4&&W]];%%GMM:	%Ne3 jjnn]B?*"((*f4X &..!,, %--!,, %**,7X%..!-- %--!--  Z4N
 ***v*VV		
07z"9- 19z":. 	

&' 3=z"<0&4?z"=1.8Gz"#45.3z"7+d"37y!-0%)-&>By!":;'2<y!,/|NA Ws   G;K5=K3>C6K5c              #   (  #    [         R                  " S5        U  HE  n[         R                  " SU5         S[        R                  " UR	                  5       5       S3v   MG     g ! [
         a     S[        R                  " U5       S3v    Ms  f = f7f)Ninside generatorzreturned chunk: %sdata: r   )r4   r`  r  r  r=  r"  )r*  chunks     r   data_generatorre  	  s|     12""#7?	34::ejjl34D99   	34::e,-T22	3s(   3B*A% B%&BBBBc                  #    [         R                  " S5         [        R                  " 5         U  IS h  vN n[        R	                  XS9I S h  vN nU  S h  vN nUR                  SS9n SU S37v   M$   NE N, N#! [         a  nS[        U5       S37v    S nAML  S nAff = f
 S S S 5      IS h  vN    O! , IS h  vN  (       d  f       O= fSnSU S37v   g ! [         Ga  n[         R                  " SR                  [        U5      5      5        [        R                  UUUS	9I S h  vN    [         R                  " S
U S35        [        U[        5      (       a  Ue[        R                  " 5       n[        U5       SU 3n[        [!        USU5      [!        USS5      [!        USS5      [!        USS5      S9n	["        R$                  " SU	R'                  5       05      n
SU
 S37v    S nAg S nAff = f7f)Nrb  r  r*  Tr  rc  r   [DONE]zTlitellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}r  original_exceptionr  [1;31mAn error occurred: P

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r   r   Noner   r     r   r   )r4   r`  r0  r  async_post_call_streaming_hookmodel_dump_jsonr"  r  rX  r*   post_call_failure_hookr<  r   r{  r|  ProxyExceptionr  r  r  to_dict)r*  r  r  rd  cr*  done_messageerror_traceback	error_msgproxy_exceptionerror_returneds              r   async_assistants_data_generatorrz  	  s     12/,		8u ,JJ"3 K  E
 ! 0a%%4%80"1#T** 
0 ! 0"3q6($///0	 ! 88888   |nD)) ,&&biiA	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a''G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++7,s   HC$ A;C$ B>A=B>B+A?B+B>0	B9B>;C$ =B>?B+
B(B#B>#B((B>,C$ 7B:8C$ >CCCC$ #H$H/AG=5D86CG=8H=HHc                  #    [         R                  " S5         [        R                  " 5         U   S h  vN n[         R                  " SR                  U5      5        [        R                  XS9I S h  vN n[        U[        5      (       a  UR                  SSS9n SU S37v   Mz   Nu N7! [         a  nS[        U5       S37v    S nAM  S nAff = f
 SnSU S37v   g ! [         Ga  n[         R                  " S	R                  [        U5      5      5        [        R                  UUUS
9I S h  vN    [         R                  " SU S35        [        U[        5      (       a  Ue[        R                  " 5       n[        U5       SU 3n[!        [#        USU5      [#        USS5      [#        USS5      [#        USS5      S9n[$        R&                  " SUR)                  5       05      n	SU	 S37v    S nAg S nAff = f7f)Nrb  3async_data_generator: received streaming chunk - {}rg  T)r  exclude_unsetrc  r   rh  Ilitellm.proxy.proxy_server.async_data_generator(): Exception occured - {}ri  rk  rl  r   r   rm  r   r   rn  r   r   )r4   r`  r0  r*   r  ro  r<  rE  rp  r"  r  rX  rq  r   r{  r|  rr  r  r  r  rs  )
r*  r  r  rd  r*  ru  rv  rw  rx  ry  s
             r   async_data_generatorr  (
  s     122,		# 	,% &&ELLUS ,JJ"3 K  E %++--4t-T,ugT**	,
  ,s1vhd+++, $$  |nD)) ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a''G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   HC& CB+C<C& 5B-6)C&  	B/)C& +C-C& /
C9CC& CC& %H&H1AG?7D:8CG?:H?HHc                  #    [         R                  " S5         [        R                  " 5         U   S h  vN n[         R                  " SR                  U5      5        [        R                  XS9I S h  vN nUR                  S5      n SU S[        R                  " U5       S37v   M}   Nx N:! [         a   nSU S[        U5       S37v    S nAM  S nAff = f
 g ! [         Ga  n[         R                  " SR                  [        U5      5      5        [        R                  UUUS	9I S h  vN    [         R                  " S
U S35        [        U[        5      (       a  Ue[        R                   " 5       n[        U5       SU 3n[#        [%        USU5      [%        USS5      [%        USS5      [%        USS5      S9n[        R                  " SUR'                  5       05      n	SU	 S37v    S nAg S nAff = f7f)Nrb  r|  rg  r   zevent: z
data:r   r~  ri  rk  rl  r   rm  r   r   rn  r   r   rc  )r4   r`  r0  r*   r  ro  rx  r  r  r"  r  rX  rq  r<  r   r{  r|  rr  r  rs  )
r*  r  r  rd  
event_typer*  rv  rw  rx  ry  s
             r   async_data_generator_anthropicr  a
  s     12-,		# 	@% &&ELLUS ,JJ"3 K  E 6*J@
|74::e3D2ETJJ	@
  @
|73q6($???@ $  ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a''G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   HC! CB.C<C! 5B06C!  B2,C! .C0C! 2
C<CC! CC!  H!G?,AG:2D53CG:5H:G??Hc                     [        U UUS9$ )Nr*  r  r  )r  r  s      r   select_data_generatorr  
  s      +! r   r  c                     U R                  S0 5      nU R                  S0 5      R                  SS 5      n SU;   a  UR                  SS 5      n[        R                  " U5      nU$ ! [         a    0 s $ f = f)Nr  r  r  azure
base_model)rx  rS  get_model_infor"  )r  r  model_to_lookuplitellm_model_infos       r   get_litellm_model_infor  
  sz    <,Jii 0"599'4HOo%(nn\4@O$33OD!!  	s   /A& &A54A5c                 8    [         R                  " SU S   5        g )Nz$Backing off... this was attempt # %stries)r4   r`  )r  s    r   
on_backoffr  
  s    EwwGWXr   c                 v   [        U [        5      =(       aF    [        U SS 5      S L=(       a0    [        U R                  [        5      =(       a    SU R                  ;   (       + n[
        R                  S5      SL a  gU(       a6  [        R                  " [        R                  " S[	        U 5      S.5      5        U$ )Nr   z"Max parallel request limit reached1disable_retry_on_max_parallel_request_limit_errorTgiveup)eventrX  )r<  rr  r  r   r  r   rx  r4   rz  r  r  )r*  results     r   r  r  
  s    1n% 	>Ay$'t3	>qyy#&	> 1AII=	F 	PQ	 !!$**xcRSf-U"VWMr   c                       \ rS rSr\S\\   S\S\\   4S j5       r	\S\
S\\   S\4S	 j5       r\S
\S\S\S\
4S j5       r\S\4S j5       r\S\
S\S\S\S\S\4S j5       r\S\\   S\S\S\\   4S j5       rSrg)ProxyStartupEventi
  r   r  r  c                 J    [        5         [        5         UR                  XS9  g)z*Initialize logging and alerting on startup)r   r  N)ra  rd  startup_event)clsr   r  r  s       r   _initialize_startup_logging-ProxyStartupEvent._initialize_startup_logging
  s'     	 	''! 	( 	
r   r   r  r  c                 >   UR                  SS5      bk  US   R                  5        HE  u  pE[        U[        5      (       d  M  UR	                  S5      (       d  M4  [        U5      US   U'   MG     [        S0 US   D6nO
[        5       n[        R                  UUUS9  g)zInitialize JWT auth on startuplitellm_jwtauthNr-  )r  r  r  r   )	rx  r   r<  r  r   r   LiteLLM_JWTAuthjwt_handlerupdate_environment)r  r   r  r  r%  r&  r  s          r   _initialize_jwt_auth&ProxyStartupEvent._initialize_jwt_auth
  s      148D():;AACa%%!,,}*E*E=G]$%67: D .T0@AR0STO-/O&&'1+ 	' 	
r   r  r  c                 F   [         R                  " SS5      b  [         R                  " SU5      nUR                  S5      SL a  [        R                  " S5        g[        SS/ 0 0 SUU[        R                  SS	[        R                  0S
9n[        R                  " U5        g)z,Adds master key hash to db for cost trackingPROXY_ADMIN_IDN$disable_adding_master_key_hash_to_dbTz&Skipping writing master key hash to dbuserr   update_data	user_role)request_typedurationmodelsaliasesr  r  rp  r  r  
query_typeupdate_key_values)
rQ  rR  rx  r4   rz  rd   LitellmUserRolesPROXY_ADMINr1  r  )r  r  r  r  r   task_1s         r   _add_master_key_hash_to_db,ProxyStartupEvent._add_master_key_hash_to_db
  s     99%t,8')yy ":($  FG4O %%&NO ,# 0*66(#.0@0L0L"MF 'r   rv  c                     [         R                  c  [        S5      e[        R                  " [        SUS/ 0 0 S[         R                  [         R                  S[         R                  [         R                  S.S95        g)z Adds a global proxy budget to dbNzPbudget_duration not set on Proxy. budget_duration is required to use max_budget.r  r   r  )ru  budget_duration)r  r  r  r  r  r  r  ru  r  r  r  )rS  r  r"  r1  r  rd   ru  )r  rv  s     r   _add_proxy_budget_to_db)ProxyStartupEvent._add_proxy_budget_to_db  sy     ""*b 
 	"#1"-- ' 7 7(")"4"4'.'>'>#	
r   r  r  r  c           
        #    [        5       n[        R                  " X45      n[        R                  " US-
  US-   5      n	UR                  SS5      SL a  UR	                  [
        SX/S9  UR	                  [        SU	U[        U/S9  [        S[        5      =(       d    [        q	[        SL a<  UR	                  [        R                  SSX&/S9  [        R                  X&S	9I S
h  vN   UGbL  UR                  R                  Gb4  UGb0  [        S5        UR                  SS5      =(       d    Sn
[        U
S
S 5      nU
S   R!                  5       S:w  a  [#        S5      eUR	                  UR                  R$                  SU[&        R(                  " 5       [+        SS9-   U
/S9  UR	                  UR                  R,                  SSS9  [.        R0                  " S5      (       aU  SSKJn  UR	                  UR                  R6                  SSSU" S5      S9  UR                  R7                  5       I S
h  vN   UR9                  5         g
 GNf N7f)z%Initializes scheduled background jobsr   disable_reset_budgetFinterval)secondsr  STORE_MODEL_IN_DBTr  r  r  Nz3Alerting: Initializing Weekly/Monthly Spend Reportsspend_report_frequency7ddzBspend_report_frequency must be specified in days, e.g., '1d', '7d')r  )daysnext_run_timer  cronr   )dayPROMETHEUS_URLr   )ZoneInfo	   zAmerica/Los_Angeles)hourminutetimezone)r!   r&   randintrx  add_jobrw   rx   r  r   r  r
  r3  r  r{  r)   r   r  r  send_weekly_spend_reportr   nowr   send_monthly_spend_reportrQ  rR  zoneinfor  #send_fallback_stats_from_prometheusstart)r  r   r  r  r  r  r  	schedulerr  batch_writing_intervalr  r  r  s                r   $initialize_scheduled_background_jobs6ProxyStartupEvent.initialize_scheduled_background_jobs*  sv     %&	>>-
 "( 1$&:Q&>"

  6>%Gj(  
 	*!13DE	 	 	
 /1BCXGX 	 $++#7	   --+ .   
 )!99BBN)GH !$$%=tDL #
 -cr23D%b)//1S8 X  !99RR&llnB'(,-   !99SS   yy)**-!!%==aa%&;< "  (??cceeegb fs%   C#I%I&EI5I6IIr  r  c                 L  #    SnUb   [        XS9nUR                  5       I Sh  vN   [        R                  " UR                  5       5        [        R                  " UR                  5       5        UR                  5       I Sh  vN   U$ ! [         a  nUeSnAff = f Nz N7f)z9
- Sets up prisma client
- Adds necessary views to proxy
N)r  r  )rl   r"  connectr1  r  check_view_exists(_set_spend_logs_row_count_in_proxy_statehealth_check)r  r  r  r  r  r*  s         r   _setup_prisma_client&ProxyStartupEvent._setup_prisma_client  s      15# ,!-!  ''))) //1 FFH
  ,,...!   * /sD   B$	B B$B AB$B"B$
BBBB$"B$r   N)r   r   r   r   classmethodr   r2   rm   r7   r  r=  rl   r6   r  r  r  r  r   r  r  r   r   r   r   r  r  
  s_   
V$
 (
 $J/	
 
" 

  -
 &	
 
( (( $( #&	(
 ( (@ 
 
 
6 bb $b ,/	b
 ,/b "b (b bH  sm  (  &	 
 
,	   r   r  startupc            	        #    SS K n [        5         [        SS 5      q[        c1  [        SS 5      n[
        R                  U[        [        S9I S h  vN q[        R                  " SR                  [        5      5        [        SL a  [        R                  5       q[        S5      n[        S5      n[        R                  " S	U5        Ubb  [         R"                  R%                  U5      (       a<  [&        R)                  US
9(       a$  [&        R+                  [,        US9I S h  vN u  qqqGO7UGb3  [3        U[4        5      (       aa  [         R"                  R%                  U5      (       a=  [&        R)                  US
9(       a%  [&        R+                  [,        US9I S h  vN u  qqqO[         R6                  R9                  S5      b:  [3        U[4        5      (       a%  [&        R+                  [,        US9I S h  vN u  qqqOc[3        U[:        5      (       a  [=        S0 UD6I S h  vN   O:U R>                  " U5      n[3        U[:        5      (       a  [=        S0 UD6I S h  vN   [
        RA                  [,        [        [B        S9  [
        RE                  [0        [        [        S9  [F        (       a  [H        RJ                  " [M        5       5        [N        b  [N        RQ                  [,        S9  [        R                  " S[        5        [        b-  [        b&  [
        RS                  [        [        [T        [0        S9  [        b+  [V        RX                  S:  a  [
        R[                  [T        S9  [        b9  [
        R]                  [0        [        [^        [`        [b        [        S9I S h  vN   g g  GN_ GN GN GN GN GN_ N7f)Nr   r  DATABASE_URL)r  r  r  zElitellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {}Fr:  CONFIG_FILE_PATHzworker_config: %srM  rE  rG  )r   r  r  )r   r  r  r?   zprisma_client: %s)r  r  r  r   )rv  )r   r  r  r  r  r  r   )2r  rN   r   r  r  r  r  r  r  r4   r`  r*   r   r  r  r   rQ  r   r  r
  r  r  r   r   r   r<  r  rO  rx  r=  r`  loadsr  r  r  r  r1  r  r  r  r  r  r  rS  ru  r  r  r  r  r  )r  _db_urlworker_configenv_config_yamls       r   r  r    sT      0$7J!+ND!A/DD /1 E 
 
 OVV	

 u%002 1;?0KM%34F%GO2MB"77>>/**|/C/C, 0D 0
 #..!O /  	  
	"}c**}--$$m$D #..!M /  	  ZZ^^89E*3K
 K
 #..!M /  	  t,,-}--- !JJ}5M-.. 1=11111++ 2  **)#- +  $#(*	
 &1&999L2MB Z%;44!'%=-	 	5 	
  W%7%7!%;11&> 	2 	

  DD-'.O.O!5/ E 
 	
 	
 !E
8 .
 2N	
s   AON/CO"N2#A:ON5AO7N88,O$N;%:ON> EO(O)O2O5O8O;O>OOz
/v1/modelszmodel management)dependenciestagsz/modelsc           
      P  #    / n[        [        5      n[        c  / nO([        R                  5       n[        R	                  5       n[        U UUS9n[        U UUS9n[        UUU[        [        R                  SS5      S9n[        U Vs/ s H
  nUSSSS.PM     snS	S
9$ s  snf 7f)z
Use `/model/info` - to get detailed model information, example - pricing, mode, etc.

This is just for compatibility with openai projects like aider.
r  proxy_model_listmodel_access_groupsinfer_model_from_keysF
key_modelsteam_modelsr  rM  r  r  ijNcopenai)r  objectcreatedowned_byr_  )r"  r  )r-   r_  r   get_model_namesget_model_access_groupsrF   rG   rE   rM  r   rx  r=  )r  
all_modelsr  r  r  r  r  s          r   r  r  .  s      J0;D0A%557(@@B+)/J "+)/K
 )).223JERJ  $
 $ !%$	 $
  
s   BB&
B!B&z/v1/chat/completionszchat/completionsz/chat/completionsz&/engines/{model:path}/chat/completionsz1/openai/deployments/{model:path}/chat/completions   r   zSuccessful response)r  r  r+  )	max_triesmax_timer  r  loggerfastapi_responsec                 Z  #    0 n U R                  5       I Sh  vN nUR                  5       n [        R                  " U5      n[        R                  " SR                  [
        R                  " USS95      5        [        UU [        U[        [        S9I Sh  vN n[        R                  SS5      =(       d    [         =(       d    U=(       d    US   US'   ["        (       a	  ["        US'   [$        (       a	  [$        US	'   [&        (       a	  [&        US
'   [(        (       a	  [(        US'   [+        US   [,        5      (       a0  US   [.        R0                  ;   a  [.        R0                  US      US'   [2        R5                  X4SS9I Sh  vN nU R6                  R                  S[-        [8        R:                  " 5       5      5      US'   [.        R<                  R>                  " S=S[.        R<                  RA                  5       [B        RD                  " 5       S.UD6u  ptXtS'   / nURG                  [2        RI                  UUSS95        [K        US[L        [         S9I Sh  vN n	URG                  U	5        [N        RP                  " U6 n
U
I Sh  vN nUS   n[S        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      nUR                  S0 5      =(       d    0 n[L        b  [L        RU                  US9US'   [N        RV                  " [2        RY                  UR                  SS5      SS 95        S!U;   aM  US!   S"L aE  [[        S=UUR\                  UUU[        U[S        US#S5      UUS$.
UD6n[_        UUUS%9n[a        US&US'9$ [2        Rc                  XCUS(9I Sh  vN n[S        US0 5      =(       d    0 nUR                  S0 5      =(       d    0 nUR6                  Re                  [[        S=UUR\                  UUU[        U[S        US#S5      UUS$.
UD65        [g        US)9I Sh  vN   U$  GN}! [         a    [
        R                  " U5      n GNwf = f GN& GN: GNg GN< N N>! [h         Ga  nURj                  n[2        Rm                  UUUS*9I Sh  vN    [.        Rn                  " 5       nURp                  URr                  S+   Rp                  l:        UR                  S!S5      bw  US!   S"L ao  [.        R<                  Rw                  US"S,9n[.        Rx                  " UUR                  SS5      S-UR                  SS5      S.9n[_        UUUS%9n[a        US&S/9s SnA$ [.        Rz                  " S+S+S+S09nUUl>        Us SnA$ SnAf[         Ga*  n[        R~                  " S1[-        U5       35        [2        Rm                  UUUS*9I Sh  vN    [S        US2S5      n[        R                  " S3UU5        [+        U[        5      (       a[  [        [S        US4[-        U5      5      [S        US5S65      [S        US7S65      [S        US8[        R                  5      [S        US90 5      S:9e[-        U5       n[        [S        US;U5      [S        US5S65      [S        US7S65      [S        US8S<5      [S        US90 5      S:9eSnAff = f7f)>a  

Follows the exact same API spec as `OpenAI's Chat API https://platform.openai.com/docs/api-reference/chat`

```bash
curl -X POST http://localhost:4000/v1/chat/completions 
-H "Content-Type: application/json" 
-H "Authorization: Bearer sk-1234" 
-d '{
    "model": "gpt-4o",
    "messages": [
        {
            "role": "user",
            "content": "Hello!"
        }
    ]
}'
```

NRequest received by LiteLLM:
{}r  r  r"  r   r   r  r   r
  completion_modelr  rH  rI  rG  r  
completionr  r"  	call_typer  litellm_call_idacompletion)original_function	rules_objr6  litellm_logging_objr"  r  r  r"  
route_typer   rM  r   _hidden_paramsr  r   r  r  r  additional_headersr  r  successr  r   streamTallowed_model_region)
r  r  r  r  r  r   r  r  r  r  r  text/event-stream
media_typer   r"  r  r*  r*  ri  r   model_responseconvert_to_deltacached_response)completion_streamr  custom_llm_providerlogging_objr  prompt_tokenscompletion_tokenstotal_tokenszBlitellm.proxy.proxy_server.chat_completion(): Exception occured - litellm_debug_infoo[1;31mAn error occurred: %s %s

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r/  r   rm  r   r   r   )r   r   r   r   r   r   rn  r   )Dbodydecodeastliteral_evalr"  r  r  r4   r`  r*   r  r`   r   r   r
  rx  rM  rU  rV  user_max_tokensrT  r<  r  rS  model_alias_mapr  pre_call_hookr   uuiduuid4utilsfunction_setupRulesr   r  rD  during_call_hookrj   r   r1  r(  r  get_deploymentr  update_request_statusr+  r  r  r   post_call_success_hookr!  rB   r8   r  rq  ModelResponser   choicesr   ModelResponseIteratorCustomStreamWrapperUsageusagerX  r   rr  r   HTTP_400_BAD_REQUEST)r   r  r  r  r"  r'  body_strr  tasksllm_callllm_responsesr+  r*  hidden_paramsr  r  r  r  r  r  custom_headersselected_data_generatorr*  _data_chat_response	_iterator_streaming_response_usager%  rw  s                                 r   chat_completionrJ  e  sH    z Dd
\\^#;;=	(##H-D 	"".55djja6PQ	
 1-/%
 
   !3T:  G}	 	W "2D&:D"#?!0D=,D
 d7mS))d7mw?V?V.V#33DMBDM '44/l 5 
 
 #*//"5"5TZZ\!2#
 $MM88 
+mm))+||~
 	
 '2"#.."3& / 	
 '$!!	
 
 	X  
 ('	Q<*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB,9,=,=/-
) $1#4#45I2#N#TRT !!+!:!:H!:!MD33 $):B ?	 4 	
 h4!7/ "3#33!#!+$%68NPRS2S! %N '<!"3!'#
 %'.&  +AAX B 
 

 H.39r 	 +../CRHNB  '' "3#33!#!+$%68NPRS2S! %	
 *8<<<G $  	(::h'D	(
D
:
 (d
0 	=    66/  7 
 	
 	

 !..045IIq!))188Hd#/DNd4J;;- < I #*"="="+hhw+$5 HH%:DA	# '<,"3"'# %'.  Q!RST% 
&&PQTUVQWPXY	
  66/ATX 7 
 	
 	
 %Q(<bA"" C	
 a'' 8SV4Q/a&1Qv/J/JK9b1  1vh	Ay)4FF+!Wf-M3/Ay"-
 	
1
s  \+S RS R AS S C.S 	S
CS S -S S	ES -\+.S SB	S SS \+S  R=9S <R==S S S 	S S S \($W0?T CW0\(\+W0*\(+\+0\(>:\#8X;9C*\##\((\+z/v1/completionscompletionsz/completionsz!/engines/{model:path}/completionsz,/openai/deployments/{model:path}/completionsc                 6
  #    0 n U R                  5       I Sh  vN nUR                  5       n [        R                  " U5      n[        R                  SS5      =(       d    [        =(       d    U=(       d    US   US'   [        (       a	  [        US'   [        UU [        U[        [        S9I Sh  vN n[        (       a	  [        US'   [        (       a	  [        US'   [        (       a	  [        US'   [         (       a	  [         US'   US   ["        R$                  ;   a  ["        R$                  US      US'   [&        R)                  X4S	S
9I Sh  vN n[+        US[,        [        S9I Sh  vN nUI Sh  vN n[/        US0 5      =(       d    0 n	U	R                  SS5      =(       d    Sn
U	R                  SS5      =(       d    SnU	R                  SS5      =(       d    SnU	R                  SS5      =(       d    SnU	R                  SS5      =(       d    Sn[0        R2                  " [&        R5                  UR                  SS5      SS95        [6        R8                  " SU5        SU;   a2  US   SL a*  [;        UUU
UU[        UUS9n[=        UUUS9n[?        USUS9$ [&        RA                  XCUS9I Sh  vN nURB                  RE                  [;        UUU
UU[        UUS95        [G        US9I Sh  vN   U$  GN! [         a    [
        R                  " U5      n GNf = f GNq GN GN GN Nz N>! [H         Ga<  nURJ                  n[&        RM                  UUUS9I Sh  vN    UR                  SS5      b  US   SL a  ["        RN                  " 5       n["        RP                  " SSSS 9nUUl)        URT                  URV                  S   RT                  l,        ["        RZ                  R]                  USS!9n["        R^                  " UUR                  SS5      S"9n[=        UUUS9n[?        US0 S9s SnA$ ["        R`                  " 5       nURT                  URV                  S   l1        Us SnA$ SnAf[         a  n[&        RM                  UUUS9I Sh  vN    [6        Rd                  " S#Rg                  [i        U5      5      5        [i        U5       n[k        [/        US$U5      [/        US%S&5      [/        US'S&5      [/        US(S)5      S*9eSnAff = f7f)+a  
Follows the exact same API spec as `OpenAI's Completions API https://platform.openai.com/docs/api-reference/completions`

```bash
curl -X POST http://localhost:4000/v1/completions 
-H "Content-Type: application/json" 
-H "Authorization: Bearer sk-1234" 
-d '{
    "model": "gpt-3.5-turbo-instruct",
    "prompt": "Once upon a time",
    "max_tokens": 50,
    "temperature": 0.7
}'
```
Nr  r  r   rH  rI  rG  r  text_completionr  atext_completionr  r  r  r   r  r  r  r  r  final response: %sr  T)r  r  r  r  r  r   r  r  r  r  r  r  r  ri  r   r!  r  r  r  z?litellm.proxy.proxy_server.completion(): Exception occured - {}r   r   rm  r   r   rn  r   )6r'  r(  r)  r*  r"  r  r  r   rx  rM  r`   r   r
  rU  rV  r+  rT  rS  r,  r  r-  rj   r   r  r1  r  r5  r4   r`  r+  r  r   r6  r   r!  rB   r8   r  rq  r7  r;  r<  r   r8  r   r0  r9  TextCompletionStreamWrapperTextCompletionResponsetextrX  r*   r  rr  )r   r  r  r  r"  r'  r>  r@  r*  rB  r  r  r  r  r  rC  rD  r*  rE  rF  rI  rG  rH  	_responserw  s                            r   r  r    s&    R Dj
\\^#;;=	(##H-D
   !3T:  G}	 	W :&DM0-/%
 
 "2D&:D"#?!0D=,D
 =G333#33DMBDM '44/FW 5 
 

 ')!!	
 
 ">*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ 	33 $):B ?	 4 	
 	""#7Bh4!7/"3'!#!+!	N '<!"3!'# %'.&  +AAX B 
 
 	  ''"3'!#!+!		
 *8<<<e $  	(::h'D	(
4


 "T
  	= &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&MTTA	

 1vh	Ay)4FF+!Wf-M3/	
 	

s  TL2 K8L2 K; A,L2 1L"2BL2 L%L2 )L(*	L2 3L+4D'L2 TL2 3L.4=L2 1L02L2 7T8L2 ; LL2 LL2 %L2 (L2 +L2 .L2 0L2 2T=$Q4!M$"CQ44T5T:4Q4.T/T4TTRA7TTTz/v1/embeddings
embeddings)r  response_classr  z/embeddingsz /engines/{model:path}/embeddingsz+/openai/deployments/{model:path}/embeddingsc                 >
  #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        R                  " S[
        R                  " USS95        [        UU [        U[        [        S9I Sh  vN n[        R                  SS5      =(       d    [        =(       d    U=(       d    US   US'   [        (       a	  [        US'   US   [        R                  ;   a  [        R                  US      US'   [        b  [        R                   O/ nSU;   a  [#        US   [$        5      (       a  ['        US   5      S	:  a  [#        US   S	   [$        5      (       a  [#        US   S	   S	   [(        5      (       a  [*        b  US   U;   a  [*         H}  nUS
   US   :X  a8  US   S   [        R,                  ;   d  US   S   R/                  S5      (       a  MG  / nUS    H'  n	UR1                  [        R2                  " SU	S95        M)     XS'     O   [4        R7                  X4SS9I Sh  vN n/ n
U
R1                  [4        R9                  UUSS95        [;        US[        [        S9I Sh  vN nU
R1                  U5        [<        R>                  " U
6 nUI Sh  vN nUS   n[<        R@                  " [4        RC                  UR                  SS5      SS95        [E        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  S0 5      =(       d    0 nURF                  RI                  [K        S-UUUU[        U[E        USS5      UUS .	UD65        [M        US!9I Sh  vN   U$  GN GNq GN GN GNV N! [N         Ga  n[4        RQ                  UUUS"9I Sh  vN    [E        US#S5      n[        R                  " S$UU5        [        RR                  " S%RU                  [W        U5      5      5        [#        U[X        5      (       aF  [[        U5      n[]        U[E        US&S'5      [E        US(S'5      [E        US)[^        R`                  5      S*9e[W        U5       n[]        [E        US+U5      [E        US&S'5      [E        US(S'5      [E        US)S,5      S*9eSnAff = f7f).al  
Follows the exact same API spec as `OpenAI's Embeddings API https://platform.openai.com/docs/api-reference/embeddings`

```bash
curl -X POST http://localhost:4000/v1/embeddings 
-H "Content-Type: application/json" 
-H "Authorization: Bearer sk-1234" 
-d '{
    "model": "text-embedding-ada-002",
    "input": "The quick brown fox jumps over the lazy dog"
}'
```

NzRequest received by LiteLLM:
%sr  r  r   embedding_modelr  inputr   r  r  zazure/zgpt-3.5-turbo)r  tokensrU  r  r
  
aembeddingr  r   r  r   r  r  r  r  r  r  r  r  r  	r  r  r  r  r   r  r  r  r  r  ri  r%  r&  z?litellm.proxy.proxy_server.embeddings(): Exception occured - {}r   rm  r   r   r   r   rn  r   )1r'  orjsonr  r4   r`  r  r  r`   r   r   r
  rx  rM  rS  r,  r   model_namesr<  r_  r  r   r   open_ai_embedding_modelsr   rD  r(  r  r-  r3  rj   r1  r(  r  r5  r  r   r!  r+  rB   r"  rq  rX  r*   r  r   rt   rr  r   r=  )r   r  r  r  r"  r'  router_model_namesr  
input_listir?  r@  rA  r+  r*  rB  r  r  r  r  r  r  r*  r%  r   rw  s                             r   rU  rU  a  s    ` D\\\^#||D!"".JJtA&	
 1-/%
 
   !2D9  G}	 	W :&DM
 =G333#33DMBDM7A7MZ33SUtO4=$//DM"Q&4=+T224=+A.44 )d7m?Q.Q'A$w-7*+G48X8XX-.w7BB8LL &(
!%gA&-- '_Q O "/ )3W (" '44/l 5 
 
 .."3& / 	
 '#!!	
 
 	X  
 ('	Q< 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'! %	
 *8<<<w $
f

 (D 	=  66/ATX 7 
 	
 	
 %Q(<bA"" C	

 	&&MTTA	

 a''+A.G Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 3s   TO: O)A!O: ;O,<F3O: /O/0AO: 1O22-O: O5 EO: "O8#O: (T)O: ,O: /O: 2O: 5O: 8O: :TTP C7TTTz/v1/images/generationsimagesz/images/generationsc                   #    0 n U R                  5       I S h  vN n[        R                  " U5      n[        UU [        U[
        [        S9I S h  vN n[        R                  SS 5      =(       d    [        =(       d    US   US'   [        (       a	  [        US'   US   [        R                  ;   a  [        R                  US      US'   [        R                  X#SS9I S h  vN n[        US[        [        S9I S h  vN nUI S h  vN n[        R                   " [        R#                  UR                  SS	5      S
S95        [%        US0 5      =(       d    0 nUR                  SS 5      =(       d    S	nUR                  SS 5      =(       d    S	n	UR                  SS 5      =(       d    S	n
UR                  SS 5      =(       d    S	nUR                  SS 5      =(       d    S	nUR&                  R)                  [+        UUU	U
[
        U[%        USS	5      UUS9	5        U$  GN GN GNA GN( GN!! [,         Ga!  n[        R/                  X-US9I S h  vN    [0        R2                  " SR5                  [7        U5      5      5        [0        R8                  " [:        R<                  " 5       5        [?        U[@        5      (       aO  [C        [%        US[7        U5      5      [%        USS5      [%        USS5      [%        US[D        RF                  5      S9e[7        U5       n[C        [%        USU5      [%        USS5      [%        USS5      [%        USS5      S9eS nAff = f7f)Nr   image_generation_modelr  image_generationr  aimage_generationr  r  r   r  r  r  r  r  r  r  r  r\  ri  zElitellm.proxy.proxy_server.image_generation(): Exception occured - {}r   r   rm  r   r   r   rn  )$r'  r]  r  r`   r   r   r
  rx  rM  rS  r,  r  r-  rj   r   r1  r  r5  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  r   rr  r   r=  )r   r  r  r"  r'  r@  r*  rB  r  r  r  r  r  r*  rw  s                  r   rf  rf  1  s7    $ D`\\^#||D! 1-/%
 
   !94@ G} 	W
 :&DM
 =G333#33DMBDM '44/FX 5 
 

 '*!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ  ''"3!#!+$%68NPRS'!
	
 K $
0


 ">  66/TX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a'' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 's   MH- H7H- H!BH- #H$$H- ?H' 	H- 	H*
DH- MH- !H- $H- 'H- *H- -M8MIDMMMz/v1/audio/speechaudioz/audio/speechc                   #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        UU [        U[
        [        S9I Sh  vN nUR                  SS5      c  UR                  b  UR                  US'   [        (       a	  [        US'   [        R                  X#SS9I Sh  vN n[        US[        [        S9I Sh  vN nUI Sh  vN n[        R                  " [        R!                  UR                  S	S
5      SS95        [#        US0 5      =(       d    0 nUR                  SS5      =(       d    S
nUR                  SS5      =(       d    S
n	UR                  SS5      =(       d    S
n
UR                  SS5      =(       d    S
nUR                  S	S5      =(       d    S
nS[$        4S jn['        UUU	U
[
        U[#        USS
5      SUUS9
n[)        UUUS9  [+        U" U5      SUS9$  GN GN GNN GN5 GN.! [,         a^  n[.        R0                  " SR3                  [5        U5      5      5        [.        R6                  " [8        R:                  " 5       5        UeSnAff = f7f)zT
Same params as:

https://platform.openai.com/docs/api-reference/audio/createSpeech
Nr   r  r  rf  r  aspeechr  r  r   r  r  r  r  r  r  r  rT  c                h   #    U R                  SS9I S h  vN nU  S h  vN nU7v   M   N N
 g 7f)Ni   )
chunk_size)aiter_bytes)rT  
_generatorrd  s      r   generateaudio_speech.<locals>.generate  s7     (444EEJ) e Fzs$   2,20.0	202r  )
r  r  r  r  r   r  r  r  r  r  r  z
audio/mpegr  zAlitellm.proxy.proxy_server.audio_speech(): Exception occured - {})r'  r]  r  r`   r   r   r
  rx  r  rM  r  r-  rj   r   r1  r  r5  r  r   r+  r  r   r"  r4   r   r*   r  r`  r{  r|  )r   r  r  r"  r'  r@  r*  rB  r  r  r  r  r  ro  rC  r*  s                   r   audio_speechrq    sp    * DU\\^#||D! 1-/%
 
 88FD!).?.G.G.S,44DL:&DM '44/FX 5 
 

 ' !!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ	&@ 	
 ,/' !24JBO.2#
 	/	

 !X<
 	
Q $
 


 "Z  ""OVVA	

 	""9#7#7#9:s   I2H G87H G;AH 0G>1H H	H HD H 7I28H ;H >H H H 
I/AI**I//I2z/v1/audio/transcriptionsz/audio/transcriptions.r   c                  	  #    0 n U R                  5       I Sh  vN nUR                  5        VVs0 s H  u  pgUS:w  d  M  Xg_M     nnn[        UU [        U[        [
        S9I Sh  vN nUR                  SS5      c  UR                  b  UR                  US'   [        R                  SS5      =(       d    [        =(       d    US   US'   [        (       a	  [        US'   [        b  [        R                  O/ nUR                  c  [        S[        R                  SSS	9e[        UUUS
9  UR!                  5       I Sh  vN n	["        R$                  " U	5      n
UR                  U
l        XS'    [(        R+                  UUSS9I Sh  vN n[-        US[        [        S9I Sh  vN nUI Sh  vN n U
R5                  5         [6        R8                  " [(        R;                  UR                  SS5      SS95        [=        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  S0 5      =(       d    0 nUR>                  RA                  [C        S%UUUU[        U[=        USS5      UUS.	UD65        U$  GNs  snnf  GN GN GNz GNa GNZ! [.         a  n[1        S[3        U5      S9eSnAff = f! U
R5                  5         f = f! [.         Ga+  n[(        RE                  X=US9I Sh  vN    [F        RH                  " SRK                  [3        U5      5      5        [F        RL                  " [N        RP                  " 5       5        [S        U[0        5      (       aY  [        [=        US[3        URT                  5      5      [=        US S!5      [=        US"S!5      [=        US#[        R                  5      S$9e[3        U5       n[        [=        USU5      [=        US S!5      [=        US"S!5      [=        US#S5      S$9eSnAff = f7f)&ze
Same params as:

https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=curl
Nr   r   r  moderation_modelr  z.File name is None. Please check your file namebad_request)r   r   r   r   )r  r   r`  audio_transcriptionr  atranscriptionr  rn  r.  r  r   r  r  r  r  r  r  r  r  r  r\  ri  zHlitellm.proxy.proxy_server.audio_transcription(): Exception occured - {}r   r   rm  r   r   r   r   )+formr   r`   r   r   r
  rx  r  rM  r   r^  r   rr  r   r=  rR   readioBytesIOr   r  r-  rj   r"  r   r  closer1  r  r5  r  r   r!  r+  rq  r4   r   r*   r`  r{  r|  r<  r/  )r   r  r   r  r"  	form_datar)  r   r`  file_contentfile_objectr@  r*  r*  rB  r  r  r  r  r  r  rw  s                         r   audio_transcriptionsr    sQ    , D|!,,.(	-6__->P->zs#-

->P 1-/%
 
 88FD!).?.G.G.S,44DL   !3T: G} 	W
 :&DM7A7MZ33SU==  H00"	  	$1	
 "YY[(jj.=="V	 *88"3/ 9  D ++%%	 H &~H  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'! %	
 C )P
J ) & 	@CA??	@ D  66/TX 7 
 	
 	
 	""V]]A	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 's   Q>M K3M K6K6 M 'K<(CM ?K? /M 0L L	L $L%	L .L/L 4D>M 2Q>3M 6M ?M L L L 
L-L((L--L0 0MM Q;Q6'M*(DQ66Q;;Q>)r   	WebSocketWebSocketDisconnect)
_arealtimez/v1/realtime	websocketc                   #    SS K nU R                  5       I S h  vN   UU S.n [        US[        [        S9I S h  vN nUI S h  vN   g  N1 N N	! UR
                  R                   aC  n[        R                  " S5        U R                  UR                  SS9I S h  vN     S nAg S nAf[         a2    [        R                  " S5        U R                  SSS9I S h  vN     g f = f7f)	Nr   )r  r  r  r  zInvalid status code)r   reasonzInternal server errori  )
websocketsacceptrj   r   rM  
exceptionsInvalidStatusCoder4   rX  r{  r   r"  )r  r  r  r  r"  r@  r*  s          r   websocket_endpointr    s      



 DI&#!!	
 
 ! 
 	  22 P&&'<=oo1==9NoOOO I&&'>?oo40GoHHHIs~   C,A	C,A A	A AA C,A A C),3B*B" B*%C,*6C) C#!C)&C,(C))C,z/v1/assistants
assistantsz/assistantsc                 H  #    0 n U R                  5       I Sh  vN   [        UU [        U[        [        S9I Sh  vN n[
        c$  [        SS[        R                  R                  0S9e[
        R                  " S0 UD6I Sh  vN n[        R                  " [        R                  UR                  SS5      SS	95        [!        US
0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR"                  R%                  ['        UUUU[        [!        USS5      US95        U$  GNH GN) N! [(         Ga+  n	[        R+                  X)US9I Sh  vN    [,        R.                  " SR1                  [3        U	5      5      5        [,        R4                  " [6        R8                  " 5       5        [;        U	[        5      (       aY  [=        [!        U	S[3        U	R>                  5      5      [!        U	SS5      [!        U	SS5      [!        U	S[@        RB                  5      S9e[3        U	5       n
[=        [!        U	SU
5      [!        U	SS5      [!        U	SS5      [!        U	SS5      S9eSn	A	ff = f7f)z~
Returns a list of assistants.

API Reference docs - https://platform.openai.com/docs/api-reference/assistants/listAssistants
Nr   rn  r   r.  r  r   r  r  r  r  r  r  r  r  r  r  r  r   r  r  ri  zClitellm.proxy.proxy_server.get_assistants(): Exception occured - {}r   r   rm  r   r   r   r   )"r'  r`   r   r   r
  r   r   r  no_llm_routerr   aget_assistantsr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r   r  r  r"  r*  rB  r  r  r  r*  rw  s              r   get_assistantsr    s    * DGlln 1-/%
 
 2C2Q2Q2W2W(X  $33;d;; 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 Y 	
 <8  66/TX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'i   J"E) E!!E) E$AE) E'CE)  J"!E) $E) 'E) )J4JFDJJJ"c                 t  #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        UU [        U[
        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " S0 UD6I Sh  vN n[        R                  " [        R!                  UR#                  SS5      SS	95        [%        US
0 5      =(       d    0 nUR#                  SS5      =(       d    SnUR#                  SS5      =(       d    SnUR#                  SS5      =(       d    Sn	UR&                  R)                  [+        UUUU	[
        [%        USS5      US95        U$  GN^ GN) N! [,         Ga+  n
[        R/                  X*US9I Sh  vN    [0        R2                  " SR5                  [7        U
5      5      5        [0        R8                  " [:        R<                  " 5       5        [?        U
[        5      (       aY  [A        [%        U
S[7        U
RB                  5      5      [%        U
SS5      [%        U
SS5      [%        U
S[D        RF                  5      S9e[7        U
5       n[A        [%        U
SU5      [%        U
SS5      [%        U
SS5      [%        U
SS5      S9eSn
A
ff = f7f)zr
Create assistant

API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
Nr   rn  r   r.  r  r   r  r  r  r  r  r  r  r  ri  zElitellm.proxy.proxy_server.create_assistant(): Exception occured - {}r   r   rm  r   r   r   r   )$r'  r]  r  r`   r   r   r
  r   r   r  r  r   acreate_assistantsr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  )r   r  r  r"  r'  r*  rB  r  r  r  r*  rw  s               r   create_assistantr  >  s    * DH\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $66>>> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 [ $
 ?8  66/TX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J8E? E77E? E:AE? E=CE? 6J87E? :E? =E? ?J5
J0!F$"DJ00J55J8z"/v1/assistants/{assistant_id:path}z/assistants/{assistant_id:path}assistant_idc                   #    0 n [        UU [        U[        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " SSU0UD6I Sh  vN n[        R                  " [        R                  UR                  SS5      S	S
95        [        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    Sn	UR                   R#                  [%        UUUU	[        [        USS5      US95        U$  GN( N! [&         Ga+  n
[        R)                  X:US9I Sh  vN    [*        R,                  " SR/                  [1        U
5      5      5        [*        R2                  " [4        R6                  " 5       5        [9        U
[
        5      (       aY  [;        [        U
S[1        U
R<                  5      5      [        U
SS5      [        U
SS5      [        U
S[>        R@                  5      S9e[1        U
5       n[;        [        U
SU5      [        U
SS5      [        U
SS5      [        U
SS5      S9eSn
A
ff = f7f)zr
Delete assistant

API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
r   Nrn  r   r.  r  r  r   r  r  r  r  r  r  r  r  ri  zElitellm.proxy.proxy_server.delete_assistant(): Exception occured - {}r   r   rm  r   r   r   r   )!r`   r   r   r
  r   r   r  r  r   adelete_assistantr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  )r   r  r  r  r"  r*  rB  r  r  r  r*  rw  s               r   delete_assistantr    sr    , DF 1-/%
 
 2C2Q2Q2W2W(X  $55X<XSWXX 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 Y8  66/TX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'Y   J	E EA
E .E/CE 
J	E E JJ2E53DJJJ	z/v1/threadsz/threadsc                 H  #    0 n U R                  5       I Sh  vN   [        UU [        U[        [        S9I Sh  vN n[
        c$  [        SS[        R                  R                  0S9e[
        R                  " S0 UD6I Sh  vN n[        R                  " [        R                  UR                  SS5      SS	95        [!        US
0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR"                  R%                  ['        UUUU[        [!        USS5      US95        U$  GNH GN) N! [(         Ga+  n	[        R+                  X)US9I Sh  vN    [,        R.                  " SR1                  [3        U	5      5      5        [,        R4                  " [6        R8                  " 5       5        [;        U	[        5      (       aY  [=        [!        U	S[3        U	R>                  5      5      [!        U	SS5      [!        U	SS5      [!        U	S[@        RB                  5      S9e[3        U	5       n
[=        [!        U	SU
5      [!        U	SS5      [!        U	SS5      [!        U	SS5      S9eSn	A	ff = f7f)zg
Create a thread.

API Reference - https://platform.openai.com/docs/api-reference/threads/createThread
Nr   rn  r   r.  r  r   r  r  r  r  r  r  r  r  ri  zClitellm.proxy.proxy_server.create_threads(): Exception occured - {}r   r   rm  r   r   r   r   )"r'  r`   r   r   r
  r   r   r  r  r   acreate_threadr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r  s              r   create_threadsr    s    * DGlln 1-/%
 
 2C2Q2Q2W2W(X  $22:T:: 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 Y 	
 ;8  66/TX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}z/threads/{thread_id}	thread_idc                   #    0 n [        UU [        U[        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " SSU0UD6I Sh  vN n[        R                  " [        R                  UR                  SS5      S	S
95        [        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    Sn	UR                   R#                  [%        UUUU	[        [        USS5      US95        U$  GN( N! [&         Ga+  n
[        R)                  X:US9I Sh  vN    [*        R,                  " SR/                  [1        U
5      5      5        [*        R2                  " [4        R6                  " 5       5        [9        U
[
        5      (       aY  [;        [        U
S[1        U
R<                  5      5      [        U
SS5      [        U
SS5      [        U
S[>        R@                  5      S9e[1        U
5       n[;        [        U
SU5      [        U
SS5      [        U
SS5      [        U
SS5      S9eSn
A
ff = f7f)zg
Retrieves a thread.

API Reference - https://platform.openai.com/docs/api-reference/threads/getThread
r   Nrn  r   r.  r  r  r   r  r  r  r  r  r  r  r  ri  z?litellm.proxy.proxy_server.get_thread(): Exception occured - {}r   r   rm  r   r   r   r   )!r`   r   r   r
  r   r   r  r  r   aget_threadr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r   r  r  r  r"  r*  rB  r  r  r  r*  rw  s               r   
get_threadr  _  sq    , DE 1-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 M8  66/TX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z /v1/threads/{thread_id}/messagesz/threads/{thread_id}/messagesc                 x  #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        UU [        U[
        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " SSU0UD6I Sh  vN n[        R                  " [        R!                  UR#                  SS5      S	S
95        [%        US0 5      =(       d    0 nUR#                  SS5      =(       d    SnUR#                  SS5      =(       d    Sn	UR#                  SS5      =(       d    Sn
UR&                  R)                  [+        UUU	U
[
        [%        USS5      US95        U$  GN` GN+ N! [,         Ga+  n[        R/                  X;US9I Sh  vN    [0        R2                  " SR5                  [7        U5      5      5        [0        R8                  " [:        R<                  " 5       5        [?        U[        5      (       aY  [A        [%        US[7        URB                  5      5      [%        USS5      [%        USS5      [%        US[D        RF                  5      S9e[7        U5       n[A        [%        USU5      [%        USS5      [%        USS5      [%        USS5      S9eSnAff = f7f)zj
Create a message.

API Reference - https://platform.openai.com/docs/api-reference/messages/createMessage
Nr   rn  r   r.  r  r  r   r  r  r  r  r  r  r  r  ri  zAlitellm.proxy.proxy_server.add_messages(): Exception occured - {}r   r   rm  r   r   r   r   )$r'  r]  r  r`   r   r   r
  r   r   r  r  r   a_add_messager1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r   r  r  r  r"  r'  r*  rB  r  r  r  r*  rw  s                r   add_messagesr    s    , DH\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 [ $
 O8  66/TX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J:F E97F E<A
F E?CF 8J:9F <F ?F J7J2#F&$DJ22J77J:c                   #    0 n [        UU [        U[        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " SSU0UD6I Sh  vN n[        R                  " [        R                  UR                  SS5      S	S
95        [        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    Sn	UR                   R#                  [%        UUUU	[        [        USS5      US95        U$  GN( N! [&         Ga+  n
[        R)                  X:US9I Sh  vN    [*        R,                  " SR/                  [1        U
5      5      5        [*        R2                  " [4        R6                  " 5       5        [9        U
[
        5      (       aY  [;        [        U
S[1        U
R<                  5      5      [        U
SS5      [        U
SS5      [        U
S[>        R@                  5      S9e[1        U
5       n[;        [        U
SU5      [        U
SS5      [        U
SS5      [        U
SS5      S9eSn
A
ff = f7f)z
Returns a list of messages for a given thread.

API Reference - https://platform.openai.com/docs/api-reference/messages/listMessages
r   Nrn  r   r.  r  r  r   r  r  r  r  r  r  r  r  ri  zAlitellm.proxy.proxy_server.get_messages(): Exception occured - {}r   r   rm  r   r   r   r   )!r`   r   r   r
  r   r   r  r  r   aget_messagesr1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r  s               r   get_messagesr     so    , DD0-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 S
 O8  66/TX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}/runsz/threads/{thread_id}/runsc                   #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        UU [        U[
        [        S9I Sh  vN n[        c$  [        SS[        R                  R                  0S9e[        R                  " SSU0UD6I Sh  vN nSU;   a  US   SL a  [        [        UUUS	9S
S9$ [        R                   " ["        R%                  UR'                  SS5      SS95        [)        US0 5      =(       d    0 nUR'                  SS5      =(       d    SnUR'                  SS5      =(       d    Sn	UR'                  SS5      =(       d    Sn
UR*                  R-                  [/        UUU	U
[
        [)        USS5      US95        U$  GN GNL GN! [0         Ga+  n["        R3                  X;US9I Sh  vN    [4        R6                  " SR9                  [;        U5      5      5        [4        R<                  " [>        R@                  " 5       5        [C        U[        5      (       aY  [E        [)        US[;        URF                  5      5      [)        USS5      [)        USS5      [)        US[H        RJ                  5      S9e[;        U5       n[E        [)        USU5      [)        USS5      [)        USS5      [)        USS5      S9eSnAff = f7f)z]
Create a run.

API Reference: https://platform.openai.com/docs/api-reference/runs/createRun
Nr   rn  r   r.  r  r  Tr  r*  r  r  r   r  r   r  r  r  r  r  r  r  r  ri  z?litellm.proxy.proxy_server.run_thread(): Exception occured - {}r   r   rm  r   r   r   r   )&r'  r]  r  r`   r   r   r
  r   r   r  r  r   arun_threadr   rz  r1  r  r  r5  rx  r  r   r!  r+  r"  rq  r4   r   r*   r  r`  r{  r|  r<  rr  r/  r   r=  r  s                r   
run_threadr  ~  s    , DR\\^#||D!0-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL h4!7$/&7%!%
 /  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 q $
 MP  66/TX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sw   KF# F7F# FA
F# F $F# KCF# KF# F#  F# #K.KGDKKKz/{provider}/v1/batchesbatchz/v1/batchesz/batchesrl  c                 n  #    0 n U R                  5       I Sh  vN nUR                  5       n [        R                  " U5      n[        R                  " SR                  [
        R                  " USS95      5        [        UU [        U[        [        S9I Sh  vN nSnSn[        R                   SL a   UR#                  SS5      n[%        U[&        S	9n[)        S"0 UD6n	U=(       d    U	R+                  S
S5      n
[        R                   SL a<  U(       a5  Ub2  [&        c  [-        SSS0S9e[&        R.                  " S"0 U	D6I Sh  vN nO%U
c  Sn
[        R.                  " S"S
U
0U	D6I Sh  vN n[0        R2                  " [4        R7                  UR#                  SS5      SS95        [9        US0 5      =(       d    0 nUR#                  SS5      =(       d    SnUR#                  SS5      =(       d    SnUR#                  SS5      =(       d    SnUR:                  R=                  [?        UUUU[        [9        USS5      US95        U$  GN/! [         a    [
        R                  " U5      n GN)f = f GN GN0 GN! [         Ga,  n[4        RA                  UUUS9I Sh  vN    [        RB                  " SR                  [E        U5      5      5        [        R                  " [F        RH                  " 5       5        [K        U[,        5      (       aY  [M        [9        US[E        URN                  5      5      [9        USS5      [9        USS5      [9        US [P        RR                  5      S!9e[E        U5       n[M        [9        USU5      [9        USS5      [9        USS5      [9        US S5      S!9eSnAff = f7f)#a  
Create large batches of API requests for asynchronous processing.
This is the equivalent of POST https://api.openai.com/v1/batch
Supports Identical Params as: https://platform.openai.com/docs/api-reference/batch

Example Curl
```
curl http://localhost:4000/v1/batches         -H "Authorization: Bearer sk-1234"         -H "Content-Type: application/json"         -d '{
        "input_file_id": "file-abc123",
        "endpoint": "/v1/chat/completions",
        "completion_window": "24h"
}'
```
Nr  r  r  r   FTr  )r  r   r  rn  r   9LLM Router not initialized. Ensure models added to proxy.r.  r  r  r   r  r  r  r  r  r  r  r  ri  zAlitellm.proxy.proxy_server.create_batch(): Exception occured - {}r   r   rm  r   r   r   r   )*r'  r(  r)  r*  r"  r  r  r4   r`  r*   r  r`   r   r   r
  rS  'enable_loadbalancing_on_batch_endpointsrx  rg   r   r/   r%  r   acreate_batchr1  r  r  r5  r  r   r!  r+  rq  rX  r  r{  r|  r<  rr  r/  r   r=  )r   r  rl  r  r"  r'  r>  router_modelis_router_model_create_batch_datar  r*  rB  r  r  r  r*  rw  s                     r   create_batchr    s    N Dg\\^#;;=	(##H-D 	"".55djja6PQ	

 1-/%
 
 '+::dB88GT2L,<JWO/7$7&]*<*@*@AVX\*] ;;tC(!# #!\  (55K8JKKH"*&.#$22 $7;M H
 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 [ $  	(::h'D	(
@ L<  66/ATX 7 
 	
 	
 	&&OVVA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 's   N5I; II; I AI; I2B*I; I5%I; +I8,CI; N5I;  I/+I; .I//I; 5I; 8I; ;N2N-J!DN--N22N5z&/{provider}/v1/batches/{batch_id:path}z/v1/batches/{batch_id:path}z/batches/{batch_id:path}zBatch ID to retrievezThe ID of the batch to retrieve)r   r   batch_idc                 L  #    0 n [        US9n[        R                  SL a2  [        c  [	        SSS0S9e[        R
                  " S0 UD6I Sh  vN nO%Uc  Sn[        R
                  " SS	U0UD6I Sh  vN n[        R                  " [        R                  UR                  S
S5      SS95        [        US0 5      =(       d    0 nUR                  SS5      =(       d    Sn	UR                  SS5      =(       d    Sn
UR                  SS5      =(       d    SnUR                  R                  [        UU	U
U[        [        USS5      US95        U$  GN N! [          Ga@  n[        R#                  X,US9I Sh  vN    [$        R&                  " SR)                  [+        U5      5      5        [$        R,                  " [.        R0                  " 5       5        [3        U[        5      (       aY  [5        [        US[+        UR6                  5      5      [        USS5      [        USS5      [        US[8        R:                  5      S9e[.        R0                  " 5         [+        U5       n[5        [        USU5      [        USS5      [        USS5      [        USS5      S9eSnAff = f7f)aT  
Retrieves a batch.
This is the equivalent of GET https://api.openai.com/v1/batches/{batch_id}
Supports Identical Params as: https://platform.openai.com/docs/api-reference/batch/retrieve

Example Curl
```
curl http://localhost:4000/v1/batches/batch_abc123     -H "Authorization: Bearer sk-1234"     -H "Content-Type: application/json" 
```
)r  TNrn  r   r  r.  r  r  r  r   r  r  r  r  r  r  r  r  ri  Clitellm.proxy.proxy_server.retrieve_batch(): Exception occured - {}r   r   rm  r   r   r   r   )r1   rS  r  r   r   aretrieve_batchr1  r  r  r5  rx  r  r   r!  r+  r   r"  rq  r4   rX  r*   r  r`  r{  r|  r<  rr  r/  r   r=  )r   r  r  rl  r  r"  _retrieve_batch_requestr*  rB  r  r  r  r*  rw  s                 r   retrieve_batchr    s    L DK #7#
 ::dB!# #!\  (77R:QRRH#$44 $,0G H
 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 C S<  66/TX 7 
 	
 	
 	&&QXXA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	    "q6(I 9i8Q/a&1Qs3	 )s[   J$AE E%E 4E5CE J$E E J!!J8E;9D#JJ!!J$limitafterc                   #    [         R                  " SR                  X25      5         Uc  Sn[        R                  " UUUS9I Sh  vN n[        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  S	S5      =(       d    Sn	U R                  R                  [        UUUU	[        [        US
S5      S95        U$  N! [         GaC  n
[        R                  UU
X2S.S9I Sh  vN    [         R                  " SR                  [        U
5      5      5        [         R                  " [         R"                  " 5       5        [%        U
[&        5      (       aY  [)        [        U
S[        U
R*                  5      5      [        U
SS5      [        U
SS5      [        U
S[,        R.                  5      S9e[         R"                  " 5         [        U
5       n[)        [        U
SU5      [        U
SS5      [        U
SS5      [        U
SS5      S9eSn
A
ff = f7f)a5  
Lists 
This is the equivalent of GET https://api.openai.com/v1/batches/
Supports Identical Params as: https://platform.openai.com/docs/api-reference/batch/list

Example Curl
```
curl http://localhost:4000/v1/batches?limit=2     -H "Authorization: Bearer sk-1234"     -H "Content-Type: application/json" 
```
z!GET /v1/batches after={} limit={}Nr  )r  r  r  r  r  r   r  r  r  )r  r  r  r  r   r  )r  r  ri  r  r   r   rm  r   r   r   rn  )r4   r`  r*   rS  alist_batchesr  rx  r   r!  r+  r   r"  r  rq  r   r  r{  r|  r<  r   rr  r/  r   r=  )r  rl  r  r  r  r*  rB  r  r  r  r*  rw  s               r   list_batchesr    s    H BII%WX6H .. (
 
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS		
 /
0  66/ #(9 7 
 	
 	

 	""QXXA	

 	""9#7#7#9:a'' 9c!((m<Q/a&1Qv/J/JK	    "q6(I 9i8Q/a&1Qs3	 -sH   &H>C- C+B"C- *H>+C- -H;8H6DD#H66H;;H>z/v1/moderationsmoderationsz/moderationsc                   #    0 n U R                  5       I Sh  vN n[        R                  " U5      n[        UU [        U[
        [        S9I Sh  vN n[        R                  SS5      =(       d    [        =(       d    UR                  S5      US'   [        (       a	  [        US'   [        R                  X#SS9I Sh  vN n[        R                  " 5         [        US[        [        S9I Sh  vN nUI Sh  vN n[        R                  " [        R!                  UR                  S	S
5      SS95        [#        US0 5      =(       d    0 nUR                  SS5      =(       d    S
nUR                  SS5      =(       d    S
n	UR                  SS5      =(       d    S
n
UR$                  R'                  [)        UUU	U
[
        [#        USS
5      US95        U$  GN GN GN N N! [*         a  n[        R-                  X+US9I Sh  vN    [.        R0                  " SR3                  [5        U5      5      5        [7        U[8        5      (       aO  [;        [#        US[5        U5      5      [#        USS5      [#        USS5      [#        US[<        R>                  5      S9e[5        U5       n[;        [#        USU5      [#        USS5      [#        USS5      [#        USS5      S9eSnAff = f7f)ag  
The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies.

Quick Start
```
curl --location 'http://0.0.0.0:4000/moderations'     --header 'Content-Type: application/json'     --header 'Authorization: Bearer sk-1234'     --data '{"input": "Sample text goes here", "model": "text-moderation-stable"}'
```
Nr   rs  r  
moderationr  amoderationr  r  r   r  r  r  r  r  r  r  r  ri  z@litellm.proxy.proxy_server.moderations(): Exception occured - {}r   r   rm  r   r   r   rn  ) r'  r]  r  r`   r   r   r
  rx  rM  r  r-  r0  rj   r   r1  r  r5  r  r   r!  r+  r"  rq  r4   rX  r*   r  r<  r   rr  r   r=  )r   r  r  r"  r'  r@  r*  rB  r  r  r  r*  rw  s                r   r  r  ]  s    : DX\\^#||D! 1-/%
 
   !3T: !!xx  	W
 :&DM '44/l 5 
 
 			 '$!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!
	
 } $
$

 "8  66/TX 7 
 	
 	
 	&&NUUA	

 a'' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 %s   K)G$ G7G$ GA-G$ ?G 0G$ 0G 1	G$ :G";CG$ K)G$ G$ G$  G$ "G$ $
K&.K!HCK!!K&&K)z/v1/messagesz[beta] Anthropic `/v1/messages`)r  r  response_modelinclude_in_schemaanthropic_datac                 L  #    SSK Jn  SSKJn  SUS./[         l        UR                  5       I Sh  vN nUR                  5       n [        R                  " U5      n0 UESS0En	 [        R                  SS5      =(       d    [        =(       d    U	S	   U	S	'   [        (       a	  [        U	S	'   [        U	U[        U[         ["        S
9I Sh  vN n	[$        (       a	  [$        U	S'   [&        (       a	  [&        U	S'   [(        (       a	  [(        U	S'   [*        (       a	  [*        U	S'   U	S	   [         R,                  ;   a  [         R,                  U	S	      U	S	'   [.        R1                  X9SS9I Sh  vN n	[2        b  [2        R4                  O/ n
SU	;   a,  [6        R8                  " [         R:                  " S30 U	D65      nGO[2        b5  U	S	   U
;   a,  [6        R8                  " [2        R:                  " S30 U	D65      nGO[2        bT  [2        R<                  bC  U	S	   [2        R<                  ;   a,  [6        R8                  " [2        R:                  " S30 U	D65      nGOm[2        bG  U	S	   [2        R>                  ;   a0  [6        R8                  " [2        R:                  " S30 U	DSS0D65      nGO[2        bF  U	S	   [2        RA                  5       ;   a+  [6        R8                  " [2        R:                  " S30 U	D65      nO[2        bl  U	S	   U
;  ac  [2        RB                  c'  [E        [2        RF                  RH                  5      S:  a+  [6        R8                  " [2        R:                  " S30 U	D65      nO_[        b+  [6        R8                  " [         R:                  " S30 U	D65      nO-[K        [L        RN                  SSU	R                  S	S5      -   0S9eUI Sh  vN n[Q        US0 5      =(       d    0 nUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    SnUR                  SS5      =(       d    Sn[6        R8                  " [.        RS                  U	R                  SS5      SS95        [T        RV                  " SU5        URX                  R[                  []        UUUU[         UU	S 95        S!U	;   a  U	S!   SL a  [_        UUU	S"9n[a        US#S$9$ [T        Rb                  " S%Re                  U5      5        U$  GN! [         a    [        R                  " U5      n GNf = f GNk GN GN|! [f         Ga<  nURh                  n[.        Rk                  UUUS&9I Sh  vN    UR                  S!S5      b  US!   SL a  [         Rl                  " 5       n[         Rn                  " SSSS'9nUUl8        URr                  URt                  S   Rr                  l;        [         Rx                  R{                  USS(9n[         R|                  " UUR                  S	S5      S)9n[        UUU	S"9n[a        US#0 S*9s SnA$ [         R                  " 5       nURr                  URt                  S   lA        Us SnA$ SnAf[         a  n[.        Rk                  UUU	S&9I Sh  vN    [T        R                  " S+Re                  [        U5      5      5        [        U5       n[        [Q        US,U5      [Q        US-S.5      [Q        US/S.5      [Q        US0S15      S29eSnAff = f7f)4u   
🚨 DEPRECATED ENDPOINT🚨

Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/anthropic_completion).

This was a BETA endpoint that calls 100+ LLMs in the anthropic format.
r   )adapter_completion)anthropic_adapter	anthropic)r  adapterN
adapter_idr  r  r   rH  rI  rG  r  rM  r  api_keyspecific_deploymentTr   z/completion: Invalid model name passed in model=r   r.  r  r  r  r  r  r  r  rO  )r  r  r  r  r   r  r  r  r  r  r   z
Response from Litellm:
{}ri  r!  r  rP  r  zGlitellm.proxy.proxy_server.anthropic_response(): Exception occured - {}r   r   rm  r   r   rn  r   r   )ErS  r  "litellm.adapters.anthropic_adapterr  adaptersr'  r(  r)  r*  r"  r  r  r   rx  rM  r`   r   r
  rU  rV  r+  rT  r,  r  r-  r   r^  r1  r  aadapter_completionmodel_group_aliasdeployment_namesr  default_deploymentr  pattern_routerpatternsr   r   r=  r  r5  r4   r`  r   r!  r+  r  r   rz  r*   r8   r  rq  r7  r;  r<  r   r8  r   r0  r9  rQ  r  rR  rS  rX  r  rr  )r  r  r   r  r  r  r'  r>  r  r"  r`  llm_responser*  rB  r  r  r  r  rD  r*  rE  rF  rI  rG  rH  rT  rw  s                              r   anthropic_responser    s    ( +D*7HIJG D{{}H, --h7 =L<,<Dw
  !3T: G} 	W
 :&DM0-/%
 
 "2D&:D"#?!0D=,D
 =G333#33DMBDM '44/FW 5 
 

 8B7MZ33SU"..w/J/J/RT/RSL"tG}8J'J"..z/M/M/UPT/UVL",,8W!=!=="..z/M/M/UPT/UVL"tG}
8S8S'S"....PP4PL "tG}
8P8P8R'R"..z/M/M/UPT/UVL"W%77--9z0099:Q> #..z/M/M/UPT/UVL#"..w/J/J/RT/RSL"77Nhhw+,  &%*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB 	33 $):B ?	 4 	
 	""#7B  ''"3!#!+!
	
 h4!7&D!"3!'#
 %'. 
 	!!"@"G"G"QRO    ,zz(+,
4
d &X   &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&U\\A	

 1vh	Ay)4FF+!Wf-M3/	
 	

s   /\$T
\$T \$&A#T= 	T4
BT= %T7&IT= >T:?D#T= "\$#&T= 	\$ T1-\$0T11\$4T= 7T= :T= =\!$Y?,U/-CY??\! \$4Y?9\!:\$?\!\$Z'%A7\\!!\$z/utils/token_counterz	llm utils)r  r  r  c                   #    SSK Jn  U R                  nU R                  nUc  Uc
  [	        SSS9eSnSn[
        b.  [
        R                   H  nUS   U R                  :X  d  M  Un  O   Ub<  UR                  S0 5      R                  S	5      nS
U;   a  UR                  S
S5      S   nU=(       d    U R                  n[         R                  R                  US9n[        US   5      n	U" UUUS9n
[        U
U R                  UU	S9$ 7f)r   r   )token_counterN  z#prompt or messages must be providedr.  r  r  r  /r   r  r   )r  rS  r    )r$  request_model
model_usedtokenizer_type)rS  r  promptr    r   r   r  r  rx  splitr0  _select_tokenizerr  TokenCountResponse)r   r  r  r    r  r  _modelmodel_to_use_tokenizer_usedtokenizer_usedr$  s              r   r  r    s,     & ^^FH~(*$I
 	
 J ++Fl#w}}4#
 , '^^,<bAEEgN$$!3!9!9#q!A!!D 	+gmm  mm55L5IO01N L
 !mm%	 s   AD$B#Dz/utils/supported_openai_params)r  r  c                    #     [         R                  " U S9u  p  nS[         R                  " XS90$ ! [         a    [	        SSSR                  U 5      0S9ef = f7f)a  
Returns supported openai params for a given litellm model name 

e.g. `gpt-4` vs `gpt-3.5-turbo` 

Example curl: 
```
curl -X GET --location 'http://localhost:4000/utils/supported_openai_params?model=gpt-3.5-turbo-16k'         --header 'Authorization: Bearer sk-1234'
```
r  supported_openai_paramsr  r  r  r   zCould not map model={}r.  )rS  get_llm_providerget_supported_openai_paramsr"  r   r*   )r  r  r  s      r   r  r    sq     "

+2+C+C%+P(Aq%w'J'J(
 	

  
W.F.M.Me.T$U
 	

s   A-2 A&AAz/budget/newzbudget management
budget_objc                 J  #    [         c$  [        SS[        R                  R                  0S9e[         R
                  R                  R                  0 U R                  SS9EUR                  =(       d    [        UR                  =(       d    [        S.ES9I Sh  vN nU$  N7f)	M
Create a new budget object. Can apply this to teams, orgs, end-users, keys.
Nrn  r   r.  Tr  )r  r  r"  )r  r   r  db_not_connected_errorr   r  litellm_budgettablecreate
model_dumpr  r  r  r  r*  s      r   
new_budgetr    s      .EEKKL
 	

 #%%99@@
###6
+33O7O+33O7O
 A  H Os   BB#B!B#z/budget/updatec                 f  #    [         c$  [        SS[        R                  R                  0S9eU R
                  c  [        SSS0S9e[         R                  R                  R                  SU R
                  00 U R                  SS	9ES
UR                  =(       d    [        0ES9I Sh  vN nU$  N7f)r  Nrn  r   r.  r  zbudget_id is required	budget_idTr  r  r  r"  )r  r   r  r  r   r  r  r  r!  r  r  r  r  s      r   update_budgetr  1  s      .EEKKL
 	
 #W>U4VWW"%%99@@J001
###6
+33O7O
 A  H Os   B&B1(B/)B1z/budget/infor"  c                   #    [         c  [        SSS0S9e[        U R                  5      S:X  a  [        SSSU R                   30S9e[         R                  R
                  R                  S	S
U R                  00S9I Sh  vN nU$  N7f)z(
Get the budget id specific information
Nrn  r   zNo db connectedr.  r   r  z0Specify list of budget id's to query. Passed in=r  inr  )r  r   r  budgetsr  r  r/  )r"  r*  s     r   info_budgetr   R  s      W>O4PQQ
4<<AKDLL>Z
 	
 #%%99CCT4<<01 D  H O	s   A>B	 BB	z/budget/settingsr  c           
        #    [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[         R                  R                  R                  SU 0S9I Sh  vN nUb  UR                  SS	9nO0 nS
S0S
S0S
S0S
S0S
S0S
S0S.n/ n[        R                  R!                  5        H_  u  pgXd;   d  M  Sn[#        UXF   S
   UR$                  =(       d    SUR'                  US5      UUR(                  S9n	UR+                  U	5        Ma     U$  N7f)zr
Get list of configurable params + current value for a budget item + description of each field

Used on Admin UI.
Nr  r   r.  {}, your role={}r  r  Tr  r   IntegerStringFloat)r  r  r  r  ru  r  r   )
field_name
field_typefield_descriptionfield_valuestored_in_dbfield_default_value)r  r   r  r  r   r  r  r  r*   not_allowed_accessr  r  r  r  	BudgetNewmodel_fieldsr   
ConfigListr   rx  r   rD  )
r  r  db_budget_rowdb_budget_row_dictallowed_args
return_valr  
field_info_stored_in_db_response_objs
             r   budget_settingsr  n  s     .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 (**>>III& J  M  *5545H #))!4i(i("H-w'(L J"+"8"8">">"@
% M&%'3F;","8"8">B.22:tD*$.$6$6M m, #A Gs   B4E,6E*7AE,AE,z/budget/listc                   #    [         c$  [        SS[        R                  R                  0S9eU R
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  U R
                  5      0S9e[         R                  R                  R                  5       I Sh  vN nU$  N7f)z;List all the created budgets in proxy db. Used on Admin UI.Nr  r   r.  r  )r  r   r  r  r   r  r  r  r*   r  r  r  r/  rg  s     r   list_budgetr    s      .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%99CCEEHO Fs   B3B>5B<6B>z/budget/deletec                   #    [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[         R                  R                  R                  SU R                  0S9I Sh  vN nU$  N7f)	zDelete budgetNrn  r   r.  r  r  r  r  )r  r   r  r  r   r  r  r  r*   r  r  r  deleter  r  s      r   delete_budgetr    s      .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%99@@DGG$ A  H O	s   B>C	 CC	model_paramsc                   #    U R                   R                  SS9nU R                   R                  nUR                  5        H  u  pV[	        US9nXpR                   U'   M     U R
                  R                  U R                  U R                   R                  SS9U R
                  R                  SS9UR                  =(       d    [        UR                  =(       d    [        S.nU R
                  R                  b  U R
                  R                  US'   UR                  R                  R                  US9I S h  vN n	U	$  N7f)NTr  ro  )r  r  r  r  r  r  r  r  )r  r=  r  r   rP   r  r  r  rp  r  r  r  r.  r  )
r  r  r  _litellm_params_dict_orignal_litellm_model_namer%  r&  encrypted_valuerE  r  s
             r   _add_model_to_dbr"    s6     (66;;;N"."="="C"C$**,.Q7)8##A& - !++.."--&55EESWEX"--== > 
 (//K3K'//K3K	E !!-(3366j(++CCJJ K  N s   D9E;E<Ec                   #    U R                   R                  nU R                  nUc  gSU S[        R                  " 5        3nXPl        [        U UUS9I Sh  vN n[        [        UXE0S9U[        SS0S9S	9I Sh  vN   U$  N+ N7f)
z
If 'team_id' is provided,

- generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid})
- store the model in the db with the unique 'model_name'
- store a team model alias mapping {"model_name": "model_name_{team_id}_{uuid}"}
Nmodel_name_r  r  r  r  )r  model_aliasesr   r   )scope)r"  r  http_request)	r  r  r  r.  r/  r"  re   UpdateTeamRequestr   )r  r  r  _team_idoriginal_model_nameunique_model_namer  s          r   _add_team_model_to_dbr-    s      &&..H&11%hZq?/ ,!+# N .B
 ,FF#34   !s$   ABB$B=B>BBc                     SnUR                   (       a  UR                   [        R                  :X  a  gU c"  UR                   [        R                  :w  a  SnU$ UR                  U :w  a  SnU$ )NTF)r  r  r  r  )r  r  can_make_calls      r   check_if_team_id_matches_keyr0  @  sl     M##''+;+G+GG&&*:*F*FF!M  $$/!Mr   z
/model/newz=Allows adding new models to the model list in the config.yaml)r   r  r  c                 B  #     SS K n[        c  [        SSS0S9e[        U R                  R
                  US9(       d  [        SSS0S9eS n[        S	L a    U R                  nU R                  R
                  c  [        U U[        S
9I S h  vN nO[        U U[        S
9I S h  vN n[        R                  [        [        S9I S h  vN   [        R                  S/ 5      =(       d    / nSU;   a;  [        R                  R!                  U R                  UU R                  S9I S h  vN   U$ [        SSS0S9e N N Nv N! ["         a$  n[$        R&                  " SU 35         S nAU$ S nAff = f! ["         Ga  n[$        R(                  " SR+                  [-        U5      5      5        [$        R.                  " [0        R2                  " 5       5        [5        U[        5      (       aV  [7        [9        USS[-        U5       S35      [:        R<                  [9        USS5      [9        US[>        R@                  5      S9e[5        U[6        5      (       a  Ue[7        S[-        U5      -   [:        R<                  [9        USS5      [>        R@                  S9eS nAff = f7f)Nr   rn  r   VNo DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keysr.  )r  r  i  z,Team ID does not match the API key's team IDTr%  r  r{  slack)r  r  passed_model_infozException in add_new_model: CSet `'STORE_MODEL_IN_DB='True'` in your env to enable this feature.zBlitellm.proxy.proxy_server.add_new_model(): Exception occured - {}r/  Authentication Error()r   rm  r   r   Authentication Error, )!r  r  r   r0  r  r  r  r  r"  r-  r
  r3  r  r   rx  r  model_added_alertr"  r4   rX  r   r*   r  r`  r{  r|  r<  rr  r  ProxyErrorTypes
auth_errorr   r=  )r  r  r  r  _original_litellm_model_name	_alertingr*  s          r   add_new_modelr>  S  s    W
 u  , ++33GX
  !OP 
 $
S/;/F/F,**22:+;%1*;&3, &N ,A%1*;&3, &N
 #11"/CT 2    -00R@FB	i'+CCUU#/#:#:+G*6*A*A V       b 7&&

  S$..1MaS/QRR S  
""PWWA	

 	""9#7#7#9:a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	
!
s   JAE4 5E D;E $D=%#E D?	AE (E)E -E4 .J/E4 ;E =E ?E E 
E1E,&E4 +J,E11E4 4J?DJJJz/model/updatezEdit existing model paramsc                 >  #     SS K n[        c  [        SSS0S9e[        SL Ga  S n[	        U SS 5      nUc  [        S5      eUR                  nUc  [        S	5      e[        R                  R                  R                  S
U0S9I S h  vN nUc2  [        b   [        R                  US9b  [        SSS0S9e[        S5      e[        UR                  5      nU R                  c  [        S5      eU R                  R                  SS9nUR                  5        H  u  p[        U	S9n
XR                  U'   M     0 nU R                  R                  5       nUR                  5        H   u  pUb  XU'   M  X;   a  Xm   b  Xm   X'   M   M"     [         R"                  " U5      UR$                  =(       d    [&        S.n[        R                  R                  R)                  S
U0US9I S h  vN nU$ g  GNe N
! [
         Ga  n[*        R,                  " SR/                  [1        U5      5      5        [*        R2                  " [4        R6                  " 5       5        [9        U[        5      (       aV  [;        [	        USS[1        U5       S35      [<        R>                  [	        USS5      [	        US[@        RB                  5      S9e[9        U[:        5      (       a  Ue[;        S[1        U5      -   [<        R>                  [	        USS5      [@        RB                  S9eS nAff = f7f)Nr   rn  r   r2  r.  Tr  zmodel_info not providedzmodel_info.id not providedr  r  r  r  zOCan't edit model. Model in config. Store model in db via `/model/new`. to edit.zmodel not foundzlitellm_params not providedr  ro  )r  r  r  zAlitellm.proxy.proxy_server.update_model(): Exception occured - {}r/  r6  r7  r   rm  r   r   r8  )"r  r  r   r  r  r"  r  r  r.  find_uniquer   r4  r=  r  r   rP   r  r  r  r  r!  r4   r   r*   r  r`  r{  r|  r<  rr  r:  r;  r   r=  )r  r  r  	_model_idr  _existing_litellm_params_existing_litellm_params_dict_new_litellm_params_dictr%  r&  r!  merged_dictionary_mpr)  r   rE  r  r*  s                     r   update_modelrG    s4    e
  u  $I!,dCK" 9::#I  <==#&&>>JJ%y1 K   %
 (/*"1191EQ'$'#%v     122,0(77-) **2 =>>'3'B'B'G'G! (H ($
 1668"6Q"?1@++A. 9
 !#--224C!iik
$-2c*85:F-J-O%* * #'**->"?/77S;SE $1#3#3#K#K#R#R!9- $S $ N
 "!E %f  
""OVVA	

 	""9#7#7#9:a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	
!
sO   LBG2 G-	EG2 %G0&G2 +L-G2 0G2 2L=DLLLz/v2/model/infozv2 - returns all the models set on the config.yaml, shows 'user_access' = True if the user has access to the model. Provides more info about each model in /models, including config.yaml descriptions (except api key and api base))r   r  r  r  z!Specify the model name (optional))r   r`  c                   #    [         b  [        [         [        5      (       d  [        SSS[          S30S9e[        R                  5       I Sh  vN   [        R                  " [         5      n[        b
  U[        /-  n[        U R                  5      S:  a   Ub  U Vs/ s H  oDS   U:X  d  M  UPM     nnU GH  nUR                  S	0 5      nUS
L a9  Sn[        b  [        R                  U0 SS9=(       d    SnOSn[        U5      nXS'   [        US9n	U	0 :X  a9  UR                  S0 5      n
U
R                  SS5      n [         R"                  " US9n	U	0 :X  aa  UR                  S0 5      n
U
R                  SS5      nUR'                  S5      n[        U5      S:  a  US   n [         R"                  " XS   S9n	U	R)                  5        H  u  pX;  d  M  XU'   M     XeS	'   US   R+                  SS5        US   R+                  SS5        US   R+                  SS5        US   R+                  SS5        GM     [,        R.                  " SU5        SU0$  GN
s  snf ! [$         a    0 n	 GNf = f! [$         a    0 n	 Nf = f7f)zI
BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now.
Nrn  r   zNo model list passed, models=zI. You can add a model through the config.yaml or on the LiteLLM Admin UI.r.  r   r  r  Trm  async)r  r#  client_typellm_router_is_Noneopenai_clientr  r  r  r  r  r  r  vertex_credentialsaws_access_key_idaws_secret_access_keyall_models: %sr"  )r   r<  r_  r   r
  rT  r  r  rM  r  r  rx  r   _get_clientr  r  rS  r  r"  r  r   r%  r4   r`  )r  r  r`  r  r  r  r  _openai_clientrL  r  r  litellm_modelsplit_modelr%  r&  s                  r   model_info_v2rU  ,  s    & Z%E%E88H  IR  S
 	
 
!
!
###~.Jzl"
 ##$q(!+HA%/Ga
H ZZb1
D=#N%**#)"' +   	  "6/M&3?# 4&A ##ZZ(8"=N*..w=M(%,%;%;-%P" ##ZZ(8"=N*..w=M'--c2K;!# +B(%,%;%;'^&"
 ',,.DA" !1 /  *| 	 $$Y5 $$%94@ $$%8$? $$%<dCk n /<JO $ I>  (%'"(  (%'"(s|   AJIAJI/I5BJI AJ%I3<JBJJ I0,J/I00J3J?JJJz/model/streaming_metricsz1View time to first token for models in spend logs)r   r  r  r  _selected_model_grouprh  ri  c                   #    [         c2  [        [        R                  R                  SS[
        R                  S9eU=(       d    [        R                  " 5       [        SS9-
  nU=(       d    [        R                  " 5       nUR                  5       UR                  5       :H  nU(       a  SnOSn[        5       n[         R                  R                  XQX#5      I S h  vN n0 nUGb  U H  n	U	S   n
U	S	   nU	S
   nSnU(       a  U	S   nUnX;  a  0 X'   OU	S   nUnU	S
   nX;  a  0 X'   [        U5      nSU
;   a  [        U
5      nSU;   a  UR                  S5      S   nUR!                  U5        XU   U'   M      / n[#        [%        UR'                  5       S S95      nU HC  nS[        U5      0nUU   R'                  5        H  u  nnUUU'   M     UR)                  U5        ME     U[+        U5      S.$ g  GN7f)Ninternal_errorrm  r      r  a  
            SELECT
                api_base,
                model_group,
                model,
                "startTime",
                request_id,
                EXTRACT(epoch FROM ("completionStartTime" - "startTime")) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
                AND DATE("startTime") = DATE($2::timestamp)
            GROUP BY
                api_base,
                model_group,
                model,
                request_id
            ORDER BY
                time_to_first_token DESC;
        a  
            SELECT
                api_base,
                model_group,
                model,
                DATE_TRUNC('day', "startTime")::DATE AS day,
                AVG(EXTRACT(epoch FROM ("completionStartTime" - "startTime"))) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "startTime" BETWEEN $2::timestamp AND $3::timestamp
                AND "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
            GROUP BY
                api_base,
                model_group,
                model,
                day
            ORDER BY
                time_to_first_token DESC;
        r  r  time_to_first_tokenr   
request_idr  https:///openai/r   c                     U S   $ Nr   r   r0  s    r   <lambda>)model_streaming_metrics.<locals>.<lambda>	  	    dSTgr   rr  dater"  all_api_bases)r  rr  r  r  r   r   r   r   r  r   re  setr  	query_rawr  r  addr=  sortedr   rD  r_  )r  rV  rh  ri  is_same_day	sql_query_all_api_basesdb_response_daily_entries
model_data	_api_baser  r[  
unique_key_request_id_day_combined_model_namer*  r  entry	model_keylatencys                         r   model_streaming_metricsrz    s*     %<<BB!66	
 	
 ?X\\^iQ.??I'G.."glln4K	0	. UN%((22) K N%J":.I(F",-B"CJ(6(
424N/!%(!
&01F&G#-+-N(#&v; Y&'*9~$11';'A'A*'Ma'P$34?R:&';<1 &4	  " f^%9%9%;AUVW!CSX&E&4S&9&?&?&A"	7#*i  'BOOE"	 " !.1
 	
_ 	s   CG<G9DG<z/model/metricsz>View number of requests & avg latency per model on config.yamlz	gpt-4-32kr  customerc                 ^  #    [         c  [        SSS[        R                  S9eU=(       d    [        R
                  " 5       [        SS9-
  nU=(       d    [        R
                  " 5       nUb  US:X  a  SnUb  US:X  a  SnS	n[        5       n[         R                  R                  XaX#XE5      I S h  vN n0 n	Ub  U Hn  n
U
S
   nU
S   nU
S   nU
S   nX;  a  0 X'   [        U5      nSU;   a  [        U5      nSU;   a  UR                  S5      S   nUR                  U5        XU   U'   Mp      / n[        [        U	R                  5       S S95      n	U	 HC  nS[        U5      0nU	U   R                  5        H  u  nnUUU'   M     UR!                  U5        ME     U[#        U5      S.$ g  N7f)N Prisma Client is not initializedrX  rm  r      rZ  	undefinednulla  
        SELECT
            api_base,
            model_group,
            model,
            DATE_TRUNC('day', "startTime")::DATE AS day,
            AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / NULLIF("completion_tokens", 0)) AS avg_latency_per_token
        FROM
            "LiteLLM_SpendLogs"
        WHERE
            "startTime" >= $2::timestamp AND "startTime" <= $3::timestamp
            AND "model_group" = $1 AND "cache_hit" != 'True'
            AND (
                CASE
                    WHEN $4 != 'null' THEN "api_key" = $4
                    ELSE TRUE
                END
            )
            AND (
                CASE
                    WHEN $5 != 'null' THEN "end_user" = $5
                    ELSE TRUE
                END
            )
        GROUP BY
            api_base,
            model_group,
            model,
            day
        HAVING
            SUM(completion_tokens) > 0
        ORDER BY
            avg_latency_per_token DESC;
    r  r  r  avg_latency_per_tokenr]  r^  r   c                     U S   $ r`  r   ra  s    r   rb  model_metrics.<locals>.<lambda>|  rd  r   rr  re  rf  )r  rr  r   r   r   r  r   rh  r  ri  r  r  rj  r=  rk  r   rD  r_  )r  rV  rh  ri  r  r{  rm  rn  ro  rp  rq  rr  r  ru  _avg_latency_per_tokenrv  r*  r  rw  rx  ry  s                        r   model_metricsr    s      6!66	
 	
 @X\\^iR.@@I'G'[08{2!ID UN%((22)g K N%J":.I(Fe$D%/0G%H")')$#&v; Y&'*9~$11';'A'A*'Ma'P$349O4 !56 & 	  " f^%9%9%;AUVW!CSX&E&4S&9&?&?&A"	7#*i  'BOOE"	 " !.1
 	
K s   B.F-0F+1C;F-z/model/metrics/slow_responsesz/View number of hanging requests per model_groupc           	      N  #    [         c  [        SSS[        R                  S9eUb  US:X  a  SnUb  US:X  a  SnU=(       d    [        R
                  " 5       [        SS9-
  nU=(       d    [        R
                  " 5       n[        R                  R                  =(       d    S	n[        U5      nS
n[         R                  R                  UUUUUUU5      I S h  vN nUbA  U H;  n	U	R                  S5      =(       d    Sn
SU
;   a  U
R                  S5      S   n
XS'   M=     U$  NJ7f)Nr}  rX  rm  r   r  r  r~  rZ  r  a  
SELECT
    api_base,
    COUNT(*) AS total_count,
    SUM(CASE
        WHEN ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) THEN 1
        ELSE 0
    END) AS slow_count
FROM
    "LiteLLM_SpendLogs"
WHERE
    "model_group" = $2
    AND "cache_hit" != 'True'
    AND "startTime" >= $3::timestamp
    AND "startTime" <= $4::timestamp
    AND (
        CASE
            WHEN $5 != 'null' THEN "api_key" = $5
            ELSE TRUE
        END
    )
    AND (
        CASE
            WHEN $6 != 'null' THEN "end_user" = $6
            ELSE TRUE
        END
    )
GROUP BY
    api_base
ORDER BY
    slow_count DESC;
    r  r   r^  r   )r  rr  r   r   r   r  r   r  r  r|  r   r  ri  rx  r  )r  rV  rh  ri  r  r{  r|  rm  ro  rowrr  s              r   model_metrics_slow_responsesr    s2      6!66	
 	
 '[08{2@X\\^iR.@@I'G 	11DDK  /0IB &((22 K C
+1rIY&%OOJ7:	'
O	 
 !s   CD%D#AD%z/model/metrics/exceptionsz7View number of failed requests per model on config.yamlc                   #    [         c  [        SSS[        R                  S9eU=(       d    [        R
                  " 5       [        SS9-
  nU=(       d    [        R
                  " 5       nUb  US:X  a  Sn S	n[         R                  R                  XbX1U5      I S h  vN n/ n[        5       n	 Ub  U H  n
U
R                  S
S5      nU
R                  SS5      nU
R                  S0 5      nUUS.nUR                  U5        UR                  U5        UR                  5        H  u  nnU	R                  U5        M     M     U[        U	5      S.$  N7f)Nr}  rX  rm  r   r~  rZ  r  r  aD  
        WITH cte AS (
            SELECT 
                CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base,
                exception_type,
                COUNT(*) AS num_rate_limit_exceptions
            FROM "LiteLLM_ErrorLogs"
            WHERE 
                "startTime" >= $1::timestamp 
                AND "endTime" <= $2::timestamp 
                AND model_group = $3
            GROUP BY combined_model_api_base, exception_type
        )
        SELECT 
            combined_model_api_base,
            COUNT(*) AS total_exceptions,
            json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts
        FROM cte
        GROUP BY combined_model_api_base
        ORDER BY total_exceptions DESC
        LIMIT 200;
    combined_model_api_baser   total_exceptionsr   exception_counts)r  r  )r"  exception_types)r  rr  r   r   r   r  r   r  ri  rh  rx  r!  rD  r   rj  r_  )r  rV  rh  ri  r  r{  rm  ro  r*  r  rq  r  r  r  curr_rowr%  r&  s                    r   model_metrics_exceptionsr    sW      6!66	
 	
 @X\\^iR.@@I'G'[0I, &((22gg K HeO %JNN#<bAE)~~.@!D)~~.@"E$4H OO,-OOH%(..01##A& 1 & o1FGG?s   BEEB4Ez/model/infoz/v1/model/infolitellm_model_idc           
        #    [         bZ   [        [        [        R                  " [         S95      n[        S[        [         S9US9nUR                  5       n[        US9nSU0$ [        c  [        SSS	0S
9e[        c  [        SSS0S
9eUbC  [        R                  US9nUc  [        SSSU S30S
9eUR                  5       n[        US9nSU0$ / n[        [        5      n[        c  / nO([        R!                  5       n[        R#                  5       n[%        U UUS9n	['        U UUS9n
[)        U	U
U[         [*        R-                  SS5      S9n[/        U5      S:  aV  Un[        R1                  5       q
[        b7  [         Vs/ s H  oS   U;   d  M  UPM     nn[2        R4                  " U5      nO/ nU H  nUR-                  S0 5      n[7        US9nU0 :X  a9  UR-                  S0 5      nUR-                  SS5      n [        R                  " US9nU0 :X  ab  UR-                  S0 5      nUR-                  SS5      nUR9                  S5      n[/        U5      S:  a  US   n [        R                  " UUS   S9nUR;                  5        H  u  nnUU;  d  M  UUU'   M     X/S'   [        US9nM     [<        R>                  " SU5        SU0$ ! [
         a    0 n GNf = fs  snf ! [
         a    0 n Nf = f! [
         a    0 n Nf = f7f)a  
Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)

Parameters:
    litellm_model_id: Optional[str] = None (this is the value of `x-litellm-model-id` returned in response headers)

    - When litellm_model_id is passed, it will return the info for that specific model
    - When litellm_model_id is not passed, it will return the info for all models

Returns:
    Returns a dictionary containing information about each model.

Example Response:
```json
{
    "data": [
                {
                    "model_name": "fake-openai-endpoint",
                    "litellm_params": {
                        "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
                        "model": "openai/fake"
                    },
                    "model_info": {
                        "id": "112f74fab24a7a5245d2ced3536dd8f5f9192c57ee6e332af0f0512e08bed5af",
                        "db_model": false
                    }
                }
            ]
}

```
Nr  r>   r  )deployment_dictr"  rn  r   zLLM Model List not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr.  zLLM Router is not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr  r   zModel id = z not found on litellm proxyr  r  Fr  r   r  r  r  r  r  r  r  rP  ) rM  r
   r   rS  r  r"  r{   r|   r  rU   r   r   r   r4  r-   r_  r  r  rF   rG   rE   r   rx  r  r  r  r  r  r  r   r4   r`  )r  r  r  _deployment_info_deployment_info_dictdeployment_infor  r  r  r  r  all_models_strr^  r  _relevant_modelsr  r  r  rS  rT  r%  r&  s                         r   model_info_v1r  :  s    ` 	#D'*@*@z*RSJ &)  "
 !1 ; ; = E1!
 -..  r
 	
   q
 	
 #$33=M3N"{+;*<<WX  !0 : : < E1!
 -..J0;D0A%557(@@B+)/J
 "+)/K
 -).223JERN >Q$#224%) )a|_-K>    '78JJYY|R0
 4%@ #"YY'7<N*..w=M(%,%;%;-%P" #"YY'7<N*..w=M'--c2K;!# +B(%,%;%;'[^&"
 ',,.DAq
" !
1 / )l5eLI L /<Jw  	J	Z ,  (%'"(  (%'"(s   L	&K EL	K K A%L	>K%AL	"K7:L	5L	KL	KL	%K41L	3K44L	7LL	LL	z/model_group/infoc           
        #    [         c  [        SSS0S9e[        c  [        SSS0S9e[        [        5      n[        c  / nO([        R                  5       n[        R                  5       n[        U UUS9n[        U UUS9n[        UUU[        [        R                  SS5      S	9n/ nU H,  n[        R                  US
9nUc  M  UR                  U5        M.     SU0$ 7f)a  
Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base)

- /models returns all deployments. Proxy Admins can use this to list all deployments setup on the proxy
- /model_group/info returns all model groups. End users of proxy should use /model_group/info since those models will be used for /chat/completions, /embeddings, etc.


```shell
curl -X 'GET'     'http://localhost:4000/model_group/info'     -H 'accept: application/json'     -H 'x-api-key: sk-1234'
```

Example Response:
```json
    {
        "data": [
            {
            "model_group": "rerank-english-v3.0",
            "providers": [
                "cohere"
            ],
            "max_input_tokens": null,
            "max_output_tokens": null,
            "input_cost_per_token": 0.0,
            "output_cost_per_token": 0.0,
            "mode": null,
            "tpm": null,
            "rpm": null,
            "supports_parallel_function_calling": false,
            "supports_vision": false,
            "supports_function_calling": false,
            "supported_openai_params": [
                "stream",
                "temperature",
                "max_tokens",
                "logit_bias",
                "top_p",
                "frequency_penalty",
                "presence_penalty",
                "stop",
                "n",
                "extra_headers"
            ]
            },
            {
            "model_group": "gpt-3.5-turbo",
            "providers": [
                "openai"
            ],
            "max_input_tokens": 16385.0,
            "max_output_tokens": 4096.0,
            "input_cost_per_token": 1.5e-06,
            "output_cost_per_token": 2e-06,
            "mode": "chat",
            "tpm": null,
            "rpm": null,
            "supports_parallel_function_calling": false,
            "supports_vision": false,
            "supports_function_calling": true,
            "supported_openai_params": [
                "frequency_penalty",
                "logit_bias",
                "logprobs",
                "top_logprobs",
                "max_tokens",
                "max_completion_tokens",
                "n",
                "presence_penalty",
                "seed",
                "stop",
                "stream",
                "stream_options",
                "temperature",
                "top_p",
                "tools",
                "tool_choice",
                "function_call",
                "functions",
                "max_retries",
                "extra_headers",
                "parallel_tool_calls",
                "response_format"
            ]
            },
            {
            "model_group": "llava-hf",
            "providers": [
                "openai"
            ],
            "max_input_tokens": null,
            "max_output_tokens": null,
            "input_cost_per_token": 0.0,
            "output_cost_per_token": 0.0,
            "mode": null,
            "tpm": null,
            "rpm": null,
            "supports_parallel_function_calling": false,
            "supports_vision": true,
            "supports_function_calling": false,
            "supported_openai_params": [
                "frequency_penalty",
                "logit_bias",
                "logprobs",
                "top_logprobs",
                "max_tokens",
                "max_completion_tokens",
                "n",
                "presence_penalty",
                "seed",
                "stop",
                "stream",
                "stream_options",
                "temperature",
                "top_p",
                "tools",
                "tool_choice",
                "function_call",
                "functions",
                "max_retries",
                "extra_headers",
                "parallel_tool_calls",
                "response_format"
            ]
            }
        ]
        }
```
rn  r   zLLM Model List not loaded inr.  zLLM Router is not loaded inr  r  Fr  )r  r"  )r   r   r   r-   r_  r  r  rF   rG   rE   rM  r   rx  get_model_group_inforD  )	r  r  r  r  r  r  model_groupsr  _model_group_infos	            r   model_group_infor    s    X W.L$M
 	
 W.K$L
 	
 1<D0A%557(@@B+)/J
 "+)/K
 -).223JERN *,L&;;;N( 12	   L!!s   B>CCz/model/deletez;Allows deleting models in the model list in the config.yamlr  c                 &  #      [         c  [        SSS0S9e[        SL a   [         R                  R                  R                  SU R                  0S9I S h  vN nUc  [        SSS	U R                   S
30S9e[        b  [        R                  U R                  S9  SSUR                   S30$ [        SSS0S9e Nc! [         a  n[        U[        5      (       aV  [        [        USS[        U5       S35      [        R                   [        USS5      [        US["        R$                  5      S9e[        U[        5      (       a  Ue[        S[        U5      -   [        R                   [        USS5      ["        R$                  S9eS nAff = f7f)Nrn  r   r2  r.  Tr  r  r  zModel with id=z not found in dbr  r   zModel: z deleted successfullyr5  r/  r6  r7  r   rm  r   r   r8  )r  r   r  r  r.  r  r  r   r  r  r"  r<  rr  r  r  r:  r;  r   r=  )r  r  r*  s      r   delete_modelr    s    ?
	  u  $
 )++CCJJ!:==1 K  F ~# ##~jmm_DT%UV  %,,
,>(99NOPPb !.  
a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	

s=   FAB> B<AB> /F0B> >
FCF		FFz/model/settingszMReturns provider name, description, and required parameters for each providerc            
         #    / n [         R                   H.  nU R                  [        U[         R                  " US9S95        M0     U $ 7f)a  
Used by UI to generate 'model add' page
{
    field_name=field_name,
    field_type=allowed_args[field_name]["type"], # string/int
    field_description=field_info.description or "", # human-friendly description
    field_value=general_settings.get(field_name, None), # example value
}
)r  )r   fields)rS  provider_listrD  ProviderInfoget_provider_fields)returned_listrl  s     r   model_settingsr    sK     $ M))22xP	
 * r<  z/alerting/settingszFReturn the configurable alerting param, description, and current valuer{  c                   #     [         c$  [        SS[        R                  R                  0S9eU R
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  U R
                  5      0S9e[         R                  R                  R                  SS0S9I S h  vN nUbF  UR                  b9  [        UR                  5      nUR                  S0 5      nUR                  S	5      nO0 nS nS
S0S
S0S
S0S
S0S
S0S
S0S
S0S
S0S
S0S.	n[         R"                  nUR$                  R'                  5       n/ nSn	[(        R                  S	5      (       a+  [+        [(        S	   [,        5      (       a  S[(        S	   ;   a  Sn	[/        SUS   S
   SU	Ub  SOSS SS9n
UR1                  U
5        [2        R4                  R7                  5        Hr  u  pX;   d  M  S nX;   a  SnOSn[/        UX[   S
   UR8                  =(       d    SUR                  US 5      UUR:                  US:X  a  SOSS9n
UR1                  U
5        Mt     U$  GN7f)Nr  r   r.  r  r  r   r  r  r{  r   Booleanr  )	slack_alertingdaily_report_frequencyreport_check_intervalbudget_alert_ttloutage_alert_ttlregion_outage_alert_ttlminor_outage_alert_thresholdmajor_outage_alert_thresholdmax_outage_alert_list_sizeFr3  Tr  zhEnable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.)r  r  r  r	  r
  r  premium_fieldr   r  )r  r   r  r  r   r  r  r  r*   r  r  r  r  r  r=  rx  r  r  r  r  r   r<  r_  r  rD  r   r  r   r   r   )r  r  db_general_settings_dictalerting_args_dictalerting_valuesr  _slack_alerting_slack_alerting_args_dictr  is_slack_enabledr  r  r  r  s                 r   alerting_settingsr    s     .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #;#?#?QS#T*B*F*Fz*R "9-#)9"5"()!4#Y/#Y/$*I#6)/(;)/(;'-y&9
L &7%N%NO / = = H H JJJ''J$d- - &z22## 01&9 E$,8Te M m$"3"@"@"F"F"H
%,0M/ $ %&%'3F;","8"8">B599*dK*$.$6$6&*CCD
M m,) #I* Ks   B5I7I8D3I/A-Iz/queue/chat/completionsrW  )r  r  r  c                   #     0 n U R                  5       I S h  vN n[        U R                  5      U R                  [	        U R
                  5      [        R                  " U5      S.US'   [        R                  " SU5        [        R                  SS 5      =(       d    [        =(       d    U=(       d    US   US'   UR                  SS 5      c  UR                  b  UR                  US'   SU;  a  0 US'   UR                  US   S'   UR                  US   S	'   [	        U R
                  5      nUR                  S
S 5        XTS   S'   [!        USS 5      US   S'   UR                  US   S'   [!        USS 5      US   S'   [        U R                  5      US   S'   ["        (       a	  ["        US'   [$        (       a	  [$        US'   [&        (       a	  [&        US'   [(        (       a	  [(        US'   [*        c$  [-        SS[.        R0                  R2                  0S9e[*        R4                  " S)0 UD6I S h  vN nSU;   a  US   SL a  [7        [9        UUUS9SS9$ UR
                  R;                  S[        US   5      05        U$  GNi NS! [<         a  n[>        RA                  X7US 9I S h  vN    [C        U[,        5      (       aV  [E        [!        US!S"[        U5       S#35      [F        RH                  [!        US$S%5      [!        US&[J        RL                  5      S'9e[C        U[D        5      (       a  Ue[E        S([        U5      -   [F        RH                  [!        US$S%5      [J        RL                  S'9eS nAff = f7f)*N)r   methodr   r'  proxy_server_requestzreceiving data: %sr  r  r  metadatauser_api_keyuser_api_key_metadataauthorizationr   r  user_api_key_aliasuser_api_key_user_idr  user_api_key_team_idendpointrH  rI  rG  r  rn  r   r.  r  Tr  r  r   zx-litellm-prioritypriorityri  r/  r6  r7  r   rm  r   r   r8  r   )'r  r  r   r  r=  r   r  r4   r`  r   rx  rM  r  r  r  r%  r  rU  rV  r+  rT  r   r   r  r  r   schedule_acompletionr   r  r!  r"  r  rq  r<  rr  r:  r;  r   r=  )r   r  r  r  r"  _headersr*  r*  s           r   async_queue_requestr    s|     D\
\\^# w{{#nnGOO,IIdO	(
#$ 	""#7>  !3T:  G}	 	W 88FD!).?.G.G.S,44DLT!!D+<+D+DZ(4E4N4NZ01(T	
 '/Z#18{D2
Z-. 4E3L3LZ/03:y$4
Z/0 (+7;;'7Z$ "2D&:D"#?!0D=,D2C2Q2Q2W2W(X  $88@4@@ h4!7$$&7%!%
 /  	  '')=s4
CS?T(UVQ $r A   
66/TX 7 
 	
 	
 a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	

sd   M8J JHJ 2J3$J M8*J M8J J 
M5M0)J,*CM00M55M8z/fallback/login)r  r  c                   #    [         R                  " S[        U R                  5      5      n[         R                  " S5      nUR	                  S5      (       a  US-  nOUS-  nUb  SSKJn  U" [        SS	9$ SSKJn  U" [        SS	9$ 7f)
z
Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
PROXY_BASE_URL should be the your deployed proxy endpoint, e.g. PROXY_BASE_URL="https://litellm-production-7002.up.railway.app/"
Example:
r   UI_USERNAMEr  zsso/callbackz/sso/callbackr   )HTMLResponser  )r   r   )rQ  rR  r  base_urlendswithfastapi.responsesr  rJ   )r   redirect_urlui_usernamer  s       r   fallback_loginr    s~      99-s73C3C/DEL))M*KS!!&' 	3I3??2I3??s   BB	z/login)r  c                 
  #     SS K n[        c(  [        S[        R                  S[        R                  S9eU R                  5       I S h  vN n[        UR                  S5      5      n[        UR                  S5      5      n[        R                  " SS	5      n[        R                  " S
S 5      nUc  [        b  [        [        5      OS nUc(  [        S[        R                  S
[        R                  S9eS nS n[        b3  [        R                   R"                  R%                  SSU00S9I S h  vN n [&        R(                  " X55      (       Ga  [&        R(                  " XF5      (       Ga  [*        R,                  n[.        n	[.        n
[        R                  " SS 5      b  [        R0                  S   U	:X  d
  U	[.        :X  a  [        R                  " S[.        5      n
[3        [5        U
US9S9I S h  vN   [        R                  " S5      b.  [7        S1SS0[*        R,                  SS/ 0 0 SU
SS.	D6I S h  vN nO([        S[        R                  S[        R                  S9eUS   n[        R                  " SS5      nUR9                  S5      (       a  US-  nOUS-  nSS KnUR=                  U	US US [>        [@        R                  S!S"5      S#.[        S$S%9nUS&U	-   -  n[C        US'S(9nURE                  SUS)9  U$ UGb   [G        US*S+5      n	[G        US,[*        RH                  5      n[G        USS+5      n[G        USS+5      n[K        US-9n[&        R(                  " UU5      (       d  [&        R(                  " UU5      (       a  [        R                  " S5      b   [7        S1SS0USS/ 0 0 SU	SS.	D6I S h  vN nO([        S[        R                  S[        R                  S9eUS   n[        R                  " SS5      nUR9                  S5      (       a  US-  nOUS-  nSS KnUR=                  U	UUUS [>        [@        R                  S!S"5      S#.[        S$S%9nUS&U	-   -  n[C        US'S(9nURE                  SUS)9  U$ [        S.U 3[        R                  S/[        RL                  S9e[        S0[        R                  S/[        RL                  S9e! [         a    [        R                  " / SQ5         GNf = f GN GN GN GN GNP7f)2Nr   )pipinstallzpython-multipartMaster Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.r  r   usernamerb  r  r	  UI_PASSWORDzset Proxy master key to use UI. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.
user_emailequalsr  r  )r  r  r  r  r  r)  24hr   litellm-dashboard	r  r  key_max_budgetr  r  r  r  r  r  z_No Database connected. Set DATABASE_URL in .env. If set, use `--detailed_debug` to debug issue.rp  r   r   r  zui/r   username_passwordlitellm_key_header_nameAuthorizationr  r)  r  r  login_methodr   auth_header_nameHS256	algorithmz?userID=i/  )r   r   )r)  r   r  unknownr  ro  zAInvalid credentials used to access UI.
Not valid credentials for invalid_credentialszVInvalid credentials used to access UI.
Check 'UI_USERNAME', 'UI_PASSWORD' in .env filer   )'	multipartImportErrorr  runr  rr  r:  r;  r   r   rw  r  rx  rQ  rR  r  r  litellm_usertabler  secretscompare_digestr  r  r  rO  ra   UpdateUserRequestrd   r  jwtencoder   r   r   
set_cookier  INTERNAL_USER_VIEW_ONLYrv   HTTP_401_UNAUTHORIZED)r   r  rw  r  rb  r  ui_password	_user_rowr  r  key_user_idr*  r)  litellm_dashboard_uir  	jwt_tokenredirect_responser  	_passwordhash_passwords                       r   loginr    sc    
?  P ++66	
 	
 D488J'(H488J'(H))M73K))M40K)3)?c*oT V ++66	
 	
 I 	   '**<<GG(H!56 H 
 
	
 h449O9O: : %00	* /II&-9

+,700))$46NOK
 "##
 	
 	
 99^$03 " "2!=!= &&' ! *2
 H !y$//$::	  w!yy)92>((-- E)  F* JJ""& 3 ,$4$8$8-%
   
	 	
W 44,1ESVW$$	$B  			
 )Y	:{$4$L$L
	 Yi@
Iz9=	 #2!!(I66':P:P9;
 ;
 yy(4!7 "!&" &/$**+"$#%"$!"#*#6
"  %}(33(>>	  7#C#%99-=r#B #,,S11$-$$.$

&",!*$7$0(8(<(<1?)
 ! # I !J$88  0(c! ((WI(F$$ \]e\fg$//+11	  m ++'--	
 	
G  ?=>?  4
6	
Fs   T
S AT

S;C%T
0S>1CT
2T3AT
6T7F T
7T8DT
"S84T
7S88T
>T
T
T
T
z/onboarding/get_tokeninvite_linkc                 .  #    [         c(  [        S[        R                  S[        R
                  S9e[        c$  [        SS[        R                  R                  0S9e[        R                  R                  R                  SU 0S	9I Sh  vN nUc  [        S
SS0S9e[        R                  R!                  5       R#                  5       nUR$                  R#                  5       nX2:  a  [        S
SS0S9e[        R                  R!                  5       n[        R                  R                  R'                  SU 0UUSUR(                  S.S9I Sh  vN n[        R                  R*                  R                  SUR(                  0S	9I Sh  vN nUc  [        S
SS0S9eUR,                  n[/        S'SS0UR0                  SS/ 0 0 SUR(                  SS.	D6I Sh  vN nUS   n	[2        R4                  " SS5      n
U
R7                  S5      (       a  U
S-  n
OU
S-  n
SSKnUR;                  UR(                  U	UR,                  UR0                  S[<        [>        RA                  S S!5      S".[         S#S$9nU
S%RC                  X5      -  n
U
UUS&.$  GN GN? GN N7f)(z
- Get the invite link
- Validate it's still 'valid'
- Invalidate the link (prevents abuse)
- Get user from db
- Pass in user_email if set
Nr  r  r   rn  r   r.  r  r    %Invitation link does not exist in db.Invitation link has expired.T)accepted_atr  is_acceptedr  r  r  User does not exist in db.r  r)  r  r  r   r  r  rp  r   r   r  zui/onboardingz/ui/onboardingr  r  r  r  r  r  z?token={}&user_email={})	login_urlrp  r  r   )"r  rr  r:  r;  r   r   r  r   r  r  r   r  litellm_invitationlinkr@  rS  r0  get_utc_datetimere  
expires_atr!  r  r  r  rd   r  rQ  rR  r  r  r  r   r   rx  r*   )r  
invite_objutc_now_dateexpires_at_datecurrent_timer  user_objr  r*  r)  r  r  r  s                r   
onboardingr    s      P ++66	
 	
 .EEKKL
 	

 %''>>JJ[! K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	

 ==113L55<<[!'&$,,	
 =  	A #%%77CC*,,- D  H W.J$K
 	
 $$J+  "++''*

 H 7
C99%5r:$$S))/ 00

''"--!++/( 0 4 4)?!
	
 	  I  5<<YSS)  c&	sK   BJ	J

CJJ:J	J
AJJB3JJJJz/onboarding/claim_tokenc                 <  #    [         c$  [        SS[        R                  R                  0S9e[         R
                  R                  R                  SU R                  0S9I Sh  vN nUc  [        SSS0S9e[        R                  R                  5       R                  5       nUR                  R                  5       nX2:  a  [        SSS	0S9eUR                  S
L a  O[        SSS0S9eUR                  U R                  :w  a0  [        SSSR!                  U R                  UR                  5      0S9e[#        U R$                  S9n[         R
                  R&                  R)                  SUR                  0SU0S9I Sh  vN nUc  [        SSS0S9eU$  GN3 N7f)a  
Special route. Allows UI link share user to update their password.

- Get the invite link
- Validate it's still 'valid'
- Check if user within initial session (prevents abuse)
- Get user from db
- Update user password

This route can only update user password.
Nrn  r   r.  r  r  r  r  r  TzThe invitation link was never validated. Please file an issue, if this is not intended - https://github.com/BerriAI/litellm/issues.zwInvalid invitation link. The user id submitted does not match the user id this link is attached to. Got={}, Expected={}ro  r  rb  r  r  )r  r   r  r  r   r  r  r@  invitation_linkrS  r0  r	  re  r
  r  r  r*   rv   rb  r  r!  )r"  r  r  r  r  r  s         r   claim_onboarding_linkr  Q  s     .EEKKL
 	

 %''>>JJT))* K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	
 %  _
 	
 T\\)  S  Z  ZLL*"4"4
 	
 T]]3M"%%77>>*,,-Z4O ?  H W.J$K
 	
 OiVs%   A"F$F%DFFFFz
/get_imagec                     [         R                  R                  [         R                  R                  [        5      5      n [         R                  R                  U S5      n[         R                  " SU5      n[        R                  " SU5        UR                  S5      (       a  [        5       nUR                  U5      nUR                  S:X  aZ  [         R                  R                  U S5      n[        US5       nUR                  UR                  5        SSS5        [!        US	S
9$ [!        US	S
9$ [!        US	S
9$ ! , (       d  f       N,= f)zGet logo to show on admin UIzlogo.jpgUI_LOGO_PATHzReading logo from path: %s)zhttp://r]  r  zcached_logo.jpgwbNz
image/jpegr   )rQ  r   r  r  __file__r  rR  r4   r`  r   r=   rx  r   r  r   r   r   )current_dirdefault_logo	logo_pathrZ  r*  
cache_pathfs          r   	get_imager    s    
 ''//"''//(";<K77<<Z8L		.,7I;YG 344::i(3&k3DEJj$'1(() (  
|DD  FF I,?? ('s   <D>>
Ez/invitation/newzInvite Linksc           	        #    [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[        R                  R                  5       nU[        SS9-   n [         R                  R                  R!                  U R"                  UUUR"                  =(       d    [$        UUR"                  =(       d    [$        S.S	9I Sh  vN nU$  N! [&         a5  nS
[)        U5      ;   a  [        SSS0S9e[        SS[)        U5      0S9eSnAff = f7f)	  
Allow admin to create invite links, to onboard new users to Admin UI.

```
curl -X POST 'http://localhost:4000/invitation/new'         -H 'Content-Type: application/json'         -d '{
        "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable'
    }'
```
Nr  r   r.  r  rY  rZ  )r  r  r
  r  r  r  r  z*Foreign key constraint failed on the fieldzYUser id does not exist in 'LiteLLM_UserTable'. Fix this by creating user via `/user/new`.rn  )r  r   r  r  r   r  r  r  r*   r  rS  r0  r	  r   r  r  r  r  r  r"  r  )r"  r  r  r
  r*  r*  s         r   new_invitationr     sh    . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 ==113L	q 11JG&))@@GG<<*(/77S;S*/77S;S H 	
 	
 	
  G73q6Ax  Wc!f4EFFGs=   B2E)5A)D' D%D' $E)%D' '
E&10E!!E&&E)z/invitation/infoinvitation_idc                   #    [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[         R                  R                  R                  SU 0S9I Sh  vN nUc  [        SSS0S9eU$  N7f)	r  Nr  r   r.  r  r  r  -Invitation id does not exist in the database.)r  r   r  r  r   r  r  r  r*   r  r  r  r@  )r!  r  r*  s      r   invitation_infor$     s     . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<HH]# I  H LM
 	
 Os   B4C6C7Cz/invitation/updatec           	        #    [         c$  [        SS[        R                  R                  0S9eUR
                  c%  [        SSSR                  UR
                  5      0S9e[        R                  R                  5       n[         R                  R                  R                  SU R                  0U R                  U R                  UUUR
                  S.S	9I Sh  vN nUc  [        SSS
0S9eU$  N7f)u6  
Update when invitation is accepted

```
curl -X POST 'http://localhost:4000/invitation/update'         -H 'Content-Type: application/json'         -d '{
        "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
        "is_accepted": True // when invitation is accepted
    }'
```
Nr  r   r.  rn  z'Unable to identify user id. Received={}r  )r  r  r  r  r  r  r#  )r  r   r  r  r   r  r*   rS  r0  r	  r  r  r!  r!  r  )r"  r  r  r*  s       r   invitation_updater&  6   s    2 .EEKKL
 	

   (BII%--
 	
 ==113L"%%<<CCT''($$++'&+33
 D 	 	H LM
 	
 O!	s   CC1C/C1z/invitation/deletec                   #    [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[         R                  R                  R                  SU R                  0S9I Sh  vN nUc  [        SSS0S9eU$  N7f)	u   
Delete invitation link

```
curl -X POST 'http://localhost:4000/invitation/delete'         -H 'Content-Type: application/json'         -d '{
        "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
    }'
```
Nr  r   r.  r  r  r  r#  )r  r   r  r  r   r  r  r  r*   r  r  r  r  r!  r  s      r   invitation_deleter(  s   s     0 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<CCT''( D  H LM
 	
 Os   B>C CCz/config/updatezconfig.yamlconfig_infoc                 0  #     SSK n [        c  [        S5      e[        SLa  [	        SSS0S9eU R                  SS	9n[        R                  U5      nUR                  5        HF  u  p4US
:X  d  M  [        R                  R                  R                  SU0X4S.SU0S.S9I Sh  vN   MH     [        R                  5       I Sh  vN n[        R                  " SU5        U R                  b  UR!                  S0 5        U R                  R#                  SS	9nUS   nUR                  5        HP  u  p4US:X  aA  SU;  a  SS/0nO5[%        US   [&        5      (       a  SUS   ;  a  US   R)                  S5        XGU'   MR     XuS'   U R*                  bh  UR!                  S0 5        U R*                  nUR                  5        H  u  p4[-        US9n	XU'   M     US   n
UR                  5        H  u  p4X   X'   M     U R.                  b  UR!                  S0 5        U R.                  n0 UEUS   EUS'   SU;   af  SUS   ;   a]  [%        US   S   [&        5      (       aB  [%        US   [&        5      (       a*  US   S   US   -   n['        [1        U5      5      nUUS   S'   [        R3                  US9I Sh  vN   [        R5                  [        [6        S9I Sh  vN   SS0$  GN< GN N2 N! [         Ga  n[        R8                  " SR;                  [=        U5      5      5        [        R                  " [>        R@                  " 5       5        [%        U[        5      (       aV  [C        [E        USS[=        U5       S 35      [F        RH                  [E        US!S"5      [E        US#[J        RL                  5      S$9e[%        U[B        5      (       a  Ue[C        S%[=        U5      -   [F        RH                  [E        US!S"5      [J        RL                  S$9eSnAff = f7f)&zw
For Admin UI - allows admin to update config via UI

Currently supports modifying General Settings + LiteLLM settings
r   NzNo DB ConnectedTrn  r   r5  r.  r  r	  r  r  r  r  r  r!  r  zLoaded config: %sr   r~  r{  r3  rN  ro  r
  rn  )r   r  r   zConfig updated successfullyzBlitellm.proxy.proxy_server.update_config(): Exception occured - {}r/  r6  r7  r   rm  r   r   r8  )'r  r  r"  r  r   r  jsonify_objectr   r  r  upsertr
  rT  r4   r`  r   
setdefaultr=  r<  r_  rD  rN  rP   r
  rh  r(  r3  r  r   r*   r  r{  r|  rr  r  r:  r;  r   r=  )r)  r  updated_settingsr%  r&  r  updated_general_settings_existing_settings_updated_environment_variablesr!  _existing_env_variablesupdated_litellm_settingscombined_success_callbackr*  s                 r   update_configr7     s}    @
	  -..D(b  '+++>(778HI$**,DA%%#&&55<<'+12"E#0!"4 =    - $..00""#6? ''30"5'2'C'C'H'H! (I ($ "((:!;0668..!);;.87)-D*#$6z$BDII"*<Z*HH.z:AA'J()1% 9 *<%&,,85r:-8-N-N* 7<<>"6Q"?4Cq1 ? '--D&E#6<<>-K-N'* ?
 ''30"5'2'C'C$***+,*F%& #&>>&&1C*DD -./ABD  !9:L!MtTT123EF23EFG . 15S9R5S0T- 2 -.*
 &&&&999))';L * 
 	
 	
 899i 1F 	:	

  
""PWWA	

 	""9#7#7#9:a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	
!
sy   PA)K+ 14K+ %K!&K+ K$G.K+ 4K'5#K+ K)K+  P!K+ $K+ 'K+ )K+ +P6DPPPz/config/field/updatec           
        #     [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a$  [        SS[        R                  R                  0S9eU R                  [        R                  ;  a%  [        SSSR                  U R                  5      0S9e [        S0 U R                  U R                  0D6  [         R                   R"                  R%                  SS0S	9I Sh  vN nUb  UR&                  c  0 nO[)        UR&                  5      nU R                  X0R                  '   [         R                   R"                  R+                  SS0S[,        R.                  " U5      S
.S[,        R.                  " U5      0S.S9I Sh  vN nU$ ! [         a/    [        SSSR                  [        U R                  5      5      0S9ef = f N ND7f)z5
Update a specific field in litellm general settings
Nr  r   r.  Invalid field={} passed in.z)Invalid type of field value={} passed in.r  r   r  r+  r  r,  r  r   )r  r   r  r  r   r  r  r  r  r  ConfigGeneralSettingsr  r*   r	  r"  r   r  r  r  r  r=  r.  r  r  r"  r  r  r   r*  s        r   update_config_general_settingsr<  B!  s     .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 3@@@:AA$//RS
 	



D$2B2B CD !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M 3 ? ?@ )-(8(8__%"%%44;;/0%7

ScHde$djj1A&BC
 <  H OC  
DKK))*
 	

s=   B2G+5!F+ ,G+G'B!G+$G)%G++9G$$G+)G+z/config/field/infor  c                   #     [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a$  [        SS[        R                  R                  0S9eU [        R                  ;  a  [        SSSR                  U 5      0S9e[         R                  R                  R                  SS0S9I S h  vN nUb  UR                  c  [        SSSR                  U 5      0S9e[!        UR                  5      nX;   a  [#        XU    S	9$ [        SSSR                  U 5      0S9e Np7f)
Nr  r   r.  r9  r  r   r  zField name={} not in DB)r  r	  )r  r   r  r  r   r  r  r  r  r:  r  r*   r  r  r  r  r=  ConfigFieldInfo)r  r  r  r   s       r   get_config_general_settingsr?  !  sm    
 .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 .;;;:AA*MN
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M6==jIJ
 	

   3 ? ?@)"%J3O   !:!A!A*!MN %s   C
D?D=A1D?z/config/listconfig_typec                   #     [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9e[         R                  R                  R                  SS0S9I Sh  vN nUb#  UR                  b  [        UR                  5      nO0 nS	S
0S	S
0S	S
0S	S
0S	S0S.n/ n[        R                   R#                  5        GH  u  pgXd;   d  M  XF   S	   nUS:X  Ga=  US:X  a  [$        /n	O/ n	U	 GH#  n
U
R&                  R#                  5        VVs/ s H0  u  p[)        UUR*                  S[,        R/                  US5      SS9PM2     nnnSnU
R                   R#                  5        H<  u  nn[1        US5      (       a   UR2                  b  UR2                  X   l        US-  nM>     SnXc;   a  SnOU[,        ;   a  Sn[7        UXF   S	   UR2                  =(       d    S[,        R/                  US5      UUR8                  US9nUR;                  U5        GM&     GMX  SnSnXc;   a  SnOU[,        ;   a  Sn[7        UXF   S	   UR2                  =(       d    S[,        R/                  US5      UUR8                  US9nUR;                  U5        GM     U$  GN,s  snnf 7f)z
List the available fields + current values for a given type of setting (currently just 'general_settings'user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),)
Nr  r   r.  r  r  r   r  r   r  PydanticModel)r  r  max_request_size_mbmax_response_size_mbr  r  r   )r  r  r  r  r
  r   r   r   TF)r  r  r  r	  r
  r  nested_fields)r  r   r  r  r   r  r  r  r*   r  r  r  r  r  r=  r:  r  r   PassThroughGenericEndpoint__annotations__FieldDetailr   r   rx  hasattrr   r  r  r   rD  )r@  r  r  r  r  r  r  r  typed_dict_typepydantic_class_listpydantic_class	sub_fieldsub_field_typerE  r  sub_field_infor  r  s                     r   get_config_listrP  !  sP     
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #%  #))!4)/(; &	2!' 3#)?";L J"7"D"D"J"J"L
% +6v>O/1!99+E*F'*,'&9N :H9W9W9]9]9_	% :`5I $'0'5'>'>.00@0D0DYPT0U)- :` " 	% C (44::<!& $NMBB . : : F !/ : : *.@ q = %)M!=(,#'77(-$.#-#/#;F#C*4*@*@*FB$4$8$8T$J%2,6,>,>&3%M %%m4U ':Z !% $9$(M#33$)M *)+7?&0&<&<&B 0 4 4Z F!.(2(:(:"/! !!-0W #MZ AD	%s-   B5K,7K#8A'K,#AK,$7K&
E	K,&K,z/config/field/deletec                   #     [         c$  [        SS[        R                  R                  0S9eUR
                  [        R                  :w  a>  [        SSSR                  [        R                  R                  UR
                  5      0S9eU R                  [        R                  ;  a%  [        SSSR                  U R                  5      0S9e[         R                  R                  R                  SS0S	9I Sh  vN nUb  UR                   c%  [        SSS
R                  U R                  5      0S9e[#        UR                   5      nUR%                  U R                  S5        [         R                  R                  R'                  SS0S[(        R*                  " U5      S.S[(        R*                  " U5      0S.S9I Sh  vN nU$  N N7f)zt
Delete the db value of this field in litellm general settings. Resets it to it's initial default value on litellm.
Nr  r   r.  r  r9  r  r   r  zField name={} not in configr+  r  r,  r  )r  r   r  r  r   r  r  r  r*   r  r  r:  r  r  r  r  r  r=  r%  r.  r  r  r;  s        r   delete_config_general_settingsrR  X"  s    
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 3@@@:AA$//RS
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M:AA$//RS
 	

   3 ? ?@ $/"%%44;;/0%7

ScHde$djj1A&BC
 <  H O3"s%   C8G:G	;CGGGGz/get/config/callbacks)r  r  r  c                  X  #     SSK n [        5       n[        R                  5       I Sh  vN nUR	                  S0 5      nUR	                  S0 5      nUR	                  S0 5      nUR	                  S/ 5      n/ n U H  nUS:w  a  US:X  a  S	/n	O6US
:X  a  S/n	O,US:X  a  S/n	O"US:X  a  S/n	OUS:X  a  / SQn	OUS:X  a  / SQn	O/ n	0 n
U	 H+  nUR	                  US5      nUc  SX'   M  [        US9nXU'   M-     UR                  XS.5        M  US:X  d  M  / SQn0 nU H+  nUR	                  US5      nUc  SX'   M  [        US9nXU'   M-     UR                  XS.5        M     UR	                  S/ 5      n/ nSU;   a  S/n0 nU HD  nUR	                  US5      nUc  [        R                  " SS5      nUUU'   M6  [        US9nUUU'   MF     [        R                  R                  n[        R                  R                  5       n[        R                  R                  nUR                  SUUUS.5        / SQn0 nU H-  nUR	                  US5      nUc  SUU'   M  [        US9nUUU'   M/     UR                  SUS.5        [        c  0 nO[        R                  5       nSUUUUS.$  GN}! [          a  n["        R$                  " SR'                  [)        U5      5      5        [+        U[,        5      (       aV  [/        [1        US S![)        U5       S"35      [2        R4                  [1        US#S$5      [1        US%[6        R8                  5      S&9e[+        U[.        5      (       a  Ue[/        S'[)        U5      -   [2        R4                  [1        US#S$5      [6        R8                  S&9eSnAff = f7f)(zr
For Admin UI - allows admin to view config via UI
# return the callbacks and the env variables for the callback

r   Nr
  r   rN  rn  langfuse	openmeterOPENMETER_API_KEY
braintrustBRAINTRUST_API_KEY	traceloopTRACELOOP_API_KEYcustom_callback_apiGENERIC_LOGGER_ENDPOINTotel)OTEL_EXPORTEROTEL_ENDPOINTOTEL_HEADERS	langsmith)LANGSMITH_API_KEYLANGSMITH_PROJECTLANGSMITH_DEFAULT_RUN_NAMEro  )r   	variables)LANGFUSE_PUBLIC_KEYLANGFUSE_SECRET_KEYLANGFUSE_HOSTr{  r3  SLACK_WEBHOOK_URL)r   re  active_alertsalerts_to_webhook)	SMTP_HOST	SMTP_PORTSMTP_USERNAMESMTP_PASSWORDSMTP_SENDER_EMAILTEST_EMAIL_ADDRESSEMAIL_LOGO_URLEMAIL_SUPPORT_CONTACTemailr  )r   rf  alertsr	  available_callbacksz?litellm.proxy.proxy_server.get_config(): Exception occured - {}r/  r6  r7  r   rm  r   r   r8  )r  AllCallbacksr
  rT  rx  rO   rD  rQ  rR  r  r  r}  _all_possible_alert_typesr~  r   get_settingsr"  r4   rX  r*   r  r<  r   rr  r  r:  r;  r   r=  )r  all_available_callbacksr  _litellm_settingsr  rN  _success_callbacks_data_to_return	_callbackenv_varsenv_vars_dict_varenv_variabler  _langfuse_vars_langfuse_env_varsr=  alerting_data_slack_vars_slack_env_varsr  _decrypted_value_alerting_types_all_alert_types_alerts_to_webhook_email_vars_email_env_varsr  r*  s                                r   rT  rT  "  s/    o
"..(3355'OO,>C'OO,>C +0G L /223ErJ	 ,IJ&++ H ,., H +- 34H"77 9:H&(QH+- H  "H "$D#8#<#<T4#HL#+.2+ +?\*R.=d+ %  &&	'VWj("
 &("*D#8#<#<T4#HL#+37*0 +?\*R3B40 +  &&&Hi ,r &))*b9	i#K !O#488tD'YY':DAF,2OD) (<,'O$,<OD) $ 0GGSSO!99SSU  "99NN    #!0%4);		
 D044T4@L#(,% $8l#K (8%   	,	
 !)668  (#/#:
 	
_ 6l  
&&MTTA	

 a'' 8/DSVHA-NO$//a&1Qv/J/JK	  >**G,s1v5 ++!Wf-,,	
 	

sA   N*%J) J&C&J) FJ) %N*&J) )
N'3C/N""N''N*z/config/yamlc                    #    SS0$ 7f)a  
This is a mock endpoint, to show what you can set in config.yaml details in the Swagger UI.

Parameters:

The config.yaml object has the following attributes:
- **model_list**: *Optional[List[ModelParams]]* - A list of supported models on the server, along with model-specific configurations. ModelParams includes "model_name" (name of the model), "litellm_params" (litellm-specific parameters for the model), and "model_info" (additional info about the model such as id, mode, cost per token, etc).

- **litellm_settings**: *Optional[dict]*: Settings for the litellm module. You can specify multiple properties like "drop_params", "set_verbose", "api_base", "cache".

- **general_settings**: *Optional[ConfigGeneralSettings]*: General settings for the server like "completion_model" (default model for chat completion calls), "use_azure_key_vault" (option to load keys from azure key vault), "master_key" (key required for all calls to proxy), and others.

Please, refer to each class's description for a better understanding of the specific attributes within them.

Note: This is a mock endpoint primarily meant for demonstration purposes, and does not actually provide or change any configurations.
helloworldr   )r)  s    r   config_yaml_endpointr  _#  s     . Ws   z/get/litellm_model_cost_map)r  r  c                     #     [         R                  n U $ ! [         a  n[        SS[	        U5       S3S9eS nAff = f7f)Nrn  zInternal Server Error (r7  r.  )rS  
model_costr"  r   r  )_model_cost_mapr*  s     r   get_litellm_model_cost_mapr  y#  sJ     
!,, 
,SVHA6
 	

s   ? ?
<7<?r  )r  c                    #    g7f)NzLiteLLM: RUNNINGr   )r   s    r   homer  #  s     s   z/routesc            	        #    / n [         R                   Hl  n[        USS5      nUc  M  [        USS5      [        USS5      [        USS5      [        USS5      (       a  UR                  OSS.nU R	                  U5        Mn     SU 0$ 7f)z<
Get a list of available routes in the FastAPI application.
r  Nr   methodsr   )r   r  r   r  r   )r   r   r  r   rD  )r   r   endpoint_route
route_infos       r   
get_routesr  #  s     
 F 
D9%vt4"5)T:vt4 uj$77 #++	J MM*%  fs   $B	AB	shutdownc                  R  #    [         R                  " S5        [        (       a2  [         R                  " S5        [        R	                  5       I S h  vN   [
        R                  b&  [
        R                  R	                  5       I S h  vN   [        R                  5       I S h  vN   [        b  [        R                  5       I S h  vN   S[
        R                  ;   a$   SSKJn   U b  U R                  R                  5         [!        5         g  N N Nn NM! [         a     Nf = f7f)Nz"Shutting down LiteLLM Proxy ServerzDisconnecting from PrismarT  r   langFuseLogger)r4   rz  r  r`  
disconnectrS  rC  r  r{  r  rn  r  r  Langfuseflushr"  cleanup_router_config_variablesr  s    r   shutdown_eventr  #  s      BC}""#>?&&(((}} mm&&(((



#$$&&& W---	4)''--/ $%/ 	) 	) 	'  		sl   AD'D6D'D	D'$D%"D'DD'!#D D'D'D'D'
D$!D'#D$$D'c                  ,    S q S qS qS qS qS qS qS qS qS q	g r  )
r  r   otel_loggingr  user_custom_auth_pathr  r  r  r  r  r   r   r   r  r  #  s<     J L #O#'  Mr   )NNr5  r  )NNNNNNNN)NNNNFFNNr-  NFTTNFFN(  r)  r1  r  r  ry  rQ  r&   r  r  sysr0  r{  r.  warningsr   r   typingr   r   r   r   r	   r
   r   r   r   opentelemetry.tracer   _Spanr   filterwarningsUserWarningr    r_  rG  r   insertr  rO  backofffastapir]  r  apscheduler.schedulers.asyncior!   r  r*  r(   r,   collectionsr-   rS  r.   r/   r0   r1   r2   rP  r4   r5   r  r6   r7   litellm.exceptionsr8   1litellm.integrations.SlackAlerting.slack_alertingr9   'litellm.litellm_core_utils.core_helpersr:   r;   &litellm.llms.custom_httpx.http_handlerr<   r=   litellm.proxy._types5litellm.proxy.analytics_endpoints.analytics_endpointsr@   analytics_routerlitellm.proxy.auth.auth_checksrA   litellm.proxy.auth.auth_utilsrB   litellm.proxy.auth.handle_jwtrC   "litellm.proxy.auth.litellm_licenserD   litellm.proxy.auth.model_checksrE   rF   rG   $litellm.proxy.auth.user_api_key_authrH   rI   litellm.proxy.caching_routescaching_router)litellm.proxy.common_utils.admin_ui_utilsrJ   )litellm.proxy.common_utils.callback_utilsrK   rL   rM   &litellm.proxy.common_utils.debug_utilsrN   debugging_endpoints_router0litellm.proxy.common_utils.encrypt_decrypt_utilsrO   rP   -litellm.proxy.common_utils.http_parsing_utilsrQ   rR   ,litellm.proxy.common_utils.load_config_utilsrS   rT   0litellm.proxy.common_utils.openai_endpoint_utilsrU   &litellm.proxy.common_utils.proxy_staterV   (litellm.proxy.common_utils.swagger_utilsrW   -litellm.proxy.fine_tuning_endpoints.endpointsfine_tuning_routerrX   (litellm.proxy.guardrails.init_guardrailsrY   rZ   litellm.proxy.health_checkr[   0litellm.proxy.health_endpoints._health_endpointshealth_router,litellm.proxy.hooks.model_max_budget_limiterr\   .litellm.proxy.hooks.prompt_injection_detectionr]   )litellm.proxy.hooks.proxy_failure_handlerr^   -litellm.proxy.hooks.proxy_track_cost_callbackr_   $litellm.proxy.litellm_pre_call_utilsr`   5litellm.proxy.management_endpoints.customer_endpointscustomer_router:litellm.proxy.management_endpoints.internal_user_endpointsinternal_user_routerra   ;litellm.proxy.management_endpoints.key_management_endpointsrb   rc   rd   key_management_router9litellm.proxy.management_endpoints.organization_endpointsorganization_router:litellm.proxy.management_endpoints.team_callback_endpointsteam_callback_router1litellm.proxy.management_endpoints.team_endpointsteam_routerre   )litellm.proxy.management_endpoints.ui_ssoui_sso_router+litellm.proxy.management_helpers.audit_logsrf   4litellm.proxy.openai_files_endpoints.files_endpointsrg   openai_files_routerrh   >litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsllm_passthrough_router;litellm.proxy.pass_through_endpoints.pass_through_endpointsri   pass_through_router(litellm.proxy.rerank_endpoints.endpointsrerank_routerlitellm.proxy.route_llm_requestrj   7litellm.proxy.spend_tracking.spend_management_endpointsspend_management_router1litellm.proxy.spend_tracking.spend_tracking_utilsrk   7litellm.proxy.ui_crud_endpoints.proxy_setting_endpointsui_crud_endpoints_routerlitellm.proxy.utilsrl   rm   rn   ro   rp   rq   rr   rs   rt   ru   rv   rw   rx   4litellm.proxy.vertex_ai_endpoints.langfuse_endpointslangfuse_router2litellm.proxy.vertex_ai_endpoints.vertex_endpointsvertex_routerry   litellm.routerrz   r{   r|   r}   litellm.schedulerr~   r   r   *litellm.secret_managers.aws_secret_managerr   "litellm.secret_managers.google_kmsr   litellm.secret_managers.mainr   r   r   r   )litellm.types.integrations.slack_alertingr   litellm.types.llms.anthropicr   r   r   r   litellm.types.llms.openair   litellm.types.routerr   r  r   r   litellm.types.utilsr   r  r   litellm._versionr   r"  suppress_debug_infor  r   r   r   r   r   r   r   r   r   r   r   r   fastapi.encodersr   fastapi.middleware.corsr   fastapi.openapi.utilsr   r  r   r   r   r   r   fastapi.routingr   fastapi.securityr   fastapi.security.api_keyr   fastapi.staticfilesr   litellm.proxy.enterpriseproxy
enterpriserR  r   r  r  r   r7  r   r  proxy_stater   r   r   r   r  ui_link
ui_messagecustom_swagger_message_title_descriptionr   r   openapienumEnumr   exception_handlerrr  r   originsr  r  r  r  ui_pathr  listdirr   r  r  folder_namefolder_pathmakedirssrcdstrenamer)   rO  
middlewarer   add_middlewarer   rT  rM  rN  r+  rV  rU  rX  user_configrS  r   local_loggingrW  r   r   r   r=  r   log_filer  r  r  r  r   r   r  model_max_budget_limiterrf  rD  r  r  r  r  r  r]  r  r  r  r  rv  r  r
  Literalr  r  r  r  r  r  r  r  r  r  async_resultcelery_app_conn	celery_fnr  UserAPIKeyAuthr+  r7  rA  rG  r\  ra  rd  SpendLogsPayloadrl  r  r  r  r  r  r
  r;  r`  re  rz  r  r  r  r  r  r  r  on_eventr  rx  r  poston_exceptionexporJ  r  rU  rf  rq  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  TokenCountRequestr  r  r  r  r  BudgetRequestr   r  r  BudgetDeleteRequestr  r"  r-  r0  r>  rG  QueryrU  rz  r  r  r  r  r  ModelInfoDeleter  r  r  r  r  r  r  InvitationClaimr  r  InvitationModelInvitationNewr   r$  InvitationUpdater&  InvitationDeleter(  
ConfigYAMLr7  ConfigFieldUpdater<  r>  r?  r  rP  ConfigFieldDeleterR  rT  r  r  r  r  r  r  include_routerr   r   r   <module>rB     s"(   
    	 	    
     (
 
 
 1DD# #    	K 8
 $  rwww	U?	 8 $   I 9 3 K Q " : E 4 ; 
 B ? 
 H W > D V P < T M T L S 
 T I M S O R M 9 R    W X  E D C >  H  A = H 6 ;( #       . 2 -    & % 1 1 1 1 + +11 99/4 #..0d 09;1: '#  l*2/0'0*-.U*V'?Ayy7@ -x}  187;-u;0551- d
#.wi7TU  T T
 N  4@<<	/] |	 II
VWmVnnrs}r~
 AAW@XX\]g\h	i  _
& 99_g&&0\\ CKTYY  ~&G .  '& 
% 	''//"''//(";<Kggll;?GIIe[7>TIJJJw'W%%(l*B''**84Q7K'',,w<KKKd3 '',,w1C'',,{L9CIIc3 ( 2FGWFXX[\	
 99%&.+;BJJ'(			,' 	, 
 	,   %%   

  '+ x} +#
HV #!% % $  4  
HSM  (,x% ,0DDJJ  B!      1 2 8J'    # 	   t 2 , */' /$' !$' !   lOS )K L S 4 '+ x} + )  	/3 (+, 3 ""#"!"&158<#%2%2 c]2 sm	2
 }2 sm2 c]2 3-2 E%*-.2 (0~2 4.2 
2jw :4 "
4 
>SH %)4))* SM& s
lwC=wc]w #w c]	w
 E?w tnwt
7>x xv% }3 

	#sn33,!/3,?C3,l6,!/6,?C6,r1,!/1,?C1,h!/?C *, $ Y
&k k\ v
 v
t (9 :;CUBV   W%678@R?S   )00A(B.%..b +,-
	  
 +,-
	  
 ,+,-
	  
 7+,-
	]$9:NoN	   	LL16  (/0A(B	D
D
D
 C=D
 &	D
<D
N W->%?$@   '*;"<!=]O   '+,-
  
 2+,-
    (/0A(B	D
D
D
 C=D
 &	D
 D
N +,-!
	   +,-!
	   &+,-!
	   1+,-!
	    (/0A(B	uuu C=u &	u0up +,-!
	   +,-!
	   )00A(Bggg &ggT +,-
  
 +,-
   )00A(Baaa &aaH +,-
  
 +,-
   Cy(/0A(B	III I &	IIb < ;  ~ 9:III IP +,-
  
 +,-
   )00A(BSSS &SSl +,-
  
 +,-
   )00A(BTTT &TTn (+,-
  
 %+,-
   )00A(B	SSS S &	SSl +,-
  
 +,-
   )00A(BSSS &SSl +,-
  
 +,-
   )00A(B	RRR R &	RRj &+,-
  
 #+,-
   )00A(B	UUU U &	UUp &+,-
  
 #+,-
   )00A(B	QQQ Q &	QQh "+,-
  
 +,-
   )00A(B	___ _ &	__P +,-
  
 +,-
  
 +,-
   #(/0A(B	AAA smA &	AAH ,+,-
  
 !+,-
  
 +,-
   )00A(B"$2Sccc &c sm	c
 ccL +,-
  
 +,-
  
 +,-
   #(/0A(BLLsmL C=L C=	L
 &LLl +,-!
	   +,-!
	   )00A(Bjjj &jj` 
+	,+,-$   )00A(B	Q
,Q
Q
 Q
 &	Q
Q
| 
+,-%	  *!2 **Z $
+,-  

 


8 
	+,-   )00A(B%
4 
	+,-   )00A(B%
8 
	+,-  
M 
. 
	+,-   )00A(B>>%>
>B 
	+,-   )00A(B%
4 
	+,-   )00A(B
%
D%  :&&%&  &Rc]/=	& O
	+,-	   )00A(B\
\
%\
\
@ ,
	+,-	   )00A(Bj
"j
%j
j
Z  w
	+,-   )00A(B"=== "\ %\ C=\ 
 D>\ \ ~ C
	+,-   )00A(B+/$("&	z
%z
#C=z
 !z
 h	z
z
z P
	+,-   )00A(B+6$("&!"i
%i
#C=i
 !i
 h	i

 c]i
 smi
i
X #A
	+,-   )00A(B+6$("&!"O%O#C=O !O h	O
 c]O smOOd I
	+,-   )00A(B+/$("&!"NH%NH#C=NH !NH h	NH
 c]NH smNHNHb 
	+,-  
 
	+,-   )00A(B&*e %e sme e P 
	+,-   )00A(Bq"%q"
q"j M
	+,-	  A
? A
A
H _
	+,-  4 X
+,-   )00A(Bd%ddP 
	+,-	    (/0A(B	k
k
k
 C=k
 &	k
k
\ 	.!1UK@' @ L@4   L
 L
L
^ 	 E:l# l ;l^ 
#u=Ho H >HV /@ 0@D 
	+,-"   >EEV=W8G
8G,:8G8Gv 
	+,-"   =DDU<V**+9**Z 
	+,-"   )00A(B3
3%33l 
	+,-"   )00A(B+
+%++^ 
+,-	  G
Z G
G
V 
+,-	   )00A(BE
E%EEP 
+,-"   )00A(B44%44n 
+,-	   )00A(BA+,A%A 
*AAH 
+,-	   )00A(B?
?%??D 
+,-	  v
v
r 
+,-	  J ( !+,-  



 Cw'89:;  < IW->%?$@A B` & &>    6    = !   % &   = !   ) *   ? #   & '   = !   ( )   ' (   ;    = !   & '   ? #   * +   > "   # $   - .   + ,   & '   ' (M^  U
+A3.RS
TTU|  GL   	\  		s   Aq) Ar Ar ?BAr4 "Ar4 +DAr4 q)Ar q/Aq;q;Ar rArrArrAr1rAr"r"Ar-r(Ar1r,Ar-r-Ar1r4Ar>r=Ar>