ȅiC %SSKrSSKrSSKrSSKrSSKrSSKJr SSKJr SSK J r SSK J r J r SSKJrJr SSKJrJr SSKJr SS KJrJr SS KJr SS KJr SS KJrJrJrJ r \RB"\"5r#\RH\%S '\ "SS9"SS55r&S\'\(S\\&4Sjr)S\'\(S\\&4Sjr*S,S\(S\(S\+\'\(\\&44Sjjr,S\(4Sjr-SSSSSS S!S"S#.r."S$S%\ 5r/"S&S'\ 5r0"S(S)5r1S*\2SS4S+jr3g)-N)Iterable)ThreadPoolExecutor) dataclass)BaseHTTPRequestHandlerThreadingHTTPServer)parse_qsurlparse) DictLoader Environment)tabulate)get_world_sizetcpstore_client)build_db) JobConfig) CollectiveGroup MembershipNCCLCallloggerT)slotsc6\rSrSr%\\S'\\S'SrSrSr g)Response status_codetextcnURS:wa%[SURSUR35eg)NzHTTP z: )r RuntimeErrorrselfs [/home/james-whalen/.local/lib/python3.13/site-packages/torch/distributed/debug/_frontend.pyraise_for_statusResponse.raise_for_status"s8   s "t'7'7&8499+FG G #cB[R"UR5$N)jsonloadsrrs r!r' Response.json&szz$))$$r$N) __name__ __module__ __qualname____firstlineno__int__annotations__strr"r'__static_attributes__r*r$r!rrs IH%r$rurlsreturnc^SSKmSnS[S[4U4Sjjn[US9nUR X 5nSSS5 U$!,(df  W$=f)Nrurlr4cf>TRU5n[URUR5$r&)postrrr)r7resprequestss r!getfetch_thread_pool..get0s(}}S!(($))44r$) max_workers)r;r1rrmap)r3r>r<executorrespsr;s @r!fetch_thread_poolrB*sTK555  4 S' 5 L 5 4 Ls A Ac^^SSKmSTRS[S[4SjmS[[S[ [4UU4Sjjn[ R"U"U55$)Nrsessionr7r4c# URU5IShvN nUR5IShvN n[URU5sSSS5IShvN $NCN-N !,IShvN (df  g=f7fr&)r9rrstatus)rDr7r:rs r!fetchfetch_aiohttp..fetch?sN<<$$$DDKK.%$$$%$$$sTA<AA<A"AA" A<A A<A" A<"A9(A+ ) A95A<r3c ># TR5IShvN n[R"UVs/sH nT"X5PM sn6IShvN sSSS5IShvN $NGs snfNN!,IShvN (df  g=f7fr&) ClientSessionasynciogather)r3rDr7aiohttprGs r!rLfetch_aiohttp..gatherDsY((**g )N#%*=)NOO+**)NO+***saBA BA+A" A+ A' A+ BA)B"A+)B+B1A4 2 B>B)rMrJr1rlistrrKrun)r3rLrMrGs @@r! fetch_aiohttprQ:sb/W22/// P49P();PP ;;vd| $$r$endpointargsc>[5n[[55Vs/sHnSU3PM nnURU5nUVs/sHofR 5SUSU3PM nn[ U5nXW4$s snfs snf![ a [U5nXW4$f=f)Nrankz /handler/?)rranger multi_getdecoderQ ImportErrorrB)rRrSstorerkeysaddrsaddrrAs r! fetch_allr`Ks  E %n&6 7 8 71d1#J 7D 8 OOD !EFK Led iz4& 9eE L)e$ < 9 L )!%( <)sA8!A=* BBBblobcX[R"U5n[R"USS9$)N)indent)r'r(dumps)raparseds r! format_jsonrgYs! ZZ F ::fQ ''r$a {% block title %}{% endblock %} - PyTorch Distributed
{% block header %}{% endblock %} {% block content %}{% endblock %}
z {% extends "base.html" %} {% block header %}

{% block title %}Index{% endblock %}

{% endblock %} {% block content %} Hi {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{title}}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %} {% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ format_json(resp.text) }}
{% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}torch.profiler{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} {% endif %} {% endfor %} {% endblock %} a {% extends "base.html" %} {% block header %}

{% block title %}TCPStore Keys{% endblock %}

{% endblock %} {% block content %} 
    {% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
    {% endfor %}
{% endblock %} ag {% extends "base.html" %} {% block header %}

{% block title %}{{ title }}{% endblock %}

{% endblock %} {% block content %}

Groups

{{ groups | safe }}

Memberships

{{ memberships | safe }}

Collectives

{{ collectives | safe }}

NCCL Calls

{{ ncclcalls | safe }} {% endblock %} aQ {% extends "base.html" %} {% block header %}

{% block title %}py-spy Stack Traces{% endblock %}

{% endblock %} {% block content %}
{% for i, (addr, resp) in enumerate(zip(addrs, resps)) %}

Rank {{ i }}: {{ addr }}

{% if resp.status_code != 200 %}

Failed to fetch: status={{ resp.status_code }}

{{ resp.text }}
{% else %} 
{{ resp.text }}
{% endif %} {% endfor %} {% endblock %} )z base.html index.html raw_resp.htmljson_resp.html profile.html tcpstore.html fr_trace.htmlpyspy_dump.htmlcZ\rSrSr%\R r\R\S'Sr \ \S'Sr g)_IPv6HTTPServeriKaddress_familyirequest_queue_sizer*N) r+r,r-r.socketAF_INET6rq AddressFamilyr0rrr/r2r*r$r!rprpKs!+1??NF((:""r$rpc \rSrSr%S\S'SrSrS\4SjrS\ \\ \44Sjr S\4S jr S \4S \S \ S \S\ 4SjjrSrg )HTTPRequestHandleriPFrontendServerfrontendcP[RSURSX-5 g)Nz%s %sr)rinfoclient_address)r formatrSs r! log_messageHTTPRequestHandler.log_messageSs$     " M r$c:URRU5 gr&)ry_handle_requestrs r!do_GETHTTPRequestHandler.do_GETZs %%d+r$r4c@[UR5R$r&)r pathrs r!get_pathHTTPRequestHandler.get_path]s "'''r$c4[UR55$r&)r get_raw_queryrs r! get_queryHTTPRequestHandler.get_query`s**,--r$c@[UR5R$r&)r rqueryrs r!r HTTPRequestHandler.get_raw_querycs "(((r$NnamedefaulttypecJUR5nX;aU$U"XAS5$)Nr)r)r rrrrs r! get_query_arg HTTPRequestHandler.get_query_argfs,   NEKN##r$r*)r+r,r-r.r0r~rr1rdictrOrrobjectrrr2r*r$r!rwrwPst ,(#(.4T#Y/.)s),0c$$"($7;$ $$r$rwc4\rSrSrS\4SjrSSjrSSjrS\SS4S jr S \ S \ S\ 4S jr S\S\ 4S jrS\S\ 4SjrS\S\ 4SjrS\\ S\\S\ 4SjrS\S\ 4SjrS\S\ 4SjrS\S\ 4SjrS\S\ 4SjrS\S\ 4SjrS\S\ 4SjrS\S\ 4SjrSrg)rxioportc ^[[5n[USS9UlURRR [ [[S9 URURURURURURURUR UR"UR$S. Ul[)S[*4SU05nSU4n[-XC5Ul[0R2"UR4SSS S 9UlUR6R95 g) NT)loader enable_async)ziprg enumerate) /z/stacksz /pyspy_dumpz /fr_tracez/fr_trace_jsonz/fr_trace_ncclz/fr_trace_nccl_jsonz/profilez/wait_countersz /tcpstorerwryr*z distributed.debug.FrontendServer)targetrSdaemonr)r templatesr _jinja_envglobalsupdaterrgr _handle_index_handle_stacks_handle_pyspy_dump_handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_json_handle_profiler_handle_wait_counters_handle_tcpstore_routesrrwrp_server threadingThread_serve_threadstart)r rrRequestHandlerClassserver_addresss r!__init__FrontendServer.__init__psI&%V$G &&# ' ##**22.."88"88#'#B#B--"88..   #  !    d&~K  '';;3   r$r4NcURR5 g![a [R S5 gf=f)Nz got exception in frontend server)r serve_forever Exceptionr exceptionrs r!rFrontendServer._serves8 A LL & & ( A   ? @ As ??c8URR5 gr&)rjoinrs r!rFrontendServer.joins r$reqcUR5nX R;aURSSU35 gURUnU"U5nURS5 URSS5 UR5 URRU5 g![[4a>n[ R SU5 URSS[U535 SnAgSnAff=f) NizHandler not found: z-Exception in frontend server when handling %siz Exception: rz Content-typez text/html) rr send_errorr SystemExitrrrepr send_response send_header end_headerswfilewrite)r rrhandlerr:es r!rFrontendServer._handle_requests||~ || # NN3"5dV < = ,,t$ 3[S5up#URSSX#S9$)Ndump_tracebackriStackstitler^rAr`rr rr^rAs r!rFrontendServer._handle_stackss. !12 $$ 85%  r$c\[SUR55up#URSUUS9$)N pyspy_dumprnr^rA)r`rrrs r!r!FrontendServer._handle_pyspy_dumps: s/@/@/BC $$ %  r$r^rAc [5nUR/S9nSUlSUl0n[ U5H?upgUR 5 UXS.UR 5EnSU;a/US'XSUS3'MA [[UR555Sn [XTU 5n URSS [U R[RS S 9[U R ["RS S 9[U R$[&RS S 9[U R([*RS S 9S 9$) N)rST)rU host_nameentriesrUz.jsonversionrmFlightRecorderhtml)headerstablefmt)rgroups memberships collectives ncclcalls)r parse_argsallow_incomplete_ranksverboserr"r'nextitervaluesrrr rr_fieldsrrrrrr) r r^rAconfigrSdetailsrUr:dumprdbs r!_render_fr_traceFrontendServer._render_fr_traces8  b )&*# #E*JD  ! ! #"[))+D $"$Y*.d4&& '+tGNN,-.y9 gW -$$ "BIIu}}vN  (:(:V! (:(:Vr||X5E5EPVW%  r$cR[S5up#URU[U55$)N fr_trace_jsonr`rrOrs r!rFrontendServer._handle_fr_traces% 1 $$UDK88r$c@[S5up#URSSUUS9$)Nrrjrrrrs r!r$FrontendServer._handle_fr_trace_jsons2 1 $$ " %  r$cT[SS5up#URU[U55$)Ndump_nccl_trace_jsononlyactive=truerrs r!r$FrontendServer._handle_fr_trace_nccls) !79JK $$UDK88r$cB[SS5up#URSSUUS9$)NrrrjzFlightRecorder NCCLrrrs r!r)FrontendServer._handle_fr_trace_nccl_jsons6 !79JK $$ ' %  r$cnURSS[S9n[SSU35up4URSX4S9$)Ndurationg?)rr torch_profilez duration=rkr)rfloatr`r)r rr r^rAs r!rFrontendServer._handle_profilersD$$Z5$I IhZ2HI $$^5$NNr$c>[S5up#URSSX#S9$)Nwait_counter_valuesrjz Wait Countersrrrs r!r$FrontendServer._handle_wait_counterss. !67 $$ O5%  r$c[SS9nUR5nUR5 URU5Vs/sHn[ U5PM nnUR SX5S9$s snf)Nr)prefixrl)r]r)r list_keyssortrXrr)r rr[r]vrs r!rFrontendServer._handle_tcpstores_r*  #(??4#89#8a$q'#89$$_4$OO:sA$)rrrr)r4N)r+r,r-r.r/rrrrwrr1rbytesrrrrrOrrrrrrrrrr2r*r$r!rxrxos=(S(TA #5$.PPP5P3!333 "4   &8 U " d3i" X" 5" H9$6959  );  9);99  .@ U O$6O5O );  P$6P5Pr$rxrc[R[R5 [ US9n[R SUR R5 UR5 g)N)rz"Frontend server started on port %d) rsetLevelloggingINFOrxr{r server_portr)rservers r!mainr %s> OOGLL!  &F KK4fnn6P6PQ KKMr$)r)4rKr'rrsrcollections.abcrconcurrent.futuresr dataclassesr http.serverrr urllib.parserr jinja2r r r torch.distributed.debug._storer r4torch.distributed.flight_recorder.components.builderr;torch.distributed.flight_recorder.components.config_managerr2torch.distributed.flight_recorder.components.typesrrrr getLoggerr+rLoggerr0rrOr1rBrQtupler`rgrrprwrxr/r r*r$r!r.sW  $1!C+*JIQ!**844  % % % DI (8*<  %S %hx&8%"  3 d3i(AS6S0T (c( GP""?@  cj  Z#)# $/$>sPsPlstr$