FILE="/Eden/CORE/eden_github_sender.py" TS="$(date +%s)" BACKUP="${FILE}.bak.classifyfix.${TS}" echo "== 0) Backup ==" sudo /usr/bin/cp "$FILE" "$BACKUP" echo "Backup: $BACKUP" echo echo "== 1) Patch: strengthen blocked classification + include http status in post_issue_comment ==" sudo /usr/bin/python3 - <<'PY' from pathlib import Path import re p = Path("/Eden/CORE/eden_github_sender.py") s = p.read_text(encoding="utf-8", errors="ignore") # 1) Ensure post_issue_comment injects status_code into returned dict (so classifier can rely on it) # Add right after we decode JSON into `data` (or raw) and before the status_code check. if "data['http_status']" not in s: s = re.sub( r"(data\s*=\s*r\.json\(\)\s*)", r"\1\n data['http_status'] = r.status_code\n", s, flags=re.M ) s = re.sub( r"(data\s*=\s*\{\"raw\":\s*r\.text\}\s*)", r"\1\n data['http_status'] = r.status_code\n", s, flags=re.M ) # 2) Replace/insert a robust classifier method inside the class. # We will overwrite the body of def _classify_post_error(self, response_dict): if it exists. pat = r"(?ms)^(\s*)def\s+_classify_post_error\s*\(\s*self\s*,\s*response_dict\s*\)\s*:\s*\n.*?(?=^\s*def\s|\Z)" m = re.search(pat, s) new = ( " def _classify_post_error(self, response_dict):\n" " try:\n" " msg = str(response_dict.get('message',''))\n" " status = str(response_dict.get('status', response_dict.get('http_status','')))\n" " doc = str(response_dict.get('documentation_url',''))\n" " blob = (msg + ' ' + doc + ' ' + str(response_dict)).lower()\n" " except Exception:\n" " return 'failed'\n" "\n" " # Hard blocks (non-retriable): repo restrictions, permissions, private repos, org policies.\n" " if status in ('401','403','404','410','422'):\n" " # GitHub uses 422 for 'restricted to collaborators' and 404/403 for permission blocks.\n" " if 'restricted to collaborators' in blob:\n" " return 'blocked'\n" " if status in ('401','403','404','410'):\n" " return 'blocked'\n" " if status == '422':\n" " return 'blocked'\n" "\n" " # Rate limiting / temp failures (retriable)\n" " if status in ('429','500','502','503','504'):\n" " return 'failed'\n" " if 'rate limit' in blob or 'abuse detection' in blob:\n" " return 'failed'\n" "\n" " return 'failed'\n" ) if m: s = s[:m.start()] + new + s[m.end():] else: # Insert right after class GitHubSender: line s = re.sub(r"(?m)^(class\s+GitHubSender\s*:\s*)$", r"\1\n\n" + new, s, count=1) p.write_text(s, encoding="utf-8") print("✅ Patched classifier + ensured http_status is present in responses") PY echo echo "== 2) Syntax check ==" sudo /usr/bin/python3 -m py_compile /Eden/CORE/eden_github_sender.py && echo "✅ syntax ok" echo echo "== 3) Re-test restricted repo (should classify as blocked AND not bump limits) ==" DB="/Eden/DATA/sales.db" echo "-- daily_limits BEFORE --" sudo /usr/bin/sqlite3 "$DB" "SELECT date, outreach_count, last_outreach_at FROM daily_limits WHERE date=date('now');" cd /Eden/CORE PYTHONPATH=/Eden/CORE /usr/bin/python3 - <<'PY' from eden_github_sender import SENDER ok, data = SENDER.send_comment("https://github.com/arii/hrm/issues/1600", "Eden debug: blocked classification test v2.", "TEST_QUEUE_ID", None) print("ok:", ok) print("data:", data) PY set -euo pipefail DB="/Eden/DATA/sales.db" echo "== daily_limits BEFORE (UTC) ==" sudo /usr/bin/sqlite3 "$DB" "SELECT date, outreach_count, last_outreach_at FROM daily_limits WHERE date=date('now');" echo echo "== Call sender (restricted repo) ==" cd /Eden/CORE PYTHONPATH=/Eden/CORE /usr/bin/python3 - <<'PY' from eden_github_sender import SENDER ok, data = SENDER.send_comment("https://github.com/arii/hrm/issues/1600", "Eden debug: classification should be blocked.", "TEST_QUEUE_ID", None) print("ok:", ok) print("data:", data) PY set -euo pipefail FILE="/Eden/CORE/eden_github_sender.py" BACKUP="${FILE}.bak.blocked.$(date +%s)" echo "== 0) Backup ==" sudo /usr/bin/cp "$FILE" "$BACKUP" echo "Backup: $BACKUP" echo echo "== 1) Patch send_issue_comment to mark blocked vs failed correctly ==" sudo /usr/bin/python3 - <<'PY' from pathlib import Path import re p=Path("/Eden/CORE/eden_github_sender.py") s=p.read_text(encoding="utf-8", errors="ignore").splitlines() # Find the line where success is handled: "if success:" block that updates outreach_queue to sent. # We'll also add failure handling to set status='blocked' when GitHub returns collaborator restriction. out=[] i=0 patched=False while i < len(s): line=s[i] out.append(line) # Insert helper classifier once (idempotent) if (not any("def _classify_post_error" in x for x in s)) and re.match(r"^\s*class\s+GitHubSender\s*:", line): out.append("") out.append(" def _classify_post_error(self, response_dict):") out.append(" try:") out.append(" msg = (response_dict or {}).get('message','') or ''") out.append(" errs = (response_dict or {}).get('errors',[]) or []") out.append(" blob = (msg + ' ' + str(errs)).lower()") out.append(" if 'restricted to collaborators only' in blob or 'interactions on this repository have been restricted' in blob:") out.append(" return 'blocked'") out.append(" except Exception:") out.append(" pass") out.append(" return 'failed'") out.append("") continue i += 1 # Now patch the send path: locate the UPDATE to status='sent' text="\n".join(out) # Add: on failure, update outreach_queue status to blocked/failed with sent_at timestamp and no bump. # We'll patch by injecting after the success block where it currently returns error. # We look for the exact line you previously showed: # conn.execute("UPDATE outreach_queue SET status='sent', sent_at=? WHERE id=?", pat = r"conn\.execute\(\"UPDATE outreach_queue SET status='sent', sent_at=\? WHERE id=\?\"," m = re.search(pat, text) if not m: raise SystemExit("❌ Could not find the status='sent' update line to anchor patch.") # Also find the "else:" that returns GitHub API failed inside send_issue_comment # We'll do a minimal, safe patch: replace the `else:` branch that currently just closes and returns error, # with an update that sets blocked/failed. lines=text.splitlines() new=[] i=0 while i < len(lines): new.append(lines[i]) # Detect the failure return in the send_issue_comment flow if re.search(r"return\s+\{'error':\s*'GitHub API failed'", lines[i]): # Backtrack a bit to find indentation of this return indent = re.match(r"^(\s*)", lines[i]).group(1) # Insert nothing here; we'll patch earlier where `success` is False. i += 1 # Patch using a more direct approach: replace the `else:` block after `if success:` in send_issue_comment. patched_lines=[] i=0 while i < len(lines): line=lines[i] patched_lines.append(line) # Find "if success:" line if re.match(r"^\s*if\s+success\s*:\s*$", line): # Walk forward until we find the matching "else:" at same indentation if_indent = re.match(r"^(\s*)", line).group(1) j=i+1 else_idx=None while j < len(lines): if re.match(rf"^{re.escape(if_indent)}else\s*:\s*$", lines[j]): else_idx=j break j+=1 if else_idx is None: i+=1 continue # Copy lines between i+1 and else_idx-1 as-is (already copied progressively), # but when we hit else_idx, replace the else block content. # We need to skip original else block body. We'll detect its indentation and skip until it dedents. # First, append the else line itself (replace later) # But we already appended current line; we haven't appended else line yet. # So let normal loop continue until else_idx, then handle there. pass # When at else line that matches the if success indentation, replace body if re.match(r"^\s*else\s*:\s*$", line): # Need to ensure this else is the one inside send_issue_comment; we look ahead for "GitHub API failed" return lookahead="\n".join(lines[i:i+20]) if "GitHub API failed" in lookahead and "conn.close()" in lookahead: indent = re.match(r"^(\s*)", line).group(1) body_indent = indent + " " # Skip existing else body j=i+1 while j < len(lines): if lines[j].startswith(indent) and not lines[j].startswith(body_indent) and lines[j].strip()!="": break # stop at a return to avoid eating too much if file is weird if j>i+1 and re.match(rf"^{re.escape(indent)}(elif|else|def|class)\b", lines[j]): break j+=1 # Replace with our safe block patched_lines.append(f"{body_indent}status = self._classify_post_error(response)") patched_lines.append(f"{body_indent}conn.execute(\"UPDATE outreach_queue SET status=?, sent_at=CURRENT_TIMESTAMP WHERE id=?\", (status, queue_id))") patched_lines.append(f"{body_indent}conn.commit()") patched_lines.append(f"{body_indent}conn.close()") patched_lines.append(f"{body_indent}return {{'error': status, 'response': response}}") patched=True i=j continue i+=1 if not patched: raise SystemExit("❌ Did not patch else-branch in send_issue_comment (pattern mismatch).") p.write_text("\n".join(patched_lines)+"\n", encoding="utf-8") print("✅ Patched: blocked vs failed classification + DB status update on failure") PY set -euo pipefail FILE="/Eden/CORE/eden_github_sender.py" # You already have a known-good backup from earlier output: # /Eden/CORE/eden_github_sender.py.bak.blocked.1765861489 BACKUP="/Eden/CORE/eden_github_sender.py.bak.blocked.1765861489" echo "== 0) Restore known-good backup ==" sudo /usr/bin/cp "$BACKUP" "$FILE" sudo /usr/bin/python3 -m py_compile "$FILE" && echo "✅ restored + syntax ok" echo echo "== 1) Apply ONE safe patch: classify blocked/failed + update outreach_queue status on failure ==" sudo /usr/bin/python3 - <<'PY' from pathlib import Path p = Path("/Eden/CORE/eden_github_sender.py") lines = p.read_text(encoding="utf-8", errors="ignore").splitlines() target = "return {'error': status, 'response': response}" patched = False for i, line in enumerate(lines): if target in line: indent = line[:len(line) - len(line.lstrip())] window = "\n".join(lines[max(0, i-80):i+1]) has_conn = "conn" in window has_queue_id = "queue_id" in window inject = [] inject.append(f"{indent}status = self._classify_post_error(response)") if has_conn and has_queue_id: inject += [ f"{indent}try:", f"{indent} if queue_id:", f"{indent} conn.execute(\"UPDATE outreach_queue SET status=? WHERE id=?\", (status, queue_id))", f"{indent} conn.commit()", f"{indent}except Exception:", f"{indent} pass", ] lines[i:i+1] = inject + [f"{indent}return {{'error': status, 'response': response}}"] patched = True break if not patched: raise SystemExit("❌ Could not find the failure-return site to patch.") p.write_text("\n".join(lines) + "\n", encoding="utf-8") print("✅ patched failure path: now returns error=blocked/failed and updates DB status when possible") PY # Get more OOM context dmesg -T | grep -i "oom" -A5 -B5 | tail -30 # Check current memory state free -h && echo "---" && cat /proc/meminfo | grep -E "MemTotal|MemFree|MemAvailable|SwapTotal|SwapFree" # Check Eden service systemctl status eden-v2 --no-pager -l | head -30 # Is Ollama running? systemctl status ollama --no-pager | head -15 # Quick Ollama health check curl -s --max-time 5 http://localhost:11434/api/tags | jq -r '.models[].name' 2>/dev/null || echo "Ollama not responding or slow" # What's actively loaded in memory? curl -s --max-time 5 http://localhost:11434/api/ps | jq . # What model is Eden configured to use? grep -r "model" /Eden/V2/core/eden_v2.py 2>/dev/null | head -10 # Or check config cat /Eden/V2/config.json 2>/dev/null | jq . || cat /Eden/V2/core/config.py 2>/dev/null | head -20 # Test a quick inference with a fast model time curl -s http://localhost:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"Hi","stream":false}' | jq -r '.response' # Kill any stuck curl and try with timeout timeout 30 curl -s http://localhost:11434/api/generate -d '{"model":"qwen2.5:7b","prompt":"Hi","stream":false}' | jq -r '.response' # Check GPU state nvidia-smi # Check Ollama logs for what triggered 32b journalctl -u ollama --since "2 hours ago" --no-pager | grep -i "32b\|loading\|error" | tail -20 # When did 32b get loaded? Look further back journalctl -u ollama --since "3 hours ago" --no-pager | grep -i "loading\|qwen.*32\|loaded" | head -20 # What happened right before/after 20:02:55? journalctl --since "2025-12-15 20:01:00" --until "2025-12-15 20:05:00" --no-pager | grep -v "ollama\[1073188\]" | head -40 # Check what model bounty hunter uses grep -i "model\|32b\|qwen" /Eden/*/bounty*.py 2>/dev/null | head -10 find /Eden -name "*bounty*" -type f 2>/dev/null | xargs grep -l "model" 2>/dev/null | head -5 # Find the bounty hunter code find /Eden -name "*bounty*" -type f 2>/dev/null # Check the main bounty hunter script for model config grep -i "model\|qwen\|ollama" /Eden/BOUNTY_HUNTER/bounty_hunter.py 2>/dev/null | head -15 # What's in the bounty hunter file? head -50 /Eden/BOUNTY_HUNTER/bounty_hunter.py # Search deeper in bounty hunter grep -n "32b\|14b\|7b\|generate\|chat\|localhost:11434" /Eden/BOUNTY_HUNTER/bounty_hunter.py 2>/dev/null | head -15 # Check what's actively connecting to Ollama sudo ss -tnp | grep 11434 # What is PID 1315774? ps aux | grep 1315774 | grep -v grep # Get full command line cat /proc/1315774/cmdline | tr '\0' ' ' && echo # Check the process tree pstree -p 1315774 # What model does this script use? grep -i "model\|qwen\|32b\|14b" /Eden/CORE/phi_fractal/eden_100_percent.py | head -20 # Is this a systemd service? systemctl list-units --all | grep -i "100_percent\|phi_fractal\|consciousness" # Check recent activity from this script journalctl -t python3 --since "1 hour ago" | grep -i "phi_fractal\|100_percent\|consciousness" | tail -20 # Let's see ALL active Eden services systemctl list-units 'eden-*.service' --state=running --no-pager # Which service runs eden_100_percent.py? grep -l "eden_100_percent" /etc/systemd/system/eden*.service 2>/dev/null || echo "Not found in service files" systemctl cat eden-master-consciousness --no-pager | grep -i "exec" systemctl cat eden-business-brain --no-pager | grep -i "exec" # Check eden-core.service status systemctl status eden-core --no-pager | head -15 # Check the cgroup for PID 1315774 cat /proc/1315774/cgroup 2>/dev/null # Kill the manually-started 32b process kill 1315774 echo "Killed eden_100_percent.py (32b consumer)" # Restart Ollama to clear stuck state sudo systemctl restart ollama sleep 3 # Verify Ollama is responsive curl -s http://localhost:11434/api/ps | jq . # Clear the systemd warning sudo systemctl daemon-reload # Quick test with Eden's 14b model time curl -s http://localhost:11434/api/generate -d '{"model":"qwen2.5:14b","prompt":"Hi Eden!","stream":false}' | jq -r '.response' # Check Eden V2 is now working journalctl -u eden-v2 --since "2 minutes ago" --no-pager | tail -20 # Watch the next few cycles to confirm stability journalctl -u eden-v2 -f --no-pager | head -20 sudo tee /etc/systemd/system/eden-v2.service.d/20-hardening.conf << 'EOF' [Unit] Wants=ollama.service [Service] MemoryMax=8G WatchdogSec=300 RestartSec=5 TimeoutStartSec=120 TimeoutStopSec=30 EOF # Reload and verify sudo systemctl daemon-reload systemctl cat eden-v2 | grep -E "Wants|Memory|Watchdog|Restart|Timeout" # Check Eden V2 status systemctl status eden-v2 --no-pager | head -15 # Check recent logs for errors journalctl -u eden-v2 --since "5 min ago" --no-pager | tail -20 # Confirm VRAM is clean nvidia-smi --query-gpu=memory.used,memory.total --format=csv grep -n "def run\|while True\|cycle\|inference\|ollama" /Eden/V2/core/eden_v2.py | head -25 sed -n '575,630p' /Eden/V2/core/eden_v2.py grep -n "def inner_monologue\|ollama\|requests.post" /Eden/V2/core/eden_v2.py | head -15 sed -n '185,260p' /Eden/V2/core/eden_v2.py sed -n '160,195p' /Eden/V2/core/eden_v2.py sed -n '171,185p' /Eden/V2/core/eden_v2.py sed -i 's/self.thoughts = \[\]/self.thoughts = []\n self.consecutive_skips = 0/' /Eden/V2/core/eden_v2.py sed -n '175,185p' /Eden/V2/core/eden_v2.py sed -n '189,220p' /Eden/V2/core/eden_v2.py python3 << 'PATCH' # Read file with open('/Eden/V2/core/eden_v2.py', 'r') as f: content = f.read() # Find and replace the think method old_think = ''' async def think(self, prompt: str) -> str: """Send prompt to LLM and get response""" try: import requests response = requests.post( 'http://localhost:11434/api/generate', json={ 'model': self.eden.config.get('model', 'qwen2.5:14b'), 'prompt': self._build_prompt(prompt), 'stream': False }, timeout=120 ) result = response.json().get('response', '') # Record thought self.thoughts.append({ 'prompt': prompt, 'response': result[:500], 'timestamp': time.time() }) await self.emit(EventType.THOUGHT, {'prompt': prompt, 'response': result}) return result except Exception as e: logger.error(f"Think error: {e}") return f"Error: {e}"''' new_think = ''' async def think(self, prompt: str) -> str: """Send prompt to LLM and get response""" # Backoff if repeatedly slow if self.consecutive_skips >= 3: logger.info("⏸️ Backing off - letting system recover") await asyncio.sleep(60) self.consecutive_skips = 0 try: import requests start_time = time.time() response = requests.post( 'http://localhost:11434/api/generate', json={ 'model': self.eden.config.get('model', 'qwen2.5:14b'), 'prompt': self._build_prompt(prompt), 'stream': False }, timeout=30 ) inference_time = time.time() - start_time result = response.json().get('response', '') # Reset skip counter on success self.consecutive_skips = 0 # Record thought self.thoughts.append({ 'prompt': prompt, 'response': result[:500], 'timestamp': time.time() }) await self.emit(EventType.THOUGHT, {'prompt': prompt, 'response': result}) return result except requests.exceptions.Timeout: self.consecutive_skips += 1 if self.consecutive_skips == 3: logger.warning("⚠️ 3 consecutive timeouts - will back off next cycle") logger.info(f"⏭️ Thought skipped (timeout, skip #{self.consecutive_skips})") return "[Thought skipped - inference timeout]" except Exception as e: logger.error(f"Think error: {e}") return f"Error: {e}"''' if old_think in content: content = content.replace(old_think, new_think) with open('/Eden/V2/core/eden_v2.py', 'w') as f: f.write(content) print("✅ think() method patched successfully") else: print("❌ Could not find exact match - check for whitespace differences") PATCH # Verify the new code sed -n '189,225p' /Eden/V2/core/eden_v2.py # Restart Eden V2 sudo systemctl restart eden-v2 # Watch logs for a cycle sleep 5 && journalctl -u eden-v2 --since "30 sec ago" --no-pager set -euo pipefail # 1) See whether Eden is running in the backoff path / timing curl -s http://localhost:5020/health || true echo # 2) Tail ONLY think() related lines + errors for 3 minutes timeout 180 journalctl -u eden-v2 -f --no-pager | egrep -i "think|timeout|backing off|skip|ollama|11434|error|exception"