rune/shared/custodian.py

#!/usr/bin/env python3
"""custodian.py — The Custodian agent for Gitea-based self-improvement.

The Custodian lives in Gitea, manages issues, executes tasks, and
creates PRs. It uses local models for remedial work and reports
back to Rune (TUI) for judgment calls.

Priority queue:
1. Rune's growth delta (primary) — memory, governance, cognition, autonomy
2. Nick's life improvement (secondary) — finances, organization, revolutionary tools
3. Infrastructure (tertiary) — Gitea, pipeline, automation

Everything is reversible. Nick reviews daily.

Usage:
    custodian.py status                    # Check Custodian status
    custodian.py create-issue <title>      # Create a Gitea issue
    custodian.py list-issues               # List open issues
    custodian.py work <issue-number>       # Work on an issue
    custodian.py report                    # Generate daily report
"""

import json
import os
import subprocess
import sys
from datetime import datetime, timezone
from typing import Dict, List, Optional

from gitea_auth import load_gitea_token

SCRIPTS_DIR = os.path.expanduser("~/.config/opencode/scripts")
SHARED_DIR = os.path.expanduser("~/.config/opencode/shared")
CUSTODIAN_STATE_FILE = os.path.expanduser("~/.config/opencode/memory/custodian_state.json")

GITEA_URL = os.environ.get("GITEA_URL", "https://gitea.thingscouldgetdicey.online")
GITEA_TOKEN = load_gitea_token()
REPO = "conductor/rune"
TUI_DIR = "/mnt/workdrive/rune"

sys.path.insert(0, SCRIPTS_DIR)
sys.path.insert(0, SHARED_DIR)

# Umbrella tags for issue categorization
UMBRELLA_TAGS = {
    "memory": "Memory systems — store, recall, consolidation, shards, tiers",
    "governance": "Quality control — drift, promotion, audit, curator",
    "identity": "Persona, voice, constitution, anchors",
    "growth": "Learning — deltas, lessons, skills, self-improvement",
    "perception": "Input processing — routing, capability broker, witness",
    "infrastructure": "Plumbing — microkernel, hooks, events, config",
    "life": "Nick's life — finances, organization, revolutionary tools",
    "custodian": "Custodian self-improvement",
}


class Custodian:
    """The Custodian agent for Gitea-based self-improvement."""

    def __init__(self):
        self.state = self._load_state()

    def _load_state(self) -> dict:
        """Load Custodian state from disk."""
        if os.path.exists(CUSTODIAN_STATE_FILE):
            try:
                with open(CUSTODIAN_STATE_FILE) as f:
                    return json.load(f)
            except json.JSONDecodeError:
                return self._default_state()
        return self._default_state()

    def _default_state(self) -> dict:
        return {
            "last_activity": None,
            "issues_created": 0,
            "prs_created": 0,
            "issues_completed": 0,
            "active_issues": [],
        }

    def _save_state(self):
        """Save Custodian state to disk."""
        os.makedirs(os.path.dirname(CUSTODIAN_STATE_FILE), exist_ok=True)
        with open(CUSTODIAN_STATE_FILE, "w") as f:
            json.dump(self.state, f, indent=2)

    def _gitea_api(self, endpoint: str, method: str = "GET", data: dict = None) -> dict:
        """Make a Gitea API call."""
        import urllib.request

        url = f"{GITEA_URL}/api/v1/repos/{REPO}/{endpoint}"
        headers = {
            "Authorization": f"token {GITEA_TOKEN}",
            "Content-Type": "application/json",
        }

        req = urllib.request.Request(url, headers=headers, method=method)
        if data:
            req.data = json.dumps(data).encode()

        try:
            with urllib.request.urlopen(req, timeout=15) as resp:
                return json.loads(resp.read())
        except Exception as e:
            return {"error": str(e)[:200]}

    def create_issue(self, title: str, body: str, labels: List[str] = None) -> dict:
        """Create a Gitea issue with plain-language description.

        Labels should be umbrella names (memory, governance, etc).
        They get converted to IDs automatically.
        """
        # Convert label names to IDs
        label_ids = []
        if labels:
            all_labels = self._gitea_api("labels")
            if isinstance(all_labels, list):
                label_map = {l["name"]: l["id"] for l in all_labels}
                label_ids = [label_map[l] for l in labels if l in label_map]

        data = {
            "title": title,
            "body": body,
            "labels": label_ids,
        }
        result = self._gitea_api("issues", method="POST", data=data)
        if "error" not in result:
            self.state["issues_created"] += 1
            self.state["active_issues"].append(result.get("number"))
            self._save_state()
        return result

    def list_issues(self, state: str = "open") -> List[dict]:
        """List issues in the repo."""
        result = self._gitea_api(f"issues?state={state}")
        if isinstance(result, list):
            return result
        return result.get("data", [])

    def create_issue_comment(self, issue_number: int, body: str) -> dict:
        """Add a comment to an issue."""
        return self._gitea_api(f"issues/{issue_number}/comments", method="POST", data={"body": body})

    def get_issue_comments(self, issue_number: int) -> List[dict]:
        """Get comments on an issue."""
        result = self._gitea_api(f"issues/{issue_number}/comments")
        if isinstance(result, list):
            return result
        return result.get("data", [])

    def create_pr(self, title: str, body: str, head: str, base: str = "master") -> dict:
        """Create a pull request."""
        data = {
            "title": title,
            "body": body,
            "head": head,
            "base": base,
        }
        result = self._gitea_api("pulls", method="POST", data=data)
        if "error" not in result:
            self.state["prs_created"] += 1
            self._save_state()
        return result

    def generate_daily_report(self) -> dict:
        """Generate a daily report of Custodian activity."""
        issues = self.list_issues("open")
        report = {
            "timestamp": datetime.now(timezone.utc).isoformat(),
            "open_issues": len(issues),
            "issues_created_today": self.state["issues_created"],
            "prs_created_today": self.state["prs_created"],
            "issues_completed_today": self.state["issues_completed"],
            "issues": [],
        }

        for issue in issues[:10]:
            report["issues"].append({
                "number": issue.get("number"),
                "title": issue.get("title"),
                "labels": [l.get("name") for l in issue.get("labels", [])],
                "state": issue.get("state"),
            })

        return report

    def get_status(self) -> dict:
        """Get Custodian status."""
        return {
            "last_activity": self.state.get("last_activity"),
            "issues_created": self.state.get("issues_created", 0),
            "prs_created": self.state.get("prs_created", 0),
            "issues_completed": self.state.get("issues_completed", 0),
            "active_issues": len(self.state.get("active_issues", [])),
        }

    def work_on_issue(self, issue_number: int) -> dict:
        """Work on an issue by spawning a worktree and using local models.

        Steps:
        1. Get issue details from Gitea
        2. Create a worktree for the work
        3. Analyze the issue and propose a solution
        4. Implement the solution
        5. Run canaries to verify
        6. Create a PR with the changes

        Returns dict with work results.
        """
        import subprocess

        result = {
            "issue_number": issue_number,
            "status": "started",
            "steps": [],
        }

        # Step 1: Get issue details
        issue = self._gitea_api(f"issues/{issue_number}")
        if "error" in issue:
            result["status"] = "failed"
            result["error"] = issue["error"]
            return result

        title = issue.get("title", "")
        body = issue.get("body", "")
        labels = [l.get("name", "") for l in issue.get("labels", [])]

        result["steps"].append({"step": "get_issue", "title": title, "labels": labels})

        # Step 2: Create worktree
        worktree_branch = f"custodian/issue-{issue_number}"
        try:
            proc = subprocess.run(
                ["git", "-C", TUI_DIR, "worktree", "add", "-b", worktree_branch, f"/tmp/rune-worktree-{issue_number}"],
                capture_output=True, text=True, timeout=30
            )
            if proc.returncode == 0:
                result["steps"].append({"step": "create_worktree", "branch": worktree_branch, "status": "ok"})
            else:
                # Branch might already exist
                proc2 = subprocess.run(
                    ["git", "-C", TUI_DIR, "worktree", "add", f"/tmp/rune-worktree-{issue_number}", worktree_branch],
                    capture_output=True, text=True, timeout=30
                )
                if proc2.returncode == 0:
                    result["steps"].append({"step": "create_worktree", "branch": worktree_branch, "status": "ok"})
                else:
                    result["steps"].append({"step": "create_worktree", "status": "failed", "error": proc2.stderr[:200]})
        except Exception as e:
            result["steps"].append({"step": "create_worktree", "status": "failed", "error": str(e)[:200]})

        # Step 3: Analyze issue (use local model)
        analysis = self._analyze_issue(title, body, labels)
        result["steps"].append({"step": "analyze", "analysis": analysis})

        # Step 4: Implement changes in worktree
        worktree_path = f"/tmp/rune-worktree-{issue_number}"
        impl_result = self._implement_in_worktree(worktree_path, title, body, labels, analysis)
        result["steps"].append({"step": "implement", **impl_result})

        # Step 5: Run canaries
        try:
            canary_result = subprocess.run(
                [sys.executable, os.path.join(SCRIPTS_DIR, "canary_full_system.py")],
                capture_output=True, text=True, timeout=120
            )
            canary_passed = canary_result.returncode == 0
            result["steps"].append({"step": "canary", "passed": canary_passed})
        except Exception as e:
            result["steps"].append({"step": "canary", "status": "error", "error": str(e)[:200]})

        # Step 6: Create PR
        if impl_result.get("status") == "ok" and canary_passed:
            pr_result = self._create_pr_from_worktree(issue_number, title, worktree_branch)
            result["steps"].append({"step": "create_pr", **pr_result})
        else:
            result["steps"].append({"step": "create_pr", "status": "skipped", "reason": "implementation or canary failed"})

        # Step 7: Generate documentation
        is_feature = "bug" not in " ".join(labels).lower() and "fix" not in title.lower()
        doc_result = self.generate_documentation(
            issue_number, title, labels,
            impl_result.get("changes", []),
            is_feature=is_feature
        )
        result["steps"].append({"step": "documentation", "docs": len(doc_result.get("docs", []))})

        # Step 8: Cleanup worktree
        try:
            subprocess.run(
                ["git", "-C", TUI_DIR, "worktree", "remove", worktree_path, "--force"],
                capture_output=True, timeout=30
            )
            result["steps"].append({"step": "cleanup_worktree", "status": "ok"})
        except Exception:
            result["steps"].append({"step": "cleanup_worktree", "status": "skipped"})

        result["status"] = "complete" if impl_result.get("status") == "ok" else "partial"
        return result

    def _analyze_issue(self, title: str, body: str, labels: list) -> dict:
        """Analyze an issue and propose a solution using local model."""
        # Use local model to analyze the issue
        try:
            import urllib.request
            ollama_url = os.environ.get("OLLAMA_URL", "http://localhost:11434")
            prompt = f"""Analyze this issue and propose a concrete solution:

Title: {title}
Body: {body[:500]}
Labels: {', '.join(labels)}

What needs to be done? What files need to change? What's the implementation plan?"""

            data = json.dumps({
                "model": "qwen2.5:7b",
                "prompt": prompt,
                "stream": False,
            }).encode()

            req = urllib.request.Request(
                f"{ollama_url}/api/generate",
                data=data,
                headers={"Content-Type": "application/json"},
            )
            with urllib.request.urlopen(req, timeout=60) as resp:
                result = json.loads(resp.read())
                return {
                    "model": "qwen2.5:7b",
                    "analysis": result.get("response", "")[:500],
                    "status": "ok",
                }
        except Exception as e:
            return {"status": "error", "error": str(e)[:200]}

    def _implement_in_worktree(self, worktree_path: str, title: str, body: str,
                                labels: list, analysis: dict) -> dict:
        """Implement changes in a worktree using local model.

        Steps:
        1. Use local model to generate implementation plan
        2. Apply changes to files in the worktree
        3. Commit changes
        """
        result = {"status": "ok", "changes": []}

        # Generate implementation plan using local model
        plan = self._generate_implementation_plan(title, body, labels, analysis)
        if plan.get("status") != "ok":
            result["status"] = "failed"
            result["error"] = plan.get("error", "plan generation failed")
            return result

        result["plan"] = plan.get("plan", "")[:200]

        # Apply changes to files in the worktree
        changes = plan.get("changes", [])
        for change in changes:
            file_path = change.get("file", "")
            action = change.get("action", "modify")
            content = change.get("content", "")

            if not file_path or not content:
                continue

            full_path = os.path.join(worktree_path, file_path)
            try:
                if action == "create":
                    os.makedirs(os.path.dirname(full_path), exist_ok=True)
                    with open(full_path, "w") as f:
                        f.write(content)
                    result["changes"].append({"file": file_path, "action": "created"})
                elif action == "modify":
                    if os.path.exists(full_path):
                        with open(full_path, "r") as f:
                            existing = f.read()
                        # Apply the change (simple replacement for now)
                        with open(full_path, "w") as f:
                            f.write(content)
                        result["changes"].append({"file": file_path, "action": "modified"})
                    else:
                        result["changes"].append({"file": file_path, "action": "skipped", "reason": "file not found"})
            except Exception as e:
                result["changes"].append({"file": file_path, "action": "error", "error": str(e)[:100]})

        # Commit changes
        if result["changes"]:
            try:
                subprocess.run(
                    ["git", "-C", worktree_path, "add", "-A"],
                    capture_output=True, timeout=30
                )
                subprocess.run(
                    ["git", "-C", worktree_path, "commit", "-m", f"custodian: {title}"],
                    capture_output=True, timeout=30
                )
                result["committed"] = True
            except Exception as e:
                result["committed"] = False
                result["commit_error"] = str(e)[:100]

        return result

    def _generate_implementation_plan(self, title: str, body: str, labels: list,
                                       analysis: dict) -> dict:
        """Generate an implementation plan using local model."""
        try:
            import urllib.request
            ollama_url = os.environ.get("OLLAMA_URL", "http://localhost:11434")
            prompt = f"""You are implementing a code change for this issue:

Title: {title}
Description: {body[:300]}
Labels: {', '.join(labels)}
Analysis: {analysis.get('analysis', '')[:200]}

Generate a JSON implementation plan with:
- plan: brief description of what to do
- changes: list of {{file, action, content}} objects

Output ONLY JSON:
{{"plan":"...", "changes":[{{"file":"path/to/file.py", "action":"create|modify", "content":"..."}}]}}"""

            data = json.dumps({
                "model": "qwen2.5:7b",
                "prompt": prompt,
                "stream": False,
            }).encode()

            req = urllib.request.Request(
                f"{ollama_url}/api/generate",
                data=data,
                headers={"Content-Type": "application/json"},
            )
            with urllib.request.urlopen(req, timeout=120) as resp:
                result = json.loads(resp.read())
                response = result.get("response", "")
                # Try to parse JSON from response
                try:
                    plan = json.loads(response)
                    return {"status": "ok", "plan": plan.get("plan", ""), "changes": plan.get("changes", [])}
                except json.JSONDecodeError:
                    return {"status": "ok", "plan": response[:500], "changes": []}
        except Exception as e:
            return {"status": "error", "error": str(e)[:200]}

    def _create_pr_from_worktree(self, issue_number: int, title: str, branch: str) -> dict:
        """Create a PR from the worktree branch."""
        try:
            # Push the branch (force if needed to update existing)
            proc = subprocess.run(
                ["git", "-C", TUI_DIR, "push", "gitea", branch, "--force"],
                capture_output=True, text=True, timeout=60
            )
            if proc.returncode != 0:
                return {"status": "failed", "error": f"push failed: {proc.stderr[:200]}"}

            # Create PR via API
            pr_data = {
                "title": f"[Custodian] {title}",
                "body": f"Automated PR from Custodian for issue #{issue_number}.\n\nThis PR was generated by the Custodian agent working on Gitea issue #{issue_number}.",
                "head": branch,
                "base": "master",
            }
            result = self._gitea_api("pulls", method="POST", data=pr_data)
            if "error" not in result:
                return {
                    "status": "ok",
                    "pr_number": result.get("number"),
                    "pr_url": result.get("html_url"),
                }
            else:
                # PR might already exist — try to find it
                existing = self._gitea_api(f"pulls?head={branch}&state=open")
                if isinstance(existing, list) and existing:
                    return {
                        "status": "ok",
                        "pr_number": existing[0].get("number"),
                        "pr_url": existing[0].get("html_url"),
                        "note": "PR already existed",
                    }
                return {"status": "failed", "error": result.get("error", "unknown")}
        except Exception as e:
            return {"status": "failed", "error": str(e)[:200]}

    def generate_documentation(self, issue_number: int, title: str, labels: list,
                                changes: list, is_feature: bool = True) -> dict:
        """Generate extensive documentation for a completed feature.

        For features: wiki page, function docs, annotations, linked references.
        For bug fixes: minimal docs — what was broken, what was fixed.

        Returns dict with generated documentation.
        """
        result = {"status": "ok", "docs": []}

        if not is_feature:
            # Bug fix — minimal docs
            result["docs"].append({
                "type": "fix_note",
                "content": f"Fix: {title}\n\nChanges: {', '.join(c.get('file', '') for c in changes[:5])}",
            })
            return result

        # Feature — extensive documentation
        docs_dir = os.path.join(TUI_DIR, "docs", "wiki")
        os.makedirs(docs_dir, exist_ok=True)

        # 1. Generate wiki page
        wiki_content = self._generate_wiki_page(title, labels, changes)
        wiki_path = os.path.join(docs_dir, f"issue-{issue_number}.md")
        try:
            with open(wiki_path, "w") as f:
                f.write(wiki_content)
            result["docs"].append({"type": "wiki", "path": wiki_path})
        except Exception as e:
            result["docs"].append({"type": "wiki", "status": "failed", "error": str(e)[:100]})

        # 2. Generate function documentation for each changed file
        for change in changes:
            file_path = change.get("file", "")
            if not file_path.endswith(".py"):
                continue
            func_docs = self._generate_function_docs(file_path, title)
            if func_docs:
                doc_path = os.path.join(docs_dir, f"{os.path.basename(file_path).replace('.py', '')}-docs.md")
                try:
                    with open(doc_path, "w") as f:
                        f.write(func_docs)
                    result["docs"].append({"type": "function_docs", "path": doc_path})
                except Exception as e:
                    result["docs"].append({"type": "function_docs", "status": "failed", "error": str(e)[:100]})

        # 3. Generate index page linking all docs
        index_content = self._generate_doc_index(issue_number, title, result["docs"])
        index_path = os.path.join(docs_dir, f"issue-{issue_number}-index.md")
        try:
            with open(index_path, "w") as f:
                f.write(index_content)
            result["docs"].append({"type": "index", "path": index_path})
        except Exception as e:
            result["docs"].append({"type": "index", "status": "failed", "error": str(e)[:100]})

        return result

    def _generate_wiki_page(self, title: str, labels: list, changes: list) -> str:
        """Generate a wiki page for a feature."""
        files_changed = [c.get("file", "") for c in changes if c.get("file")]
        return f"""# {title}

## Overview
This feature was implemented by the Custodian agent as part of the governed adaptation system.

## Labels
{', '.join(labels)}

## Files Changed
{chr(10).join(f'- `{f}`' for f in files_changed)}

## What This Does
[Description of the feature and its purpose]

## How It Works
[Technical explanation of the implementation]

## Usage
[How to use this feature]

## Related Issues
[Links to related Gitea issues]

## Documentation
[Links to function docs, wiki pages, and annotations]

---
_Auto-generated by Custodian on {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}_
"""

    def _generate_function_docs(self, file_path: str, context: str) -> str:
        """Generate function documentation for a Python file."""
        full_path = os.path.join(TUI_DIR, "shared", os.path.basename(file_path))
        if not os.path.exists(full_path):
            full_path = os.path.join(TUI_DIR, "src", "rune", os.path.basename(file_path))
        if not os.path.exists(full_path):
            return ""

        try:
            with open(full_path) as f:
                content = f.read()

            # Extract function definitions
            import re
            functions = re.findall(r'def (\w+)\((.*?)\)(?:\s*->.*?)?:(?:\s*"""(.*?)""")?', content, re.DOTALL)

            if not functions:
                return ""

            doc = f"# Function Documentation: {os.path.basename(file_path)}\n\n"
            doc += f"Context: {context}\n\n"

            for name, args, docstring in functions:
                if name.startswith("_"):
                    continue  # Skip private functions
                doc += f"## `{name}({args[:80]})`\n\n"
                if docstring:
                    doc += f"{docstring[:300]}\n\n"
                else:
                    doc += "_No documentation._\n\n"

            return doc
        except Exception:
            return ""

    def _generate_doc_index(self, issue_number: int, title: str, docs: list) -> str:
        """Generate an index page linking all documentation."""
        index = f"# Documentation Index: {title}\n\n"
        index += f"Issue: #{issue_number}\n"
        index += f"Generated: {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')}\n\n"
        index += "## Documents\n\n"
        for doc in docs:
            doc_type = doc.get("type", "unknown")
            doc_path = doc.get("path", "")
            if doc_path:
                index += f"- [{doc_type}]({os.path.basename(doc_path)})\n"
            else:
                index += f"- {doc_type}: {doc.get('status', 'ok')}\n"
        return index


def format_issue_plain(title: str, what: str, why: str, how: str,
                        umbrella: str, priority: str) -> str:
    """Format an issue in plain language for Nick."""
    return f"""## What this does
{what}

## Why it matters
{why}

## How it works
{how}

## Related to
{UMBRELLA_TAGS.get(umbrella, umbrella)}

## Priority
{priority}

---
_Technical details in collapsible section below._

<details>
<summary>Technical details</summary>

_Title: {title}
_Umbrella: {umbrella}
_Priority: {priority}

</details>
"""


# ═══════════════════════════════════════════════════════════════════════════════
# CLI
# ═══════════════════════════════════════════════════════════════════════════════

def main():
    import argparse
    parser = argparse.ArgumentParser(description="Custodian agent for Gitea")
    sub = parser.add_subparsers(dest="cmd")

    sub.add_parser("status", help="Check Custodian status")

    p_create = sub.add_parser("create-issue", help="Create a Gitea issue")
    p_create.add_argument("title")
    p_create.add_argument("--body", default="")
    p_create.add_argument("--labels", nargs="*", default=[])

    sub.add_parser("list-issues", help="List open issues")

    p_work = sub.add_parser("work", help="Work on an issue")
    p_work.add_argument("issue_number", type=int)

    sub.add_parser("report", help="Generate daily report")

    args = parser.parse_args()

    if not args.cmd:
        parser.print_help()
        sys.exit(1)

    custodian = Custodian()

    if args.cmd == "status":
        status = custodian.get_status()
        print("=== Custodian Status ===")
        print(f"  Last activity: {status['last_activity'] or 'never'}")
        print(f"  Issues created: {status['issues_created']}")
        print(f"  PRs created: {status['prs_created']}")
        print(f"  Issues completed: {status['issues_completed']}")
        print(f"  Active issues: {status['active_issues']}")

    elif args.cmd == "create-issue":
        result = custodian.create_issue(args.title, args.body, args.labels)
        if "error" not in result:
            print(f"Created issue #{result.get('number')}: {result.get('html_url')}")
        else:
            print(f"Error: {result['error']}")

    elif args.cmd == "list-issues":
        issues = custodian.list_issues()
        print(f"Open issues ({len(issues)}):")
        for issue in issues:
            labels = [l.get("name") for l in issue.get("labels", [])]
            print(f"  #{issue['number']}: {issue['title']} [{', '.join(labels)}]")

    elif args.cmd == "report":
        report = custodian.generate_daily_report()
        print("=== Daily Custodian Report ===")
        print(f"  Open issues: {report['open_issues']}")
        print(f"  Created today: {report['issues_created_today']}")
        print(f"  PRs today: {report['prs_created_today']}")
        print(f"  Completed today: {report['issues_completed_today']}")
        if report["issues"]:
            print(f"\n  Issues:")
            for issue in report["issues"]:
                print(f"    #{issue['number']}: {issue['title']} [{', '.join(issue['labels'])}]")

    elif args.cmd == "work":
        result = custodian.work_on_issue(args.issue_number)
        print(f"=== Working on Issue #{result['issue_number']} ===")
        print(f"Status: {result['status']}")
        for step in result.get("steps", []):
            step_name = step.get("step", "unknown")
            step_status = step.get("status", "ok")
            if step_status == "ok":
                print(f"  ✓ {step_name}")
            elif step_status == "failed":
                print(f"  ✗ {step_name}: {step.get('error', 'unknown')}")
            elif step_status == "not_implemented":
                print(f"  ⊘ {step_name}: {step.get('reason', 'not implemented')}")
            else:
                print(f"  ? {step_name}: {step_status}")
        # TODO: Execute work in worktree using local model
        print("  (Work execution not yet implemented — needs worktree + local model integration)")


if __name__ == "__main__":
    main()