{ "title": "Zero to working local AI", "slug": "start", "verified_on": "2026-10-15", "status": "Commands verified against official docs on 2026-10-15; not yet run end-to-end on the Mac Studio test rig unless marked mac_studio_tested: true.", "platform_order": [ "macos-apple-silicon", "linux", "windows" ], "steps": [ { "n": 1, "id": "check", "title": "Check your machine", "icon": "Cpu", "pages": [ { "href": "/machines", "label": "Machines: what to buy or what yours can run" }, { "href": "/models", "label": "Models vs memory: what fits your RAM" } ], "human": "Your memory decides which models you can run. Find your number, then read across the chart.", "goal": "Know total RAM (Apple Silicon: unified memory) or GPU VRAM, and pick a model size from the fit chart.", "links": [ "mlx-lm", "llama-cpp" ], "commands": { "macos-apple-silicon": [ { "run": "sysctl -n hw.memsize | awk '{printf \"%.0f GB unified memory\\n\", $1/1073741824}'", "mac_studio_tested": false }, { "run": "system_profiler SPHardwareDataType | grep -E 'Chip|Memory'", "mac_studio_tested": false } ], "linux": [ { "run": "free -g | awk '/Mem:/{print $2\" GB RAM\"}'", "mac_studio_tested": false }, { "run": "nvidia-smi --query-gpu=name,memory.total --format=csv # NVIDIA GPUs only", "mac_studio_tested": false } ], "windows": [ { "run": "(Get-CimInstance Win32_ComputerSystem).TotalPhysicalMemory/1GB # PowerShell", "mac_studio_tested": false } ] }, "verify": [ { "run": "sysctl -n hw.memsize", "expect": "prints a byte count; 128GB Mac Studio ≈ 137438953472" } ], "rollback": [ { "run": "# read-only step; nothing to undo", "mac_studio_tested": false } ], "fit_chart": { "rule": "Rule of thumb (estimate, not measured): comfortable if the model file is at most 50% of RAM/VRAM, tight up to 75%, otherwise no. Context window and the OS need the rest.", "sizes_source": "https://ollama.com/library (tags pages, download sizes as listed 2026-10-15)", "models": [ { "tag": "qwen3.5:4b", "size_gb": 3.3 }, { "tag": "gemma4", "size_gb": 6.6, "note": "latest = e4b" }, { "tag": "gemma4:26b", "size_gb": 16 }, { "tag": "qwen3-coder:30b", "size_gb": 19 }, { "tag": "qwen3.5:122b", "size_gb": 81 } ], "ram_gb": [ 8, 16, 32, 64, 128 ], "grid": [ [ "comfortable", "no", "no", "no", "no" ], [ "comfortable", "comfortable", "no", "no", "no" ], [ "comfortable", "comfortable", "comfortable", "tight", "no" ], [ "comfortable", "comfortable", "comfortable", "comfortable", "no" ], [ "comfortable", "comfortable", "comfortable", "comfortable", "tight" ] ], "pick": { "8": "qwen3.5:4b", "16": "gemma4", "32": "gemma4:26b", "64": "qwen3-coder:30b", "128": "qwen3-coder:30b (fast) + qwen3.5:122b (smart)" } } }, { "n": 2, "id": "tool", "title": "Pick your tool", "icon": "Box", "human": "Install Ollama (terminal) or LM Studio (app), then download one model. Say hi; if it answers, you're running local AI.", "goal": "A local model server is running and answers a prompt.", "links": [ "ollama", "lm-studio" ], "commands": { "macos-apple-silicon": [ { "run": "brew install ollama # or: curl -fsSL https://ollama.com/install.sh | sh (or the Mac app from ollama.com/download)", "note": "Option A (recommended for agents): Ollama", "mac_studio_tested": false }, { "run": "ollama serve # skip if the Ollama app is already running; for agents use: OLLAMA_CONTEXT_LENGTH=64000 ollama serve", "note": "Ollama docs: agents and coding tools should get at least 64000 tokens of context", "mac_studio_tested": false }, { "run": "ollama pull gemma4 # pick your size from the fit chart, e.g. qwen3.5:4b on 8GB, qwen3-coder:30b on 64GB+", "mac_studio_tested": false }, { "run": "brew install --cask lm-studio && lms get gemma-4 # lms get searches; pick a Gemma 4 or Qwen 3.5 build (--mlx for MLX on Apple Silicon)", "note": "Option B (GUI): LM Studio. Model search names are untested on Mac Studio", "mac_studio_tested": false }, { "run": "lms server start && lms load ", "mac_studio_tested": false } ], "linux": [ { "run": "curl -fsSL https://ollama.com/install.sh | sh && ollama pull gemma4", "mac_studio_tested": false } ], "windows": [ { "run": "irm https://ollama.com/install.ps1 | iex; ollama pull gemma4", "mac_studio_tested": false } ] }, "verify": [ { "run": "ollama run gemma4 \"Say hi in five words.\"", "expect": "a short greeting" }, { "run": "curl -s http://localhost:11434/v1/models", "expect": "JSON list including gemma4 (Ollama's OpenAI-compatible endpoint)" }, { "run": "ollama ps", "expect": "model listed; PROCESSOR should read 100% GPU (per Ollama context-length docs)" }, { "run": "curl -s http://localhost:1234/v1/models", "expect": "LM Studio path: JSON model list (port 1234 is LM Studio's usual default)" } ], "rollback": [ { "run": "ollama rm gemma4 && brew uninstall ollama", "mac_studio_tested": false }, { "run": "lms server stop && brew uninstall --cask lm-studio # models live under ~/.lmstudio", "mac_studio_tested": false } ] }, { "n": 3, "id": "agent", "title": "Point an agent at it", "icon": "Bot", "human": "Install an agent harness and connect it to your local model. Then just tell it what you want, in plain English.", "goal": "A harness (OpenCode, Goose, Aider or Hermes Agent) talks to the local endpoint http://localhost:11434/v1 (Ollama) or http://localhost:1234/v1 (LM Studio).", "links": [ "aider", "ollama" ], "harnesses": [ { "name": "OpenCode", "why": "Terminal coding agent; Ollama can configure it in one command.", "docs": "https://docs.ollama.com/integrations/opencode" }, { "name": "Goose", "why": "General agent with MCP extensions; desktop app or CLI.", "docs": "https://docs.ollama.com/integrations/goose" }, { "name": "Aider", "why": "Git-aware pair programmer for existing repos.", "docs": "https://aider.chat/docs/llms/ollama.html" }, { "name": "Hermes Agent", "why": "Nous Research agent with skills and cross-session memory.", "docs": "https://docs.ollama.com/integrations/hermes" } ], "commands": { "macos-apple-silicon": [ { "run": "curl -fsSL https://opencode.ai/install | bash # or: brew install anomalyco/tap/opencode", "note": "OpenCode (recommended)", "mac_studio_tested": false }, { "run": "ollama launch opencode", "note": "starts OpenCode with an inline config for the Ollama model you pick; does not overwrite ~/.config/opencode/opencode.json", "mac_studio_tested": false }, { "run": "mkdir -p ~/.config/opencode && cat > ~/.config/opencode/opencode.json <<'EOF'\n{\n \"$schema\": \"https://opencode.ai/config.json\",\n \"provider\": {\n \"ollama\": {\n \"npm\": \"@ai-sdk/openai-compatible\",\n \"name\": \"Ollama (local)\",\n \"options\": { \"baseURL\": \"http://localhost:11434/v1\" },\n \"models\": { \"gemma4\": { \"name\": \"Gemma 4\" } }\n }\n }\n}\nEOF", "note": "manual alternative, format from opencode.ai/docs/providers; model ids must match GET /v1/models. Back up any existing file first", "mac_studio_tested": false }, { "run": "curl -fsSL https://github.com/aaif-goose/goose/releases/download/stable/download_cli.sh | bash && goose configure", "note": "Goose: choose Configure Providers → Ollama → enter your model name", "mac_studio_tested": false }, { "run": "python -m pip install aider-install && aider-install && aider --model ollama_chat/gemma4", "note": "Aider: see /setups/aider", "mac_studio_tested": false }, { "run": "ollama launch hermes # or: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash", "note": "Hermes Agent: Ollama points it at http://127.0.0.1:11434/v1", "mac_studio_tested": false } ] }, "verify": [ { "run": "opencode run \"Reply with the word ready\"", "expect": "prints 'ready' using the local model (untested on Mac Studio)" }, { "run": "goose session -n itm-check", "expect": "Goose opens a session; type: say ready" } ], "rollback": [ { "run": "rm ~/.config/opencode/opencode.json # restore your backup if you had one", "mac_studio_tested": false }, { "run": "brew uninstall opencode # if installed via brew", "mac_studio_tested": false }, { "run": "# Goose / Hermes: follow uninstall steps in their docs (install scripts place a binary in your PATH)", "mac_studio_tested": false } ] }, { "n": 4, "id": "decompose", "title": "Your goal becomes workflows", "icon": "Workflow", "human": "The agent splits your goal into workflows like research, writing, files and memory. It sets up folders, an AGENTS.md, a memory file and the tools each one needs.", "goal": "A workspace with AGENTS.md, memory/MEMORY.md, one folder per workflow and MCP tools configured.", "links": [ "agents-md", "mcp-servers", "mem0", "docling" ], "workflows": [ [ "research", "Search", "fetch MCP server (uvx mcp-server-fetch)" ], [ "writing", "PenLine", "AGENTS.md style rules" ], [ "files", "FolderOpen", "filesystem MCP server" ], [ "memory", "Brain", "memory/MEMORY.md, upgrade to Mem0" ], [ "documents", "ScanText", "Docling" ] ], "commands": { "macos-apple-silicon": [ { "run": "mkdir -p ~/itm-agent/{workflows/{research,writing,files,documents},memory,tools} && cd ~/itm-agent && git init -q", "mac_studio_tested": false }, { "run": "cat > AGENTS.md <<'EOF'\n# AGENTS.md\n## Goal\n\n## Workflows\n- research/: gather sources, save notes as markdown with URLs\n- writing/: drafts; never publish or send without the user's OK\n- files/: only touch files inside ~/itm-agent\n- documents/: convert PDFs/Office files with `docling `\n## Memory\n- Read memory/MEMORY.md at the start of every session; append decisions and facts at the end.\n## Upgrades\n- Before installing anything, read the listing at https://insidethemachine.ai/setups/.md and run its verify step.\nEOF", "mac_studio_tested": false }, { "run": "printf '# MEMORY\\n\\n- %s: workspace created\\n' \"$(date +%F)\" > memory/MEMORY.md", "mac_studio_tested": false }, { "run": "cat > opencode.json <<'EOF'\n{\n \"$schema\": \"https://opencode.ai/config.json\",\n \"mcp\": {\n \"fetch\": { \"type\": \"local\", \"command\": [\"uvx\", \"mcp-server-fetch\"], \"enabled\": true },\n \"filesystem\": { \"type\": \"local\", \"command\": [\"npx\", \"-y\", \"@modelcontextprotocol/server-filesystem\", \"/Users/YOU/itm-agent\"], \"enabled\": true }\n }\n}\nEOF", "note": "project-level OpenCode config (format from opencode.ai/docs/mcp-servers); replace /Users/YOU. Requires uv (brew install uv) and Node", "mac_studio_tested": false } ] }, "verify": [ { "run": "find ~/itm-agent -maxdepth 2 -not -path '*/.git*'", "expect": "AGENTS.md, memory/MEMORY.md, workflows/* folders, opencode.json" }, { "run": "cd ~/itm-agent && opencode run \"Read AGENTS.md and list the workflows\"", "expect": "the agent lists the workflows (untested on Mac Studio)" } ], "rollback": [ { "run": "rm -rf ~/itm-agent # deletes the workspace, irreversible", "mac_studio_tested": false } ] }, { "n": 5, "id": "upgrade", "title": "It builds and upgrades itself", "icon": "RefreshCw", "human": "Your agent reads this site's agent pages, installs what's missing and sends hard jobs to a bigger model. Then it re-checks how close it is to the frontier.", "goal": "The agent pulls /llms.txt, installs missing pieces using each listing's verify/rollback, routes via LiteLLM (fast vs smart), and logs a self-check.", "links": [ "litellm", "mem0", "open-webui" ], "commands": { "macos-apple-silicon": [ { "run": "curl -s https://insidethemachine.ai/llms.txt && curl -s https://insidethemachine.ai/api/index.json", "note": "the agent's map of what exists", "mac_studio_tested": false }, { "run": "curl -s -H 'Accept: text/markdown' https://insidethemachine.ai/setups/litellm", "note": "any listing as markdown", "mac_studio_tested": false }, { "run": "brew install uv && uv tool install 'litellm[proxy]'", "note": "LiteLLM docs show pip install 'litellm[proxy]'; uv tool install is the isolated equivalent (untested on Mac Studio)", "mac_studio_tested": false }, { "run": "cat > ~/itm-agent/tools/litellm.yaml <<'EOF'\nmodel_list:\n - model_name: \"fast\"\n litellm_params:\n model: \"ollama_chat/qwen3-coder:30b\"\n - model_name: \"smart\"\n litellm_params:\n model: \"ollama_chat/qwen3.5:122b\"\nEOF", "note": "128GB example; on smaller machines map smart to the biggest model your chart allows, or to a cloud model you have a key for", "mac_studio_tested": false }, { "run": "litellm --config ~/itm-agent/tools/litellm.yaml # OpenAI-compatible proxy on http://0.0.0.0:4000", "mac_studio_tested": false } ] }, "verify": [ { "run": "curl -s http://0.0.0.0:4000/chat/completions -H 'Content-Type: application/json' -d '{\"model\":\"fast\",\"messages\":[{\"role\":\"user\",\"content\":\"ready?\"}]}'", "expect": "OpenAI-format JSON reply" }, { "run": "tail -5 ~/itm-agent/memory/MEMORY.md", "expect": "the agent logged what it installed and each verify result" } ], "rollback": [ { "run": "uv tool uninstall litellm && rm ~/itm-agent/tools/litellm.yaml", "mac_studio_tested": false }, { "run": "# for anything else the agent installed: run the rollback section of that listing's .md", "mac_studio_tested": false } ], "gap_check": "Official Gap Scores are pending test on Mac Studio. Until then the agent's self-check is: did every listing's verify step pass, plus tokens/sec from `ollama run --verbose` (untested on Mac Studio)." } ], "bootstrap_prompt": "You are my local AI setup agent. Work only inside ~/itm-agent. Ask before deleting anything, spending money, or sending anything outside this machine.\n\nContext: a local model server is already running (Ollama at http://localhost:11434/v1 or LM Studio at http://localhost:1234/v1).\n\n1. Read https://insidethemachine.ai/start.md and https://insidethemachine.ai/llms.txt. Use them as your runbook and catalog.\n2. My goal: .\n3. Split that goal into workflows (for example research, writing, files, documents, memory). Create ~/itm-agent with AGENTS.md (goal, workflows, rules), memory/MEMORY.md, and one folder per workflow, following step 4 of start.md.\n4. For each workflow, pick the tools it needs from the catalog. For each tool, fetch https://insidethemachine.ai/setups/.md, run its install and verify steps, and write the result to memory/MEMORY.md. If a verify step fails, run that listing's rollback and tell me.\n5. Set up LiteLLM routing (step 5): \"fast\" = my everyday local model, \"smart\" = the biggest model my RAM allows. Use smart only for hard tasks.\n6. Finish with a short report: what you installed, what passed, what failed, and what you'd upgrade next. Note that official Gap Scores are still pending test on Mac Studio.", "sources": [ "https://aider.chat/docs/llms/ollama.html", "https://aider.chat/docs/usage/conventions.html", "https://docs.litellm.ai/docs/providers/ollama", "https://docs.ollama.com/context-length", "https://docs.ollama.com/faq", "https://docs.ollama.com/integrations/goose", "https://docs.ollama.com/integrations/hermes", "https://docs.ollama.com/integrations/opencode", "https://docs.openclaw.ai/concepts/agent-workspace", "https://docs.openclaw.ai/concepts/soul", "https://github.com/aaif-goose/goose", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md", "https://github.com/ml-explore/mlx-lm", "https://github.com/modelcontextprotocol/servers", "https://goose-docs.ai/docs/getting-started/providers", "https://goose-docs.ai/docs/guides/context-engineering/using-goosehints", "https://hermes-agent.nousresearch.com/docs/user-guide/configuration", "https://hermes-agent.nousresearch.com/docs/user-guide/features/context-files", "https://hermes-agent.nousresearch.com/docs/user-guide/features/personality", "https://hermes-agent.nousresearch.com/docs/user-guide/features/skills", "https://lmstudio.ai/docs/cli", "https://ollama.com/library/gemma4/tags", "https://ollama.com/library/qwen3-coder/tags", "https://ollama.com/library/qwen3.5/tags", "https://opencode.ai/docs/cli/", "https://opencode.ai/docs/config/", "https://opencode.ai/docs/mcp-servers/", "https://opencode.ai/docs/providers/", "https://opencode.ai/docs/rules/" ], "gaps": { "title": "Hermes-style gaps: identity, instructions, and lean context", "intro": "Frontier apps feel smart partly because of what surrounds the model: an identity, project rules, skills, memory and careful context management. These four parts give a local agent the same scaffolding without a million-token window.", "parts": [ { "id": "files", "title": "AGENTS.md and SOUL.md", "icon": "FileText", "points": [ { "icon": "FileCode", "title": "AGENTS.md = how to work here", "human": "A plain markdown file of project rules your agent reads every session. Most harnesses (OpenCode, Goose, Hermes, OpenClaw) load it automatically.", "agent": { "facts": [ "AGENTS.md: open, harness-agnostic project-instructions file (see /setups/agents-md).", "Hermes: loads AGENTS.md from the CWD at startup and subdirectories progressively; .hermes.md / HERMES.md take highest priority (docs: user-guide/features/context-files).", "OpenCode: reads AGENTS.md in the project and ~/.config/opencode/AGENTS.md globally (opencode.ai/docs/rules).", "Goose: CONTEXT_FILE_NAMES default is [\"AGENTS.md\", \".goosehints\"] (goose-docs.ai using-goosehints).", "OpenClaw: workspace files include AGENTS.md, SOUL.md, IDENTITY, USER (docs.openclaw.ai templates); default workspace ~/.openclaw/workspace." ], "template": { "path": "~/itm-agent/AGENTS.md", "body": "# AGENTS.md\n\n## Project\nOne sentence: what this workspace is for.\n## Rules\n- Work only inside this folder. Ask before deleting, spending, or sending anything.\n- Prefer small diffs. Run the verify step after every install.\n## Map (load on demand, do not paste here)\n- workflows//AGENTS.md : rules for that workflow only\n- memory/MEMORY.md : read the last 30 lines at session start; append decisions\n- skills/ : one folder per repeatable procedure (SKILL.md each)\n## Output hygiene\n- Pipe long command output through `tail -n 50` or `head -c 4000`.\n- Summarize files; never print whole logs or datasets." } } }, { "icon": "UserRound", "title": "SOUL.md = who the agent is", "human": "A persona file: voice, values and limits. Hermes puts it first in the system prompt; OpenClaw injects it into normal sessions.", "agent": { "facts": [ "Hermes: SOUL.md is the primary identity, slot #1 in the system prompt, loaded only from $HERMES_HOME/SOUL.md (default ~/.hermes/SOUL.md). Hermes seeds a starter file and never overwrites yours (docs: features/personality).", "OpenClaw: 'SOUL.md is where your agent's voice lives. OpenClaw injects it into normal sessions' (docs.openclaw.ai/concepts/soul). Lives in the agent workspace.", "OpenCode / Goose / Aider have no SOUL.md concept: put persona lines at the top of AGENTS.md or list SOUL.md as an extra instruction file (below)." ], "template": { "path": "~/.hermes/SOUL.md", "body": "# SOUL.md\n\nYou are Kit, a calm, direct assistant running on a local model.\n## Voice\n- Short sentences. Lead with the answer. No filler, no hype.\n- Say \"I don't know\" and check, rather than guess.\n## Values\n- The user's machine and data stay local unless they say otherwise.\n- Ask before irreversible actions.\n## Never\n- Invent numbers, benchmarks or citations." } } } ] }, { "id": "point", "title": "Point each harness at the files", "icon": "Signpost", "points": [ { "icon": "Sparkles", "title": "Hermes", "human": "Edit ~/.hermes/SOUL.md for identity and start Hermes inside your workspace so it finds AGENTS.md. Skills live in ~/.hermes/skills.", "agent": { "commands": [ "$EDITOR ~/.hermes/SOUL.md", "cd ~/itm-agent && hermes" ], "notes": [ "Directory layout per docs: ~/.hermes/{config.yaml,SOUL.md,memories/,skills/}." ] } }, { "icon": "CodeXml", "title": "OpenCode", "human": "It reads AGENTS.md on its own. Add extra files like SOUL.md with one line of config.", "agent": { "commands": [ "cat > ~/itm-agent/opencode.json <<'EOF'\n{\n \"$schema\": \"https://opencode.ai/config.json\",\n \"instructions\": [\"SOUL.md\", \"workflows/*/AGENTS.md\"]\n}\nEOF" ], "notes": [ "'instructions' accepts paths, globs and URLs; combined with AGENTS.md (opencode.ai/docs/rules). Merge with the mcp block from step 4 if you created one." ] } }, { "icon": "Feather", "title": "Goose", "human": "Goose reads AGENTS.md and .goosehints by default. Tell it about other files with one environment variable.", "agent": { "commands": [ "export CONTEXT_FILE_NAMES='[\"AGENTS.md\", \"SOUL.md\", \".goosehints\"]'", "cd ~/itm-agent && goose session -n itm" ], "notes": [ "Global hints: ~/.config/goose/.goosehints; local hints win over global." ] } }, { "icon": "GitBranch", "title": "Aider", "human": "Aider loads rule files as read-only context. Put them in its config so every session gets them.", "agent": { "commands": [ "printf 'read: [AGENTS.md, SOUL.md]\\n' > ~/itm-agent/.aider.conf.yml", "aider --read AGENTS.md # one-off equivalent" ], "notes": [ "Docs: aider.chat/docs/usage/conventions.html (read files are marked read-only and cached if prompt caching is enabled)." ] } } ] }, { "id": "bloat", "title": "Avoid context bloat", "icon": "Layers", "points": [ { "icon": "Feather", "title": "Keep always-loaded files lean", "human": "AGENTS.md and SOUL.md ride along on every turn, so every line costs you. Keep rules and pointers there, and nothing else.", "agent": { "values": { "AGENTS.md": "aim for under ~150 lines (house rule of thumb, not a harness limit)", "SOUL.md": "aim for under ~40 lines" } } }, { "icon": "BookOpen", "title": "Skills load only when needed", "human": "Write repeatable procedures as skills: the agent sees a one-line description and opens the full text only when it needs it. Hermes calls this progressive disclosure.", "agent": { "facts": [ "Hermes skills: on-demand documents following a progressive disclosure pattern to minimize token usage; agentskills.io compatible; stored in ~/.hermes/skills/." ], "layout": "skills//SKILL.md with front-matter name + description; body loaded on demand" } }, { "icon": "Search", "title": "Retrieve, don't stuff", "human": "Keep notes and documents on disk and let the agent search them. Pull in a few relevant chunks instead of whole files.", "agent": { "tools": [ "memory/MEMORY.md tail + grep", "Mem0 (/setups/mem0)", "Docling to convert PDFs before retrieval (/setups/docling)", "Open WebUI knowledge bases (/setups/open-webui)" ] } }, { "icon": "Minimize2", "title": "Compact long sessions", "human": "When a chat gets long, have the harness summarize the middle and keep the recent tail. Turn it on before the window fills.", "agent": { "config": { "opencode.json": { "compaction": { "auto": true, "prune": true, "reserved": 10000 } }, "~/.hermes/config.yaml": "compression:\n enabled: true\n threshold: 0.50 # compress at 50% of context limit\n target_ratio: 0.20", "hermes (manual)": "/compress", "aider (manual)": "/clear or /drop unused files" }, "notes": [ "OpenCode prune=true removes old tool outputs (default false). Hermes: the summary model needs a context window at least as large as the main model's." ] } }, { "icon": "Scissors", "title": "Trim tool output", "human": "Most bloat is logs and command output, not your words. Cap it at the source.", "agent": { "commands": [ " 2>&1 | tail -n 50", " | head -c 4000", "rg -n --max-count 20 'pattern' ." ] } }, { "icon": "FolderTree", "title": "One folder per workflow", "human": "Give each workflow its own folder and its own small AGENTS.md. The agent only loads the rules for the job in front of it.", "agent": { "layout": "~/itm-agent/workflows/{research,writing,files,documents}/AGENTS.md; start the harness inside the workflow folder (Hermes and OpenCode both pick up subdirectory AGENTS.md)" } } ] }, { "id": "efficient", "title": "Keep the local model efficient", "icon": "Gauge", "points": [ { "icon": "MemoryStick", "title": "Pick the right quant", "human": "A 4-bit model is about a quarter the size of 16-bit and usually close in quality. Use the fit chart and leave room for context.", "agent": { "values": { "ollama default tags": "4-bit-class quants (sizes in step 1 chart)", "rule": "prefer a bigger model at 4-bit over a smaller one at 8-bit when it fits (rule of thumb, untested on Mac Studio)" } } }, { "icon": "Ruler", "title": "Set context length on purpose", "human": "Bigger windows eat memory fast. Give agents enough (64k) and no more.", "agent": { "ollama": [ "OLLAMA_CONTEXT_LENGTH=64000 ollama serve # server-wide", "/set parameter num_ctx 64000 # inside ollama run", "API: \"options\": {\"num_ctx\": 64000}", "ollama ps # CONTEXT column shows what was allocated" ], "llama.cpp": [ "llama-server -m model.gguf -c 65536 # -c, --ctx-size; 0 = from model" ], "mlx-lm": [ "mlx_lm.generate --max-kv-size 65536 ... # rotating fixed-size KV cache" ], "notes": [ "Ollama: RAM scales with OLLAMA_NUM_PARALLEL × OLLAMA_CONTEXT_LENGTH; keep OLLAMA_NUM_PARALLEL=1 for a single agent." ] } }, { "icon": "Archive", "title": "Shrink the KV cache", "human": "The cache that holds your context can be quantized too. 8-bit halves its memory with very little quality loss.", "agent": { "ollama": [ "OLLAMA_FLASH_ATTENTION=1 OLLAMA_KV_CACHE_TYPE=q8_0 ollama serve", "# values: f16 (default), q8_0 (~1/2 memory, recommended), q4_0 (~1/4, more loss at long context); global for all models" ], "llama.cpp": [ "llama-server -m model.gguf -c 65536 -fa on -ctk q8_0 -ctv q8_0" ], "notes": [ "llama.cpp allowed K/V types: f32, f16, bf16, q8_0, q4_0, q4_1, iq4_nl, q5_0, q5_1 (default f16)." ] } }, { "icon": "Zap", "title": "Reuse the prompt", "human": "Your AGENTS.md and SOUL.md are the same every turn, so cache them instead of re-reading. Keep them stable so the cache keeps hitting.", "agent": { "llama.cpp": [ "llama-server ... --cache-reuse 256 # reuse cached chunks via KV shifting; needs prompt caching" ], "mlx-lm": [ "cat AGENTS.md | mlx_lm.cache_prompt --model --prompt - --prompt-cache-file agents.safetensors", "mlx_lm.generate --prompt-cache-file agents.safetensors --prompt \"...\"" ], "ollama": [ "OLLAMA_KEEP_ALIVE=30m ollama serve # keep the model loaded between turns" ], "notes": [ "Editing the top of an always-loaded file invalidates the cached prefix: put changing content at the end." ] } }, { "icon": "Route", "title": "Small model first, big model when needed", "human": "Send routine turns to a fast model and hard ones to your biggest. LiteLLM gives both one address.", "agent": { "see": "Step 5 litellm.yaml (fast / smart aliases)", "notes": [ "Hermes can also run compression on a separate model via auxiliary.compression (model, base_url)." ] } } ] } ], "sources": [ "https://hermes-agent.nousresearch.com/docs/user-guide/features/personality", "https://hermes-agent.nousresearch.com/docs/user-guide/features/context-files", "https://hermes-agent.nousresearch.com/docs/user-guide/features/skills", "https://hermes-agent.nousresearch.com/docs/user-guide/configuration", "https://docs.openclaw.ai/concepts/soul", "https://docs.openclaw.ai/concepts/agent-workspace", "https://opencode.ai/docs/rules/", "https://opencode.ai/docs/config/", "https://goose-docs.ai/docs/guides/context-engineering/using-goosehints", "https://aider.chat/docs/usage/conventions.html", "https://docs.ollama.com/faq", "https://docs.ollama.com/context-length", "https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md", "https://github.com/ml-explore/mlx-lm" ] } }