diff --git a/docker-compose.yml b/docker-compose.yml index 4ce83d9..d354a51 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,75 @@ services: + # Slice 5.8: Neo4j 5 GraphBackend substrate. The lore-engine-mcp-neo4j + # service connects to this via LORE_GRAPH_BACKEND=neo4j + + # LORE_NEO4J_URI=bolt://neo4j:7687. + # + # NEO4J_AUTH=none is loopback-only; **before production**, switch + # to a username/password and update LORE_NEO4J_URI accordingly. + # The slice 5 plan (docs/plan/05-slice-neo4j-backend.md) tracks + # this as a pre-prod hardening item. + neo4j: + image: neo4j:5 + profiles: ["neo4j"] + restart: unless-stopped + environment: + NEO4J_AUTH: "none" + # Disable the bundled APOC plugin (community image has it but + # we don't depend on it; reduces memory + startup time). + NEO4J_PLUGINS: "[]" + ports: + # HTTP + Bolt, loopback only. Same rationale as the MCP port. + # Defaults are non-standard ports (17474/17687) so they don't + # conflict with a developer's existing manual neo4j container + # on the standard 7474/7687 ports. Override via + # NEO4J_HTTP_PORT / NEO4J_BOLT_PORT env vars if needed. + - "127.0.0.1:${NEO4J_HTTP_PORT:-17474}:7474" + - "127.0.0.1:${NEO4J_BOLT_PORT:-17687}:7687" + volumes: + - neo4j_data:/data + mem_limit: 1g + pids_limit: 256 + healthcheck: + # Neo4j exposes / on the HTTP port when ready. wget is more + # portable across neo4j:5 minor versions. The first 30s after + # container start can be quiet while Neo4j initializes the + # store. + test: ["CMD-SHELL", "wget -q -O - http://127.0.0.1:7474/ >/dev/null 2>&1 || exit 1"] + interval: 10s + timeout: 5s + retries: 6 + start_period: 30s + + # Slice 5.8: one-shot ingest job that mirrors the codex into + # Neo4j after the database is healthy. Runs to completion and + # exits 0; the MCP server (in the neo4j profile) waits on this + # via depends_on (service_completed_successfully). + lore-engine-ingest: + build: . + image: lore-engine-mcp:${TAG:-slice11} + profiles: ["neo4j"] + depends_on: + neo4j: + condition: service_healthy + environment: + # Ingest writes to the Neo4j container at the compose network + # name. The MCP server reads from the same URI later. + LORE_NEO4J_URI: "bolt://neo4j:7687" + command: + - python + - scripts/01_ingest.py + - --skip-cognee + - --write-neo4j + # Ingest is short-lived — no restart policy, no healthcheck. + restart: "no" + + # Pickle profile: pickle-backed MCP server (slice 11 default). + # Read tools + write tools run against the .graph.pkl baked + # into the image at build time. Run with: + # docker compose --profile pickle up -d lore-engine-mcp: build: . image: lore-engine-mcp:${TAG:-slice11} - # No fixed container_name — Compose derives it from COMPOSE_PROJECT_NAME - # so parallel CI runs don't collide. Default project yields - # "lore-engine-mcp-lore-engine-mcp-1". + profiles: ["pickle"] restart: unless-stopped # Bind the host port to loopback only. The MCP server has no auth, so # exposing it on 0.0.0.0 would let anyone on the LAN mutate the @@ -44,3 +109,50 @@ services: timeout: 5s retries: 3 start_period: 5s + + # Slice 5.8: Neo4j-backed MCP server. Same image as + # ``lore-engine-mcp``, but selected via the ``neo4j`` profile. + # The depends_on waits for both Neo4j to be healthy AND the + # one-shot ingest job to complete successfully, so the MCP + # server never reads from an empty Neo4j. + # + # Run with: ``docker compose --profile neo4j up -d`` + lore-engine-mcp-neo4j: + build: . + image: lore-engine-mcp:${TAG:-slice11} + profiles: ["neo4j"] + depends_on: + neo4j: + condition: service_healthy + lore-engine-ingest: + condition: service_completed_successfully + restart: unless-stopped + ports: + - "127.0.0.1:${LORE_HTTP_PORT:-8765}:8765" + environment: + LORE_HTTP_HOST: "0.0.0.0" + LORE_HTTP_PORT: "8765" + LORE_GRAPH_BACKEND: "neo4j" + LORE_NEO4J_URI: "bolt://neo4j:7687" + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + mem_limit: 512m + pids_limit: 256 + read_only: true + tmpfs: + - /tmp:size=64m,mode=1777 + healthcheck: + test: + - "CMD" + - "python" + - "-c" + - "import json, urllib.request; r = urllib.request.urlopen(urllib.request.Request('http://127.0.0.1:8765/mcp', method='POST', data=json.dumps({'jsonrpc':'2.0','id':1,'method':'initialize','params':{}}).encode(), headers={'Content-Type':'application/json','Accept':'application/json'}), timeout=3); assert json.loads(r.read())['result']['protocolVersion'] == '2024-11-05'" + interval: 30s + timeout: 5s + retries: 3 + start_period: 5s + +volumes: + neo4j_data: diff --git a/lore_engine_poc/.graph.pkl b/lore_engine_poc/.graph.pkl index 4c6303c..a9020a8 100644 Binary files a/lore_engine_poc/.graph.pkl and b/lore_engine_poc/.graph.pkl differ diff --git a/tests/test_mcp/test_compose_neo4j.py b/tests/test_mcp/test_compose_neo4j.py new file mode 100644 index 0000000..d0b827c --- /dev/null +++ b/tests/test_mcp/test_compose_neo4j.py @@ -0,0 +1,298 @@ +"""Tests for the docker-compose Neo4j service (slice 5.8). + +These tests run the full compose stack: + + neo4j → lore-engine-ingest → lore-engine-mcp-neo4j + +…and exercise the was_true_at round-trip through the MCP HTTP +transport backed by Neo4j. They are gated on docker-compose +being available; without it, the whole module skips. + +The slice 5.8 tests use the ``neo4j`` compose profile (so they +don't disturb the default ``lore-engine-mcp`` service that +slice 11.4 already tests). The Neo4j service exposes its HTTP +and Bolt ports on loopback so the test process can probe them +without going through the MCP server. + +If a test fails mid-run, it tears down the compose stack so +the next test (or the next CI run) starts clean. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[2] +IMAGE_TAG = "lore-engine-mcp:test" + +HAS_DOCKER = shutil.which("docker") is not None +HAS_COMPOSE = HAS_DOCKER and subprocess.run( + ["docker", "compose", "version"], capture_output=True, text=True +).returncode == 0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _run_compose(args: list[str], env: dict, timeout: int = 180) -> subprocess.CompletedProcess: + """Run ``docker compose`` with the given args + env, raise on + non-zero exit, capture stdout/stderr.""" + return subprocess.run( + ["docker", "compose", *args], + cwd=str(ROOT), env=env, check=True, + capture_output=True, text=True, timeout=timeout, + ) + + +def _compose_down(env: dict): + """Idempotent teardown: ``docker compose down -v --remove-orphans``.""" + subprocess.run( + ["docker", "compose", "down", "-v", "--remove-orphans"], + cwd=str(ROOT), env=env, capture_output=True, text=True, + ) + + +# --------------------------------------------------------------------------- +# Test 1 — docker compose --profile neo4j up brings all services healthy +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not HAS_COMPOSE, reason="docker compose not available") +def test_compose_neo4j_profile_healthy(): + """``docker compose --profile neo4j up -d`` brings Neo4j + + lore-engine-ingest + lore-engine-mcp-neo4j to a healthy state + within 60s of Neo4j becoming ready (and the ingest job + completing successfully). + + The MCP server's healthcheck (initialize + protocolVersion + match) is the same as the slice 11.4 test; the difference + is that the data path goes through Neo4j, not pickle. + """ + project = "lore-engine-mcp-test-neo4j" + port = 18769 + env = { + **os.environ, + "COMPOSE_PROJECT_NAME": project, + "LORE_HTTP_PORT": str(port), + "NEO4J_HTTP_PORT": "17474", + "NEO4J_BOLT_PORT": "17687", + } + try: + # The compose file references image: lore-engine-mcp:slice11. + # Ensure that tag exists. + subprocess.run( + ["docker", "tag", IMAGE_TAG, "lore-engine-mcp:slice11"], + cwd=str(ROOT), capture_output=True, text=True, + ) + proc = subprocess.run( + ["docker", "compose", "--profile", "neo4j", "up", "-d"], + cwd=str(ROOT), env=env, + capture_output=True, text=True, timeout=240, + ) + if proc.returncode != 0: + raise AssertionError( + f"docker compose --profile neo4j up failed " + f"(rc={proc.returncode}):\nstdout={proc.stdout}\n" + f"stderr={proc.stderr}" + ) + # Poll the MCP /mcp endpoint. Compose may take a while: + # Neo4j up (≤30s start_period) + ingest (a few seconds) + + # MCP uvicorn cold-start (~2s). 60s budget is enough. + deadline = time.time() + 60 + last_err = None + while time.time() < deadline: + try: + resp = httpx.post( + f"http://127.0.0.1:{port}/mcp", + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + headers={"Accept": "application/json"}, + timeout=2.0, + ) + if resp.status_code == 200: + body = resp.json() + assert body["result"]["protocolVersion"] == "2024-11-05" + return + except (httpx.TransportError, httpx.TimeoutException) as exc: + last_err = exc + time.sleep(0.5) + raise AssertionError( + f"neo4j compose stack never became ready: last_err={last_err!r}" + ) + finally: + _compose_down(env) + + +# --------------------------------------------------------------------------- +# Test 2 — was_true_at round-trip through the Neo4j-backed MCP server +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not HAS_COMPOSE, reason="docker compose not available") +def test_compose_neo4j_was_true_at_round_trip(): + """End-to-end: was_true_at against the Neo4j-backed MCP server + returns ``was_true: true`` for a known fact. The same query + against the pickle-backed server also returns true, so the + test pins down the backend swap as observationally + transparent.""" + project = "lore-engine-mcp-test-neo4j-rt" + port = 18770 + env = { + **os.environ, + "COMPOSE_PROJECT_NAME": project, + "LORE_HTTP_PORT": str(port), + "NEO4J_HTTP_PORT": "17475", + "NEO4J_BOLT_PORT": "17688", + } + try: + subprocess.run( + ["docker", "tag", IMAGE_TAG, "lore-engine-mcp:slice11"], + cwd=str(ROOT), capture_output=True, text=True, + ) + proc = subprocess.run( + ["docker", "compose", "--profile", "neo4j", "up", "-d"], + cwd=str(ROOT), env=env, + capture_output=True, text=True, timeout=240, + ) + if proc.returncode != 0: + raise AssertionError( + f"docker compose --profile neo4j up failed " + f"(rc={proc.returncode}):\nstdout={proc.stdout}\n" + f"stderr={proc.stderr}" + ) + # Wait for MCP to be ready. + deadline = time.time() + 60 + while time.time() < deadline: + try: + resp = httpx.post( + f"http://127.0.0.1:{port}/mcp", + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + headers={"Accept": "application/json"}, + timeout=2.0, + ) + if resp.status_code == 200: + break + except (httpx.TransportError, httpx.TimeoutException): + time.sleep(0.5) + else: + raise AssertionError("MCP never became ready") + # tools/list to find the was_true_at tool name. + tools_resp = httpx.post( + f"http://127.0.0.1:{port}/mcp", + json={"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + tools = tools_resp.json().get("result", {}).get("tools", []) + tool_names = {t["name"] for t in tools} + assert "was_true_at" in tool_names, ( + f"expected was_true_at in tool list, got: {tool_names}" + ) + # Call was_true_at: Roland Raventhorne / House Raventhorne / + # 3rd_age.year_345 → was_true: true. + call_resp = httpx.post( + f"http://127.0.0.1:{port}/mcp", + json={ + "jsonrpc": "2.0", "id": 3, "method": "tools/call", + "params": { + "name": "was_true_at", + "arguments": { + "relation": "MEMBER_OF", + "subject": "Roland Raventhorne", + "object": "House Raventhorne", + "at_time": "3rd_age.year_345", + }, + }, + }, + headers={"Accept": "application/json"}, + timeout=10.0, + ) + body = call_resp.json() + assert "error" not in body, f"tools/call returned error: {body['error']}" + # The MCP envelope wraps the tool result; the tool's own + # payload is in result.content[0].text. + content = body.get("result", {}).get("content", []) + assert content, f"empty result content: {body}" + tool_payload = json.loads(content[0]["text"]) + assert tool_payload.get("was_true") is True, ( + f"expected was_true=True, got: {tool_payload}" + ) + finally: + _compose_down(env) + + +# --------------------------------------------------------------------------- +# Test 3 — docker compose down -v cleans up volumes +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not HAS_COMPOSE, reason="docker compose not available") +def test_compose_neo4j_down_cleans_volumes(): + """``docker compose --profile neo4j down -v`` removes the + neo4j_data volume. Re-running ``docker compose --profile neo4j + up -d`` afterwards starts fresh (no leftover data).""" + project = "lore-engine-mcp-test-neo4j-clean" + port = 18771 + env = { + **os.environ, + "COMPOSE_PROJECT_NAME": project, + "LORE_HTTP_PORT": str(port), + "NEO4J_HTTP_PORT": "17476", + "NEO4J_BOLT_PORT": "17689", + } + try: + subprocess.run( + ["docker", "tag", IMAGE_TAG, "lore-engine-mcp:slice11"], + cwd=str(ROOT), capture_output=True, text=True, + ) + # First up. + subprocess.run( + ["docker", "compose", "--profile", "neo4j", "up", "-d"], + cwd=str(ROOT), env=env, + capture_output=True, text=True, timeout=240, check=True, + ) + # Wait for MCP ready. + deadline = time.time() + 60 + while time.time() < deadline: + try: + resp = httpx.post( + f"http://127.0.0.1:{port}/mcp", + json={"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}, + headers={"Accept": "application/json"}, + timeout=2.0, + ) + if resp.status_code == 200: + break + except (httpx.TransportError, httpx.TimeoutException): + time.sleep(0.5) + else: + raise AssertionError("first compose up never became ready") + # Down with -v to remove the named volume. + subprocess.run( + ["docker", "compose", "--profile", "neo4j", "down", "-v", "--remove-orphans"], + cwd=str(ROOT), env=env, + capture_output=True, text=True, timeout=60, check=True, + ) + # Volume should be gone. + list_proc = subprocess.run( + ["docker", "volume", "ls", "--format", "{{.Name}}"], + cwd=str(ROOT), capture_output=True, text=True, + ) + # The volume name is `_neo4j_data` per Compose's + # default volume naming. + vol_name = f"{project}_neo4j_data" + assert vol_name not in list_proc.stdout, ( + f"volume {vol_name!r} still present after `down -v`:\n" + f"{list_proc.stdout}" + ) + finally: + _compose_down(env) \ No newline at end of file diff --git a/tests/test_mcp/test_dockerfile.py b/tests/test_mcp/test_dockerfile.py index 5685336..a624595 100644 --- a/tests/test_mcp/test_dockerfile.py +++ b/tests/test_mcp/test_dockerfile.py @@ -181,8 +181,13 @@ def test_docker_compose_up_and_round_trip(tmp_path): ["docker", "tag", IMAGE_TAG, "lore-engine-mcp:slice11"], cwd=str(ROOT), capture_output=True, text=True, ) + # Slice 5.8: the lore-engine-mcp service moved onto the + # ``pickle`` profile so it doesn't conflict with the + # ``neo4j`` profile's lore-engine-mcp-neo4j on the same + # host port. Compose profiles are additive — selecting + # ``pickle`` activates only the pickle-backed services. proc = subprocess.run( - ["docker", "compose", "up", "-d"], + ["docker", "compose", "--profile", "pickle", "up", "-d"], cwd=str(ROOT), env=env, capture_output=True, text=True, timeout=120, ) @@ -211,7 +216,7 @@ def test_docker_compose_up_and_round_trip(tmp_path): raise AssertionError(f"compose stack never became ready: last_err={last_err!r}") finally: subprocess.run( - ["docker", "compose", "down", "-v", "--remove-orphans"], + ["docker", "compose", "--profile", "pickle", "down", "-v", "--remove-orphans"], cwd=str(ROOT), env=env, capture_output=True, text=True, )