diff --git a/README.md b/README.md index b5f54e1..ed2bd1d 100644 --- a/README.md +++ b/README.md @@ -180,7 +180,7 @@ curl -s -X POST http://localhost:8765/mcp \ - **No LLM in the loop.** The MCP gateway is a tool server; the LLM client (Claude, GPT, anything) is the consumer. This is intentional — the POC validates the data and tool layers, not the LLM reasoning. The reasoning harness is in the design docs (`lore-engine/docs/07-reasoning-harness.md`) and would be added as a system prompt in a real deployment. -- **Consistency detection rules are not implemented.** The `consistency` plugin and its 4 violation tools are live (v2.T3), but every tool returns an empty list. The actual detection logic per `lore-engine/docs/04-consistency.md` lands in T5. +- **Consistency detection is real (v2.T5).** The 4 tools (`find_contradictions`, `find_anachronisms`, `find_orphans`, `find_ontology_violations`) query pre-materialized violation nodes in Neo4j. The seed (`seed.py:seed_violations`) computes the violations from the same heuristics (overlapping `MEMBER_OF` windows, `Person.born > event_year`, world entities with no relations, and `:OntologyRule`-driven checks) so the math is visible in plain Python — not hidden in Cypher. - **No world-builder UI.** Everything is `curl` and `cypher-shell`. The UI is a v2 feature. @@ -188,7 +188,7 @@ curl -s -X POST http://localhost:8765/mcp \ ## Next steps after this POC -- Implement the consistency detection rules behind the 4 stub tools (T5). +- ~~Implement the consistency detection rules behind the 4 stub tools (T5).~~ **Done.** - Add the embedding-based semantic search plugin (uses the `Image.caption` and any future `Person.summary` text). - Add an LLM client that consumes the gateway with the reasoning harness system prompt and runs the 5 question types from the design. diff --git a/examples/test_consistency.sh b/examples/test_consistency.sh new file mode 100755 index 0000000..1dfb5c2 --- /dev/null +++ b/examples/test_consistency.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# lore-engine-poc — consistency engine end-to-end test (v2.T5) +# +# Calls each of the 4 consistency tools against the running gateway and +# asserts the violation count matches the seeded expectations: +# find_contradictions -> 1 +# find_anachronisms -> 1 +# find_orphans -> 1 +# find_ontology_violations -> 2 +# total -> 5 +# +# Run with: bash examples/test_consistency.sh +set -e +GATEWAY=${GATEWAY:-http://localhost:8765/mcp} + +# ─── helpers ──────────────────────────────────────────────────────────────── + +# call +# Returns the raw response text (one line, the tool's JSON envelope). +call() { + local name=$1; shift + local args=$1; shift + curl -s -X POST "$GATEWAY" \ + -H "Content-Type: application/json" \ + -d "{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{\"name\":\"$name\",\"arguments\":$args}}" +} + +# extract_count -> prints just the count field +extract_count() { + local raw=$1 + echo "$raw" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['result']['content'][0]['text'])" \ + | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['count'])" +} + +# pretty -> prints the tool envelope as pretty JSON +pretty() { + local raw=$1 + echo "$raw" | python3 -c "import json,sys; d=json.load(sys.stdin); print(d['result']['content'][0]['text'])" \ + | python3 -m json.tool +} + +# assert_count +assert_count() { + local tool=$1; shift + local args=$1; shift + local expected=$1 + local got + got=$(extract_count "$(call "$tool" "$args")") + if [ "$got" = "$expected" ]; then + echo " ✓ $tool -> count=$got (expected $expected)" + else + echo " ✗ $tool -> count=$got (expected $expected)" + pretty "$(call "$tool" "$args")" >&2 + exit 1 + fi +} + +# ─── tests ────────────────────────────────────────────────────────────────── + +echo "=== v2.T5 consistency engine — end-to-end check ===" +echo + +echo "1. find_contradictions" +assert_count "find_contradictions" '{"severity":"any"}' 1 +assert_count "find_contradictions" '{"severity":"error"}' 1 +assert_count "find_contradictions" '{"severity":"warn"}' 0 +echo + +echo "2. find_anachronisms" +assert_count "find_anachronisms" '{"severity":"any"}' 1 +assert_count "find_anachronisms" '{"severity":"error"}' 1 +assert_count "find_anachronisms" '{"severity":"warn"}' 0 +echo + +echo "3. find_orphans" +assert_count "find_orphans" '{}' 1 +echo + +echo "4. find_ontology_violations" +assert_count "find_ontology_violations" '{"severity":"any"}' 2 +assert_count "find_ontology_violations" '{"severity":"warn"}' 2 +assert_count "find_ontology_violations" '{"severity":"error"}' 0 +echo + +echo "=== violation details (sanity) ===" +echo +echo "Contradiction:" +pretty "$(call find_contradictions '{"severity":"any"}')" +echo +echo "Anachronism:" +pretty "$(call find_anachronisms '{"severity":"any"}')" +echo +echo "Orphan:" +pretty "$(call find_orphans '{}')" +echo +echo "Ontology violations:" +pretty "$(call find_ontology_violations '{"severity":"any"}')" +echo + +echo "=== all 4 tools passed, total = 5 violations ===" diff --git a/plugins/consistency.py b/plugins/consistency.py index 77d484a..56cb618 100644 --- a/plugins/consistency.py +++ b/plugins/consistency.py @@ -1,34 +1,101 @@ """ -consistency plugin — violation detection surface. +consistency plugin — violation detection surface (v2.T5). -Tools (all stubbed for v2.T3; real implementations land in T5): -- find_contradictions(severity): find Contradiction nodes. -- find_anachronisms(severity): find Anachronism nodes. -- find_orphans(): find Orphan nodes. -- find_ontology_violations(): find OntologyViolation nodes. +Four tools, each returning {"violations": [...], "count": N}. Violations +are Neo4j nodes with the labels Contradiction, Anachronism, Orphan, and +OntologyViolation, pre-materialized by the seed (see seed.py) using the +same heuristics the tools re-run defensively. This gives the LLM caller +stable violation ids and the seed.py reviewer a clear, inspectable +detection surface — no hidden magic in the cypher. -Each tool returns a stubbed {"violations": [], "count": 0} today. T5 wires -the actual detection rules per lore-engine/docs/04-consistency.md. +Tools: +- find_contradictions(severity): surfaced Contradiction nodes. +- find_anachronisms(severity): surfaced Anachronism nodes. +- find_orphans(): orphan Person/Item/Location/Event + (live query; no severity filter). +- find_ontology_violations(severity): OntologyRule-driven checks plus + surfaced OntologyViolation nodes. + +Severities: "any" (default), "error", "warn". """ +import re from server import get_neo4j, REGISTRY +# ─── Helpers ──────────────────────────────────────────────────────────────── + def _q(query, params=None): - """Run a single read query against Neo4j, return list of dicts.""" + """Run a read query against Neo4j, return list of dicts.""" driver = get_neo4j() with driver.session() as s: result = s.run(query, params or {}) return [dict(r) for r in result] -def _empty(): - """Stub envelope shared by all 4 tools until T5 wires the real Cypher.""" - return {"violations": [], "count": 0} +# Canonical time string -> year. e.g. "2nd_age.year_230" -> 230. +# Cypher doesn't have a built-in "extract trailing int" but apoc.text.regex +# groups could do it; for the POC we keep detection in the seed (see +# seed.py:_year_from_time) so the math is visible in plain Python. +_YEAR_RE = re.compile(r"year_(\d+)$") +def _year(time_str): + if not isinstance(time_str, str): + return None + m = _YEAR_RE.search(time_str) + return int(m.group(1)) if m else None + + +def _envelope(rows, label): + """Shape a list of Neo4j-node dicts into the {violations, count} envelope. + + The Cypher queries return `n` (the node) plus a few computed fields + (rule_id, person_id, etc.) so the LLM/operator can see *why* the + violation exists without re-querying. + """ + violations = [] + for r in rows: + n = r.get("n") or {} + v = { + "id": n.get("id"), + "label": label, + "severity": n.get("severity"), + "status": n.get("status"), + "details": n.get("details"), + "detected_at": n.get("detected_at"), + } + # Optional link-back fields (rule_id, person_id, etc.) when present. + for opt in ("rule_id", "entity_id", "person_id", "event_id"): + if opt in r and r[opt] is not None: + v[opt] = r[opt] + violations.append(v) + return {"violations": violations, "count": len(violations)} + + +def _severity_where(severity): + """Return (cypher_clause, params) for a leading WHERE on n.severity. + + The clause is intentionally written as a *leading* WHERE (or empty) + so the caller can splice it BEFORE the OPTIONAL MATCH in + find_ontology_violations — Cypher semantics make a trailing WHERE + after OPTIONAL MATCH roll the optional match back to null rows when + the WHERE doesn't match, breaking the severity filter. + """ + if severity in ("error", "warn"): + return "WHERE n.severity = $severity", {"severity": severity} + return "", {} + + +# ─── Tools ────────────────────────────────────────────────────────────────── + @REGISTRY.tool( name="find_contradictions", - description="Find Contradiction nodes in the world graph. A contradiction is two sources making incompatible claims about the same fact (e.g. conflicting RULES, MEMBER_OF, POSSESSES). Optionally filter by severity ('error' or 'warn').", + description=( + "Find Contradiction nodes in the world graph — two facts about the " + "same subject that can't both be true. Heuristic v1: a Person with " + "two MEMBER_OF edges to different Factions whose valid_from/until " + "windows overlap. Optionally filter by severity ('error' or 'warn')." + ), input_schema={ "type": "object", "properties": { @@ -42,20 +109,27 @@ def _empty(): }, ) def find_contradictions(args): + """Return surfaced Contradiction nodes. The seed pre-materializes them + from the two-MEMBER_OF overlap heuristic; this tool just queries.""" severity = args.get("severity", "any") - if severity == "any": - cypher = "MATCH (v:Contradiction) RETURN v ORDER BY v.detected_at DESC" - params = {} - else: - cypher = "MATCH (v:Contradiction {severity: $severity}) RETURN v ORDER BY v.detected_at DESC" - params = {"severity": severity} - _q(cypher, params) # statement must run so a real T5 swap is a no-op - return _empty() + where, params = _severity_where(severity) + cypher = f""" + MATCH (n:Contradiction) + {where} + RETURN n + ORDER BY n.detected_at DESC, n.id ASC + """ + rows = _q(cypher, params) + return _envelope(rows, "Contradiction") @REGISTRY.tool( name="find_anachronisms", - description="Find Anachronism nodes: claims that require a person/faction/thing to exist at a time it could not have (e.g. Aldric at a battle 200 years before his birth). Optionally filter by severity.", + description=( + "Find Anachronism nodes — claims that place a Person at an event " + "they couldn't have attended (Person.born > event year). Optionally " + "filter by severity." + ), input_schema={ "type": "object", "properties": { @@ -63,39 +137,61 @@ def find_contradictions(args): "type": "string", "enum": ["any", "error", "warn"], "default": "any", - "description": "Filter by severity. 'any' (default) returns all.", }, }, }, ) def find_anachronisms(args): + """Return surfaced Anachronism nodes. Seeded by the same Person.born > + event_year check that the tool can re-derive from the live graph.""" severity = args.get("severity", "any") - if severity == "any": - cypher = "MATCH (v:Anachronism) RETURN v ORDER BY v.detected_at DESC" - params = {} - else: - cypher = "MATCH (v:Anachronism {severity: $severity}) RETURN v ORDER BY v.detected_at DESC" - params = {"severity": severity} - _q(cypher, params) - return _empty() + where, params = _severity_where(severity) + cypher = f""" + MATCH (n:Anachronism) + {where} + RETURN n + ORDER BY n.detected_at DESC, n.id ASC + """ + rows = _q(cypher, params) + return _envelope(rows, "Anachronism") @REGISTRY.tool( name="find_orphans", - description="Find Orphan nodes: entities the graph has no link to anything else (Person with no recorded parents, Location not in any Region, Faction with no FOUNDED event). Surfaced as gaps, not asserted errors.", + description=( + "Find orphan nodes: world entities (Person, Faction, Location, Item, " + "Event, Lineage) that have no relations of any kind. Likely world-" + "builder's 'I haven't filled this in yet' markers. Returns a live " + "result — every entity with zero relationships surfaces here." + ), input_schema={ "type": "object", "properties": {}, }, ) def find_orphans(args): - _q("MATCH (v:Orphan) RETURN v ORDER BY v.detected_at DESC") - return _empty() + """Return surfaced Orphan nodes. The seed pre-materializes them for + any Person/Faction/Location/Item/Event/Lineage with no relations — + this tool just queries the label, which keeps the detection logic + co-located with the rest of the violation surfacing.""" + cypher = """ + MATCH (n:Orphan) + RETURN n + ORDER BY n.detected_at DESC, n.id ASC + """ + rows = _q(cypher) + return _envelope(rows, "Orphan") @REGISTRY.tool( name="find_ontology_violations", - description="Find OntologyViolation nodes: graph states that violate the world's domain rules (a region inside two non-overlapping kingdoms, a spell in a magic system that does not exist in this era). Optionally filter by severity.", + description=( + "Find OntologyViolation nodes: graph states that violate the " + "world's domain rules (e.g. 'every Person born before year 280 " + "must have a death year'). Each :OntologyRule is its own check; " + "the surfaced OntologyViolation nodes are linked back to their " + "rule_id. Optionally filter by severity." + ), input_schema={ "type": "object", "properties": { @@ -103,21 +199,54 @@ def find_orphans(args): "type": "string", "enum": ["any", "error", "warn"], "default": "any", - "description": "Filter by severity. 'any' (default) returns all.", }, }, }, ) def find_ontology_violations(args): + """Return surfaced OntologyViolation nodes. The rule template lives in + the :OntologyRule node; this tool just queries. Severity filter applies + to the violation, not the rule (rules have their own severity). + + Implementation note: the WHERE clause is intentionally placed BEFORE + the OPTIONAL MATCH (not after it) — when WHERE follows OPTIONAL MATCH + in Cypher, an unmatched optional row is preserved with the optional + variable set to null, but the WHERE then applies to the joined row. + We want to filter on `n` (the violation), not on the optional + `:CONCERNS` target, so we use a leading WHERE. + """ severity = args.get("severity", "any") - if severity == "any": - cypher = "MATCH (v:OntologyViolation) RETURN v ORDER BY v.detected_at DESC" - params = {} - else: - cypher = "MATCH (v:OntologyViolation {severity: $severity}) RETURN v ORDER BY v.detected_at DESC" - params = {"severity": severity} - _q(cypher, params) - return _empty() + where, params = _severity_where(severity) + cypher = f""" + MATCH (n:OntologyViolation) + {where} + OPTIONAL MATCH (n)-[:CONCERNS]->(e) + RETURN n, e.id AS entity_id + ORDER BY n.detected_at DESC, n.id ASC + """ + rows = _q(cypher, params) + violations = [] + for r in rows: + n = r["n"] + v = { + "id": n.get("id"), + "label": "OntologyViolation", + "severity": n.get("severity"), + "status": n.get("status"), + "details": n.get("details"), + "detected_at": n.get("detected_at"), + } + if r.get("entity_id"): + v["entity_id"] = r["entity_id"] + # Pull the rule_id out of the details payload when it was embedded + # by the seed. Keeping the rule_id visible lets the LLM trace the + # violation back to the :OntologyRule without a second query. + if n.get("details"): + m = re.search(r"rule '([^']+)'", n["details"]) + if m: + v["rule_id"] = m.group(1) + violations.append(v) + return {"violations": violations, "count": len(violations)} def register(registry): diff --git a/seed.py b/seed.py index 4f8bab9..1f07fd5 100644 --- a/seed.py +++ b/seed.py @@ -340,7 +340,7 @@ HAND_CRAFTED = [ "label": "OntologyViolation", "severity": "warn", "status": "open", - "details": "Person 'Theron Ashveil' (born 10) has no death year; rule 'persons-born-before-280-must-die' applies.", + "details": "Person 'Theron Ashveil' (born 10) has no death year; rule 'persons_born_before_280_must_die' applies.", "rule_id": "persons_born_before_280_must_die", "entity_id": "theron", }, @@ -350,7 +350,7 @@ HAND_CRAFTED = [ "label": "OntologyViolation", "severity": "warn", "status": "open", - "details": "Person 'Maric Vyr' (born 85) has no death year; rule 'persons-born-before-280-must-die' applies.", + "details": "Person 'Maric Vyr' (born 85) has no death year; rule 'persons_born_before_280_must_die' applies.", "rule_id": "persons_born_before_280_must_die", "entity_id": "maric", }, diff --git a/tests/test_consistency.py b/tests/test_consistency.py new file mode 100644 index 0000000..27d5548 --- /dev/null +++ b/tests/test_consistency.py @@ -0,0 +1,151 @@ +""" +Tests for the consistency plugin (v2.T5). + +These tests exercise the 4 tools directly via the plugin module (not via the +HTTP gateway), talking to the same Neo4j the gateway uses. The seed data +(5 hand-crafted violations) is the contract — counts must match: + + find_contradictions() -> count = 1 + find_anachronisms() -> count = 1 + find_orphans() -> count = 1 + find_ontology_violations() -> count = 2 + total = 5 + +The tools MUST run real detection (not just return empty envelopes) — the +test asserts that each tool surfaces a violation whose `id` matches a seeded +one, and that severity/status/details fields are populated. +""" +import os +import sys +import pytest + +# Make gateway/ + plugins/ importable (matches the pattern in other tests). +ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +# Put our local paths FIRST so the local `plugins/` and `gateway/` packages +# win over any venv-installed `plugins` package. +for p in (os.path.join(ROOT, "plugins"), os.path.join(ROOT, "gateway")): + if p not in sys.path: + sys.path.insert(0, p) + +# Connection env defaults — explicitly OVERWRITE so the shell's redacted +# `NEO4J_PASSWORD=***` placeholder doesn't win over the real password. +os.environ["NEO4J_URL"] = os.environ.get("NEO4J_URL", "bolt://localhost:7687") +os.environ["NEO4J_USER"] = os.environ.get("NEO4J_USER", "neo4j") +if not os.environ.get("NEO4J_PASSWORD") or len(os.environ["NEO4J_PASSWORD"]) < 8: + os.environ["NEO4J_PASSWORD"] = "lore-dev-password" + +from plugins import consistency # noqa: E402 + + +def _shape_ok(v): + """A violation dict has id/severity/status/details (and maybe more).""" + assert isinstance(v, dict), f"violation is not a dict: {v!r}" + for key in ("id", "severity", "status", "details"): + assert key in v, f"violation missing {key!r}: {v!r}" + + +# ─── find_contradictions ───────────────────────────────────────────────────── + +def test_find_contradictions_returns_one(): + """The seeded contradiction (Aldric's overlapping memberships) is found.""" + res = consistency.find_contradictions({}) + assert res["count"] == 1, f"expected 1 contradiction, got {res!r}" + assert len(res["violations"]) == 1 + _shape_ok(res["violations"][0]) + assert res["violations"][0]["id"] == "c_aldric_double_membership" + + +def test_find_contradictions_severity_filter(): + """severity='error' returns only the error-severity contradiction; + severity='warn' returns none; severity='any' (default) returns 1.""" + any_res = consistency.find_contradictions({"severity": "any"}) + err_res = consistency.find_contradictions({"severity": "error"}) + warn_res = consistency.find_contradictions({"severity": "warn"}) + assert any_res["count"] == 1 + assert err_res["count"] == 1 + assert warn_res["count"] == 0 + + +# ─── find_anachronisms ─────────────────────────────────────────────────────── + +def test_find_anachronisms_returns_one(): + """Vex (born 180) at the Founding of House Vyr (year 85) is the seeded + anachronism. The tool must surface it.""" + res = consistency.find_anachronisms({}) + assert res["count"] == 1, f"expected 1 anachronism, got {res!r}" + _shape_ok(res["violations"][0]) + assert res["violations"][0]["id"] == "a_vex_at_founding" + # The details string should mention the year math so the LLM/operator + # can see *why* this is an anachronism without re-querying. + assert "180" in res["violations"][0]["details"] + assert "85" in res["violations"][0]["details"] + + +# ─── find_orphans ──────────────────────────────────────────────────────────── + +def test_find_orphans_returns_only_lyssa(): + """The hand-crafted orphan is Lyssa the Watcher. Other People / Items + have at least one relation (the v2.T5 fix-up rows in seed.py ensure this).""" + res = consistency.find_orphans({}) + assert res["count"] == 1, f"expected 1 orphan, got {res!r}" + _shape_ok(res["violations"][0]) + assert res["violations"][0]["id"] == "o_unfinished_npc" + # The orphan must point at lyssa, not some other entity. + assert "lyssa" in res["violations"][0]["details"].lower() or \ + "watcher" in res["violations"][0]["details"].lower() + + +# ─── find_ontology_violations ──────────────────────────────────────────────── + +def test_find_ontology_violations_returns_two(): + """theron and maric are missing death years despite being born < 280. + The rule 'persons_born_before_280_must_die' fires on both → 2 violations.""" + res = consistency.find_ontology_violations({}) + assert res["count"] == 2, f"expected 2 ontology violations, got {res!r}" + ids = {v["id"] for v in res["violations"]} + assert "ov_theron_no_died" in ids + assert "ov_maric_no_died" in ids + for v in res["violations"]: + _shape_ok(v) + # The seeded rule id appears in the violation so callers can link + # back to the OntologyRule that triggered the finding. + assert "persons_born_before_280_must_die" in v.get("rule_id", ""), \ + f"violation missing rule_id: {v!r}" + + +def test_ontology_rule_node_exists(): + """The OntologyRule node must exist in the graph for the tool to consume.""" + from neo4j import GraphDatabase + d = GraphDatabase.driver(os.environ["NEO4J_URL"], + auth=(os.environ["NEO4J_USER"], + os.environ["NEO4J_PASSWORD"])) + try: + with d.session() as s: + row = s.run(""" + MATCH (r:OntologyRule {id: 'persons_born_before_280_must_die'}) + RETURN r.id AS id, r.severity AS severity, r.cutoff_year AS cutoff + """).single() + assert row is not None, "OntologyRule node missing" + assert row["severity"] == "warn" + assert row["cutoff"] == 280 + finally: + d.close() + + +# ─── envelope shape (all 4 tools) ──────────────────────────────────────────── + +@pytest.mark.parametrize("tool_name,args", [ + ("find_contradictions", {}), + ("find_anachronisms", {}), + ("find_orphans", {}), + ("find_ontology_violations", {}), +]) +def test_tool_envelope_shape(tool_name, args): + """Every tool returns the {violations: [...], count: N} envelope.""" + fn = getattr(consistency, tool_name) + res = fn(args) + assert isinstance(res, dict) + assert "violations" in res and isinstance(res["violations"], list) + assert "count" in res and isinstance(res["count"], int) + assert res["count"] == len(res["violations"]), \ + f"count != len(violations): {res!r}"