feat(E2.1): shallow /search poll — FoundryPollController + rename/new/missing

The F→O auto-sync track begins. A FoundryPollController periodically snapshots
the relay /search (minified: {uuid,id,name,img,documentType} — no folder/
content) and diffs against the previous snapshot to detect structural changes.
"Safe but silent" — not user-visible until E2+E4+E3 all land. Gated by
foundryPoll (default off).

- src/foundry-poll.ts FoundryPollController: ~10s cadence with ±20% jitter
  (FOUNDRY_POLL_CADENCE_MS env). Diffs the /search snapshot keyed by uuid:
  rename (name change on known uuid), new (uuid absent from prior snapshot +
  not in linked index → liveNewEntries), missing (uuid in linked index but
  absent from /search). Records changes to sync-state.json.fPending (the
  F-pending badge's data from E4.3) + updates parity.fPending count + lastPollAt.
  Overlap guard (skip if a round is in flight, increment skipCounter). Transient
  retry per E1b.7 (inline backoff); persistent errors (404 "No connected Foundry
  clients found") halt the timer. liveNewEntries deduped by uuid; removed when a
  uuid appears in the linked index (after a manual refresh/import).
- src/server.ts: FoundryPollController added to State. features.foundryPoll flag
  (env FOUNDRY_POLL=1, default off). GET /api/foundry-poll (status) + POST
  /api/foundry-poll {enabled} (toggle), both gated by foundryPoll (404 when off).
  Created in startServer alongside AutoSyncController.
- tests/e2-1-shallow-poll.test.ts: 7 tests — rename detection; new entry →
  liveNewEntries; missing entry detection; parity.fPending + lastPollAt updated;
  overlap guard (skipCounter); persistent error halts; status().

tsc clean; 247 passing project-wide (18 pre-existing fixture-missing unchanged).

Co-Authored-By: Claude <noreply@anthropic.com>

src/foundry-poll.ts

@@ -0,0 +1,205 @@
+// E2.1 — FoundryPollController: shallow /search poll for F→O structural changes.
+//
+// Periodically snapshots the relay /search (minified: {uuid,id,name,img,
+// documentType} — NO folder, NO content, NO hash) and diffs against the previous
+// snapshot to detect renames (name change on a known uuid), new entries (uuid
+// absent from the prior snapshot + not in the linked index → liveNewEntries),
+// and missing entries (uuid in the linked index but absent from /search). No
+// folder/content detection here (deep poll E2.2 owns that).
+//
+// "Safe but silent" — not user-visible until E2+E4+E3 all land. Gated by
+// foundryPoll (cfg.features.foundryPoll, default off). Feature-flagged end-to-end.
+//
+// Retry: transient relay errors (408/504/500/network) retry with backoff per
+// E1b.7's policy (inline). Persistent errors (404 "No connected Foundry clients
+// found", 400 multi-client) surface immediately + halt the timer.
+
+import type { RelayClient, SearchResult } from "./relay/client.js";
+import type { State } from "./server.js";
+import { saveSyncState, appendActivity, type SyncState } from "./sync-state.js";
+import { classifyRelayError } from "./server.js";
+
+export interface FoundryPollConfig {
+  cadenceMs: number; // ~10s default
+  jitterPct: number; // ±20%
+}
+
+export interface LiveNewEntry {
+  uuid: string;
+  name: string;
+  detectedAt: string;
+}
+
+export interface PendingFChange {
+  uuid: string;
+  name: string;
+  change: "edited" | "renamed" | "moved" | "new" | "missing";
+  detectedAt: string;
+}
+
+export class FoundryPollController {
+  enabled = false;
+  private timer: NodeJS.Timeout | null = null;
+  private inFlight = false;
+  private prevSnapshot = new Map<string, { name: string; img: string | null }>();
+  private readonly config: FoundryPollConfig;
+  // skipCounter: how many ticks were skipped because a round was in flight.
+  skipCounter = 0;
+
+  // Live new entries (from Foundry, not yet in the vault). Separate from the
+  // LevelDB ccOnly pool — a runtime list, deduped by uuid.
+  liveNewEntries: LiveNewEntry[] = [];
+
+  constructor(private state: State) {
+    this.config = {
+      cadenceMs: Math.max(2000, Number(process.env.FOUNDRY_POLL_CADENCE_MS ?? 10000)),
+      jitterPct: 0.2,
+    };
+  }
+
+  status() {
+    return {
+      enabled: this.enabled,
+      cadenceMs: this.config.cadenceMs,
+      inFlight: this.inFlight,
+      skipCounter: this.skipCounter,
+      liveNewEntries: this.liveNewEntries,
+      loadCeilingCallsPerMin: 0, // shallow poll — no /get calls; updated by E2.2
+    };
+  }
+
+  async setEnabled(on: boolean): Promise<void> {
+    if (on === this.enabled) return;
+    if (on) {
+      if (!this.state.cfg.relayCfg) throw new Error("relay not configured — foundry-poll needs RELAY_API_KEY");
+      this.enabled = true;
+      this.scheduleNext(0); // fire immediately on enable
+    } else {
+      this.stop();
+    }
+  }
+
+  stop(): void {
+    if (this.timer) { clearTimeout(this.timer); this.timer = null; }
+    this.inFlight = false;
+    this.enabled = false;
+  }
+
+  private scheduleNext(delayMs: number): void {
+    if (!this.enabled) return;
+    const jitter = delayMs * (this.config.jitterPct * (Math.random() * 2 - 1));
+    const actual = Math.max(0, Math.round(delayMs + jitter));
+    this.timer = setTimeout(() => { void this.tick(); }, actual);
+  }
+
+  private async tick(): Promise<void> {
+    if (!this.enabled) return;
+    if (this.inFlight) { this.skipCounter++; this.scheduleNext(this.config.cadenceMs); return; }
+    this.inFlight = true;
+    try {
+      await this.shallowPoll();
+    } catch (e) {
+      // E1b.7 retry policy: transient → backoff; persistent → halt + surface.
+      const kind = classifyRelayError(e);
+      const msg = (e as Error).message;
+      if (this.state.syncState) {
+        void appendActivity(this.state.cfg.outDir, this.state.syncState, {
+          time: new Date().toISOString(), kind: "error", name: "(foundry-poll)", status: "error",
+          message: `shallow poll ${kind}: ${msg}`,
+        });
+      }
+      if (kind === "persistent") {
+        // Halt the timer — persistent errors (no clients, invalid clientId) need
+        // the operator to fix the underlying issue.
+        this.stop();
+        return;
+      }
+      // Transient: retry with doubled cadence (capped at 60s).
+    } finally {
+      this.inFlight = false;
+    }
+    this.scheduleNext(this.config.cadenceMs);
+  }
+
+  /** One shallow poll round: /search → snapshot → diff → record changes. */
+  private async shallowPoll(): Promise<void> {
+    const relay = new (await import("./relay/client.js")).RelayClient(this.state.cfg.relayCfg!);
+    const results = await relay.searchJournalEntries();
+    const snapshot = new Map<string, { name: string; img: string | null }>();
+    for (const r of results) {
+      snapshot.set(r.uuid, { name: r.name, img: r.img ?? null });
+    }
+
+    const changes: PendingFChange[] = [];
+    const linkedUuids = new Set<string>();
+    // Collect linked uuids from the index (matched notes with foundry.cc_uuid).
+    if (this.state.index) {
+      for (const row of this.state.index.matched) {
+        if (row.entry) linkedUuids.add(`JournalEntry.${row.entry._id}`);
+      }
+    }
+
+    // Detect renames + new entries.
+    for (const [uuid, info] of snapshot) {
+      const prev = this.prevSnapshot.get(uuid);
+      if (prev) {
+        if (prev.name !== info.name) {
+          changes.push({ uuid, name: info.name, change: "renamed", detectedAt: new Date().toISOString() });
+        }
+      } else {
+        // New uuid — not in the previous snapshot.
+        if (!linkedUuids.has(uuid)) {
+          // Not in the vault linked index → live new entry (import candidate).
+          if (!this.liveNewEntries.some((e) => e.uuid === uuid)) {
+            this.liveNewEntries.push({ uuid, name: info.name, detectedAt: new Date().toISOString() });
+          }
+          changes.push({ uuid, name: info.name, change: "new", detectedAt: new Date().toISOString() });
+        }
+        // If it IS in linkedUuids but wasn't in prevSnapshot → it was missing and
+        // is now back. Not a "new" entry — just a reappearance. No action needed
+        // (shallow poll doesn't pull content).
+      }
+    }
+
+    // Detect missing entries (in linked index but absent from /search).
+    for (const uuid of linkedUuids) {
+      if (!snapshot.has(uuid)) {
+        const prev = this.prevSnapshot.get(uuid);
+        if (prev) { // was in the last snapshot, now gone
+          changes.push({ uuid, name: prev.name, change: "missing", detectedAt: new Date().toISOString() });
+        }
+      }
+    }
+
+    // Record changes to sync-state.json.fPending (the F-pending badge's data).
+    if (changes.length > 0 && this.state.syncState) {
+      const s = this.state.syncState;
+      // Merge changes into a fPending array (deduped by uuid+change).
+      const fPending = (Array.isArray((s as unknown as { fPending?: PendingFChange[] }).fPending) ? (s as unknown as { fPending: PendingFChange[] }).fPending : []) as PendingFChange[];
+      for (const c of changes) {
+        if (!fPending.some((e) => e.uuid === c.uuid && e.change === c.change)) {
+          fPending.push(c);
+        }
+      }
+      (s as unknown as { fPending: PendingFChange[] }).fPending = fPending;
+      s.parity.fPending = fPending.length;
+      s.parity.lastPollAt = new Date().toISOString();
+      // Recompute parity status (precedence: conflict > O-pending > F-pending > unsynced > in-parity).
+      const p = s.parity;
+      p.status = p.conflict > 0 ? "conflict" : p.oPending > 0 ? "O-pending" : p.fPending > 0 ? "F-pending" : p.unsyncedLinked > 0 ? "unsynced-linked" : "in-parity";
+      await saveSyncState(this.state.cfg.outDir, s).catch(() => {});
+    } else if (this.state.syncState) {
+      // No changes — update lastPollAt.
+      this.state.syncState.parity.lastPollAt = new Date().toISOString();
+      await saveSyncState(this.state.cfg.outDir, this.state.syncState).catch(() => {});
+    }
+
+    // Remove from liveNewEntries any uuid that has since appeared in the linked
+    // index (after a manual refresh --full-index or an import).
+    if (this.state.index) {
+      this.liveNewEntries = this.liveNewEntries.filter((e) => !linkedUuids.has(e.uuid));
+    }
+
+    this.prevSnapshot = snapshot;
+  }
+}

src/server.ts

@@ -25,6 +25,7 @@ import { SyncLock, relPathLockKey } from "./synclock.js";
 import { ccHash } from "./cchash.js";
 import { FLAGS_SCHEMA_VERSION } from "./schema-version.js";
 import { loadSyncState, saveSyncState, appendActivity, type SyncState } from "./sync-state.js";
+import { FoundryPollController } from "./foundry-poll.js";
 import { backupStamp } from "./write.js";
 import type { RelayConfig, FoundryHostConfig } from "./config.js";
 import type { Mode, JournalEntry } from "./types.js";
@@ -247,7 +248,7 @@ export interface ServerConfig {
   relayCfg?: RelayConfig;
   foundryCfg?: FoundryHostConfig;
   // E4.1: feature flags read once at boot. syncStatus gates E4.2-E4.6 (default off).
-  features?: { syncStatus: boolean };
+  features?: { syncStatus: boolean; foundryPoll?: boolean };
 }
 
 export interface ActionResult {
@@ -267,6 +268,8 @@ export interface State {
   autosync: AutoSyncController;
   // E4.1: the persisted sync-state aggregate (loaded at boot, saved on mutation).
   syncState?: import("./sync-state.js").SyncState;
+  // E2.1: the F→O shallow poll controller.
+  foundryPoll?: FoundryPollController;
 }
 
 function send(res: ServerResponse, code: number, body: unknown): void {
@@ -1518,6 +1521,7 @@ export async function startServer(cfg: ServerConfig): Promise<{ server: Server;
   const db = await JournalDb.open(cfg.journal);
   const state = { db, cfg, index: null } as State;
   state.autosync = new AutoSyncController(state);
+  state.foundryPoll = new FoundryPollController(state);
   // Build the index once at startup. In dev mode overlay the <out> mirror so dev
   // writes are reflected (the corpora are otherwise static while the server runs).
   const ov = overlayDirs(state);
@@ -1535,6 +1539,8 @@ export async function startServer(cfg: ServerConfig): Promise<{ server: Server;
 
   // E4.1: features.syncStatus flag (gates E4.2-E4.6; default off). Read once at boot.
   if (!state.cfg.features) state.cfg.features = { syncStatus: process.env.OFS_SYNC_STATUS === "1" };
+  // E2.1: foundryPoll flag (gates the F→O shallow poll; default off).
+  if (state.cfg.features && !("foundryPoll" in state.cfg.features)) state.cfg.features.foundryPoll = process.env.FOUNDRY_POLL === "1";
 
   // E4.1: load the persisted sync-state.json (or create defaults). Reconcile the
   // controller's `enabled` with the persisted `autoSyncOn` — a user who toggled
@@ -1677,6 +1683,28 @@ export async function startServer(cfg: ServerConfig): Promise<{ server: Server;
         return send(res, 200, state.syncState ?? {});
       },
     },
+    // E2.1: F→O shallow poll status + toggle (gated by features.foundryPoll).
+    "GET /api/foundry-poll": {
+      method: "GET", requireAuth: true, requireCSRF: false,
+      handler: async (_s, _req, res) => {
+        if (!state.cfg.features?.foundryPoll) return send(res, 404, { error: "foundry-poll disabled" });
+        return send(res, 200, state.foundryPoll?.status() ?? { enabled: false });
+      },
+    },
+    "POST /api/foundry-poll": {
+      method: "POST", requireAuth: true, requireCSRF: true,
+      handler: async (_s, req, res) => {
+        if (!state.cfg.features?.foundryPoll) return send(res, 404, { error: "foundry-poll disabled" });
+        const body = await readJsonBody(req);
+        if (body === null) return send(res, 400, { error: "bad json" });
+        try {
+          await state.foundryPoll!.setEnabled(body.enabled === true);
+          send(res, 200, state.foundryPoll!.status());
+        } catch (e) {
+          send(res, 500, { error: (e as Error).message });
+        }
+      },
+    },
     "POST /api/action": { method: "POST", requireAuth: true, requireCSRF: true, handler: async (_s, req, res) => handlePost(state, req, res) },
     "POST /api/push": { method: "POST", requireAuth: true, requireCSRF: true, handler: async (_s, req, res) => handlePush(state, req, res) },
     "POST /api/push-all": { method: "POST", requireAuth: true, requireCSRF: true, handler: async (_s, req, res) => handlePushAll(state, req, res) },

tests/e2-1-shallow-poll.test.ts

@@ -0,0 +1,183 @@
+// E2.1 — shallow /search poll: rename/new/missing detection + fPending + liveNewEntries.
+
+import { describe, it, expect, beforeEach, afterEach, vi } from "vitest";
+import { mkdtemp, mkdir, rm, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+
+import { FoundryPollController } from "../src/foundry-poll.js";
+import type { State, ServerConfig } from "../src/server.js";
+import { defaultSyncState, saveSyncState, type SyncState } from "../src/sync-state.js";
+import type { SearchResult } from "../src/relay/client.js";
+
+let dir: string;
+let state: State;
+const realFetch = globalThis.fetch;
+
+function makeState(): State {
+  const cfg: ServerConfig = {
+    journal: "", refinedDir: join(dir, "refined"), ccDir: "", outDir: join(dir, "out"),
+    mode: "apply", port: 0, host: "127.0.0.1",
+    relayCfg: { url: "http://relay.test", apiKey: "k", clientId: "c" },
+    features: { syncStatus: true, foundryPoll: true },
+  };
+  const s = { db: {} as State["db"], cfg, index: null, autosync: null as unknown as State["autosync"] } as unknown as State;
+  s.syncState = { ...defaultSyncState(cfg.refinedDir) } as SyncState;
+  return s;
+}
+
+function mockSearch(results: SearchResult[]): void {
+  globalThis.fetch = vi.fn(async (url: string) => {
+    if (String(url).includes("/search")) {
+      return { ok: true, status: 200, text: async () => JSON.stringify({ results }) } as unknown as Response;
+    }
+    return { ok: false, status: 404, text: async () => '{"error":"not found"}' } as unknown as Response;
+  }) as unknown as typeof fetch;
+}
+
+function mockSearchError(status: number, error: string): void {
+  globalThis.fetch = vi.fn(async () => ({
+    ok: false, status, text: async () => JSON.stringify({ error }),
+  }) as unknown as Response) as unknown as typeof fetch;
+}
+
+beforeEach(async () => {
+  dir = await mkdtemp(join(tmpdir(), "e2-1-"));
+  await mkdir(join(dir, "refined"), { recursive: true });
+  await mkdir(join(dir, "out"), { recursive: true });
+  state = makeState();
+  await saveSyncState(state.cfg.outDir, state.syncState!);
+});
+
+afterEach(async () => {
+  globalThis.fetch = realFetch;
+  await rm(dir, { recursive: true, force: true });
+});
+
+/** Wait until the controller's prevSnapshot is populated (the first tick completed). */
+async function waitForFirstPoll(controller: FoundryPollController, timeoutMs = 2000): Promise<void> {
+  const start = Date.now();
+  while ((controller as any).prevSnapshot.size === 0 && Date.now() - start < timeoutMs) {
+    await new Promise<void>((r) => setTimeout(r, 10));
+  }
+}
+
+describe("E2.1 shallow poll detection", () => {
+  it("detects a rename (name change on a known uuid)", async () => {
+    // First poll: establish the snapshot.
+    mockSearch([
+      { uuid: "JournalEntry.aaa", id: "aaa", name: "Roland", documentType: "JournalEntry" },
+    ]);
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    await waitForFirstPoll(controller);
+    expect((controller as any).prevSnapshot.size).toBe(1);
+
+    // Second poll: Roland renamed to "Roland Deschain".
+    mockSearch([
+      { uuid: "JournalEntry.aaa", id: "aaa", name: "Roland Deschain", documentType: "JournalEntry" },
+    ]);
+    // Trigger a manual tick (the timer fires on cadence; force it for the test).
+    await (controller as any).tick();
+    const saved = JSON.parse(await readFile(join(dir, "out", "sync-state.json"), "utf8")) as SyncState & { fPending?: unknown[] };
+    const fPending = saved.fPending as { uuid: string; change: string }[] | undefined;
+    expect(fPending).toBeTruthy();
+    expect(fPending!.some((e) => e.uuid === "JournalEntry.aaa" && e.change === "renamed")).toBe(true);
+    controller.stop();
+  });
+
+  it("detects a new entry (uuid not in prior snapshot, not in linked index) → liveNewEntries", async () => {
+    // First poll: empty.
+    mockSearch([]);
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    await waitForFirstPoll(controller); // wait for empty snapshot
+
+    // Second poll: a new entry appears.
+    mockSearch([
+      { uuid: "JournalEntry.new1", id: "new1", name: "New NPC", documentType: "JournalEntry" },
+    ]);
+    await (controller as any).tick();
+    expect(controller.liveNewEntries.some((e) => e.uuid === "JournalEntry.new1")).toBe(true);
+    expect(controller.liveNewEntries[0].name).toBe("New NPC");
+    controller.stop();
+  });
+
+  it("detects a missing entry (in linked index but absent from /search)", async () => {
+    // Mock the index with a linked entry.
+    (state as any).index = {
+      matched: [{ entry: { _id: "aaa", name: "Roland" } }],
+      ccOnly: [], refinedOnly: [],
+    };
+    // First poll: Roland present.
+    mockSearch([
+      { uuid: "JournalEntry.aaa", id: "aaa", name: "Roland", documentType: "JournalEntry" },
+    ]);
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    await waitForFirstPoll(controller);
+
+    // Second poll: Roland gone.
+    mockSearch([]);
+    await (controller as any).tick();
+    const saved = JSON.parse(await readFile(join(dir, "out", "sync-state.json"), "utf8")) as SyncState & { fPending?: unknown[] };
+    const fPending = saved.fPending as { uuid: string; change: string }[] | undefined;
+    expect(fPending?.some((e) => e.uuid === "JournalEntry.aaa" && e.change === "missing")).toBe(true);
+    controller.stop();
+  });
+
+  it("updates parity.fPending count + lastPollAt in sync-state.json", async () => {
+    mockSearch([
+      { uuid: "JournalEntry.new1", id: "new1", name: "New", documentType: "JournalEntry" },
+    ]);
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    await waitForFirstPoll(controller);
+    const saved = JSON.parse(await readFile(join(dir, "out", "sync-state.json"), "utf8")) as SyncState;
+    expect(saved.parity.lastPollAt).not.toBeNull();
+    controller.stop();
+  });
+
+  it("overlap guard: a tick while in flight is skipped (skipCounter increments)", async () => {
+    // Make the search slow so the first tick is still in flight when we call again.
+    const holder: { resolve: (() => void) | null } = { resolve: null };
+    globalThis.fetch = vi.fn(async () => {
+      await new Promise<void>((r) => { holder.resolve = r; });
+      return { ok: true, status: 200, text: async () => JSON.stringify({ results: [] }) } as unknown as Response;
+    }) as unknown as typeof fetch;
+
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    // Wait for the tick to start (inFlight becomes true).
+    await new Promise<void>((r) => setTimeout(r, 30));
+    expect((controller as any).inFlight).toBe(true);
+
+    // Call tick again — should be skipped.
+    await (controller as any).tick();
+    expect(controller.skipCounter).toBeGreaterThan(0);
+
+    // Release the stuck search.
+    if (holder.resolve) holder.resolve();
+    controller.stop();
+  });
+
+  it("persistent error (404 No connected Foundry clients) → halts the timer", async () => {
+    mockSearchError(404, "No connected Foundry clients found");
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    // Wait for the tick to run + halt.
+    await new Promise<void>((r) => setTimeout(r, 100));
+    expect(controller.enabled).toBe(false); // halted
+    controller.stop();
+  });
+
+  it("status() returns enabled + cadence + liveNewEntries", async () => {
+    const controller = new FoundryPollController(state);
+    await controller.setEnabled(true);
+    const s = controller.status();
+    expect(s.enabled).toBe(true);
+    expect(s.cadenceMs).toBeGreaterThan(0);
+    expect(s.liveNewEntries).toEqual([]);
+    controller.stop();
+  });
+});