lore-engine-poc/workers/lore-watcher/main.go

package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/fsnotify/fsnotify"
)

type Config struct {
	WatchDir   string
	IngestURL  string
	DebounceMS int
}

func configFromEnv() Config {
	ms := 500
	if v := os.Getenv("DEBOUNCE_MS"); v != "" {
		fmt.Sscan(v, &ms)
	}
	return Config{
		WatchDir:   getEnv("WATCH_DIR", "/data/lore"),
		IngestURL:  getEnv("INGEST_URL", "http://ingestion-worker:8080/ingest/lore"),
		DebounceMS: ms,
	}
}

func getEnv(key, fallback string) string {
	if v := os.Getenv(key); v != "" {
		return v
	}
	return fallback
}

// stateFile is stored inside WatchDir so it persists across container restarts
// when the directory is a bind mount.
const stateFileName = ".lore-watcher-state.json"

type watchState struct {
	Hashes map[string]string `json:"hashes"`
}

func loadState(dir string) watchState {
	s := watchState{Hashes: make(map[string]string)}
	data, err := os.ReadFile(filepath.Join(dir, stateFileName))
	if err != nil {
		return s
	}
	json.Unmarshal(data, &s)
	return s
}

func saveState(dir string, s watchState) {
	data, _ := json.Marshal(s)
	os.WriteFile(filepath.Join(dir, stateFileName), data, 0644)
}

func hashFile(path string) (string, error) {
	f, err := os.Open(path)
	if err != nil {
		return "", err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return "", err
	}
	return hex.EncodeToString(h.Sum(nil)), nil
}

func shouldIgnore(path string) bool {
	base := filepath.Base(path)
	if strings.HasPrefix(base, ".") {
		return true
	}
	for _, suffix := range []string{".swp", "~", ".tmp"} {
		if strings.HasSuffix(base, suffix) {
			return true
		}
	}
	if base == "4913" { // vim temp file number
		return true
	}
	return filepath.Ext(path) != ".md"
}

func upload(ingestURL, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	body := &bytes.Buffer{}
	w := multipart.NewWriter(body)
	part, err := w.CreateFormFile("file", filepath.Base(path))
	if err != nil {
		return err
	}
	if _, err := io.Copy(part, f); err != nil {
		return err
	}
	w.Close()

	resp, err := http.Post(ingestURL, w.FormDataContentType(), body)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("upload returned HTTP %d", resp.StatusCode)
	}
	return nil
}

func processFile(cfg Config, s watchState, absPath string) {
	if _, err := os.Stat(absPath); err != nil {
		return // file may have been deleted before debounce fired
	}
	rel, _ := filepath.Rel(cfg.WatchDir, absPath)

	hash, err := hashFile(absPath)
	if err != nil {
		slog.Error("hash failed", "path", rel, "err", err)
		return
	}
	if existing, ok := s.Hashes[rel]; ok && existing == hash {
		return // content unchanged since last upload
	}

	slog.Info("uploading", "file", rel)
	if err := upload(cfg.IngestURL, absPath); err != nil {
		slog.Error("upload failed", "file", rel, "err", err)
		return
	}
	s.Hashes[rel] = hash
	saveState(cfg.WatchDir, s)
	slog.Info("uploaded", "file", rel)
}

func main() {
	slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
	cfg := configFromEnv()

	if _, err := os.Stat(cfg.WatchDir); err != nil {
		slog.Error("watch directory not found", "dir", cfg.WatchDir, "err", err)
		os.Exit(1)
	}

	s := loadState(cfg.WatchDir)
	debounce := time.Duration(cfg.DebounceMS) * time.Millisecond

	// Scan on startup — upload anything new or changed since last run
	slog.Info("scanning directory", "dir", cfg.WatchDir)
	filepath.Walk(cfg.WatchDir, func(path string, info os.FileInfo, err error) error {
		if err != nil || info.IsDir() || shouldIgnore(path) {
			return nil
		}
		processFile(cfg, s, path)
		return nil
	})

	watcher, err := fsnotify.NewWatcher()
	if err != nil {
		slog.Error("failed to create watcher", "err", err)
		os.Exit(1)
	}
	defer watcher.Close()

	// Watch all subdirectories recursively
	filepath.Walk(cfg.WatchDir, func(path string, info os.FileInfo, err error) error {
		if err == nil && info.IsDir() {
			watcher.Add(path)
		}
		return nil
	})

	slog.Info("watching for changes", "dir", cfg.WatchDir, "debounce_ms", cfg.DebounceMS)

	// Per-file debounce timers — noisy editors (vim, VS Code) fire multiple events per save
	timers := make(map[string]*time.Timer)

	for {
		select {
		case event, ok := <-watcher.Events:
			if !ok {
				return
			}
			path := event.Name

			// Add newly created subdirectories to the watch set
			if event.Op&fsnotify.Create != 0 {
				if info, err := os.Stat(path); err == nil && info.IsDir() {
					watcher.Add(path)
					continue
				}
			}

			if shouldIgnore(path) {
				continue
			}
			if event.Op&(fsnotify.Write|fsnotify.Create) == 0 {
				continue
			}

			if t, ok := timers[path]; ok {
				t.Stop()
			}
			p := path // capture for closure
			timers[p] = time.AfterFunc(debounce, func() {
				processFile(cfg, s, p)
				delete(timers, p)
			})

		case err, ok := <-watcher.Errors:
			if !ok {
				return
			}
			slog.Error("watcher error", "err", err)
		}
	}
}