feat: persist passive browser activity batches

2026-07-12 17:29:07 +08:00
parent 39052ffe39
commit bcf49ed32c
10 changed files with 787 additions and 43 deletions
@@ -20,6 +20,17 @@ type Storage struct {
 	vault *vault.Vault
 }

+// NDJSONRetention bounds append-only plugin data. Records are compacted after
+// a successful append so settings.json never becomes an event log.
+type NDJSONRetention struct {
+	TimestampField   string
+	MaxAge           time.Duration
+	MaxEntries       int
+	MaxBytes         int64
+	DeduplicateField string
+	DeduplicateValue string
+}
+
 // New creates a new Storage instance backed by the given vault.
 func New(v *vault.Vault) *Storage {
 	return &Storage{vault: v}
@@ -234,6 +245,204 @@ func (s *Storage) WritePluginDataJSON(pluginID, name string, data map[string]int
 	return atomicWrite(path, encoded)
 }

+// ReadPluginDataNDJSON reads an append-only named data file. A missing file is
+// represented by an empty slice.
+func (s *Storage) ReadPluginDataNDJSON(pluginID, name string) ([]map[string]interface{}, error) {
+	if err := validatePluginID(pluginID); err != nil {
+		return nil, err
+	}
+	if err := validateStorageName("data", name); err != nil {
+		return nil, err
+	}
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	var records []map[string]interface{}
+	err := s.withOpenVault(func(vaultPath string) error {
+		var err error
+		records, err = readPluginDataNDJSONAt(vaultPath, pluginID, name)
+		return err
+	})
+	return records, err
+}
+
+// AppendPluginDataNDJSON appends records durably and then applies bounded
+// retention. It returns false without writing when the supplied idempotency
+// value is already present in the retained log.
+func (s *Storage) AppendPluginDataNDJSON(pluginID, name string, records []map[string]interface{}, retention NDJSONRetention) (bool, error) {
+	if err := validatePluginID(pluginID); err != nil {
+		return false, err
+	}
+	if err := validateStorageName("data", name); err != nil {
+		return false, err
+	}
+	if len(records) == 0 {
+		return false, fmt.Errorf("NDJSON records are empty")
+	}
+	if retention.MaxEntries < 0 || retention.MaxBytes < 0 || retention.MaxAge < 0 {
+		return false, fmt.Errorf("NDJSON retention values must not be negative")
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	stored := false
+	err := s.withOpenVault(func(vaultPath string) error {
+		existing, err := readPluginDataNDJSONAt(vaultPath, pluginID, name)
+		if err != nil {
+			return err
+		}
+		if retention.DeduplicateField != "" && retention.DeduplicateValue != "" {
+			for _, record := range existing {
+				if fmt.Sprint(record[retention.DeduplicateField]) == retention.DeduplicateValue {
+					return nil
+				}
+			}
+		}
+		path := pluginDataNDJSONPath(vaultPath, pluginID, name)
+		if err := appendNDJSON(path, records); err != nil {
+			return err
+		}
+		stored = true
+		compacted := compactNDJSONRecords(append(existing, records...), retention, time.Now().UTC())
+		if !sameNDJSONRecords(append(existing, records...), compacted) {
+			return writeNDJSON(path, compacted)
+		}
+		return nil
+	})
+	return stored, err
+}
+
+// WritePluginDataNDJSON replaces a named append-only data file. It is reserved
+// for explicit user actions such as clearing Activity, never normal event
+// ingestion.
+func (s *Storage) WritePluginDataNDJSON(pluginID, name string, records []map[string]interface{}) error {
+	if err := validatePluginID(pluginID); err != nil {
+		return err
+	}
+	if err := validateStorageName("data", name); err != nil {
+		return err
+	}
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.withOpenVault(func(vaultPath string) error {
+		return writeNDJSON(pluginDataNDJSONPath(vaultPath, pluginID, name), records)
+	})
+}
+
+func pluginDataNDJSONPath(vaultPath, pluginID, name string) string {
+	return filepath.Join(vaultPath, ".verstak", "plugin-data", pluginID, name+".ndjson")
+}
+
+func readPluginDataNDJSONAt(vaultPath, pluginID, name string) ([]map[string]interface{}, error) {
+	data, err := os.ReadFile(pluginDataNDJSONPath(vaultPath, pluginID, name))
+	if err != nil {
+		if os.IsNotExist(err) {
+			return []map[string]interface{}{}, nil
+		}
+		return nil, fmt.Errorf("failed to read NDJSON data %s for plugin %s: %w", name, pluginID, err)
+	}
+	if len(data) == 0 {
+		return []map[string]interface{}{}, nil
+	}
+	lines := strings.Split(strings.TrimSuffix(string(data), "\n"), "\n")
+	records := make([]map[string]interface{}, 0, len(lines))
+	for index, line := range lines {
+		if strings.TrimSpace(line) == "" {
+			continue
+		}
+		var record map[string]interface{}
+		if err := json.Unmarshal([]byte(line), &record); err != nil {
+			return nil, fmt.Errorf("corrupt NDJSON data %s line %d for plugin %s: %w", name, index+1, pluginID, err)
+		}
+		records = append(records, record)
+	}
+	return records, nil
+}
+
+func appendNDJSON(path string, records []map[string]interface{}) error {
+	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
+		return fmt.Errorf("failed to create NDJSON data directory: %w", err)
+	}
+	file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o644)
+	if err != nil {
+		return fmt.Errorf("failed to open NDJSON data file: %w", err)
+	}
+	defer file.Close()
+	for _, record := range records {
+		encoded, err := json.Marshal(record)
+		if err != nil {
+			return fmt.Errorf("failed to marshal NDJSON record: %w", err)
+		}
+		if _, err := file.Write(append(encoded, '\n')); err != nil {
+			return fmt.Errorf("failed to append NDJSON record: %w", err)
+		}
+	}
+	if err := file.Sync(); err != nil {
+		return fmt.Errorf("failed to sync NDJSON data: %w", err)
+	}
+	return nil
+}
+
+func writeNDJSON(path string, records []map[string]interface{}) error {
+	data := make([]byte, 0)
+	for _, record := range records {
+		encoded, err := json.Marshal(record)
+		if err != nil {
+			return fmt.Errorf("failed to marshal NDJSON record: %w", err)
+		}
+		data = append(data, encoded...)
+		data = append(data, '\n')
+	}
+	return atomicWrite(path, data)
+}
+
+func compactNDJSONRecords(records []map[string]interface{}, retention NDJSONRetention, now time.Time) []map[string]interface{} {
+	kept := make([]map[string]interface{}, 0, len(records))
+	cutoff := now.Add(-retention.MaxAge)
+	for _, record := range records {
+		if retention.MaxAge > 0 && retention.TimestampField != "" {
+			if raw, ok := record[retention.TimestampField].(string); ok {
+				if timestamp, err := time.Parse(time.RFC3339, raw); err == nil && timestamp.Before(cutoff) {
+					continue
+				}
+			}
+		}
+		kept = append(kept, record)
+	}
+	if retention.MaxEntries > 0 && len(kept) > retention.MaxEntries {
+		kept = kept[len(kept)-retention.MaxEntries:]
+	}
+	if retention.MaxBytes > 0 {
+		for len(kept) > 0 && ndjsonSize(kept) > retention.MaxBytes {
+			kept = kept[1:]
+		}
+	}
+	return kept
+}
+
+func ndjsonSize(records []map[string]interface{}) int64 {
+	var total int64
+	for _, record := range records {
+		encoded, err := json.Marshal(record)
+		if err == nil {
+			total += int64(len(encoded) + 1)
+		}
+	}
+	return total
+}
+
+func sameNDJSONRecords(left, right []map[string]interface{}) bool {
+	if len(left) != len(right) {
+		return false
+	}
+	for index := range left {
+		leftJSON, leftErr := json.Marshal(left[index])
+		rightJSON, rightErr := json.Marshal(right[index])
+		if leftErr != nil || rightErr != nil || string(leftJSON) != string(rightJSON) {
+			return false
+		}
+	}
+	return true
+}
+
 // ─── Cache JSON API ───────────────────────────────────────

 // ReadPluginCacheJSON reads a named JSON cache file for a plugin.
@@ -0,0 +1,91 @@
+package storage
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestAppendPluginDataNDJSONCompactsAndDeduplicates(t *testing.T) {
+	s, vaultDir := newTestStorage(t)
+	retention := NDJSONRetention{
+		TimestampField:   "occurredAt",
+		MaxAge:           60 * 24 * time.Hour,
+		MaxEntries:       2,
+		MaxBytes:         16 * 1024,
+		DeduplicateField: "sourceBatchId",
+		DeduplicateValue: "batch-1",
+	}
+	old := time.Now().UTC().Add(-61 * 24 * time.Hour).Format(time.RFC3339)
+	now := time.Now().UTC().Format(time.RFC3339)
+
+	stored, err := s.AppendPluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+		{"activityId": "old", "sourceBatchId": "old-batch", "occurredAt": old},
+	}, retention)
+	if err != nil || !stored {
+		t.Fatalf("old append = (%v, %v), want (true, nil)", stored, err)
+	}
+
+	stored, err = s.AppendPluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+		{"activityId": "current-1", "sourceBatchId": "batch-1", "occurredAt": now},
+	}, retention)
+	if err != nil || !stored {
+		t.Fatalf("first current append = (%v, %v), want (true, nil)", stored, err)
+	}
+
+	stored, err = s.AppendPluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+		{"activityId": "duplicate", "sourceBatchId": "batch-1", "occurredAt": now},
+	}, retention)
+	if err != nil || stored {
+		t.Fatalf("duplicate append = (%v, %v), want (false, nil)", stored, err)
+	}
+
+	for _, item := range []struct {
+		id    string
+		batch string
+	}{
+		{id: "current-2", batch: "batch-3"},
+		{id: "current-3", batch: "batch-4"},
+	} {
+		retention.DeduplicateValue = item.batch
+		if stored, err := s.AppendPluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+			{"activityId": item.id, "sourceBatchId": item.batch, "occurredAt": now},
+		}, retention); err != nil || !stored {
+			t.Fatalf("append %s = (%v, %v), want (true, nil)", item.id, stored, err)
+		}
+	}
+
+	records, err := s.ReadPluginDataNDJSON("verstak.activity", "activity-events")
+	if err != nil {
+		t.Fatalf("ReadPluginDataNDJSON: %v", err)
+	}
+	if len(records) != 2 || records[0]["activityId"] != "current-2" || records[1]["activityId"] != "current-3" {
+		t.Fatalf("records = %+v, want current-2 and current-3", records)
+	}
+	if _, err := os.Stat(filepath.Join(vaultDir, "VerstakVault", ".verstak", "plugin-data", "verstak.activity", "activity-events.ndjson")); err != nil {
+		t.Fatalf("activity data file missing: %v", err)
+	}
+}
+
+func TestWritePluginDataNDJSONReplacesRecordsForExplicitUserClear(t *testing.T) {
+	s, _ := newTestStorage(t)
+	if _, err := s.AppendPluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+		{"activityId": "one", "occurredAt": time.Now().UTC().Format(time.RFC3339)},
+		{"activityId": "two", "occurredAt": time.Now().UTC().Format(time.RFC3339)},
+	}, NDJSONRetention{}); err != nil {
+		t.Fatalf("AppendPluginDataNDJSON: %v", err)
+	}
+	if err := s.WritePluginDataNDJSON("verstak.activity", "activity-events", []map[string]interface{}{
+		{"activityId": "two", "occurredAt": time.Now().UTC().Format(time.RFC3339)},
+	}); err != nil {
+		t.Fatalf("WritePluginDataNDJSON: %v", err)
+	}
+	records, err := s.ReadPluginDataNDJSON("verstak.activity", "activity-events")
+	if err != nil {
+		t.Fatalf("ReadPluginDataNDJSON: %v", err)
+	}
+	if len(records) != 1 || records[0]["activityId"] != "two" {
+		t.Fatalf("records = %+v, want only two", records)
+	}
+}