Use paginated blob transport for file sync

This commit is contained in:
mirivlad 2026-07-17 05:09:41 +08:00
parent e3d8078ad5
commit 4ae08ef88e
11 changed files with 708 additions and 133 deletions

View File

@ -233,14 +233,23 @@ deletes, and incompatible files become visible conflicts rather than silent
overwrites. Pairing can optionally use an existing remote vault ID to restore overwrites. Pairing can optionally use an existing remote vault ID to restore
that scope on a new device. that scope on a new device.
The current operation transport supports ordinary files and folders plus The current transport supports ordinary files and folders plus workspace (Deal)
workspace (Deal) create/rename/trash/restore with a durable workspace UUID. create/rename/trash/restore with a durable workspace UUID. `.verstak`, trash,
`.verstak`, trash, temporary files, and symlinks are excluded from ordinary temporary files, and symlinks are excluded from ordinary file sync. UTF-8 text
file sync. Text remains bounded to 2 MB and binary payloads to 8 MB; a larger remains inline only within the existing 2 MB bound; binary and larger content
or unsupported file is left as a visible unresolved sync warning instead of is staged privately under `.verstak/sync/blobs`, uploaded through the scoped
being treated as synchronized. Blob transfer, quotas, pagination, retention, Blob API, and represented in the operation log only by SHA-256 and size. The
and Secrets, plugin settings, Todo, Journal, Activity, and Browser Inbox sync download is streamed, hash/size-verified, and atomically applied. A file over
are future milestones. the configured server blob limit or otherwise unsupported remains a visible
unresolved warning instead of being treated as synchronized.
Pull is paginated. Desktop applies operations in increasing server sequence,
persists its cursor only after each successful operation, stops before later
pages at the first failure, and retries after restart. Blob references are
authorized per user/vault; a missing or corrupt blob likewise leaves the cursor
unchanged. The server is optional and file bytes are not end-to-end encrypted.
Operation-log retention/checkpoints and synchronization of Secrets, plugin
settings, Todo, Journal, Activity, and Browser Inbox remain future milestones.
## Build from source ## Build from source

View File

@ -511,12 +511,14 @@ contributions summary.
lifecycle is a separate core `workspace` entity (`create`, `rename`, lifecycle is a separate core `workspace` entity (`create`, `rename`,
`trash`, `restore`) carrying the durable `workspaceId`; plugins never obtain `trash`, `restore`) carrying the durable `workspaceId`; plugins never obtain
access to the nested workspace marker. access to the nested workspace marker.
- File transport remains bounded: UTF-8 text uses the 2 MB Files API limit and - File transport remains bounded: UTF-8 text uses the 2 MB Files API limit;
binary/other regular files use the existing base64 path up to 8 MB. Larger or binary and larger regular files are staged privately, sent through Blob API,
unsupported files remain visible unresolved warnings and are not added to a and referenced by SHA-256/size rather than base64 in `payload_json`. Desktop
successful snapshot. Blob transport, quotas, pagination, retention, and streams and verifies downloads before atomic apply. Pull is paginated and the
synchronization of Secrets, plugin settings, Todo, Journal, Activity, and durable cursor advances only after every applied sequence. Files above the
Browser Inbox are future work. configured blob limit or otherwise unsupported remain unresolved warnings.
Operation retention/checkpoints and synchronization of Secrets, plugin
settings, Todo, Journal, Activity, and Browser Inbox are future work.
- Transport push/pull uses bounded retry/backoff for transient HTTP/network - Transport push/pull uses bounded retry/backoff for transient HTTP/network
failures. Client/auth errors are not retried. failures. Client/auth errors are not retried.

View File

@ -3459,6 +3459,15 @@ func (a *App) syncNow() (map[string]interface{}, error) {
} }
pushResult := &syncsvc.PushResponse{} pushResult := &syncsvc.PushResponse{}
if len(unpushed) > 0 { if len(unpushed) > 0 {
if err := a.uploadPendingBlobs(client, unpushed); err != nil {
message := fmt.Sprintf("blob upload: %v", err)
_ = a.updateSyncError(message)
// A server-side file/quota limit is unresolved scanner input from the
// user's perspective: retain it visibly and keep the operation pending
// so a later sync retries rather than silently accepting the snapshot.
_ = a.syncSvc.SetLastWarning(message)
return nil, fmt.Errorf("blob upload: %w", err)
}
pushResult, err = client.Push(unpushed) pushResult, err = client.Push(unpushed)
if err != nil { if err != nil {
_ = a.updateSyncError(fmt.Sprintf("push: %v", err)) _ = a.updateSyncError(fmt.Sprintf("push: %v", err))
@ -3505,58 +3514,96 @@ func (a *App) syncNow() (map[string]interface{}, error) {
return result, nil return result, nil
} }
func (a *App) pullRemoteOps(client *syncsvc.Client, cursor int, initialReconciliation bool) (pulled, nextCursor, serverSequence int, err error) { // uploadPendingBlobs ensures every operation referring to binary content has
pullResult, err := client.Pull(cursor) // its immutable local cache uploaded before the operation reaches the log.
if err != nil { func (a *App) uploadPendingBlobs(client *syncsvc.Client, ops []syncsvc.Op) error {
_ = a.updateSyncError(fmt.Sprintf("pull: %v", err)) for _, op := range ops {
return 0, cursor, cursor, fmt.Errorf("pull: %w", err) if op.EntityType != syncsvc.EntityFile || (op.OpType != syncsvc.OpCreate && op.OpType != syncsvc.OpUpdate) {
}
nextCursor = cursor
lastSequenceInBatch := cursor
for _, op := range pullResult.Ops {
if op.ServerSequence <= cursor {
continue continue
} }
if op.ServerSequence <= lastSequenceInBatch { payload, err := parseSyncFilePayload(op.PayloadJSON)
if err != nil {
return fmt.Errorf("parse operation %s: %w", op.OpID, err)
}
if payload.Blob == nil {
continue
}
if payload.Blob.Size < 0 || payload.Blob.SHA256 == "" {
return fmt.Errorf("invalid blob reference in operation %s", op.OpID)
}
cachePath := syncsvc.BlobCachePath(a.vaultPath(), payload.Blob.SHA256)
ref, err := client.UploadBlob(cachePath)
if err != nil {
return fmt.Errorf("%s: %w", syncPayloadPath(op, payload), err)
}
if ref.SHA256 != payload.Blob.SHA256 || ref.Size != payload.Blob.Size {
return fmt.Errorf("%s: uploaded blob does not match operation reference", syncPayloadPath(op, payload))
}
}
return nil
}
func (a *App) pullRemoteOps(client *syncsvc.Client, cursor int, initialReconciliation bool) (pulled, nextCursor, serverSequence int, err error) {
nextCursor = cursor
for {
pageStartCursor := nextCursor
pullResult, pullErr := client.PullPage(nextCursor, 0)
if pullErr != nil {
_ = a.updateSyncError(fmt.Sprintf("pull: %v", pullErr))
return pulled, nextCursor, serverSequence, fmt.Errorf("pull: %w", pullErr)
}
serverSequence = pullResult.ServerSequence
lastSequenceInPage := nextCursor
for _, op := range pullResult.Ops {
if op.ServerSequence <= nextCursor {
continue
}
if op.ServerSequence <= lastSequenceInPage {
errMsg := fmt.Sprintf("pull response is not strictly ordered at sequence %d (%s)", op.ServerSequence, op.OpID) errMsg := fmt.Sprintf("pull response is not strictly ordered at sequence %d (%s)", op.ServerSequence, op.OpID)
_ = a.updateSyncError(errMsg) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("%s", errMsg) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} }
lastSequenceInBatch = op.ServerSequence lastSequenceInPage = op.ServerSequence
if err := a.applyRemoteOpForReconciliation(op, initialReconciliation); err != nil { if err := a.applyRemoteOpWithClient(client, op, initialReconciliation); err != nil {
path := syncOperationPath(op) path := syncOperationPath(op)
errMsg := fmt.Sprintf("pull apply failed at sequence %d for %s %s (%s): %v", op.ServerSequence, op.EntityType, path, op.OpID, err) errMsg := fmt.Sprintf("pull apply failed at sequence %d for %s %s (%s): %v", op.ServerSequence, op.EntityType, path, op.OpID, err)
_ = a.updateSyncError(errMsg) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("%s", errMsg) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} }
if err := a.syncSvc.RecordRemoteOp(op); err != nil { if err := a.syncSvc.RecordRemoteOp(op); err != nil {
errMsg := fmt.Sprintf("record applied remote operation at sequence %d (%s): %v", op.ServerSequence, op.OpID, err) errMsg := fmt.Sprintf("record applied remote operation at sequence %d (%s): %v", op.ServerSequence, op.OpID, err)
_ = a.updateSyncError(errMsg) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("%s", errMsg) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} }
if err := a.syncSvc.SetLastPullSeq(op.ServerSequence); err != nil { if err := a.syncSvc.SetLastPullSeq(op.ServerSequence); err != nil {
errMsg := fmt.Sprintf("save pull cursor at sequence %d (%s): %v", op.ServerSequence, op.OpID, err) errMsg := fmt.Sprintf("save pull cursor at sequence %d (%s): %v", op.ServerSequence, op.OpID, err)
_ = a.updateSyncError(errMsg) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("%s", errMsg) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} }
if warnings, err := a.syncSvc.RebaseSnapshot(); err != nil { if warnings, err := a.syncSvc.RebaseSnapshot(); err != nil {
errMsg := fmt.Sprintf("rebase snapshot after sequence %d (%s): %v", op.ServerSequence, op.OpID, err) errMsg := fmt.Sprintf("rebase snapshot after sequence %d (%s): %v", op.ServerSequence, op.OpID, err)
_ = a.updateSyncError(errMsg) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("%s", errMsg) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} else if err := a.syncSvc.SetLastWarning(strings.Join(warnings, "\n")); err != nil { } else if err := a.syncSvc.SetLastWarning(strings.Join(warnings, "\n")); err != nil {
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("save sync warning: %w", err) return pulled, nextCursor, serverSequence, fmt.Errorf("save sync warning: %w", err)
} }
nextCursor = op.ServerSequence nextCursor = op.ServerSequence
pulled++ pulled++
} }
if pullResult.ServerSequence > nextCursor { if pullResult.PageLastSequence != lastSequenceInPage {
if err := a.syncSvc.SetLastPullSeq(pullResult.ServerSequence); err != nil { errMsg := fmt.Sprintf("pull page cursor mismatch: got %d, applied %d", pullResult.PageLastSequence, lastSequenceInPage)
_ = a.updateSyncError(fmt.Sprintf("save pull cursor at sequence %d: %v", pullResult.ServerSequence, err)) _ = a.updateSyncError(errMsg)
return pulled, nextCursor, pullResult.ServerSequence, fmt.Errorf("save pull cursor: %w", err) return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
}
if !pullResult.HasMore {
return pulled, nextCursor, serverSequence, nil
}
if pullResult.PageLastSequence <= pageStartCursor {
errMsg := "pull page declared more operations without advancing cursor"
_ = a.updateSyncError(errMsg)
return pulled, nextCursor, serverSequence, fmt.Errorf("%s", errMsg)
} }
nextCursor = pullResult.ServerSequence
} }
return pulled, nextCursor, pullResult.ServerSequence, nil
} }
func syncOperationPath(op syncsvc.Op) string { func syncOperationPath(op syncsvc.Op) string {
@ -3601,10 +3648,14 @@ func (a *App) updateSyncSuccess(lastSyncAt string) error {
} }
func (a *App) applyRemoteOp(op syncsvc.Op) error { func (a *App) applyRemoteOp(op syncsvc.Op) error {
return a.applyRemoteOpForReconciliation(op, false) return a.applyRemoteOpWithClient(nil, op, false)
} }
func (a *App) applyRemoteOpForReconciliation(op syncsvc.Op, initialReconciliation bool) error { func (a *App) applyRemoteOpForReconciliation(op syncsvc.Op, initialReconciliation bool) error {
return a.applyRemoteOpWithClient(nil, op, initialReconciliation)
}
func (a *App) applyRemoteOpWithClient(client *syncsvc.Client, op syncsvc.Op, initialReconciliation bool) error {
if a.debug { if a.debug {
log.Printf("[sync] applyRemoteOp: type=%s entity=%s/%s", op.OpType, op.EntityType, op.EntityID) log.Printf("[sync] applyRemoteOp: type=%s entity=%s/%s", op.OpType, op.EntityType, op.EntityID)
} }
@ -3628,7 +3679,7 @@ func (a *App) applyRemoteOpForReconciliation(op syncsvc.Op, initialReconciliatio
} }
switch op.EntityType { switch op.EntityType {
case syncsvc.EntityFile: case syncsvc.EntityFile:
return a.applyRemoteFileOp(op, payload, initialReconciliation) return a.applyRemoteFileOp(client, op, payload, initialReconciliation)
case syncsvc.EntityFolder: case syncsvc.EntityFolder:
return a.applyRemoteFolderOp(op, payload, initialReconciliation) return a.applyRemoteFolderOp(op, payload, initialReconciliation)
default: default:
@ -3640,6 +3691,7 @@ type syncFilePayload struct {
Path string `json:"path"` Path string `json:"path"`
Content string `json:"content"` Content string `json:"content"`
DataBase64 *string `json:"dataBase64"` DataBase64 *string `json:"dataBase64"`
Blob *syncsvc.BlobReference `json:"blob"`
ContentHash string `json:"contentHash"` ContentHash string `json:"contentHash"`
FromPath string `json:"fromPath"` FromPath string `json:"fromPath"`
ToPath string `json:"toPath"` ToPath string `json:"toPath"`
@ -3705,7 +3757,7 @@ func (a *App) applyRemoteWorkspaceOp(op syncsvc.Op, payload syncWorkspacePayload
} }
} }
func (a *App) applyRemoteFileOp(op syncsvc.Op, payload syncFilePayload, initialReconciliation bool) error { func (a *App) applyRemoteFileOp(client *syncsvc.Client, op syncsvc.Op, payload syncFilePayload, initialReconciliation bool) error {
switch op.OpType { switch op.OpType {
case syncsvc.OpCreate: case syncsvc.OpCreate:
path := syncPayloadPath(op, payload) path := syncPayloadPath(op, payload)
@ -3722,6 +3774,9 @@ func (a *App) applyRemoteFileOp(op syncsvc.Op, payload syncFilePayload, initialR
if exists { if exists {
return fmt.Errorf("conflict: remote create would replace local file %s", path) return fmt.Errorf("conflict: remote create would replace local file %s", path)
} }
if payload.Blob != nil {
return a.applyRemoteBlobFile(client, path, payload, corefiles.WriteOptions{CreateIfMissing: true})
}
if payload.DataBase64 != nil { if payload.DataBase64 != nil {
return a.files.WriteVaultFileBytes(path, *payload.DataBase64, corefiles.WriteOptions{CreateIfMissing: true}) return a.files.WriteVaultFileBytes(path, *payload.DataBase64, corefiles.WriteOptions{CreateIfMissing: true})
} }
@ -3746,6 +3801,9 @@ func (a *App) applyRemoteFileOp(op syncsvc.Op, payload syncFilePayload, initialR
} else if pending { } else if pending {
return fmt.Errorf("conflict: remote update would replace unpushed local file %s", path) return fmt.Errorf("conflict: remote update would replace unpushed local file %s", path)
} }
if payload.Blob != nil {
return a.applyRemoteBlobFile(client, path, payload, corefiles.WriteOptions{CreateIfMissing: !exists, Overwrite: exists})
}
if payload.DataBase64 != nil { if payload.DataBase64 != nil {
return a.files.WriteVaultFileBytes(path, *payload.DataBase64, corefiles.WriteOptions{CreateIfMissing: !exists, Overwrite: exists}) return a.files.WriteVaultFileBytes(path, *payload.DataBase64, corefiles.WriteOptions{CreateIfMissing: !exists, Overwrite: exists})
} }
@ -3810,6 +3868,26 @@ func (a *App) applyRemoteFileOp(op syncsvc.Op, payload syncFilePayload, initialR
} }
} }
func (a *App) applyRemoteBlobFile(client *syncsvc.Client, path string, payload syncFilePayload, options corefiles.WriteOptions) error {
if client == nil {
return fmt.Errorf("blob operation requires an active sync client")
}
if payload.Blob == nil || payload.Blob.Size < 0 || payload.Blob.SHA256 == "" {
return fmt.Errorf("invalid remote blob reference")
}
if payload.ContentHash != "" && payload.ContentHash != payload.Blob.SHA256 {
return fmt.Errorf("remote blob hash does not match file content hash")
}
cachePath := syncsvc.BlobCachePath(a.vaultPath(), payload.Blob.SHA256)
if err := client.DownloadBlobVerified(payload.Blob.SHA256, payload.Blob.Size, cachePath); err != nil {
return fmt.Errorf("download blob: %w", err)
}
if err := a.files.WriteVaultFileFromPath(path, cachePath, options); err != nil {
return err
}
return nil
}
func (a *App) remoteFileMatches(path string, payload syncFilePayload) (matches, exists bool, err error) { func (a *App) remoteFileMatches(path string, payload syncFilePayload) (matches, exists bool, err error) {
normalized, err := corefiles.NormalizeRelativeFile(path) normalized, err := corefiles.NormalizeRelativeFile(path)
if err != nil { if err != nil {
@ -3845,6 +3923,9 @@ func remotePayloadHash(payload syncFilePayload) (string, error) {
if payload.ContentHash != "" { if payload.ContentHash != "" {
return payload.ContentHash, nil return payload.ContentHash, nil
} }
if payload.Blob != nil {
return payload.Blob.SHA256, nil
}
data := []byte(payload.Content) data := []byte(payload.Content)
if payload.DataBase64 != nil { if payload.DataBase64 != nil {
decoded, err := base64.StdEncoding.DecodeString(*payload.DataBase64) decoded, err := base64.StdEncoding.DecodeString(*payload.DataBase64)

View File

@ -1794,7 +1794,7 @@ func TestFileBridgeRecordsSyncOps(t *testing.T) {
{syncsvc.EntityFolder, "Docs", syncsvc.OpCreate, `"path":"Docs"`}, {syncsvc.EntityFolder, "Docs", syncsvc.OpCreate, `"path":"Docs"`},
{syncsvc.EntityFile, "Docs/one.txt", syncsvc.OpCreate, `"content":"hello"`}, {syncsvc.EntityFile, "Docs/one.txt", syncsvc.OpCreate, `"content":"hello"`},
{syncsvc.EntityFile, "Docs/one.txt", syncsvc.OpUpdate, `"content":"updated"`}, {syncsvc.EntityFile, "Docs/one.txt", syncsvc.OpUpdate, `"content":"updated"`},
{syncsvc.EntityFile, "Docs/image.bin", syncsvc.OpCreate, `"dataBase64":"AQID"`}, {syncsvc.EntityFile, "Docs/image.bin", syncsvc.OpCreate, `"blob":{"sha256":`},
// Snapshot reconciliation represents external and API renames uniformly // Snapshot reconciliation represents external and API renames uniformly
// as a create at the destination followed by a delete at the source. // as a create at the destination followed by a delete at the source.
{syncsvc.EntityFile, "Docs/two.txt", syncsvc.OpCreate, `"content":"updated"`}, {syncsvc.EntityFile, "Docs/two.txt", syncsvc.OpCreate, `"content":"updated"`},
@ -1948,6 +1948,52 @@ func TestSyncNowPushesLocalOpsAndAppliesPulledFileOps(t *testing.T) {
} }
} }
func TestSyncNowAppliesEveryPullPageInOrder(t *testing.T) {
app, root := newSyncFilesTestApp(t, []string{"files.read", "files.write", "files.delete"}, "local-device")
server := newLocalHTTPTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("Authorization") != "Bearer device-token" {
http.Error(w, "missing auth", http.StatusUnauthorized)
return
}
w.Header().Set("Content-Type", "application/json")
switch r.URL.Path {
case "/api/v1/sync/pull":
var request syncsvc.PullRequest
if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
t.Fatal(err)
}
pages := map[int]map[string]interface{}{
0: {"server_sequence": 3, "page_last_sequence": 1, "has_more": true, "ops": []map[string]interface{}{{"op_id": "folder", "server_sequence": 1, "device_id": "remote", "entity_type": syncsvc.EntityFolder, "entity_id": "Remote", "op_type": syncsvc.OpCreate, "payload_json": `{"path":"Remote"}`}}},
1: {"server_sequence": 3, "page_last_sequence": 2, "has_more": true, "ops": []map[string]interface{}{{"op_id": "one", "server_sequence": 2, "device_id": "remote", "entity_type": syncsvc.EntityFile, "entity_id": "Remote/one.txt", "op_type": syncsvc.OpCreate, "payload_json": `{"path":"Remote/one.txt","content":"one"}`}}},
2: {"server_sequence": 3, "page_last_sequence": 3, "has_more": false, "ops": []map[string]interface{}{{"op_id": "two", "server_sequence": 3, "device_id": "remote", "entity_type": syncsvc.EntityFile, "entity_id": "Remote/two.txt", "op_type": syncsvc.OpCreate, "payload_json": `{"path":"Remote/two.txt","content":"two"}`}}},
3: {"server_sequence": 3, "page_last_sequence": 3, "has_more": false, "ops": []map[string]interface{}{}},
}
_ = json.NewEncoder(w).Encode(pages[request.SinceSequence])
case "/api/v1/sync/push":
_ = json.NewEncoder(w).Encode(map[string]interface{}{"accepted": []string{}, "count": 0, "conflicts": []map[string]interface{}{}})
default:
http.NotFound(w, r)
}
}))
defer server.Close()
if err := app.syncSvc.SetState(server.URL, ""); err != nil {
t.Fatal(err)
}
if err := syncsvc.SaveDeviceToken(root, "device-token"); err != nil {
t.Fatal(err)
}
result, err := app.syncNow()
if err != nil {
t.Fatal(err)
}
if result["pulled"] != 3 {
t.Fatalf("pull result = %#v, want three operations", result)
}
expectText(t, app, "Remote/one.txt", "one")
expectText(t, app, "Remote/two.txt", "two")
assertLastPullSequence(t, app.syncSvc, 3)
}
func TestSyncNowStopsAtFailedRemoteOperationAndRetriesAfterRestart(t *testing.T) { func TestSyncNowStopsAtFailedRemoteOperationAndRetriesAfterRestart(t *testing.T) {
app, root := newSyncFilesTestApp(t, []string{"files.read", "files.write", "files.delete"}, "local-device") app, root := newSyncFilesTestApp(t, []string{"files.read", "files.write", "files.delete"}, "local-device")
pullCalls := 0 pullCalls := 0

View File

@ -1,6 +1,7 @@
package api package api
import ( import (
"bytes"
"os" "os"
"path/filepath" "path/filepath"
"strings" "strings"
@ -104,6 +105,26 @@ func TestSyncNowAgainstRealServerTwoVaults(t *testing.T) {
expectText(t, appB, "Shared/external.txt", "external while running") expectText(t, appB, "Shared/external.txt", "external while running")
assertNoUnpushedOps(t, appB) assertNoUnpushedOps(t, appB)
// This exceeds the former 8 MiB inline/base64 ceiling. The operation must
// contain only a blob reference; the actual bytes travel through Blob API.
binary := make([]byte, corefiles.MaxBinaryReadBytes+1)
for i := range binary {
binary[i] = byte(i % 251)
}
if err := os.WriteFile(filepath.Join(rootA, "Shared", "large.bin"), binary, 0o644); err != nil {
t.Fatalf("write large binary: %v", err)
}
expectSyncCounts(t, appA, 1, 1)
expectSyncCounts(t, appB, 0, 1)
received, err := os.ReadFile(filepath.Join(rootB, "Shared", "large.bin"))
if err != nil {
t.Fatalf("read synced large binary: %v", err)
}
if !bytes.Equal(received, binary) {
t.Fatal("large binary content differs after blob sync")
}
assertNoUnpushedOps(t, appB)
if err := os.WriteFile(filepath.Join(rootB, "Shared", "external.txt"), []byte("external while closed"), 0o644); err != nil { if err := os.WriteFile(filepath.Join(rootB, "Shared", "external.txt"), []byte("external while closed"), 0o644); err != nil {
t.Fatalf("offline external update: %v", err) t.Fatalf("offline external update: %v", err)
} }

View File

@ -4,6 +4,7 @@ import (
"encoding/base64" "encoding/base64"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io"
"io/fs" "io/fs"
"mime" "mime"
"os" "os"
@ -201,6 +202,85 @@ func (s *Service) WriteVaultFileBytes(relativePath string, dataBase64 string, op
return s.writeVaultFileData(relativePath, data, options) return s.writeVaultFileData(relativePath, data, options)
} }
// WriteVaultFileFromPath streams a verified temporary file into the vault and
// replaces the destination atomically. It is used by core sync so a Blob never
// needs to be base64-decoded or held in memory.
func (s *Service) WriteVaultFileFromPath(relativePath, sourcePath string, options WriteOptions) error {
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
info, err := source.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("blob source is not a regular file")
}
root, rel, full, err := s.resolveFile(relativePath)
if err != nil {
return err
}
if err := rejectSymlinkPath(root, rel, true); err != nil {
return err
}
parent := filepath.Dir(full)
if info, err := os.Stat(parent); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("parent-not-found: %s", pathDir(rel))
}
return err
} else if !info.IsDir() {
return fmt.Errorf("parent-not-directory: %s", pathDir(rel))
}
existing, err := os.Lstat(full)
if err == nil {
if existing.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("symlink-not-allowed: %s", rel)
}
if !existing.Mode().IsRegular() {
return fmt.Errorf("not-regular-file: %s", rel)
}
if !options.Overwrite {
return fmt.Errorf("conflict: %s", rel)
}
} else if os.IsNotExist(err) {
if !options.CreateIfMissing {
return fmt.Errorf("not-found: %s", rel)
}
} else {
return err
}
tmp, err := os.CreateTemp(parent, ".verstak-write-*")
if err != nil {
return err
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
if _, err := io.Copy(tmp, source); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, full); err != nil {
return err
}
cleanup = false
return nil
}
func (s *Service) writeVaultFileData(relativePath string, data []byte, options WriteOptions) error { func (s *Service) writeVaultFileData(relativePath string, data []byte, options WriteOptions) error {
root, rel, full, err := s.resolveFile(relativePath) root, rel, full, err := s.resolveFile(relativePath)
if err != nil { if err != nil {

View File

@ -2,6 +2,8 @@ package sync
import ( import (
"bytes" "bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
@ -216,68 +218,147 @@ func (c *Client) Push(ops []Op) (*PushResponse, error) {
// PullRequest is the payload for POST /sync/pull. // PullRequest is the payload for POST /sync/pull.
type PullRequest struct { type PullRequest struct {
SinceSequence int `json:"since_sequence"` SinceSequence int `json:"since_sequence"`
PageLimit int `json:"page_limit,omitempty"`
} }
// PullResponse is the response from POST /sync/pull. // PullResponse is the response from POST /sync/pull.
type PullResponse struct { type PullResponse struct {
ServerSequence int `json:"server_sequence"` ServerSequence int `json:"server_sequence"`
PageLastSequence int `json:"page_last_sequence"`
HasMore bool `json:"has_more"`
Ops []Op `json:"ops"` Ops []Op `json:"ops"`
} }
// BlobReference is the only binary content representation permitted inside a
// sync operation. The bytes travel via the Blob API, never payload_json.
type BlobReference struct {
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
}
// ServerError carries a public stable error code. UI layers map Code to their
// own localized wording and must not rely on a server diagnostic string.
type ServerError struct {
Status int
Code string
}
func (e *ServerError) Error() string {
if e.Code == "" {
return fmt.Sprintf("sync-server:request_failed (HTTP %d)", e.Status)
}
return fmt.Sprintf("sync-server:%s (HTTP %d)", e.Code, e.Status)
}
// Pull fetches remote operations since a given sequence. // Pull fetches remote operations since a given sequence.
func (c *Client) Pull(sinceSequence int) (*PullResponse, error) { func (c *Client) Pull(sinceSequence int) (*PullResponse, error) {
req := PullRequest{SinceSequence: sinceSequence} return c.PullPage(sinceSequence, 0)
}
// PullPage fetches one bounded ordered page. The caller advances its durable
// cursor only after every returned operation has applied successfully.
func (c *Client) PullPage(sinceSequence, pageLimit int) (*PullResponse, error) {
req := PullRequest{SinceSequence: sinceSequence, PageLimit: pageLimit}
var resp PullResponse var resp PullResponse
if err := c.post("/api/v1/sync/pull", req, &resp); err != nil { if err := c.post("/api/v1/sync/pull", req, &resp); err != nil {
return nil, err return nil, err
} }
// Servers before pull pagination did not include page_last_sequence. Keep
// the desktop compatible during rolling upgrades without ever advancing
// beyond an operation actually present in the response.
if resp.PageLastSequence == 0 && len(resp.Ops) > 0 {
resp.PageLastSequence = resp.Ops[len(resp.Ops)-1].ServerSequence
}
return &resp, nil return &resp, nil
} }
// UploadBlob uploads a file to the server and returns its SHA-256. // UploadBlob streams a local file through a multipart pipe. It keeps the
func (c *Client) UploadBlob(localPath string) (sha256 string, err error) { // process memory bounded even when the file is many times larger than an
var b bytes.Buffer // inline sync payload.
w := multipart.NewWriter(&b) func (c *Client) UploadBlob(localPath string) (BlobReference, error) {
fw, err := w.CreateFormFile("file", filepath.Base(localPath)) info, err := os.Stat(localPath)
if err != nil { if err != nil {
return "", err return BlobReference{}, err
}
if !info.Mode().IsRegular() {
return BlobReference{}, fmt.Errorf("blob source is not a regular file")
}
reader, writer := io.Pipe()
multipartWriter := multipart.NewWriter(writer)
writeDone := make(chan error, 1)
go func() {
defer func() {
_ = writer.Close()
}()
part, err := multipartWriter.CreateFormFile("file", filepath.Base(localPath))
if err == nil {
file, openErr := os.Open(localPath)
if openErr != nil {
err = openErr
} else {
_, err = io.Copy(part, file)
closeErr := file.Close()
if err == nil {
err = closeErr
}
}
}
if closeErr := multipartWriter.Close(); err == nil {
err = closeErr
} }
f, err := os.Open(localPath)
if err != nil { if err != nil {
return "", err _ = writer.CloseWithError(err)
} }
defer f.Close() writeDone <- err
if _, err := io.Copy(fw, f); err != nil { }()
return "", err
}
w.Close()
req, err := http.NewRequest("POST", c.ServerURL+"/api/v1/blobs/", &b) req, err := http.NewRequest("POST", c.ServerURL+"/api/v1/blobs/", reader)
if err != nil { if err != nil {
return "", err _ = reader.Close()
return BlobReference{}, err
} }
req.Header.Set("Content-Type", w.FormDataContentType()) req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+c.bearerToken()) req.Header.Set("Authorization", "Bearer "+c.bearerToken())
resp, err := c.HTTP.Do(req) resp, err := c.HTTP.Do(req)
if err != nil { if err != nil {
return "", err _ = reader.Close()
<-writeDone
return BlobReference{}, err
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode >= http.StatusBadRequest {
var result struct { writeErr := <-writeDone
SHA256 string `json:"sha256"` if writeErr != nil {
Size int `json:"size"` return BlobReference{}, writeErr
} }
return BlobReference{}, c.readErrorBody(resp, resp.StatusCode)
}
var result BlobReference
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", err return BlobReference{}, err
} }
return result.SHA256, nil if err := <-writeDone; err != nil {
return BlobReference{}, err
}
if result.Size != info.Size() || !validBlobSHA256(result.SHA256) {
return BlobReference{}, fmt.Errorf("invalid blob upload response")
}
return result, nil
} }
// DownloadBlob downloads a blob by SHA-256 hash. // DownloadBlob downloads a blob by SHA-256 hash.
func (c *Client) DownloadBlob(sha256, destPath string) error { func (c *Client) DownloadBlob(shaHex, destPath string) error {
req, err := http.NewRequest("GET", c.ServerURL+"/api/v1/blobs/"+sha256, nil) return c.DownloadBlobVerified(shaHex, -1, destPath)
}
// DownloadBlobVerified streams to a temporary file, verifies the announced
// hash and size, and only then atomically makes the file visible to the vault.
func (c *Client) DownloadBlobVerified(shaHex string, expectedSize int64, destPath string) error {
if !validBlobSHA256(shaHex) || expectedSize < -1 {
return fmt.Errorf("invalid blob reference")
}
req, err := http.NewRequest("GET", c.ServerURL+"/api/v1/blobs/"+shaHex, nil)
if err != nil { if err != nil {
return err return err
} }
@ -289,18 +370,66 @@ func (c *Client) DownloadBlob(sha256, destPath string) error {
} }
defer resp.Body.Close() defer resp.Body.Close()
if resp.StatusCode != 200 { if resp.StatusCode != http.StatusOK {
return fmt.Errorf("download blob: HTTP %d", resp.StatusCode) return c.readErrorBody(resp, resp.StatusCode)
} }
out, err := os.Create(destPath) if expectedSize >= 0 && resp.ContentLength >= 0 && resp.ContentLength != expectedSize {
return fmt.Errorf("download blob: size mismatch")
}
if err := os.MkdirAll(filepath.Dir(destPath), 0o750); err != nil {
return err
}
out, err := os.CreateTemp(filepath.Dir(destPath), ".verstak-blob-*")
if err != nil { if err != nil {
return err return err
} }
defer out.Close() tmpPath := out.Name()
_, err = io.Copy(out, resp.Body) cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
hash := sha256.New()
limit := int64(1<<63 - 1)
if expectedSize >= 0 {
limit = expectedSize + 1
}
written, err := io.Copy(io.MultiWriter(out, hash), io.LimitReader(resp.Body, limit))
if err != nil {
_ = out.Close()
return err return err
} }
if expectedSize >= 0 && written != expectedSize {
_ = out.Close()
return fmt.Errorf("download blob: size mismatch")
}
if actual := hex.EncodeToString(hash.Sum(nil)); actual != shaHex {
_ = out.Close()
return fmt.Errorf("download blob: SHA-256 mismatch")
}
if err := out.Sync(); err != nil {
_ = out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, destPath); err != nil {
return err
}
cleanup = false
return nil
}
func validBlobSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}
func minInt(a, b int) int { func minInt(a, b int) int {
if a < b { if a < b {
@ -430,5 +559,11 @@ func (c *Client) readErrorBody(resp *http.Response, statusCode int) error {
if strings.Contains(lower, "<html") || strings.Contains(lower, "<!doctype") { if strings.Contains(lower, "<html") || strings.Contains(lower, "<!doctype") {
return fmt.Errorf("not a Verstak Sync server (HTTP %d)", statusCode) return fmt.Errorf("not a Verstak Sync server (HTTP %d)", statusCode)
} }
return fmt.Errorf("server error (HTTP %d)", statusCode) var payload struct {
Code string `json:"code"`
}
if err := json.Unmarshal([]byte(body), &payload); err == nil && payload.Code != "" {
return &ServerError{Status: statusCode, Code: payload.Code}
}
return &ServerError{Status: statusCode, Code: "request_failed"}
} }

View File

@ -1,9 +1,14 @@
package sync package sync
import ( import (
"crypto/sha256"
"encoding/json" "encoding/json"
"fmt"
"io"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
"os"
"path/filepath"
"testing" "testing"
"time" "time"
) )
@ -119,3 +124,95 @@ func TestPairDeviceSendsVaultID(t *testing.T) {
t.Fatalf("paired vault ID = %q, want vault-123", pairedVaultID) t.Fatalf("paired vault ID = %q, want vault-123", pairedVaultID)
} }
} }
func TestPullPageReadsPaginationMetadata(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/sync/pull" {
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"server_sequence": 9, "page_last_sequence": 4, "has_more": true,
"ops": []map[string]interface{}{{"op_id": "op-4", "server_sequence": 4, "device_id": "other", "entity_type": "file", "entity_id": "a.bin", "op_type": "update", "payload_json": `{}`, "created_at": "2026-01-01T00:00:00Z"}},
})
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
response, err := client.PullPage(2, 2)
if err != nil {
t.Fatal(err)
}
if response.PageLastSequence != 4 || !response.HasMore || response.ServerSequence != 9 {
t.Fatalf("pagination response = %+v", response)
}
}
func TestUploadBlobStreamsMultipartAndChecksReturnedSize(t *testing.T) {
data := []byte("streamed binary payload")
path := filepath.Join(t.TempDir(), "blob.bin")
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1024); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
file, _, err := r.FormFile("file")
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer file.Close()
received, err := io.ReadAll(file)
if err != nil || string(received) != string(data) {
http.Error(w, "bad upload", http.StatusBadRequest)
return
}
hash := sha256.Sum256(data)
_ = json.NewEncoder(w).Encode(map[string]interface{}{"sha256": fmt.Sprintf("%x", hash[:]), "size": len(data)})
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
ref, err := client.UploadBlob(path)
if err != nil {
t.Fatal(err)
}
if ref.Size != int64(len(data)) {
t.Fatalf("uploaded reference = %+v", ref)
}
}
func TestDownloadBlobVerifiesHashAndLeavesNoCorruptDestination(t *testing.T) {
dest := filepath.Join(t.TempDir(), "received.bin")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", "7")
_, _ = w.Write([]byte("corrupt"))
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
err := client.DownloadBlobVerified("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 7, dest)
if err == nil {
t.Fatal("corrupt blob was accepted")
}
if _, err := os.Stat(dest); !os.IsNotExist(err) {
t.Fatalf("corrupt destination remains: %v", err)
}
}
func TestClientPreservesStableServerErrorCode(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusRequestEntityTooLarge)
_, _ = w.Write([]byte(`{"error":"internal wording must not reach UI","code":"quota_exceeded"}`))
}))
defer server.Close()
client := NewClient(server.URL, "token", "device", t.TempDir())
err := client.post("/api/v1/sync/push", map[string]string{}, nil)
serverErr, ok := err.(*ServerError)
if !ok || serverErr.Code != "quota_exceeded" || serverErr.Status != http.StatusRequestEntityTooLarge {
t.Fatalf("error = %#v, want quota server error", err)
}
}

View File

@ -2,7 +2,6 @@ package sync
import ( import (
"crypto/sha256" "crypto/sha256"
"encoding/base64"
"encoding/hex" "encoding/hex"
"encoding/json" "encoding/json"
"fmt" "fmt"
@ -21,7 +20,17 @@ import (
) )
const snapshotVersion = 1 const snapshotVersion = 1
const maxOperationFileBytes = corefiles.MaxBinaryReadBytes
// maxOperationFileBytes is an explicit desktop-side safety ceiling. Binary
// content is streamed through the Blob API rather than embedded in operations,
// so it is intentionally higher than the plugin Files API read limit.
const maxOperationFileBytes int64 = 256 * 1024 * 1024
// BlobCachePath is core-private durable staging for a local operation's
// immutable binary content. It remains excluded from ordinary file sync.
func BlobCachePath(vaultRoot, hash string) string {
return filepath.Join(vaultRoot, ".verstak", "sync", "blobs", hash)
}
// Snapshot is the durable local view of the synchronizable part of a vault. // Snapshot is the durable local view of the synchronizable part of a vault.
// Its entries are only files and folders whose latest state was successfully // Its entries are only files and folders whose latest state was successfully
@ -721,24 +730,93 @@ func entriesEqual(left, right SnapshotEntry) bool {
return left.Type == right.Type && left.Size == right.Size && left.Hash == right.Hash return left.Type == right.Type && left.Size == right.Size && left.Hash == right.Hash
} }
func payloadForEntry(vaultRoot string, entry SnapshotEntry) (map[string]string, error) { func payloadForEntry(vaultRoot string, entry SnapshotEntry) (map[string]interface{}, error) {
payload := map[string]string{"path": entry.Path, "contentHash": entry.Hash} payload := map[string]interface{}{"path": entry.Path, "contentHash": entry.Hash}
if entry.Type == EntityFolder { if entry.Type == EntityFolder {
return payload, nil return payload, nil
} }
data, err := os.ReadFile(filepath.Join(vaultRoot, filepath.FromSlash(entry.Path))) path := filepath.Join(vaultRoot, filepath.FromSlash(entry.Path))
if err != nil { if hash, err := sha256File(path); err != nil {
return nil, err
}
if int64(len(data)) > maxOperationFileBytes {
return nil, fmt.Errorf("file-too-large: %s", entry.Path)
}
if hash, err := sha256File(filepath.Join(vaultRoot, filepath.FromSlash(entry.Path))); err != nil {
return nil, err return nil, err
} else if hash != entry.Hash { } else if hash != entry.Hash {
return nil, fmt.Errorf("file changed during scan: %s", entry.Path) return nil, fmt.Errorf("file changed during scan: %s", entry.Path)
} }
return filePayload(entry.Path, data, entry.Hash), nil if entry.Size > corefiles.MaxTextFileBytes {
if err := cacheBlob(path, BlobCachePath(vaultRoot, entry.Hash), entry.Hash); err != nil {
return nil, err
}
payload["blob"] = map[string]interface{}{"sha256": entry.Hash, "size": entry.Size}
return payload, nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
if isSyncText(data) {
payload["content"] = string(data)
return payload, nil
}
if err := cacheBlob(path, BlobCachePath(vaultRoot, entry.Hash), entry.Hash); err != nil {
return nil, err
}
payload["blob"] = map[string]interface{}{"sha256": entry.Hash, "size": entry.Size}
return payload, nil
}
func cacheBlob(source, destination, wantHash string) error {
if info, err := os.Lstat(destination); err == nil {
if !info.Mode().IsRegular() {
return fmt.Errorf("blob cache target is not a regular file")
}
if hash, err := sha256File(destination); err == nil && hash == wantHash {
return nil
}
if err := os.Remove(destination); err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
if err := os.MkdirAll(filepath.Dir(destination), 0o750); err != nil {
return err
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
tmp, err := os.CreateTemp(filepath.Dir(destination), ".blob-*")
if err != nil {
return err
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
hash := sha256.New()
if _, err := io.Copy(io.MultiWriter(tmp, hash), in); err != nil {
_ = tmp.Close()
return err
}
if actual := hex.EncodeToString(hash.Sum(nil)); actual != wantHash {
_ = tmp.Close()
return fmt.Errorf("file changed during blob staging")
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, destination); err != nil {
return err
}
cleanup = false
return nil
} }
func newSnapshotOp(deviceID, entityType, entityID, opType string, payload interface{}) Op { func newSnapshotOp(deviceID, entityType, entityID, opType string, payload interface{}) Op {
@ -763,19 +841,6 @@ func pathDepth(path string) int {
return strings.Count(path, "/") + 1 return strings.Count(path, "/") + 1
} }
// filePayload encodes an already bounded file without changing its content
// representation. UTF-8 files use plain text up to the existing text limit;
// other supported files use the current bounded base64 transport.
func filePayload(path string, data []byte, hash string) map[string]string {
payload := map[string]string{"path": path, "contentHash": hash}
if int64(len(data)) <= corefiles.MaxTextFileBytes && isSyncText(data) {
payload["content"] = string(data)
return payload
}
payload["dataBase64"] = base64.StdEncoding.EncodeToString(data)
return payload
}
func isSyncText(data []byte) bool { func isSyncText(data []byte) bool {
if !utf8.Valid(data) { if !utf8.Valid(data) {
return false return false

View File

@ -9,6 +9,7 @@ import (
"testing" "testing"
"github.com/google/uuid" "github.com/google/uuid"
corefiles "github.com/verstak/verstak-desktop/internal/core/files"
) )
func TestScanAndRecordTracksExternalWorkspaceLifecycleByIdentity(t *testing.T) { func TestScanAndRecordTracksExternalWorkspaceLifecycleByIdentity(t *testing.T) {
@ -191,6 +192,41 @@ func TestScanAndRecordBaselinesThenRecordsExternalChanges(t *testing.T) {
} }
} }
func TestSnapshotUsesBlobReferenceForBinaryFileBeyondInlineLimit(t *testing.T) {
root := t.TempDir()
service := NewService(root, "device-a")
if _, err := service.ScanAndRecord(); err != nil {
t.Fatalf("baseline: %v", err)
}
data := make([]byte, corefiles.MaxBinaryReadBytes+1)
for i := range data {
data[i] = byte(i % 251)
}
if err := os.WriteFile(filepath.Join(root, "large.bin"), data, 0o600); err != nil {
t.Fatal(err)
}
if _, err := service.ScanAndRecord(); err != nil {
t.Fatalf("scan binary: %v", err)
}
ops := unpushedOps(t, service)
if len(ops) != 1 {
t.Fatalf("operations = %#v, want one file create", ops)
}
var payload struct {
DataBase64 *string `json:"dataBase64"`
Blob *struct {
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
} `json:"blob"`
}
if err := json.Unmarshal([]byte(ops[0].PayloadJSON), &payload); err != nil {
t.Fatal(err)
}
if payload.DataBase64 != nil || payload.Blob == nil || payload.Blob.Size != int64(len(data)) || payload.Blob.SHA256 == "" {
t.Fatalf("binary sync payload = %s, want blob reference without base64", ops[0].PayloadJSON)
}
}
func TestScanAndRecordNeverTreatsInitialFilesAsDeletes(t *testing.T) { func TestScanAndRecordNeverTreatsInitialFilesAsDeletes(t *testing.T) {
root := t.TempDir() root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "Existing"), 0o755); err != nil { if err := os.Mkdir(filepath.Join(root, "Existing"), 0o755); err != nil {
@ -362,7 +398,10 @@ func TestScanAndRecordSkipsReservedTemporaryAndSymlinkPaths(t *testing.T) {
func TestScanAndRecordKeepsUnsupportedFileUnresolved(t *testing.T) { func TestScanAndRecordKeepsUnsupportedFileUnresolved(t *testing.T) {
root := t.TempDir() root := t.TempDir()
path := filepath.Join(root, "too-large.bin") path := filepath.Join(root, "too-large.bin")
if err := os.WriteFile(path, make([]byte, maxOperationFileBytes+1), 0o600); err != nil { if err := os.WriteFile(path, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Truncate(path, maxOperationFileBytes+1); err != nil {
t.Fatal(err) t.Fatal(err)
} }
service := NewService(root, "device-a") service := NewService(root, "device-a")

View File

@ -50,10 +50,10 @@ NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
sqlite3 "$DATA_DIR/server.db" " sqlite3 "$DATA_DIR/server.db" "
INSERT INTO server_users (id, username, email, password_hash, confirmed, created_at) INSERT INTO server_users (id, username, email, password_hash, confirmed, created_at)
VALUES ('smoke-user', 'smoke-user', 'smoke@example.test', 'unused', 1, '$NOW'); VALUES ('smoke-user', 'smoke-user', 'smoke@example.test', 'unused', 1, '$NOW');
INSERT INTO server_devices (id, name, api_key, user_id, vault_id, last_seen, created_at) INSERT INTO server_devices (id, name, api_key, legacy_api_key, user_id, vault_id, last_seen, created_at)
VALUES ('smoke-device-a', 'Smoke Device A', 'smoke-key-a', 'smoke-user', 'smoke-vault', '$NOW', '$NOW'); VALUES ('smoke-device-a', 'Smoke Device A', 'smoke-key-a', 1, 'smoke-user', 'smoke-vault', '$NOW', '$NOW');
INSERT INTO server_devices (id, name, api_key, user_id, vault_id, last_seen, created_at) INSERT INTO server_devices (id, name, api_key, legacy_api_key, user_id, vault_id, last_seen, created_at)
VALUES ('smoke-device-b', 'Smoke Device B', 'smoke-key-b', 'smoke-user', 'smoke-vault', '$NOW', '$NOW'); VALUES ('smoke-device-b', 'Smoke Device B', 'smoke-key-b', 1, 'smoke-user', 'smoke-vault', '$NOW', '$NOW');
" "
( (