Use paginated blob transport for file sync

This commit is contained in:
2026-07-17 05:09:41 +08:00
parent e3d8078ad5
commit 4ae08ef88e
11 changed files with 708 additions and 133 deletions
+80
View File
@@ -4,6 +4,7 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
"io"
"io/fs"
"mime"
"os"
@@ -201,6 +202,85 @@ func (s *Service) WriteVaultFileBytes(relativePath string, dataBase64 string, op
return s.writeVaultFileData(relativePath, data, options)
}
// WriteVaultFileFromPath streams a verified temporary file into the vault and
// replaces the destination atomically. It is used by core sync so a Blob never
// needs to be base64-decoded or held in memory.
func (s *Service) WriteVaultFileFromPath(relativePath, sourcePath string, options WriteOptions) error {
source, err := os.Open(sourcePath)
if err != nil {
return err
}
defer source.Close()
info, err := source.Stat()
if err != nil {
return err
}
if !info.Mode().IsRegular() {
return fmt.Errorf("blob source is not a regular file")
}
root, rel, full, err := s.resolveFile(relativePath)
if err != nil {
return err
}
if err := rejectSymlinkPath(root, rel, true); err != nil {
return err
}
parent := filepath.Dir(full)
if info, err := os.Stat(parent); err != nil {
if os.IsNotExist(err) {
return fmt.Errorf("parent-not-found: %s", pathDir(rel))
}
return err
} else if !info.IsDir() {
return fmt.Errorf("parent-not-directory: %s", pathDir(rel))
}
existing, err := os.Lstat(full)
if err == nil {
if existing.Mode()&os.ModeSymlink != 0 {
return fmt.Errorf("symlink-not-allowed: %s", rel)
}
if !existing.Mode().IsRegular() {
return fmt.Errorf("not-regular-file: %s", rel)
}
if !options.Overwrite {
return fmt.Errorf("conflict: %s", rel)
}
} else if os.IsNotExist(err) {
if !options.CreateIfMissing {
return fmt.Errorf("not-found: %s", rel)
}
} else {
return err
}
tmp, err := os.CreateTemp(parent, ".verstak-write-*")
if err != nil {
return err
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
if _, err := io.Copy(tmp, source); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, full); err != nil {
return err
}
cleanup = false
return nil
}
func (s *Service) writeVaultFileData(relativePath string, data []byte, options WriteOptions) error {
root, rel, full, err := s.resolveFile(relativePath)
if err != nil {
+171 -36
View File
@@ -2,6 +2,8 @@ package sync
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
@@ -216,68 +218,147 @@ func (c *Client) Push(ops []Op) (*PushResponse, error) {
// PullRequest is the payload for POST /sync/pull.
type PullRequest struct {
SinceSequence int `json:"since_sequence"`
PageLimit int `json:"page_limit,omitempty"`
}
// PullResponse is the response from POST /sync/pull.
type PullResponse struct {
ServerSequence int `json:"server_sequence"`
Ops []Op `json:"ops"`
ServerSequence int `json:"server_sequence"`
PageLastSequence int `json:"page_last_sequence"`
HasMore bool `json:"has_more"`
Ops []Op `json:"ops"`
}
// BlobReference is the only binary content representation permitted inside a
// sync operation. The bytes travel via the Blob API, never payload_json.
type BlobReference struct {
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
}
// ServerError carries a public stable error code. UI layers map Code to their
// own localized wording and must not rely on a server diagnostic string.
type ServerError struct {
Status int
Code string
}
func (e *ServerError) Error() string {
if e.Code == "" {
return fmt.Sprintf("sync-server:request_failed (HTTP %d)", e.Status)
}
return fmt.Sprintf("sync-server:%s (HTTP %d)", e.Code, e.Status)
}
// Pull fetches remote operations since a given sequence.
func (c *Client) Pull(sinceSequence int) (*PullResponse, error) {
req := PullRequest{SinceSequence: sinceSequence}
return c.PullPage(sinceSequence, 0)
}
// PullPage fetches one bounded ordered page. The caller advances its durable
// cursor only after every returned operation has applied successfully.
func (c *Client) PullPage(sinceSequence, pageLimit int) (*PullResponse, error) {
req := PullRequest{SinceSequence: sinceSequence, PageLimit: pageLimit}
var resp PullResponse
if err := c.post("/api/v1/sync/pull", req, &resp); err != nil {
return nil, err
}
// Servers before pull pagination did not include page_last_sequence. Keep
// the desktop compatible during rolling upgrades without ever advancing
// beyond an operation actually present in the response.
if resp.PageLastSequence == 0 && len(resp.Ops) > 0 {
resp.PageLastSequence = resp.Ops[len(resp.Ops)-1].ServerSequence
}
return &resp, nil
}
// UploadBlob uploads a file to the server and returns its SHA-256.
func (c *Client) UploadBlob(localPath string) (sha256 string, err error) {
var b bytes.Buffer
w := multipart.NewWriter(&b)
fw, err := w.CreateFormFile("file", filepath.Base(localPath))
// UploadBlob streams a local file through a multipart pipe. It keeps the
// process memory bounded even when the file is many times larger than an
// inline sync payload.
func (c *Client) UploadBlob(localPath string) (BlobReference, error) {
info, err := os.Stat(localPath)
if err != nil {
return "", err
return BlobReference{}, err
}
f, err := os.Open(localPath)
if err != nil {
return "", err
if !info.Mode().IsRegular() {
return BlobReference{}, fmt.Errorf("blob source is not a regular file")
}
defer f.Close()
if _, err := io.Copy(fw, f); err != nil {
return "", err
}
w.Close()
reader, writer := io.Pipe()
multipartWriter := multipart.NewWriter(writer)
writeDone := make(chan error, 1)
go func() {
defer func() {
_ = writer.Close()
}()
part, err := multipartWriter.CreateFormFile("file", filepath.Base(localPath))
if err == nil {
file, openErr := os.Open(localPath)
if openErr != nil {
err = openErr
} else {
_, err = io.Copy(part, file)
closeErr := file.Close()
if err == nil {
err = closeErr
}
}
}
if closeErr := multipartWriter.Close(); err == nil {
err = closeErr
}
if err != nil {
_ = writer.CloseWithError(err)
}
writeDone <- err
}()
req, err := http.NewRequest("POST", c.ServerURL+"/api/v1/blobs/", &b)
req, err := http.NewRequest("POST", c.ServerURL+"/api/v1/blobs/", reader)
if err != nil {
return "", err
_ = reader.Close()
return BlobReference{}, err
}
req.Header.Set("Content-Type", w.FormDataContentType())
req.Header.Set("Content-Type", multipartWriter.FormDataContentType())
req.Header.Set("Authorization", "Bearer "+c.bearerToken())
resp, err := c.HTTP.Do(req)
if err != nil {
return "", err
_ = reader.Close()
<-writeDone
return BlobReference{}, err
}
defer resp.Body.Close()
var result struct {
SHA256 string `json:"sha256"`
Size int `json:"size"`
if resp.StatusCode >= http.StatusBadRequest {
writeErr := <-writeDone
if writeErr != nil {
return BlobReference{}, writeErr
}
return BlobReference{}, c.readErrorBody(resp, resp.StatusCode)
}
var result BlobReference
if err := json.NewDecoder(resp.Body).Decode(&result); err != nil {
return "", err
return BlobReference{}, err
}
return result.SHA256, nil
if err := <-writeDone; err != nil {
return BlobReference{}, err
}
if result.Size != info.Size() || !validBlobSHA256(result.SHA256) {
return BlobReference{}, fmt.Errorf("invalid blob upload response")
}
return result, nil
}
// DownloadBlob downloads a blob by SHA-256 hash.
func (c *Client) DownloadBlob(sha256, destPath string) error {
req, err := http.NewRequest("GET", c.ServerURL+"/api/v1/blobs/"+sha256, nil)
func (c *Client) DownloadBlob(shaHex, destPath string) error {
return c.DownloadBlobVerified(shaHex, -1, destPath)
}
// DownloadBlobVerified streams to a temporary file, verifies the announced
// hash and size, and only then atomically makes the file visible to the vault.
func (c *Client) DownloadBlobVerified(shaHex string, expectedSize int64, destPath string) error {
if !validBlobSHA256(shaHex) || expectedSize < -1 {
return fmt.Errorf("invalid blob reference")
}
req, err := http.NewRequest("GET", c.ServerURL+"/api/v1/blobs/"+shaHex, nil)
if err != nil {
return err
}
@@ -289,17 +370,65 @@ func (c *Client) DownloadBlob(sha256, destPath string) error {
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return fmt.Errorf("download blob: HTTP %d", resp.StatusCode)
if resp.StatusCode != http.StatusOK {
return c.readErrorBody(resp, resp.StatusCode)
}
out, err := os.Create(destPath)
if expectedSize >= 0 && resp.ContentLength >= 0 && resp.ContentLength != expectedSize {
return fmt.Errorf("download blob: size mismatch")
}
if err := os.MkdirAll(filepath.Dir(destPath), 0o750); err != nil {
return err
}
out, err := os.CreateTemp(filepath.Dir(destPath), ".verstak-blob-*")
if err != nil {
return err
}
defer out.Close()
_, err = io.Copy(out, resp.Body)
return err
tmpPath := out.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
hash := sha256.New()
limit := int64(1<<63 - 1)
if expectedSize >= 0 {
limit = expectedSize + 1
}
written, err := io.Copy(io.MultiWriter(out, hash), io.LimitReader(resp.Body, limit))
if err != nil {
_ = out.Close()
return err
}
if expectedSize >= 0 && written != expectedSize {
_ = out.Close()
return fmt.Errorf("download blob: size mismatch")
}
if actual := hex.EncodeToString(hash.Sum(nil)); actual != shaHex {
_ = out.Close()
return fmt.Errorf("download blob: SHA-256 mismatch")
}
if err := out.Sync(); err != nil {
_ = out.Close()
return err
}
if err := out.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, destPath); err != nil {
return err
}
cleanup = false
return nil
}
func validBlobSHA256(value string) bool {
if len(value) != sha256.Size*2 {
return false
}
_, err := hex.DecodeString(value)
return err == nil
}
func minInt(a, b int) int {
@@ -430,5 +559,11 @@ func (c *Client) readErrorBody(resp *http.Response, statusCode int) error {
if strings.Contains(lower, "<html") || strings.Contains(lower, "<!doctype") {
return fmt.Errorf("not a Verstak Sync server (HTTP %d)", statusCode)
}
return fmt.Errorf("server error (HTTP %d)", statusCode)
var payload struct {
Code string `json:"code"`
}
if err := json.Unmarshal([]byte(body), &payload); err == nil && payload.Code != "" {
return &ServerError{Status: statusCode, Code: payload.Code}
}
return &ServerError{Status: statusCode, Code: "request_failed"}
}
+97
View File
@@ -1,9 +1,14 @@
package sync
import (
"crypto/sha256"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"testing"
"time"
)
@@ -119,3 +124,95 @@ func TestPairDeviceSendsVaultID(t *testing.T) {
t.Fatalf("paired vault ID = %q, want vault-123", pairedVaultID)
}
}
func TestPullPageReadsPaginationMetadata(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/api/v1/sync/pull" {
http.NotFound(w, r)
return
}
_ = json.NewEncoder(w).Encode(map[string]interface{}{
"server_sequence": 9, "page_last_sequence": 4, "has_more": true,
"ops": []map[string]interface{}{{"op_id": "op-4", "server_sequence": 4, "device_id": "other", "entity_type": "file", "entity_id": "a.bin", "op_type": "update", "payload_json": `{}`, "created_at": "2026-01-01T00:00:00Z"}},
})
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
response, err := client.PullPage(2, 2)
if err != nil {
t.Fatal(err)
}
if response.PageLastSequence != 4 || !response.HasMore || response.ServerSequence != 9 {
t.Fatalf("pagination response = %+v", response)
}
}
func TestUploadBlobStreamsMultipartAndChecksReturnedSize(t *testing.T) {
data := []byte("streamed binary payload")
path := filepath.Join(t.TempDir(), "blob.bin")
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatal(err)
}
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(1024); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
file, _, err := r.FormFile("file")
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
defer file.Close()
received, err := io.ReadAll(file)
if err != nil || string(received) != string(data) {
http.Error(w, "bad upload", http.StatusBadRequest)
return
}
hash := sha256.Sum256(data)
_ = json.NewEncoder(w).Encode(map[string]interface{}{"sha256": fmt.Sprintf("%x", hash[:]), "size": len(data)})
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
ref, err := client.UploadBlob(path)
if err != nil {
t.Fatal(err)
}
if ref.Size != int64(len(data)) {
t.Fatalf("uploaded reference = %+v", ref)
}
}
func TestDownloadBlobVerifiesHashAndLeavesNoCorruptDestination(t *testing.T) {
dest := filepath.Join(t.TempDir(), "received.bin")
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Length", "7")
_, _ = w.Write([]byte("corrupt"))
}))
defer server.Close()
client := NewClient(server.URL, "", "device", t.TempDir())
client.DeviceToken = "token"
err := client.DownloadBlobVerified("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 7, dest)
if err == nil {
t.Fatal("corrupt blob was accepted")
}
if _, err := os.Stat(dest); !os.IsNotExist(err) {
t.Fatalf("corrupt destination remains: %v", err)
}
}
func TestClientPreservesStableServerErrorCode(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusRequestEntityTooLarge)
_, _ = w.Write([]byte(`{"error":"internal wording must not reach UI","code":"quota_exceeded"}`))
}))
defer server.Close()
client := NewClient(server.URL, "token", "device", t.TempDir())
err := client.post("/api/v1/sync/push", map[string]string{}, nil)
serverErr, ok := err.(*ServerError)
if !ok || serverErr.Code != "quota_exceeded" || serverErr.Status != http.StatusRequestEntityTooLarge {
t.Fatalf("error = %#v, want quota server error", err)
}
}
+91 -26
View File
@@ -2,7 +2,6 @@ package sync
import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
@@ -21,7 +20,17 @@ import (
)
const snapshotVersion = 1
const maxOperationFileBytes = corefiles.MaxBinaryReadBytes
// maxOperationFileBytes is an explicit desktop-side safety ceiling. Binary
// content is streamed through the Blob API rather than embedded in operations,
// so it is intentionally higher than the plugin Files API read limit.
const maxOperationFileBytes int64 = 256 * 1024 * 1024
// BlobCachePath is core-private durable staging for a local operation's
// immutable binary content. It remains excluded from ordinary file sync.
func BlobCachePath(vaultRoot, hash string) string {
return filepath.Join(vaultRoot, ".verstak", "sync", "blobs", hash)
}
// Snapshot is the durable local view of the synchronizable part of a vault.
// Its entries are only files and folders whose latest state was successfully
@@ -721,24 +730,93 @@ func entriesEqual(left, right SnapshotEntry) bool {
return left.Type == right.Type && left.Size == right.Size && left.Hash == right.Hash
}
func payloadForEntry(vaultRoot string, entry SnapshotEntry) (map[string]string, error) {
payload := map[string]string{"path": entry.Path, "contentHash": entry.Hash}
func payloadForEntry(vaultRoot string, entry SnapshotEntry) (map[string]interface{}, error) {
payload := map[string]interface{}{"path": entry.Path, "contentHash": entry.Hash}
if entry.Type == EntityFolder {
return payload, nil
}
data, err := os.ReadFile(filepath.Join(vaultRoot, filepath.FromSlash(entry.Path)))
if err != nil {
return nil, err
}
if int64(len(data)) > maxOperationFileBytes {
return nil, fmt.Errorf("file-too-large: %s", entry.Path)
}
if hash, err := sha256File(filepath.Join(vaultRoot, filepath.FromSlash(entry.Path))); err != nil {
path := filepath.Join(vaultRoot, filepath.FromSlash(entry.Path))
if hash, err := sha256File(path); err != nil {
return nil, err
} else if hash != entry.Hash {
return nil, fmt.Errorf("file changed during scan: %s", entry.Path)
}
return filePayload(entry.Path, data, entry.Hash), nil
if entry.Size > corefiles.MaxTextFileBytes {
if err := cacheBlob(path, BlobCachePath(vaultRoot, entry.Hash), entry.Hash); err != nil {
return nil, err
}
payload["blob"] = map[string]interface{}{"sha256": entry.Hash, "size": entry.Size}
return payload, nil
}
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
if isSyncText(data) {
payload["content"] = string(data)
return payload, nil
}
if err := cacheBlob(path, BlobCachePath(vaultRoot, entry.Hash), entry.Hash); err != nil {
return nil, err
}
payload["blob"] = map[string]interface{}{"sha256": entry.Hash, "size": entry.Size}
return payload, nil
}
func cacheBlob(source, destination, wantHash string) error {
if info, err := os.Lstat(destination); err == nil {
if !info.Mode().IsRegular() {
return fmt.Errorf("blob cache target is not a regular file")
}
if hash, err := sha256File(destination); err == nil && hash == wantHash {
return nil
}
if err := os.Remove(destination); err != nil {
return err
}
} else if !os.IsNotExist(err) {
return err
}
if err := os.MkdirAll(filepath.Dir(destination), 0o750); err != nil {
return err
}
in, err := os.Open(source)
if err != nil {
return err
}
defer in.Close()
tmp, err := os.CreateTemp(filepath.Dir(destination), ".blob-*")
if err != nil {
return err
}
tmpPath := tmp.Name()
cleanup := true
defer func() {
if cleanup {
_ = os.Remove(tmpPath)
}
}()
hash := sha256.New()
if _, err := io.Copy(io.MultiWriter(tmp, hash), in); err != nil {
_ = tmp.Close()
return err
}
if actual := hex.EncodeToString(hash.Sum(nil)); actual != wantHash {
_ = tmp.Close()
return fmt.Errorf("file changed during blob staging")
}
if err := tmp.Sync(); err != nil {
_ = tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
if err := os.Rename(tmpPath, destination); err != nil {
return err
}
cleanup = false
return nil
}
func newSnapshotOp(deviceID, entityType, entityID, opType string, payload interface{}) Op {
@@ -763,19 +841,6 @@ func pathDepth(path string) int {
return strings.Count(path, "/") + 1
}
// filePayload encodes an already bounded file without changing its content
// representation. UTF-8 files use plain text up to the existing text limit;
// other supported files use the current bounded base64 transport.
func filePayload(path string, data []byte, hash string) map[string]string {
payload := map[string]string{"path": path, "contentHash": hash}
if int64(len(data)) <= corefiles.MaxTextFileBytes && isSyncText(data) {
payload["content"] = string(data)
return payload
}
payload["dataBase64"] = base64.StdEncoding.EncodeToString(data)
return payload
}
func isSyncText(data []byte) bool {
if !utf8.Valid(data) {
return false
+40 -1
View File
@@ -9,6 +9,7 @@ import (
"testing"
"github.com/google/uuid"
corefiles "github.com/verstak/verstak-desktop/internal/core/files"
)
func TestScanAndRecordTracksExternalWorkspaceLifecycleByIdentity(t *testing.T) {
@@ -191,6 +192,41 @@ func TestScanAndRecordBaselinesThenRecordsExternalChanges(t *testing.T) {
}
}
func TestSnapshotUsesBlobReferenceForBinaryFileBeyondInlineLimit(t *testing.T) {
root := t.TempDir()
service := NewService(root, "device-a")
if _, err := service.ScanAndRecord(); err != nil {
t.Fatalf("baseline: %v", err)
}
data := make([]byte, corefiles.MaxBinaryReadBytes+1)
for i := range data {
data[i] = byte(i % 251)
}
if err := os.WriteFile(filepath.Join(root, "large.bin"), data, 0o600); err != nil {
t.Fatal(err)
}
if _, err := service.ScanAndRecord(); err != nil {
t.Fatalf("scan binary: %v", err)
}
ops := unpushedOps(t, service)
if len(ops) != 1 {
t.Fatalf("operations = %#v, want one file create", ops)
}
var payload struct {
DataBase64 *string `json:"dataBase64"`
Blob *struct {
SHA256 string `json:"sha256"`
Size int64 `json:"size"`
} `json:"blob"`
}
if err := json.Unmarshal([]byte(ops[0].PayloadJSON), &payload); err != nil {
t.Fatal(err)
}
if payload.DataBase64 != nil || payload.Blob == nil || payload.Blob.Size != int64(len(data)) || payload.Blob.SHA256 == "" {
t.Fatalf("binary sync payload = %s, want blob reference without base64", ops[0].PayloadJSON)
}
}
func TestScanAndRecordNeverTreatsInitialFilesAsDeletes(t *testing.T) {
root := t.TempDir()
if err := os.Mkdir(filepath.Join(root, "Existing"), 0o755); err != nil {
@@ -362,7 +398,10 @@ func TestScanAndRecordSkipsReservedTemporaryAndSymlinkPaths(t *testing.T) {
func TestScanAndRecordKeepsUnsupportedFileUnresolved(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "too-large.bin")
if err := os.WriteFile(path, make([]byte, maxOperationFileBytes+1), 0o600); err != nil {
if err := os.WriteFile(path, nil, 0o600); err != nil {
t.Fatal(err)
}
if err := os.Truncate(path, maxOperationFileBytes+1); err != nil {
t.Fatal(err)
}
service := NewService(root, "device-a")