| 1 | diff --git a/internal/auth/auth.go b/internal/auth/auth.go |
| 2 | index b1f0ac5..169a968 100644 |
| 3 | --- a/internal/auth/auth.go |
| 4 | +++ b/internal/auth/auth.go |
| 5 | @@ -105,16 +105,16 @@ func (v *Validator) verifyBlossom(header, action, server, blobHash string, stric |
| 6 | if expiration == 0 || expiration <= now.Unix() { |
| 7 | return event.Event{}, authError("token expired") |
| 8 | } |
| 9 | - if e.CreatedAt > now.Unix()+int64(5*time.Minute/time.Second) { |
| 10 | + // BUD-11 authorization timestamps may not be from the future. The |
| 11 | + // expiration tag, rather than an arbitrary maximum age, controls how long |
| 12 | + // an otherwise valid token remains usable. |
| 13 | + if e.CreatedAt > now.Unix() { |
| 14 | return event.Event{}, authError("token is from the future") |
| 15 | } |
| 16 | - if strict && e.CreatedAt < now.Unix()-int64(5*time.Minute/time.Second) { |
| 17 | - return event.Event{}, authError("token is expired") |
| 18 | - } |
| 19 | if strict && requiresBlobScope(action) && len(event.TagValues(e, "x")) == 0 { |
| 20 | return event.Event{}, authError("token must include an x tag") |
| 21 | } |
| 22 | - if strict && blobHash != "" && !contains(event.TagValues(e, "x"), blobHash) { |
| 23 | + if strict && blobHash != "" && len(event.TagValues(e, "x")) > 0 && !contains(event.TagValues(e, "x"), blobHash) { |
| 24 | return event.Event{}, authError("token x tag does not name this blob") |
| 25 | } |
| 26 | return e, nil |
| 27 | diff --git a/internal/auth/auth_test.go b/internal/auth/auth_test.go |
| 28 | index 908a0f7..1e6185b 100644 |
| 29 | --- a/internal/auth/auth_test.go |
| 30 | +++ b/internal/auth/auth_test.go |
| 31 | @@ -122,6 +122,25 @@ func TestVerifyBlossomRequestRequiresScopeAndServer(t *testing.T) { |
| 32 | } |
| 33 | } |
| 34 | |
| 35 | +func TestVerifyBlossomRequestAllowsUnscopedGet(t *testing.T) { |
| 36 | + now := time.Unix(1_700_000_000, 0) |
| 37 | + validator := NewValidator(func() time.Time { return now }) |
| 38 | + hash := sha256Hex("blob") |
| 39 | + e := signedEvent(t, 24242, now.Add(-24*time.Hour).Unix(), "", [][]string{{"t", "get"}, {"expiration", "1700086400"}}) |
| 40 | + if _, err := validator.VerifyBlossomRequest(token(t, e), "get", "relay.example", hash); err != nil { |
| 41 | + t.Fatalf("unscoped GET token rejected: %v", err) |
| 42 | + } |
| 43 | +} |
| 44 | + |
| 45 | +func TestVerifyBlossomRequestRejectsFutureCreatedAt(t *testing.T) { |
| 46 | + now := time.Unix(1_700_000_000, 0) |
| 47 | + validator := NewValidator(func() time.Time { return now }) |
| 48 | + e := signedEvent(t, 24242, now.Add(time.Second).Unix(), "", [][]string{{"t", "get"}, {"expiration", "1700000300"}}) |
| 49 | + if _, err := validator.VerifyBlossomRequest(token(t, e), "get", "relay.example", ""); err == nil || !strings.Contains(err.Error(), "future") { |
| 50 | + t.Fatalf("future token accepted: %v", err) |
| 51 | + } |
| 52 | +} |
| 53 | + |
| 54 | func TestChallengeManagerBindsRelayAndConsumesChallenge(t *testing.T) { |
| 55 | now := time.Unix(1_700_000_000, 0) |
| 56 | m := NewChallengeManager("wss://relay.example/tenant", func() time.Time { return now }) |
| 57 | diff --git a/internal/blob/blob.go b/internal/blob/blob.go |
| 58 | index 57e300a..134df32 100644 |
| 59 | --- a/internal/blob/blob.go |
| 60 | +++ b/internal/blob/blob.go |
| 61 | @@ -19,6 +19,7 @@ import ( |
| 62 | "sort" |
| 63 | "strconv" |
| 64 | "strings" |
| 65 | + "sync" |
| 66 | "time" |
| 67 | |
| 68 | "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 69 | @@ -57,6 +58,14 @@ type Config struct { |
| 70 | // RecordReport joins native blob reporting to the host moderation ledger. |
| 71 | // It runs inside the blob report transaction. |
| 72 | RecordReport ReportRecorder |
| 73 | + // Limits is read for each upload so policy changes apply without reopening |
| 74 | + // the blob service. Zero values mean unlimited. |
| 75 | + Limits func() Limits |
| 76 | +} |
| 77 | + |
| 78 | +type Limits struct { |
| 79 | + MaxFileBytes int64 |
| 80 | + UserStorageBytes int64 |
| 81 | } |
| 82 | |
| 83 | type Blob struct { |
| 84 | @@ -79,10 +88,18 @@ type Service struct { |
| 85 | root string |
| 86 | store *storage.Store |
| 87 | resolve ResolveIP |
| 88 | + quotaMu sync.Mutex |
| 89 | + // multipartMu protects partial files and their range metadata. Lock it |
| 90 | + // before quotaMu; network bodies are never read while either is held. |
| 91 | + multipartMu sync.Mutex |
| 92 | + activeParts map[string]int |
| 93 | } |
| 94 | |
| 95 | var ErrBlocked = errors.New("blocked: this blob was removed by a moderator") |
| 96 | var ErrHashMismatch = errors.New("conflict: mirrored blob hash does not match the requested hash") |
| 97 | +var ErrFileTooLarge = errors.New("invalid: blob exceeds the per-file size limit") |
| 98 | +var ErrQuotaExceeded = errors.New("restricted: uploader storage quota exceeded") |
| 99 | +var ErrMultipartConflict = errors.New("conflict: upload metadata differs from existing session") |
| 100 | |
| 101 | var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) |
| 102 | var blobPathPattern = regexp.MustCompile(`^/([0-9a-f]{64})(?:\.[a-z0-9]{1,8})?$`) |
| 103 | @@ -99,6 +116,7 @@ var extensions = map[string]string{ |
| 104 | "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp", "image/avif": "avif", "image/svg+xml": "svg", |
| 105 | "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/wav": "wav", "audio/mp4": "m4a", |
| 106 | "application/pdf": "pdf", "application/json": "json", "text/plain": "txt", "text/markdown": "md", |
| 107 | + "application/vnd.blossom.directory+msgpack": "bdir", |
| 108 | } |
| 109 | |
| 110 | func New(ctx context.Context, config Config) (*Service, error) { |
| 111 | @@ -126,6 +144,8 @@ func New(ctx context.Context, config Config) (*Service, error) { |
| 112 | WHEN NEW.uploader != '' BEGIN INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(NEW.sha256,NEW.uploader,NEW.uploaded); END; |
| 113 | CREATE TABLE IF NOT EXISTS blob_blocks (sha256 TEXT PRIMARY KEY, reason TEXT NOT NULL DEFAULT '', blocked_at INTEGER NOT NULL DEFAULT 0); |
| 114 | CREATE TABLE IF NOT EXISTS blob_tombstones (sha256 TEXT PRIMARY KEY, deleted_at INTEGER NOT NULL); |
| 115 | + CREATE TABLE IF NOT EXISTS multipart_uploads (id TEXT PRIMARY KEY, sha256 TEXT NOT NULL, uploader TEXT NOT NULL, length INTEGER NOT NULL, type TEXT NOT NULL, created INTEGER NOT NULL, last_seen INTEGER NOT NULL); |
| 116 | + CREATE TABLE IF NOT EXISTS multipart_parts (upload_id TEXT NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY(upload_id, offset), FOREIGN KEY(upload_id) REFERENCES multipart_uploads(id) ON DELETE CASCADE); |
| 117 | INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) |
| 118 | SELECT sha256,uploader,uploaded FROM blobs WHERE uploader != ''`); err != nil { |
| 119 | return nil, fmt.Errorf("initialize blob metadata: %w", err) |
| 120 | @@ -139,7 +159,11 @@ func New(ctx context.Context, config Config) (*Service, error) { |
| 121 | return net.DefaultResolver.LookupIP(ctx, "ip", host) |
| 122 | } |
| 123 | } |
| 124 | - return &Service{config: config, root: filepath.Join(config.Root, "blobs"), store: config.Store, resolve: resolve}, nil |
| 125 | + s := &Service{config: config, root: filepath.Join(config.Root, "blobs"), store: config.Store, resolve: resolve, activeParts: make(map[string]int)} |
| 126 | + if err := s.reconcileMultipart(ctx); err != nil { |
| 127 | + return nil, err |
| 128 | + } |
| 129 | + return s, nil |
| 130 | } |
| 131 | |
| 132 | func (s *Service) Handler() http.HandlerFunc { return s.serveHTTP } |
| 133 | @@ -320,9 +344,13 @@ func (s *Service) Descriptor(baseURL string, entry Blob) map[string]any { |
| 134 | |
| 135 | func (s *Service) serveHTTP(w http.ResponseWriter, r *http.Request) { |
| 136 | w.Header().Set("access-control-allow-origin", "*") |
| 137 | - w.Header().Set("access-control-allow-headers", "authorization, content-type, x-sha-256, x-content-length") |
| 138 | - w.Header().Set("access-control-allow-methods", "GET, HEAD, PUT, POST, DELETE, OPTIONS") |
| 139 | + w.Header().Set("access-control-allow-headers", "authorization, content-type, x-sha-256, x-content-length, x-content-type, upload-type, upload-length, upload-offset") |
| 140 | + w.Header().Set("access-control-allow-methods", "GET, HEAD, PUT, PATCH, POST, DELETE, OPTIONS") |
| 141 | + w.Header().Set("Access-Control-Expose-Headers", "Allow, X-Reason") |
| 142 | if r.Method == http.MethodOptions { |
| 143 | + if blobPathPattern.MatchString(r.URL.Path) { |
| 144 | + w.Header().Set("Allow", "GET, HEAD, PUT, PATCH, DELETE, OPTIONS") |
| 145 | + } |
| 146 | w.WriteHeader(http.StatusNoContent) |
| 147 | return |
| 148 | } |
| 149 | @@ -342,6 +370,14 @@ func (s *Service) serveHTTP(w http.ResponseWriter, r *http.Request) { |
| 150 | s.mirror(w, r) |
| 151 | return |
| 152 | } |
| 153 | + if match := blobPathPattern.FindStringSubmatch(r.URL.Path); match != nil && (r.Method == http.MethodPut || r.Method == http.MethodPatch || r.Method == http.MethodOptions) { |
| 154 | + if r.Method == http.MethodPatch || r.Method == http.MethodOptions { |
| 155 | + s.multipart(w, r, match[1]) |
| 156 | + return |
| 157 | + } |
| 158 | + s.pathUpload(w, r, match[1]) |
| 159 | + return |
| 160 | + } |
| 161 | if r.URL.Path == "/report" && r.Method == http.MethodPut { |
| 162 | s.report(w, r) |
| 163 | return |
| 164 | @@ -388,12 +424,18 @@ func (s *Service) uploadHead(w http.ResponseWriter, r *http.Request) { |
| 165 | s.fail(w, http.StatusBadRequest, errors.New("invalid: X-SHA-256 must be lowercase hex")) |
| 166 | return |
| 167 | } |
| 168 | - length, err := strconv.ParseInt(strings.TrimSpace(r.Header.Get("x-content-length")), 10, 64) |
| 169 | + rawLength := strings.TrimSpace(r.Header.Get("x-content-length")) |
| 170 | + if rawLength == "" { |
| 171 | + s.fail(w, http.StatusLengthRequired, errors.New("invalid: X-Content-Length is required")) |
| 172 | + return |
| 173 | + } |
| 174 | + length, err := strconv.ParseInt(rawLength, 10, 64) |
| 175 | if err != nil || length <= 0 { |
| 176 | s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Length must be a positive integer")) |
| 177 | return |
| 178 | } |
| 179 | - if _, err := s.authorize(r, ActionUpload); err != nil { |
| 180 | + pubkey, err := s.authorize(r, ActionUpload) |
| 181 | + if err != nil { |
| 182 | s.fail(w, http.StatusUnauthorized, err) |
| 183 | return |
| 184 | } |
| 185 | @@ -405,6 +447,26 @@ func (s *Service) uploadHead(w http.ResponseWriter, r *http.Request) { |
| 186 | s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Type is required")) |
| 187 | return |
| 188 | } |
| 189 | + if existing, lookupErr := s.lookup(r.Context(), sha); lookupErr == nil && existing.SHA256 != "" && s.hasClaim(r.Context(), sha, pubkey) { |
| 190 | + w.WriteHeader(http.StatusOK) |
| 191 | + return |
| 192 | + } |
| 193 | + limits := s.currentLimits() |
| 194 | + if limits.MaxFileBytes > 0 && length > limits.MaxFileBytes { |
| 195 | + s.fail(w, http.StatusRequestEntityTooLarge, ErrFileTooLarge) |
| 196 | + return |
| 197 | + } |
| 198 | + // HEAD is advisory for deduplicated uploads, whose final physical size is |
| 199 | + // already present, but it should still reflect the current quota budget. |
| 200 | + if limits.UserStorageBytes > 0 { |
| 201 | + if used, quotaErr := s.quotaUsage(r.Context(), pubkey); quotaErr != nil { |
| 202 | + s.fail(w, http.StatusInternalServerError, quotaErr) |
| 203 | + return |
| 204 | + } else if length > limits.UserStorageBytes-used { |
| 205 | + s.fail(w, http.StatusForbidden, ErrQuotaExceeded) |
| 206 | + return |
| 207 | + } |
| 208 | + } |
| 209 | if existing, err := s.lookup(r.Context(), sha); err == nil && existing.SHA256 != "" { |
| 210 | w.WriteHeader(http.StatusOK) |
| 211 | return |
| 212 | @@ -516,6 +578,16 @@ func (s *Service) put(ctx context.Context, source io.Reader, typ, uploader, clai |
| 213 | } |
| 214 | |
| 215 | func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploader, claimed string, validate func(string) error) (Blob, bool, error) { |
| 216 | + limits := s.currentLimits() |
| 217 | + streamLimit := limits.MaxFileBytes |
| 218 | + quotaStream := false |
| 219 | + if streamLimit == 0 && limits.UserStorageBytes > 0 && uploader != "" { |
| 220 | + streamLimit = limits.UserStorageBytes |
| 221 | + quotaStream = true |
| 222 | + } |
| 223 | + if streamLimit > 0 && streamLimit < int64(^uint64(0)>>1) { |
| 224 | + source = io.LimitReader(source, streamLimit+1) |
| 225 | + } |
| 226 | tmp, err := os.CreateTemp(s.root, ".upload-") |
| 227 | if err != nil { |
| 228 | return Blob{}, false, fmt.Errorf("create upload temporary file: %w", err) |
| 229 | @@ -528,9 +600,13 @@ func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploa |
| 230 | _ = tmp.Close() |
| 231 | return Blob{}, false, fmt.Errorf("read upload: %w", err) |
| 232 | } |
| 233 | - if count == 0 { |
| 234 | + if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 235 | _ = tmp.Close() |
| 236 | - return Blob{}, false, errors.New("invalid: empty body") |
| 237 | + return Blob{}, false, ErrFileTooLarge |
| 238 | + } |
| 239 | + if quotaStream && count > streamLimit { |
| 240 | + _ = tmp.Close() |
| 241 | + return Blob{}, false, ErrQuotaExceeded |
| 242 | } |
| 243 | if err := tmp.Sync(); err != nil { |
| 244 | _ = tmp.Close() |
| 245 | @@ -548,21 +624,66 @@ func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploa |
| 246 | return Blob{}, false, err |
| 247 | } |
| 248 | } |
| 249 | + return s.installUpload(ctx, uploadCandidate{path: tmpName, hash: sha, size: count, typ: typ, uploader: uploader}) |
| 250 | +} |
| 251 | + |
| 252 | +type uploadCandidate struct { |
| 253 | + path, hash, typ, uploader, reservation string |
| 254 | + size int64 |
| 255 | +} |
| 256 | + |
| 257 | +func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate) (Blob, bool, error) { |
| 258 | + sha, count, typ, uploader := candidate.hash, candidate.size, candidate.typ, candidate.uploader |
| 259 | + // Serialize the lookup, quota check, installation and metadata transaction. |
| 260 | + // This closes the race where concurrent uploads could each observe the same |
| 261 | + // remaining quota before one of them installs a new physical object. |
| 262 | + s.quotaMu.Lock() |
| 263 | + defer s.quotaMu.Unlock() |
| 264 | if s.blocked(ctx, sha) { |
| 265 | return Blob{}, false, ErrBlocked |
| 266 | } |
| 267 | + limits := s.currentLimits() |
| 268 | + if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 269 | + return Blob{}, false, ErrFileTooLarge |
| 270 | + } |
| 271 | + if limits.UserStorageBytes > 0 && uploader != "" && count > limits.UserStorageBytes { |
| 272 | + return Blob{}, false, ErrQuotaExceeded |
| 273 | + } |
| 274 | if existing, err := s.lookup(ctx, sha); err == nil { |
| 275 | if uploader != "" { |
| 276 | + if limits.UserStorageBytes > 0 && !s.hasClaim(ctx, sha, uploader) { |
| 277 | + used, usageErr := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 278 | + if usageErr != nil { |
| 279 | + return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", usageErr) |
| 280 | + } |
| 281 | + if existing.Size > limits.UserStorageBytes-used { |
| 282 | + return Blob{}, false, ErrQuotaExceeded |
| 283 | + } |
| 284 | + } |
| 285 | if err := s.claim(ctx, sha, uploader); err != nil { |
| 286 | return Blob{}, false, err |
| 287 | } |
| 288 | } |
| 289 | + if candidate.reservation != "" { |
| 290 | + if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 291 | + return Blob{}, false, err |
| 292 | + } |
| 293 | + } |
| 294 | return existing, false, nil |
| 295 | } else if !errors.Is(err, sql.ErrNoRows) { |
| 296 | return Blob{}, false, err |
| 297 | } |
| 298 | + if limits.UserStorageBytes > 0 && uploader != "" { |
| 299 | + used, err := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 300 | + if err != nil { |
| 301 | + return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", err) |
| 302 | + } |
| 303 | + if count > limits.UserStorageBytes-used { |
| 304 | + return Blob{}, false, ErrQuotaExceeded |
| 305 | + } |
| 306 | + } |
| 307 | final := filepath.Join(s.root, sha) |
| 308 | - if err := os.Rename(tmpName, final); err != nil && !errors.Is(err, os.ErrExist) { |
| 309 | + if err := os.Rename(candidate.path, final); err != nil && !errors.Is(err, os.ErrExist) { |
| 310 | return Blob{}, false, fmt.Errorf("install blob: %w", err) |
| 311 | } |
| 312 | if err := syncDirectory(s.root); err != nil { |
| 313 | @@ -574,7 +695,7 @@ func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploa |
| 314 | typ = detectFileType(final) |
| 315 | } |
| 316 | entry := Blob{SHA256: sha, Size: count, Type: typ, Uploader: uploader, Uploaded: now} |
| 317 | - err = s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 318 | + err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 319 | if _, err := tx.ExecContext(ctx, "DELETE FROM blob_tombstones WHERE sha256=?", entry.SHA256); err != nil { |
| 320 | return fmt.Errorf("clear blob deletion tombstone: %w", err) |
| 321 | } |
| 322 | @@ -586,6 +707,11 @@ func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploa |
| 323 | return fmt.Errorf("record blob claim: %w", err) |
| 324 | } |
| 325 | } |
| 326 | + if candidate.reservation != "" { |
| 327 | + if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 328 | + return fmt.Errorf("release upload reservation: %w", err) |
| 329 | + } |
| 330 | + } |
| 331 | return nil |
| 332 | }) |
| 333 | if err != nil { |
| 334 | @@ -597,6 +723,21 @@ func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploa |
| 335 | return entry, true, nil |
| 336 | } |
| 337 | |
| 338 | +func (s *Service) quotaUsage(ctx context.Context, uploader string) (int64, error) { |
| 339 | + return s.quotaUsageExcept(ctx, uploader, "") |
| 340 | +} |
| 341 | + |
| 342 | +func (s *Service) quotaUsageExcept(ctx context.Context, uploader, reservation string) (int64, error) { |
| 343 | + var used sql.NullInt64 |
| 344 | + err := s.store.DB().QueryRowContext(ctx, `SELECT COALESCE((SELECT SUM(b.size) |
| 345 | + FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=?),0) + |
| 346 | + COALESCE((SELECT SUM(length) FROM multipart_uploads WHERE uploader=? AND id != ?),0)`, uploader, uploader, reservation).Scan(&used) |
| 347 | + if err != nil { |
| 348 | + return 0, err |
| 349 | + } |
| 350 | + return used.Int64, nil |
| 351 | +} |
| 352 | + |
| 353 | func (s *Service) claim(ctx context.Context, sha, uploader string) error { |
| 354 | _, err := s.store.DB().ExecContext(ctx, "INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(?,?,?)", sha, uploader, time.Now().UTC().Unix()) |
| 355 | if err != nil { |
| 356 | @@ -605,6 +746,20 @@ func (s *Service) claim(ctx context.Context, sha, uploader string) error { |
| 357 | return nil |
| 358 | } |
| 359 | |
| 360 | +func (s *Service) currentLimits() Limits { |
| 361 | + if s.config.Limits == nil { |
| 362 | + return Limits{} |
| 363 | + } |
| 364 | + limits := s.config.Limits() |
| 365 | + if limits.MaxFileBytes < 0 { |
| 366 | + limits.MaxFileBytes = 0 |
| 367 | + } |
| 368 | + if limits.UserStorageBytes < 0 { |
| 369 | + limits.UserStorageBytes = 0 |
| 370 | + } |
| 371 | + return limits |
| 372 | +} |
| 373 | + |
| 374 | func (s *Service) hasClaim(ctx context.Context, sha, uploader string) bool { |
| 375 | var found int |
| 376 | err := s.store.DB().QueryRowContext(ctx, "SELECT 1 FROM blob_claims WHERE sha256=? AND uploader=?", sha, uploader).Scan(&found) |
| 377 | @@ -634,7 +789,7 @@ func reconcile(ctx context.Context, store *storage.Store, root string) error { |
| 378 | continue |
| 379 | } |
| 380 | name := entry.Name() |
| 381 | - if strings.HasPrefix(name, ".upload-") { |
| 382 | + if strings.HasPrefix(name, ".upload-") || strings.HasPrefix(name, ".multipart-chunk-") { |
| 383 | if err := os.Remove(filepath.Join(root, name)); err != nil { |
| 384 | return fmt.Errorf("remove abandoned upload: %w", err) |
| 385 | } |
| 386 | @@ -1138,9 +1293,21 @@ func statusFor(err error) int { |
| 387 | if errors.Is(err, ErrHashMismatch) { |
| 388 | return http.StatusConflict |
| 389 | } |
| 390 | + if errors.Is(err, ErrMultipartConflict) { |
| 391 | + return http.StatusConflict |
| 392 | + } |
| 393 | + if errors.Is(err, ErrFileTooLarge) { |
| 394 | + return http.StatusRequestEntityTooLarge |
| 395 | + } |
| 396 | + if errors.Is(err, ErrQuotaExceeded) { |
| 397 | + return http.StatusForbidden |
| 398 | + } |
| 399 | if errors.Is(err, ErrBlocked) { |
| 400 | return http.StatusForbidden |
| 401 | } |
| 402 | + if strings.HasPrefix(err.Error(), "auth-required:") { |
| 403 | + return http.StatusUnauthorized |
| 404 | + } |
| 405 | if strings.HasPrefix(err.Error(), "invalid:") { |
| 406 | return http.StatusBadRequest |
| 407 | } |
| 408 | diff --git a/internal/blob/blob_test.go b/internal/blob/blob_test.go |
| 409 | index 07f7c40..f524ebf 100644 |
| 410 | --- a/internal/blob/blob_test.go |
| 411 | +++ b/internal/blob/blob_test.go |
| 412 | @@ -4,6 +4,7 @@ import ( |
| 413 | "context" |
| 414 | "crypto/sha256" |
| 415 | "encoding/hex" |
| 416 | + "errors" |
| 417 | "io" |
| 418 | "net" |
| 419 | "net/http" |
| 420 | @@ -11,7 +12,9 @@ import ( |
| 421 | "os" |
| 422 | "path/filepath" |
| 423 | "strings" |
| 424 | + "sync" |
| 425 | "testing" |
| 426 | + "time" |
| 427 | |
| 428 | "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 429 | ) |
| 430 | @@ -46,6 +49,116 @@ func TestUploadDeduplicatesAndServesByHash(t *testing.T) { |
| 431 | } |
| 432 | } |
| 433 | |
| 434 | +func TestUploadEnforcesFileAndUploaderLimits(t *testing.T) { |
| 435 | + service := testService(t) |
| 436 | + service.config.Limits = func() Limits { return Limits{MaxFileBytes: 4, UserStorageBytes: 6} } |
| 437 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("12345"), Type: "text/plain", Uploader: testUploader}); !errors.Is(err, ErrFileTooLarge) { |
| 438 | + t.Fatalf("oversized upload error = %v", err) |
| 439 | + } |
| 440 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("1234"), Type: "text/plain", Uploader: testUploader}); err != nil { |
| 441 | + t.Fatalf("first limited upload: %v", err) |
| 442 | + } |
| 443 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("5678"), Type: "text/plain", Uploader: testUploader}); !errors.Is(err, ErrQuotaExceeded) { |
| 444 | + t.Fatalf("quota upload error = %v", err) |
| 445 | + } |
| 446 | + // A deduplicated claim does not consume additional physical storage. |
| 447 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("1234"), Type: "text/plain", Uploader: strings.Repeat("b", 64)}); err != nil { |
| 448 | + t.Fatalf("deduplicated upload: %v", err) |
| 449 | + } |
| 450 | + second := strings.Repeat("b", 64) |
| 451 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("abc"), Type: "text/plain", Uploader: strings.Repeat("c", 64)}); err != nil { |
| 452 | + t.Fatal(err) |
| 453 | + } |
| 454 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("56"), Type: "text/plain", Uploader: second}); err != nil { |
| 455 | + t.Fatalf("second uploader upload: %v", err) |
| 456 | + } |
| 457 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("xyz"), Type: "text/plain", Uploader: second}); !errors.Is(err, ErrQuotaExceeded) { |
| 458 | + t.Fatalf("second uploader remaining quota error = %v", err) |
| 459 | + } |
| 460 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("abc"), Type: "text/plain", Uploader: second}); !errors.Is(err, ErrQuotaExceeded) { |
| 461 | + t.Fatalf("existing blob claim bypassed remaining quota: %v", err) |
| 462 | + } |
| 463 | + digest := sha256.Sum256([]byte("56")) |
| 464 | + sha := hex.EncodeToString(digest[:]) |
| 465 | + if err := service.DeleteForUploader(context.Background(), sha, second); err != nil { |
| 466 | + t.Fatalf("remove second uploader claim: %v", err) |
| 467 | + } |
| 468 | + if _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("ab"), Type: "text/plain", Uploader: second}); err != nil { |
| 469 | + t.Fatalf("quota after claim deletion: %v", err) |
| 470 | + } |
| 471 | +} |
| 472 | + |
| 473 | +func TestConcurrentUploadsRespectUploaderQuota(t *testing.T) { |
| 474 | + service := testService(t) |
| 475 | + service.config.Limits = func() Limits { return Limits{UserStorageBytes: 4} } |
| 476 | + var wg sync.WaitGroup |
| 477 | + errs := make(chan error, 2) |
| 478 | + for _, body := range []string{"aaaa", "bbbb"} { |
| 479 | + wg.Add(1) |
| 480 | + go func(body string) { |
| 481 | + defer wg.Done() |
| 482 | + _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader(body), Type: "text/plain", Uploader: testUploader}) |
| 483 | + errs <- err |
| 484 | + }(body) |
| 485 | + } |
| 486 | + wg.Wait() |
| 487 | + close(errs) |
| 488 | + var success, quota int |
| 489 | + for err := range errs { |
| 490 | + if err == nil { |
| 491 | + success++ |
| 492 | + } else if errors.Is(err, ErrQuotaExceeded) { |
| 493 | + quota++ |
| 494 | + } |
| 495 | + } |
| 496 | + if success != 1 || quota != 1 { |
| 497 | + t.Fatalf("concurrent quota results success=%d quota=%d", success, quota) |
| 498 | + } |
| 499 | +} |
| 500 | + |
| 501 | +func TestQuotaOnlyLimitReportsQuotaBeforeHashMismatch(t *testing.T) { |
| 502 | + service := testService(t) |
| 503 | + service.config.Limits = func() Limits { return Limits{UserStorageBytes: 4} } |
| 504 | + _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("12345"), Type: "text/plain", Uploader: testUploader, Hash: strings.Repeat("a", 64)}) |
| 505 | + if !errors.Is(err, ErrQuotaExceeded) { |
| 506 | + t.Fatalf("quota-only oversized upload error = %v", err) |
| 507 | + } |
| 508 | +} |
| 509 | + |
| 510 | +func TestSlowUploadDoesNotHoldQuotaLockWhileReading(t *testing.T) { |
| 511 | + service := testService(t) |
| 512 | + reader, writer := io.Pipe() |
| 513 | + done := make(chan error, 1) |
| 514 | + go func() { |
| 515 | + _, err := service.Put(context.Background(), PutOptions{Reader: reader, Type: "text/plain", Uploader: testUploader}) |
| 516 | + done <- err |
| 517 | + }() |
| 518 | + select { |
| 519 | + case <-done: |
| 520 | + t.Fatal("stalled upload returned before receiving data") |
| 521 | + case <-time.After(25 * time.Millisecond): |
| 522 | + } |
| 523 | + second := strings.Repeat("b", 64) |
| 524 | + finished := make(chan error, 1) |
| 525 | + go func() { |
| 526 | + _, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader("fast"), Type: "text/plain", Uploader: second}) |
| 527 | + finished <- err |
| 528 | + }() |
| 529 | + select { |
| 530 | + case err := <-finished: |
| 531 | + if err != nil { |
| 532 | + t.Fatalf("second upload: %v", err) |
| 533 | + } |
| 534 | + case <-time.After(time.Second): |
| 535 | + t.Fatal("second upload blocked behind stalled stream") |
| 536 | + } |
| 537 | + _, _ = writer.Write([]byte("slow")) |
| 538 | + _ = writer.Close() |
| 539 | + if err := <-done; err != nil { |
| 540 | + t.Fatalf("stalled upload completion: %v", err) |
| 541 | + } |
| 542 | +} |
| 543 | + |
| 544 | func TestPrivateBlobRequiresReadAuthorization(t *testing.T) { |
| 545 | service := testService(t) |
| 546 | service.config.CanRead = func(context.Context, string, []string) bool { return false } |
| 547 | diff --git a/internal/blob/multipart.go b/internal/blob/multipart.go |
| 548 | new file mode 100644 |
| 549 | index 0000000..8b2a480 |
| 550 | --- /dev/null |
| 551 | +++ b/internal/blob/multipart.go |
| 552 | @@ -0,0 +1,525 @@ |
| 553 | +package blob |
| 554 | + |
| 555 | +import ( |
| 556 | + "context" |
| 557 | + "crypto/sha256" |
| 558 | + "database/sql" |
| 559 | + "encoding/hex" |
| 560 | + "errors" |
| 561 | + "fmt" |
| 562 | + "io" |
| 563 | + "mime" |
| 564 | + "net/http" |
| 565 | + "os" |
| 566 | + "path/filepath" |
| 567 | + "sort" |
| 568 | + "strconv" |
| 569 | + "strings" |
| 570 | + "time" |
| 571 | +) |
| 572 | + |
| 573 | +const ( |
| 574 | + multipartExpiry = 60 * time.Second |
| 575 | + multipartChunkLimit = 8 << 20 |
| 576 | + multipartStorageLimit = 1 << 30 |
| 577 | + multipartSessionLimit = 64 |
| 578 | + multipartUserSessionLimit = 16 |
| 579 | + multipartActiveLimit = 32 |
| 580 | + multipartRangeLimit = 1024 |
| 581 | +) |
| 582 | + |
| 583 | +type multipartRequest struct { |
| 584 | + id, hash, uploader, typ string |
| 585 | + length, offset, count int64 |
| 586 | +} |
| 587 | + |
| 588 | +type multipartError struct { |
| 589 | + status int |
| 590 | + message string |
| 591 | +} |
| 592 | + |
| 593 | +func (e *multipartError) Error() string { return e.message } |
| 594 | +func multipartFailure(status int, message string) error { return &multipartError{status, message} } |
| 595 | +func (s *Service) failMultipart(w http.ResponseWriter, err error) { |
| 596 | + var failure *multipartError |
| 597 | + if errors.As(err, &failure) { |
| 598 | + s.fail(w, failure.status, err) |
| 599 | + return |
| 600 | + } |
| 601 | + s.fail(w, statusFor(err), err) |
| 602 | +} |
| 603 | + |
| 604 | +func (s *Service) multipart(w http.ResponseWriter, r *http.Request, hash string) { |
| 605 | + uploader, err := s.authorize(r, ActionUpload) |
| 606 | + if err != nil { |
| 607 | + s.fail(w, http.StatusUnauthorized, err) |
| 608 | + return |
| 609 | + } |
| 610 | + part, err := parseMultipart(r, hash, uploader) |
| 611 | + if err != nil { |
| 612 | + s.failMultipart(w, err) |
| 613 | + return |
| 614 | + } |
| 615 | + if err := s.reserveMultipart(r.Context(), part); err != nil { |
| 616 | + s.failMultipart(w, err) |
| 617 | + return |
| 618 | + } |
| 619 | + defer s.releaseMultipart(part.id) |
| 620 | + chunk, err := s.stageMultipart(r, part) |
| 621 | + if err != nil { |
| 622 | + s.failMultipart(w, err) |
| 623 | + return |
| 624 | + } |
| 625 | + defer os.Remove(chunk) |
| 626 | + entry, created, complete, err := s.commitMultipart(r.Context(), part, chunk) |
| 627 | + if err != nil { |
| 628 | + s.failMultipart(w, err) |
| 629 | + return |
| 630 | + } |
| 631 | + if !complete { |
| 632 | + w.WriteHeader(http.StatusNoContent) |
| 633 | + return |
| 634 | + } |
| 635 | + s.writeJSON(w, chooseStatus(created, http.StatusCreated), s.descriptor(r, entry)) |
| 636 | +} |
| 637 | + |
| 638 | +func parseMultipart(r *http.Request, hash, uploader string) (multipartRequest, error) { |
| 639 | + part := multipartRequest{id: multipartID(hash, uploader), hash: hash, uploader: uploader, count: r.ContentLength} |
| 640 | + if r.ContentLength < 0 { |
| 641 | + return part, multipartFailure(411, "Content-Length is required") |
| 642 | + } |
| 643 | + if r.Header.Get("Content-Type") != "application/octet-stream" { |
| 644 | + return part, multipartFailure(415, "Content-Type must be application/octet-stream") |
| 645 | + } |
| 646 | + for _, item := range []struct { |
| 647 | + name string |
| 648 | + target *int64 |
| 649 | + }{{"Upload-Length", &part.length}, {"Upload-Offset", &part.offset}} { |
| 650 | + raw := r.Header.Get(item.name) |
| 651 | + value, err := strconv.ParseInt(raw, 10, 64) |
| 652 | + if err != nil || value < 0 || raw == "" || strings.Trim(raw, "0123456789") != "" { |
| 653 | + return part, multipartFailure(400, item.name+" must be a nonnegative integer") |
| 654 | + } |
| 655 | + *item.target = value |
| 656 | + } |
| 657 | + if part.offset > part.length || part.count > part.length-part.offset { |
| 658 | + return part, multipartFailure(416, "chunk range exceeds Upload-Length") |
| 659 | + } |
| 660 | + if part.count == 0 && part.length != 0 { |
| 661 | + return part, multipartFailure(400, "nonempty uploads require nonempty chunks") |
| 662 | + } |
| 663 | + if part.count > multipartChunkLimit || part.length > multipartStorageLimit { |
| 664 | + return part, multipartFailure(413, "multipart upload exceeds temporary storage limits") |
| 665 | + } |
| 666 | + typ, _, err := mime.ParseMediaType(r.Header.Get("Upload-Type")) |
| 667 | + if err != nil || !strings.Contains(typ, "/") { |
| 668 | + return part, multipartFailure(400, "Upload-Type must be a MIME type") |
| 669 | + } |
| 670 | + part.typ = contentType(typ) |
| 671 | + return part, nil |
| 672 | +} |
| 673 | + |
| 674 | +func multipartID(hash, uploader string) string { |
| 675 | + digest := sha256.Sum256([]byte(uploader)) |
| 676 | + return hash + "-" + hex.EncodeToString(digest[:]) |
| 677 | +} |
| 678 | +func (s *Service) multipartPath(id string) string { return filepath.Join(s.root, ".multipart-"+id) } |
| 679 | + |
| 680 | +// Admission reserves the final length before reading any request bytes. Each |
| 681 | +// active chunk is also bounded, so unlimited durable storage does not mean |
| 682 | +// unlimited temporary files or sessions. |
| 683 | +func (s *Service) reserveMultipart(ctx context.Context, part multipartRequest) error { |
| 684 | + s.multipartMu.Lock() |
| 685 | + defer s.multipartMu.Unlock() |
| 686 | + s.quotaMu.Lock() |
| 687 | + defer s.quotaMu.Unlock() |
| 688 | + if err := s.cleanupMultipartLocked(ctx); err != nil { |
| 689 | + return err |
| 690 | + } |
| 691 | + active := 0 |
| 692 | + for _, count := range s.activeParts { |
| 693 | + active += count |
| 694 | + } |
| 695 | + if active >= multipartActiveLimit { |
| 696 | + return multipartFailure(429, "too many active upload chunks") |
| 697 | + } |
| 698 | + if err := s.multipartAdmission(ctx, part); err != nil { |
| 699 | + return err |
| 700 | + } |
| 701 | + s.activeParts[part.id]++ |
| 702 | + return nil |
| 703 | +} |
| 704 | + |
| 705 | +func (s *Service) multipartAdmission(ctx context.Context, part multipartRequest) error { |
| 706 | + if s.blocked(ctx, part.hash) { |
| 707 | + return ErrBlocked |
| 708 | + } |
| 709 | + limits := s.currentLimits() |
| 710 | + if limits.MaxFileBytes > 0 && part.length > limits.MaxFileBytes { |
| 711 | + return ErrFileTooLarge |
| 712 | + } |
| 713 | + existing, lookupErr := s.lookup(ctx, part.hash) |
| 714 | + if lookupErr != nil && !errors.Is(lookupErr, sql.ErrNoRows) { |
| 715 | + return lookupErr |
| 716 | + } |
| 717 | + if lookupErr == nil { |
| 718 | + if existing.Size != part.length { |
| 719 | + return ErrMultipartConflict |
| 720 | + } |
| 721 | + return s.checkMultipartQuota(ctx, part, limits) |
| 722 | + } |
| 723 | + var length int64 |
| 724 | + var typ string |
| 725 | + err := s.store.DB().QueryRowContext(ctx, "SELECT length,type FROM multipart_uploads WHERE id=?", part.id).Scan(&length, &typ) |
| 726 | + if err != nil && !errors.Is(err, sql.ErrNoRows) { |
| 727 | + return err |
| 728 | + } |
| 729 | + if err == nil && (length != part.length || typ != part.typ) { |
| 730 | + return ErrMultipartConflict |
| 731 | + } |
| 732 | + if err := s.checkMultipartQuota(ctx, part, limits); err != nil { |
| 733 | + return err |
| 734 | + } |
| 735 | + if errors.Is(err, sql.ErrNoRows) { |
| 736 | + return s.createMultipart(ctx, part) |
| 737 | + } |
| 738 | + _, err = s.store.DB().ExecContext(ctx, "UPDATE multipart_uploads SET last_seen=? WHERE id=?", time.Now().Unix(), part.id) |
| 739 | + return err |
| 740 | +} |
| 741 | + |
| 742 | +func (s *Service) checkMultipartQuota(ctx context.Context, part multipartRequest, limits Limits) error { |
| 743 | + if limits.UserStorageBytes == 0 || s.hasClaim(ctx, part.hash, part.uploader) { |
| 744 | + return nil |
| 745 | + } |
| 746 | + used, err := s.quotaUsageExcept(ctx, part.uploader, part.id) |
| 747 | + if err != nil { |
| 748 | + return err |
| 749 | + } |
| 750 | + if part.length > limits.UserStorageBytes-used { |
| 751 | + return ErrQuotaExceeded |
| 752 | + } |
| 753 | + return nil |
| 754 | +} |
| 755 | + |
| 756 | +func (s *Service) createMultipart(ctx context.Context, part multipartRequest) error { |
| 757 | + var count, userCount int |
| 758 | + var total int64 |
| 759 | + err := s.store.DB().QueryRowContext(ctx, `SELECT COUNT(*),COALESCE(SUM(length),0),COALESCE(SUM(uploader=?),0) FROM multipart_uploads`, part.uploader).Scan(&count, &total, &userCount) |
| 760 | + if err != nil { |
| 761 | + return err |
| 762 | + } |
| 763 | + if count >= multipartSessionLimit || userCount >= multipartUserSessionLimit || part.length > multipartStorageLimit-total { |
| 764 | + return multipartFailure(429, "temporary upload capacity is full") |
| 765 | + } |
| 766 | + path := s.multipartPath(part.id) |
| 767 | + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0o600) |
| 768 | + if err != nil { |
| 769 | + return err |
| 770 | + } |
| 771 | + err = errors.Join(file.Truncate(part.length), file.Sync(), file.Close()) |
| 772 | + if err != nil { |
| 773 | + _ = os.Remove(path) |
| 774 | + return err |
| 775 | + } |
| 776 | + now := time.Now().Unix() |
| 777 | + _, err = s.store.DB().ExecContext(ctx, "INSERT INTO multipart_uploads(id,sha256,uploader,length,type,created,last_seen) VALUES(?,?,?,?,?,?,?)", part.id, part.hash, part.uploader, part.length, part.typ, now, now) |
| 778 | + if err != nil { |
| 779 | + _ = os.Remove(path) |
| 780 | + } |
| 781 | + return err |
| 782 | +} |
| 783 | + |
| 784 | +func (s *Service) stageMultipart(r *http.Request, part multipartRequest) (string, error) { |
| 785 | + chunk, err := os.CreateTemp(s.root, ".multipart-chunk-") |
| 786 | + if err != nil { |
| 787 | + return "", err |
| 788 | + } |
| 789 | + path := chunk.Name() |
| 790 | + keep := false |
| 791 | + defer func() { |
| 792 | + _ = chunk.Close() |
| 793 | + if !keep { |
| 794 | + _ = os.Remove(path) |
| 795 | + } |
| 796 | + }() |
| 797 | + hasher := sha256.New() |
| 798 | + if _, err := io.CopyN(io.MultiWriter(chunk, hasher), r.Body, part.count); err != nil { |
| 799 | + return "", multipartFailure(400, "chunk body is shorter than Content-Length") |
| 800 | + } |
| 801 | + var extra [1]byte |
| 802 | + if n, err := io.ReadFull(r.Body, extra[:]); n != 0 || (err != nil && !errors.Is(err, io.EOF)) { |
| 803 | + return "", multipartFailure(400, "chunk body is longer than Content-Length") |
| 804 | + } |
| 805 | + if err := r.Context().Err(); err != nil { |
| 806 | + return "", err |
| 807 | + } |
| 808 | + if s.config.ValidateUpload != nil { |
| 809 | + if err := s.config.ValidateUpload(r, hex.EncodeToString(hasher.Sum(nil))); err != nil { |
| 810 | + return "", err |
| 811 | + } |
| 812 | + } |
| 813 | + if err := chunk.Close(); err != nil { |
| 814 | + return "", err |
| 815 | + } |
| 816 | + keep = true |
| 817 | + return path, nil |
| 818 | +} |
| 819 | + |
| 820 | +func (s *Service) releaseMultipart(id string) { |
| 821 | + s.multipartMu.Lock() |
| 822 | + defer s.multipartMu.Unlock() |
| 823 | + s.activeParts[id]-- |
| 824 | + if s.activeParts[id] > 0 { |
| 825 | + return |
| 826 | + } |
| 827 | + delete(s.activeParts, id) |
| 828 | + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 829 | + defer cancel() |
| 830 | + var parts int |
| 831 | + if err := s.store.DB().QueryRowContext(ctx, "SELECT COUNT(*) FROM multipart_parts WHERE upload_id=?", id).Scan(&parts); err == nil && parts == 0 { |
| 832 | + s.quotaMu.Lock() |
| 833 | + defer s.quotaMu.Unlock() |
| 834 | + _ = s.discardMultipart(ctx, id) |
| 835 | + } |
| 836 | +} |
| 837 | + |
| 838 | +func (s *Service) commitMultipart(ctx context.Context, part multipartRequest, chunk string) (Blob, bool, bool, error) { |
| 839 | + s.multipartMu.Lock() |
| 840 | + defer s.multipartMu.Unlock() |
| 841 | + // Recheck policy after streaming, including uploads whose next chunk does |
| 842 | + // not complete the file. Policy may have changed while the body arrived. |
| 843 | + s.quotaMu.Lock() |
| 844 | + admissionErr := s.multipartAdmission(ctx, part) |
| 845 | + s.quotaMu.Unlock() |
| 846 | + if admissionErr != nil { |
| 847 | + return Blob{}, false, false, admissionErr |
| 848 | + } |
| 849 | + candidate := uploadCandidate{hash: part.hash, size: part.length, typ: part.typ, uploader: part.uploader, reservation: part.id} |
| 850 | + if existing, err := s.lookup(ctx, part.hash); err == nil { |
| 851 | + if existing.Size != part.length { |
| 852 | + return Blob{}, false, false, ErrMultipartConflict |
| 853 | + } |
| 854 | + entry, _, err := s.installUpload(ctx, candidate) |
| 855 | + return entry, false, err == nil, err |
| 856 | + } else if !errors.Is(err, sql.ErrNoRows) { |
| 857 | + return Blob{}, false, false, err |
| 858 | + } |
| 859 | + ranges, err := s.multipartRanges(ctx, part) |
| 860 | + if err != nil { |
| 861 | + return Blob{}, false, false, err |
| 862 | + } |
| 863 | + candidate.path = s.multipartPath(part.id) |
| 864 | + if err := writeMultipartChunk(candidate.path, chunk, part.offset); err != nil { |
| 865 | + _ = s.discardMultipart(ctx, part.id) |
| 866 | + return Blob{}, false, false, err |
| 867 | + } |
| 868 | + if err := s.recordMultipartRanges(ctx, part.id, ranges); err != nil { |
| 869 | + _ = s.discardMultipart(ctx, part.id) |
| 870 | + return Blob{}, false, false, err |
| 871 | + } |
| 872 | + if len(ranges) != 1 || ranges[0].start != 0 || ranges[0].end != part.length { |
| 873 | + return Blob{}, false, false, nil |
| 874 | + } |
| 875 | + if err := verifyMultipart(candidate.path, part.hash, part.length); err != nil { |
| 876 | + return Blob{}, false, false, errors.Join(err, s.discardMultipart(ctx, part.id)) |
| 877 | + } |
| 878 | + entry, created, err := s.installUpload(ctx, candidate) |
| 879 | + cleanupErr := s.discardMultipart(ctx, part.id) |
| 880 | + return entry, created, err == nil && cleanupErr == nil, errors.Join(err, cleanupErr) |
| 881 | +} |
| 882 | + |
| 883 | +type multipartRange struct{ start, end int64 } |
| 884 | + |
| 885 | +func (s *Service) multipartRanges(ctx context.Context, part multipartRequest) ([]multipartRange, error) { |
| 886 | + rows, err := s.store.DB().QueryContext(ctx, "SELECT offset,length FROM multipart_parts WHERE upload_id=? ORDER BY offset", part.id) |
| 887 | + if err != nil { |
| 888 | + return nil, err |
| 889 | + } |
| 890 | + defer rows.Close() |
| 891 | + ranges := []multipartRange{{part.offset, part.offset + part.count}} |
| 892 | + for rows.Next() { |
| 893 | + var offset, length int64 |
| 894 | + if err := rows.Scan(&offset, &length); err != nil { |
| 895 | + return nil, err |
| 896 | + } |
| 897 | + ranges = append(ranges, multipartRange{offset, offset + length}) |
| 898 | + } |
| 899 | + if err := rows.Err(); err != nil { |
| 900 | + return nil, err |
| 901 | + } |
| 902 | + sort.Slice(ranges, func(i, j int) bool { return ranges[i].start < ranges[j].start }) |
| 903 | + merged := []multipartRange{ranges[0]} |
| 904 | + for _, span := range ranges[1:] { |
| 905 | + last := &merged[len(merged)-1] |
| 906 | + if span.start <= last.end { |
| 907 | + last.end = max(last.end, span.end) |
| 908 | + } else { |
| 909 | + merged = append(merged, span) |
| 910 | + } |
| 911 | + } |
| 912 | + if len(merged) > multipartRangeLimit { |
| 913 | + return nil, multipartFailure(429, "too many disjoint upload ranges") |
| 914 | + } |
| 915 | + return merged, nil |
| 916 | +} |
| 917 | + |
| 918 | +func writeMultipartChunk(path, chunk string, offset int64) error { |
| 919 | + source, err := os.Open(chunk) |
| 920 | + if err != nil { |
| 921 | + return err |
| 922 | + } |
| 923 | + defer source.Close() |
| 924 | + target, err := os.OpenFile(path, os.O_RDWR, 0o600) |
| 925 | + if err != nil { |
| 926 | + return err |
| 927 | + } |
| 928 | + defer target.Close() |
| 929 | + if _, err := target.Seek(offset, io.SeekStart); err != nil { |
| 930 | + return err |
| 931 | + } |
| 932 | + if _, err := io.Copy(target, source); err != nil { |
| 933 | + return err |
| 934 | + } |
| 935 | + return target.Sync() |
| 936 | +} |
| 937 | +func (s *Service) recordMultipartRanges(ctx context.Context, id string, ranges []multipartRange) error { |
| 938 | + return s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 939 | + if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_parts WHERE upload_id=?", id); err != nil { |
| 940 | + return err |
| 941 | + } |
| 942 | + for _, span := range ranges { |
| 943 | + if _, err := tx.ExecContext(ctx, "INSERT INTO multipart_parts(upload_id,offset,length) VALUES(?,?,?)", id, span.start, span.end-span.start); err != nil { |
| 944 | + return err |
| 945 | + } |
| 946 | + } |
| 947 | + _, err := tx.ExecContext(ctx, "UPDATE multipart_uploads SET last_seen=? WHERE id=?", time.Now().Unix(), id) |
| 948 | + return err |
| 949 | + }) |
| 950 | +} |
| 951 | +func verifyMultipart(path, hash string, size int64) error { |
| 952 | + file, err := os.Open(path) |
| 953 | + if err != nil { |
| 954 | + return err |
| 955 | + } |
| 956 | + defer file.Close() |
| 957 | + hasher := sha256.New() |
| 958 | + count, err := io.Copy(hasher, file) |
| 959 | + if err != nil { |
| 960 | + return err |
| 961 | + } |
| 962 | + if count != size || hex.EncodeToString(hasher.Sum(nil)) != hash { |
| 963 | + return ErrHashMismatch |
| 964 | + } |
| 965 | + return nil |
| 966 | +} |
| 967 | +func (s *Service) discardMultipart(ctx context.Context, id string) error { |
| 968 | + _, err := s.store.DB().ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", id) |
| 969 | + if err != nil { |
| 970 | + return err |
| 971 | + } |
| 972 | + if err := os.Remove(s.multipartPath(id)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 973 | + return err |
| 974 | + } |
| 975 | + return nil |
| 976 | +} |
| 977 | + |
| 978 | +// CleanupMultipart releases inactive reservations and removes partial bytes. |
| 979 | +// The tenant maintenance scheduler calls it even when no uploads arrive. |
| 980 | +func (s *Service) CleanupMultipart(ctx context.Context) error { |
| 981 | + s.multipartMu.Lock() |
| 982 | + defer s.multipartMu.Unlock() |
| 983 | + s.quotaMu.Lock() |
| 984 | + defer s.quotaMu.Unlock() |
| 985 | + return s.cleanupMultipartLocked(ctx) |
| 986 | +} |
| 987 | +func (s *Service) cleanupMultipartLocked(ctx context.Context) error { |
| 988 | + rows, err := s.store.DB().QueryContext(ctx, "SELECT id FROM multipart_uploads WHERE last_seen <= ?", time.Now().Add(-multipartExpiry).Unix()) |
| 989 | + if err != nil { |
| 990 | + return err |
| 991 | + } |
| 992 | + var expired []string |
| 993 | + for rows.Next() { |
| 994 | + var id string |
| 995 | + if err := rows.Scan(&id); err != nil { |
| 996 | + _ = rows.Close() |
| 997 | + return err |
| 998 | + } |
| 999 | + if s.activeParts[id] == 0 { |
| 1000 | + expired = append(expired, id) |
| 1001 | + } |
| 1002 | + } |
| 1003 | + err = errors.Join(rows.Err(), rows.Close()) |
| 1004 | + if err != nil { |
| 1005 | + return err |
| 1006 | + } |
| 1007 | + for _, id := range expired { |
| 1008 | + if err := s.discardMultipart(ctx, id); err != nil { |
| 1009 | + return err |
| 1010 | + } |
| 1011 | + } |
| 1012 | + return nil |
| 1013 | +} |
| 1014 | +func (s *Service) reconcileMultipart(ctx context.Context) error { |
| 1015 | + if err := s.CleanupMultipart(ctx); err != nil { |
| 1016 | + return err |
| 1017 | + } |
| 1018 | + entries, err := os.ReadDir(s.root) |
| 1019 | + if err != nil { |
| 1020 | + return err |
| 1021 | + } |
| 1022 | + for _, entry := range entries { |
| 1023 | + if !strings.HasPrefix(entry.Name(), ".multipart-") || entry.IsDir() { |
| 1024 | + continue |
| 1025 | + } |
| 1026 | + id := strings.TrimPrefix(entry.Name(), ".multipart-") |
| 1027 | + var length int64 |
| 1028 | + err := s.store.DB().QueryRowContext(ctx, "SELECT length FROM multipart_uploads WHERE id=?", id).Scan(&length) |
| 1029 | + if errors.Is(err, sql.ErrNoRows) { |
| 1030 | + if err := os.Remove(filepath.Join(s.root, entry.Name())); err != nil { |
| 1031 | + return err |
| 1032 | + } |
| 1033 | + continue |
| 1034 | + } |
| 1035 | + if err != nil { |
| 1036 | + return err |
| 1037 | + } |
| 1038 | + info, err := entry.Info() |
| 1039 | + if err != nil { |
| 1040 | + return err |
| 1041 | + } |
| 1042 | + if info.Size() != length { |
| 1043 | + if err := s.discardMultipart(ctx, id); err != nil { |
| 1044 | + return err |
| 1045 | + } |
| 1046 | + } |
| 1047 | + } |
| 1048 | + // A reservation with a missing file cannot represent acknowledged bytes. |
| 1049 | + rows, err := s.store.DB().QueryContext(ctx, "SELECT id FROM multipart_uploads") |
| 1050 | + if err != nil { |
| 1051 | + return err |
| 1052 | + } |
| 1053 | + var missing []string |
| 1054 | + for rows.Next() { |
| 1055 | + var id string |
| 1056 | + if err := rows.Scan(&id); err != nil { |
| 1057 | + _ = rows.Close() |
| 1058 | + return err |
| 1059 | + } |
| 1060 | + if _, err := os.Stat(s.multipartPath(id)); errors.Is(err, os.ErrNotExist) { |
| 1061 | + missing = append(missing, id) |
| 1062 | + } else if err != nil { |
| 1063 | + _ = rows.Close() |
| 1064 | + return err |
| 1065 | + } |
| 1066 | + } |
| 1067 | + err = errors.Join(rows.Err(), rows.Close()) |
| 1068 | + if err != nil { |
| 1069 | + return err |
| 1070 | + } |
| 1071 | + for _, id := range missing { |
| 1072 | + if err := s.discardMultipart(ctx, id); err != nil { |
| 1073 | + return fmt.Errorf("reconcile multipart upload: %w", err) |
| 1074 | + } |
| 1075 | + } |
| 1076 | + return nil |
| 1077 | +} |
| 1078 | diff --git a/internal/blob/multipart_review_test.go b/internal/blob/multipart_review_test.go |
| 1079 | new file mode 100644 |
| 1080 | index 0000000..cc91b2f |
| 1081 | --- /dev/null |
| 1082 | +++ b/internal/blob/multipart_review_test.go |
| 1083 | @@ -0,0 +1,192 @@ |
| 1084 | +package blob |
| 1085 | + |
| 1086 | +import ( |
| 1087 | + "context" |
| 1088 | + "crypto/sha256" |
| 1089 | + "encoding/hex" |
| 1090 | + "errors" |
| 1091 | + "io" |
| 1092 | + "net/http" |
| 1093 | + "net/http/httptest" |
| 1094 | + "path/filepath" |
| 1095 | + "strings" |
| 1096 | + "sync" |
| 1097 | + "testing" |
| 1098 | + "time" |
| 1099 | +) |
| 1100 | + |
| 1101 | +func multipartReviewRequest(sha string, length, offset int64, body string) *http.Request { |
| 1102 | + r := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+sha, strings.NewReader(body)) |
| 1103 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1104 | + r.Header.Set("Upload-Type", "application/octet-stream") |
| 1105 | + r.Header.Set("Upload-Length", formatInt(length)) |
| 1106 | + r.Header.Set("Upload-Offset", formatInt(offset)) |
| 1107 | + return r |
| 1108 | +} |
| 1109 | + |
| 1110 | +func TestMultipartReviewConcurrentChunksProduceOneBlob(t *testing.T) { |
| 1111 | + s := testService(t) |
| 1112 | + body := []byte(strings.Repeat("a", 70000) + strings.Repeat("b", 70000)) |
| 1113 | + digest := sha256.Sum256(body) |
| 1114 | + sha := hex.EncodeToString(digest[:]) |
| 1115 | + parts := []struct { |
| 1116 | + offset int64 |
| 1117 | + body string |
| 1118 | + }{ |
| 1119 | + {70000, string(body[70000:])}, |
| 1120 | + {0, string(body[:70000])}, |
| 1121 | + } |
| 1122 | + responses := make([]*httptest.ResponseRecorder, len(parts)) |
| 1123 | + var wg sync.WaitGroup |
| 1124 | + for i, part := range parts { |
| 1125 | + wg.Add(1) |
| 1126 | + go func() { |
| 1127 | + defer wg.Done() |
| 1128 | + responses[i] = record(s.Handler(), multipartReviewRequest(sha, int64(len(body)), part.offset, part.body)) |
| 1129 | + }() |
| 1130 | + } |
| 1131 | + wg.Wait() |
| 1132 | + var created, partial int |
| 1133 | + for _, response := range responses { |
| 1134 | + switch response.Code { |
| 1135 | + case http.StatusCreated: |
| 1136 | + created++ |
| 1137 | + case http.StatusNoContent: |
| 1138 | + partial++ |
| 1139 | + default: |
| 1140 | + t.Fatalf("concurrent chunk status = %d: %s", response.Code, response.Body.String()) |
| 1141 | + } |
| 1142 | + } |
| 1143 | + if created != 1 || partial != 1 { |
| 1144 | + t.Fatalf("concurrent chunk results created=%d partial=%d", created, partial) |
| 1145 | + } |
| 1146 | + entry, file, err := s.Get(context.Background(), sha) |
| 1147 | + if err != nil { |
| 1148 | + t.Fatal(err) |
| 1149 | + } |
| 1150 | + defer file.Close() |
| 1151 | + if entry.Size != int64(len(body)) { |
| 1152 | + t.Fatalf("stored size = %d, want %d", entry.Size, len(body)) |
| 1153 | + } |
| 1154 | + got, err := io.ReadAll(file) |
| 1155 | + if err != nil || string(got) != string(body) { |
| 1156 | + t.Fatalf("stored body mismatch: %v", err) |
| 1157 | + } |
| 1158 | +} |
| 1159 | + |
| 1160 | +func TestMultipartReviewStalledBodyDoesNotBlockOrdinaryUpload(t *testing.T) { |
| 1161 | + s := testService(t) |
| 1162 | + body := "stalled" |
| 1163 | + digest := sha256.Sum256([]byte(body)) |
| 1164 | + sha := hex.EncodeToString(digest[:]) |
| 1165 | + started := make(chan struct{}) |
| 1166 | + release := make(chan struct{}) |
| 1167 | + r := multipartReviewRequest(sha, int64(len(body)), 0, body) |
| 1168 | + r.Body = &multipartBlockingBody{Reader: strings.NewReader(body), started: started, release: release} |
| 1169 | + r.ContentLength = int64(len(body)) |
| 1170 | + result := make(chan *httptest.ResponseRecorder, 1) |
| 1171 | + go func() { result <- record(s.Handler(), r) }() |
| 1172 | + select { |
| 1173 | + case <-started: |
| 1174 | + case <-time.After(time.Second): |
| 1175 | + t.Fatal("multipart body was not read") |
| 1176 | + } |
| 1177 | + if _, err := s.Put(context.Background(), PutOptions{Reader: strings.NewReader("ordinary"), Type: "text/plain", Uploader: "uploader"}); err != nil { |
| 1178 | + t.Fatalf("ordinary upload blocked by stalled multipart body: %v", err) |
| 1179 | + } |
| 1180 | + close(release) |
| 1181 | + select { |
| 1182 | + case response := <-result: |
| 1183 | + if response.Code != http.StatusCreated { |
| 1184 | + t.Fatalf("multipart completion status = %d", response.Code) |
| 1185 | + } |
| 1186 | + case <-time.After(time.Second): |
| 1187 | + t.Fatal("stalled multipart request did not finish") |
| 1188 | + } |
| 1189 | +} |
| 1190 | + |
| 1191 | +type multipartBlockingBody struct { |
| 1192 | + io.Reader |
| 1193 | + started chan<- struct{} |
| 1194 | + release <-chan struct{} |
| 1195 | + once sync.Once |
| 1196 | +} |
| 1197 | + |
| 1198 | +func (b *multipartBlockingBody) Read(p []byte) (int, error) { |
| 1199 | + b.once.Do(func() { close(b.started) }) |
| 1200 | + <-b.release |
| 1201 | + return b.Reader.Read(p) |
| 1202 | +} |
| 1203 | + |
| 1204 | +func (b *multipartBlockingBody) Close() error { return nil } |
| 1205 | + |
| 1206 | +func TestMultipartReviewReservationsBlockUploadsAndReleaseOnExpiry(t *testing.T) { |
| 1207 | + s := testService(t) |
| 1208 | + s.config.Limits = func() Limits { return Limits{UserStorageBytes: 8} } |
| 1209 | + full := "12345678" |
| 1210 | + digest := sha256.Sum256([]byte(full)) |
| 1211 | + sha := hex.EncodeToString(digest[:]) |
| 1212 | + partial := multipartReviewRequest(sha, int64(len(full)), 0, full[:4]) |
| 1213 | + partial.ContentLength = 4 |
| 1214 | + if response := record(s.Handler(), partial); response.Code != http.StatusNoContent { |
| 1215 | + t.Fatalf("partial reservation status = %d: %s", response.Code, response.Body.String()) |
| 1216 | + } |
| 1217 | + if _, err := s.Put(context.Background(), PutOptions{Reader: strings.NewReader("1234"), Type: "text/plain", Uploader: "uploader"}); !errors.Is(err, ErrQuotaExceeded) { |
| 1218 | + t.Fatalf("ordinary upload after reservation = %v", err) |
| 1219 | + } |
| 1220 | + other := "abcdefgh" |
| 1221 | + otherDigest := sha256.Sum256([]byte(other)) |
| 1222 | + second := multipartReviewRequest(hex.EncodeToString(otherDigest[:]), int64(len(other)), 0, other[:4]) |
| 1223 | + second.ContentLength = 4 |
| 1224 | + if response := record(s.Handler(), second); response.Code != http.StatusForbidden { |
| 1225 | + t.Fatalf("second reserved upload status = %d", response.Code) |
| 1226 | + } |
| 1227 | + if _, err := s.store.DB().Exec("UPDATE multipart_uploads SET last_seen=0"); err != nil { |
| 1228 | + t.Fatal(err) |
| 1229 | + } |
| 1230 | + if err := s.CleanupMultipart(context.Background()); err != nil { |
| 1231 | + t.Fatal(err) |
| 1232 | + } |
| 1233 | + matches, err := filepath.Glob(filepath.Join(s.root, ".multipart-*")) |
| 1234 | + if err != nil { |
| 1235 | + t.Fatal(err) |
| 1236 | + } |
| 1237 | + if len(matches) != 0 { |
| 1238 | + t.Fatalf("expired multipart files remain: %v", matches) |
| 1239 | + } |
| 1240 | + if _, err := s.Put(context.Background(), PutOptions{Reader: strings.NewReader("1234"), Type: "text/plain", Uploader: "uploader"}); err != nil { |
| 1241 | + t.Fatalf("ordinary upload after expiry: %v", err) |
| 1242 | + } |
| 1243 | +} |
| 1244 | + |
| 1245 | +func TestMultipartReviewHashMismatchReleasesReservation(t *testing.T) { |
| 1246 | + s := testService(t) |
| 1247 | + s.config.Limits = func() Limits { return Limits{UserStorageBytes: 5} } |
| 1248 | + wrong := strings.Repeat("a", 64) |
| 1249 | + r := multipartReviewRequest(wrong, 5, 0, "wrong") |
| 1250 | + r.ContentLength = 5 |
| 1251 | + if response := record(s.Handler(), r); response.Code != http.StatusConflict { |
| 1252 | + t.Fatalf("hash mismatch status = %d: %s", response.Code, response.Body.String()) |
| 1253 | + } |
| 1254 | + if _, err := s.Put(context.Background(), PutOptions{Reader: strings.NewReader("right"), Type: "text/plain", Uploader: "uploader"}); err != nil { |
| 1255 | + t.Fatalf("upload after mismatch reservation: %v", err) |
| 1256 | + } |
| 1257 | +} |
| 1258 | + |
| 1259 | +func TestMultipartReviewExistingBlobCannotBypassModerationBlock(t *testing.T) { |
| 1260 | + s := testService(t) |
| 1261 | + body := "blocked" |
| 1262 | + digest := sha256.Sum256([]byte(body)) |
| 1263 | + sha := hex.EncodeToString(digest[:]) |
| 1264 | + if _, err := s.Put(context.Background(), PutOptions{Reader: strings.NewReader(body), Type: "text/plain"}); err != nil { |
| 1265 | + t.Fatal(err) |
| 1266 | + } |
| 1267 | + if _, err := s.store.DB().Exec("INSERT INTO blob_blocks(sha256,reason,blocked_at) VALUES(?,?,?)", sha, "review", time.Now().Unix()); err != nil { |
| 1268 | + t.Fatal(err) |
| 1269 | + } |
| 1270 | + r := multipartReviewRequest(sha, int64(len(body)), 0, body) |
| 1271 | + r.ContentLength = int64(len(body)) |
| 1272 | + if response := record(s.Handler(), r); response.Code != http.StatusForbidden { |
| 1273 | + t.Fatalf("blocked existing blob status = %d: %s", response.Code, response.Body.String()) |
| 1274 | + } |
| 1275 | +} |
| 1276 | diff --git a/internal/blob/multipart_test.go b/internal/blob/multipart_test.go |
| 1277 | new file mode 100644 |
| 1278 | index 0000000..6ad4f95 |
| 1279 | --- /dev/null |
| 1280 | +++ b/internal/blob/multipart_test.go |
| 1281 | @@ -0,0 +1,171 @@ |
| 1282 | +package blob |
| 1283 | + |
| 1284 | +import ( |
| 1285 | + "context" |
| 1286 | + "crypto/sha256" |
| 1287 | + "encoding/hex" |
| 1288 | + "io" |
| 1289 | + "net/http" |
| 1290 | + "net/http/httptest" |
| 1291 | + "strconv" |
| 1292 | + "strings" |
| 1293 | + "testing" |
| 1294 | +) |
| 1295 | + |
| 1296 | +func TestMultipartAcceptsOutOfOrderLargeChunks(t *testing.T) { |
| 1297 | + s := testService(t) |
| 1298 | + body := []byte(strings.Repeat("a", 70000) + strings.Repeat("b", 70000)) |
| 1299 | + digest := sha256.Sum256(body) |
| 1300 | + sha := hex.EncodeToString(digest[:]) |
| 1301 | + patch := func(offset int64, data []byte) *httptest.ResponseRecorder { |
| 1302 | + r := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+sha+".bin", strings.NewReader(string(data))) |
| 1303 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1304 | + r.Header.Set("Upload-Type", "application/octet-stream") |
| 1305 | + r.Header.Set("Upload-Length", "140000") |
| 1306 | + r.Header.Set("Upload-Offset", formatInt(offset)) |
| 1307 | + return record(s.Handler(), r) |
| 1308 | + } |
| 1309 | + if got := patch(70000, body[70000:]); got.Code != http.StatusNoContent { |
| 1310 | + t.Fatalf("second chunk status = %d", got.Code) |
| 1311 | + } |
| 1312 | + if got := patch(0, body[:70000]); got.Code != http.StatusCreated { |
| 1313 | + t.Fatalf("first chunk status = %d: %s", got.Code, got.Body.String()) |
| 1314 | + } |
| 1315 | + entry, file, err := s.Get(context.Background(), sha) |
| 1316 | + if err != nil || entry.Size != int64(len(body)) { |
| 1317 | + t.Fatalf("stored multipart blob = %#v, %v", entry, err) |
| 1318 | + } |
| 1319 | + defer file.Close() |
| 1320 | +} |
| 1321 | + |
| 1322 | +func formatInt(value int64) string { return strconv.FormatInt(value, 10) } |
| 1323 | + |
| 1324 | +func TestMultipartRejectsMalformedRangesAndLength(t *testing.T) { |
| 1325 | + s := testService(t) |
| 1326 | + digest := sha256.Sum256([]byte("abc")) |
| 1327 | + sha := hex.EncodeToString(digest[:]) |
| 1328 | + tests := []struct { |
| 1329 | + name string |
| 1330 | + contentLength int64 |
| 1331 | + uploadLength string |
| 1332 | + offset string |
| 1333 | + want int |
| 1334 | + }{ |
| 1335 | + {"missing content length", -1, "3", "0", http.StatusLengthRequired}, |
| 1336 | + {"offset outside upload", 1, "3", "3", http.StatusRequestedRangeNotSatisfiable}, |
| 1337 | + {"chunk crosses upload", 2, "3", "2", http.StatusRequestedRangeNotSatisfiable}, |
| 1338 | + } |
| 1339 | + for _, tc := range tests { |
| 1340 | + t.Run(tc.name, func(t *testing.T) { |
| 1341 | + r := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+sha, strings.NewReader("ab")) |
| 1342 | + r.ContentLength = tc.contentLength |
| 1343 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1344 | + r.Header.Set("Upload-Type", "text/plain") |
| 1345 | + r.Header.Set("Upload-Length", tc.uploadLength) |
| 1346 | + r.Header.Set("Upload-Offset", tc.offset) |
| 1347 | + if got := record(s.Handler(), r).Code; got != tc.want { |
| 1348 | + t.Fatalf("status = %d, want %d", got, tc.want) |
| 1349 | + } |
| 1350 | + }) |
| 1351 | + } |
| 1352 | +} |
| 1353 | + |
| 1354 | +func TestMultipartMismatchDoesNotPublishBlob(t *testing.T) { |
| 1355 | + s := testService(t) |
| 1356 | + want := strings.Repeat("0", 64) |
| 1357 | + r := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+want, strings.NewReader("wrong")) |
| 1358 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1359 | + r.Header.Set("Upload-Type", "text/plain") |
| 1360 | + r.Header.Set("Upload-Length", "5") |
| 1361 | + r.Header.Set("Upload-Offset", "0") |
| 1362 | + if got := record(s.Handler(), r).Code; got != http.StatusConflict { |
| 1363 | + t.Fatalf("status = %d, want 409", got) |
| 1364 | + } |
| 1365 | + if _, _, err := s.Get(context.Background(), want); err == nil { |
| 1366 | + t.Fatal("mismatched multipart upload was published") |
| 1367 | + } |
| 1368 | +} |
| 1369 | + |
| 1370 | +func TestMultipartOverlappingChunksReconstructExactBytes(t *testing.T) { |
| 1371 | + s := testService(t) |
| 1372 | + body := []byte("abcdefghij") |
| 1373 | + digest := sha256.Sum256(body) |
| 1374 | + sha := hex.EncodeToString(digest[:]) |
| 1375 | + request := func(offset int, chunk string) int { |
| 1376 | + r := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+sha, strings.NewReader(chunk)) |
| 1377 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1378 | + r.Header.Set("Upload-Type", "text/plain") |
| 1379 | + r.Header.Set("Upload-Length", "10") |
| 1380 | + r.Header.Set("Upload-Offset", formatInt(int64(offset))) |
| 1381 | + return record(s.Handler(), r).Code |
| 1382 | + } |
| 1383 | + if got := request(0, "abcdef"); got != http.StatusNoContent { |
| 1384 | + t.Fatalf("first status = %d", got) |
| 1385 | + } |
| 1386 | + if got := request(4, "efghij"); got != http.StatusCreated { |
| 1387 | + t.Fatalf("overlap status = %d", got) |
| 1388 | + } |
| 1389 | + _, file, err := s.Get(context.Background(), sha) |
| 1390 | + if err != nil { |
| 1391 | + t.Fatal(err) |
| 1392 | + } |
| 1393 | + defer file.Close() |
| 1394 | + got, err := io.ReadAll(file) |
| 1395 | + if err != nil || string(got) != string(body) { |
| 1396 | + t.Fatalf("body = %q, %v", got, err) |
| 1397 | + } |
| 1398 | +} |
| 1399 | + |
| 1400 | +func TestMultipartResumesAfterServiceRestart(t *testing.T) { |
| 1401 | + original := testService(t) |
| 1402 | + body := "survives restart" |
| 1403 | + digest := sha256.Sum256([]byte(body)) |
| 1404 | + hash := hex.EncodeToString(digest[:]) |
| 1405 | + patch := func(service *Service, offset int, chunk string) *httptest.ResponseRecorder { |
| 1406 | + request := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+hash, strings.NewReader(chunk)) |
| 1407 | + request.Header.Set("Content-Type", "application/octet-stream") |
| 1408 | + request.Header.Set("Upload-Type", "text/plain") |
| 1409 | + request.Header.Set("Upload-Length", strconv.Itoa(len(body))) |
| 1410 | + request.Header.Set("Upload-Offset", strconv.Itoa(offset)) |
| 1411 | + return record(service.Handler(), request) |
| 1412 | + } |
| 1413 | + if result := patch(original, 0, body[:5]); result.Code != 204 { |
| 1414 | + t.Fatalf("partial: %d %s", result.Code, result.Body.String()) |
| 1415 | + } |
| 1416 | + reopened, err := New(context.Background(), original.config) |
| 1417 | + if err != nil { |
| 1418 | + t.Fatal(err) |
| 1419 | + } |
| 1420 | + if result := patch(reopened, 5, body[5:]); result.Code != 201 { |
| 1421 | + t.Fatalf("resumed: %d %s", result.Code, result.Body.String()) |
| 1422 | + } |
| 1423 | +} |
| 1424 | + |
| 1425 | +func TestMultipartAcceptsEmptyFile(t *testing.T) { |
| 1426 | + service := testService(t) |
| 1427 | + digest := sha256.Sum256(nil) |
| 1428 | + request := httptest.NewRequest(http.MethodPatch, "https://relay.test/"+hex.EncodeToString(digest[:]), http.NoBody) |
| 1429 | + request.Header.Set("Content-Type", "application/octet-stream") |
| 1430 | + request.Header.Set("Upload-Type", "application/octet-stream") |
| 1431 | + request.Header.Set("Upload-Length", "0") |
| 1432 | + request.Header.Set("Upload-Offset", "0") |
| 1433 | + if result := record(service.Handler(), request); result.Code != 201 { |
| 1434 | + t.Fatalf("empty: %d %s", result.Code, result.Body.String()) |
| 1435 | + } |
| 1436 | +} |
| 1437 | + |
| 1438 | +func TestMultipartExistingBlobAddsUploaderClaim(t *testing.T) { |
| 1439 | + service := testService(t) |
| 1440 | + const content = "deduplicated" |
| 1441 | + entry, err := service.Put(context.Background(), PutOptions{Reader: strings.NewReader(content), Uploader: "first"}) |
| 1442 | + if err != nil { |
| 1443 | + t.Fatal(err) |
| 1444 | + } |
| 1445 | + request := multipartReviewRequest(entry.SHA256, int64(len(content)), 0, content[:2]) |
| 1446 | + if result := record(service.Handler(), request); result.Code != 200 { |
| 1447 | + t.Fatalf("duplicate: %d %s", result.Code, result.Body.String()) |
| 1448 | + } |
| 1449 | + if !service.hasClaim(context.Background(), entry.SHA256, "uploader") { |
| 1450 | + t.Fatal("existing blob did not grant the uploader a claim") |
| 1451 | + } |
| 1452 | +} |
| 1453 | diff --git a/internal/blob/path_upload.go b/internal/blob/path_upload.go |
| 1454 | new file mode 100644 |
| 1455 | index 0000000..35fabf3 |
| 1456 | --- /dev/null |
| 1457 | +++ b/internal/blob/path_upload.go |
| 1458 | @@ -0,0 +1,128 @@ |
| 1459 | +package blob |
| 1460 | + |
| 1461 | +import ( |
| 1462 | + "context" |
| 1463 | + "errors" |
| 1464 | + "fmt" |
| 1465 | + "io" |
| 1466 | + "net/http" |
| 1467 | + "net/url" |
| 1468 | +) |
| 1469 | + |
| 1470 | +// maxPathUploadSources bounds the remote sources one BUD-13 upload may name. |
| 1471 | +const maxPathUploadSources = 8 |
| 1472 | + |
| 1473 | +// pathUpload implements the draft BUD-13 path-addressed upload endpoint. |
| 1474 | +// The route itself is dispatched by serveHTTP after validating the hash path. |
| 1475 | +func (s *Service) pathUpload(w http.ResponseWriter, r *http.Request, expectedHash string) { |
| 1476 | + // Authenticate before inspecting the body. This is important for signed |
| 1477 | + // uploads: an invalid request must not consume a potentially large stream. |
| 1478 | + pubkey, err := s.authorize(r, ActionUpload) |
| 1479 | + if err != nil { |
| 1480 | + s.fail(w, http.StatusUnauthorized, err) |
| 1481 | + return |
| 1482 | + } |
| 1483 | + |
| 1484 | + urls := r.URL.Query()["url"] |
| 1485 | + if len(urls) == 0 { |
| 1486 | + entry, created, putErr := s.putValidated(r.Context(), r.Body, |
| 1487 | + contentType(r.Header.Get("content-type")), pubkey, expectedHash, |
| 1488 | + func(actual string) error { |
| 1489 | + if s.config.ValidateUpload != nil { |
| 1490 | + return s.config.ValidateUpload(r, actual) |
| 1491 | + } |
| 1492 | + return nil |
| 1493 | + }) |
| 1494 | + if putErr != nil { |
| 1495 | + s.fail(w, statusFor(putErr), putErr) |
| 1496 | + return |
| 1497 | + } |
| 1498 | + s.writeJSON(w, chooseStatus(created, http.StatusCreated), s.descriptor(r, entry)) |
| 1499 | + return |
| 1500 | + } |
| 1501 | + |
| 1502 | + // BUD-13 remote uploads must have an empty request body. Check this after |
| 1503 | + // authorization, including chunked requests whose ContentLength is unknown. |
| 1504 | + if r.Body != nil { |
| 1505 | + var probe [1]byte |
| 1506 | + n, readErr := io.ReadFull(r.Body, probe[:]) |
| 1507 | + if n != 0 || (readErr != nil && !errors.Is(readErr, io.EOF)) { |
| 1508 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: remote upload body must be empty")) |
| 1509 | + return |
| 1510 | + } |
| 1511 | + } |
| 1512 | + if len(urls) > maxPathUploadSources { |
| 1513 | + s.fail(w, http.StatusBadRequest, fmt.Errorf("invalid: at most %d remote sources per upload", maxPathUploadSources)) |
| 1514 | + return |
| 1515 | + } |
| 1516 | + for _, raw := range urls { |
| 1517 | + if err := s.validatePathUploadURL(r.Context(), raw); err != nil { |
| 1518 | + s.fail(w, http.StatusBadRequest, err) |
| 1519 | + return |
| 1520 | + } |
| 1521 | + } |
| 1522 | + |
| 1523 | + client := s.config.HTTPClient |
| 1524 | + if client == nil { |
| 1525 | + client = s.safeClient() |
| 1526 | + } |
| 1527 | + var mismatch bool |
| 1528 | + for _, raw := range urls { |
| 1529 | + fetchRequest, requestErr := http.NewRequestWithContext(r.Context(), http.MethodGet, raw, nil) |
| 1530 | + if requestErr != nil { |
| 1531 | + continue |
| 1532 | + } |
| 1533 | + response, fetchErr := client.Do(fetchRequest) |
| 1534 | + if fetchErr != nil { |
| 1535 | + continue |
| 1536 | + } |
| 1537 | + if response.StatusCode < 200 || response.StatusCode >= 300 { |
| 1538 | + _ = response.Body.Close() |
| 1539 | + continue |
| 1540 | + } |
| 1541 | + typ := contentType(response.Header.Get("content-type")) |
| 1542 | + if typ == "application/octet-stream" { |
| 1543 | + typ = typeForExtension(raw) |
| 1544 | + } |
| 1545 | + entry, created, putErr := s.putValidated(r.Context(), response.Body, typ, |
| 1546 | + pubkey, expectedHash, func(actual string) error { |
| 1547 | + if s.config.ValidateUpload != nil { |
| 1548 | + return s.config.ValidateUpload(r, actual) |
| 1549 | + } |
| 1550 | + return nil |
| 1551 | + }) |
| 1552 | + _ = response.Body.Close() |
| 1553 | + if putErr == nil { |
| 1554 | + s.writeJSON(w, chooseStatus(created, http.StatusCreated), s.descriptor(r, entry)) |
| 1555 | + return |
| 1556 | + } |
| 1557 | + if errors.Is(putErr, ErrHashMismatch) { |
| 1558 | + mismatch = true |
| 1559 | + continue |
| 1560 | + } |
| 1561 | + s.fail(w, statusFor(putErr), putErr) |
| 1562 | + return |
| 1563 | + } |
| 1564 | + if mismatch { |
| 1565 | + s.fail(w, http.StatusConflict, ErrHashMismatch) |
| 1566 | + return |
| 1567 | + } |
| 1568 | + s.fail(w, http.StatusBadGateway, errors.New("error: no usable remote source")) |
| 1569 | +} |
| 1570 | + |
| 1571 | +func (s *Service) validatePathUploadURL(ctx context.Context, raw string) error { |
| 1572 | + u, err := url.Parse(raw) |
| 1573 | + if err != nil || u == nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.Fragment != "" { |
| 1574 | + return errors.New("invalid: url must be an absolute http or https url") |
| 1575 | + } |
| 1576 | + ips, err := s.resolve(ctx, u.Hostname()) |
| 1577 | + if err != nil || len(ips) == 0 { |
| 1578 | + return errors.New("invalid: origin hostname did not resolve") |
| 1579 | + } |
| 1580 | + for _, ip := range ips { |
| 1581 | + if !publicIP(ip) { |
| 1582 | + return errors.New("invalid: origin resolves to a private address") |
| 1583 | + } |
| 1584 | + } |
| 1585 | + return nil |
| 1586 | +} |
| 1587 | diff --git a/internal/blob/path_upload_test.go b/internal/blob/path_upload_test.go |
| 1588 | new file mode 100644 |
| 1589 | index 0000000..a3f84d7 |
| 1590 | --- /dev/null |
| 1591 | +++ b/internal/blob/path_upload_test.go |
| 1592 | @@ -0,0 +1,131 @@ |
| 1593 | +package blob |
| 1594 | + |
| 1595 | +import ( |
| 1596 | + "context" |
| 1597 | + "crypto/sha256" |
| 1598 | + "encoding/hex" |
| 1599 | + "errors" |
| 1600 | + "io" |
| 1601 | + "net" |
| 1602 | + "net/http" |
| 1603 | + "net/http/httptest" |
| 1604 | + "strings" |
| 1605 | + "testing" |
| 1606 | +) |
| 1607 | + |
| 1608 | +func TestPathUploadStoresExactBytesAndReturnsCreatedThenOK(t *testing.T) { |
| 1609 | + s := testService(t) |
| 1610 | + body := []byte("BUD-13 bytes") |
| 1611 | + digest := sha256.Sum256(body) |
| 1612 | + sha := hex.EncodeToString(digest[:]) |
| 1613 | + request := httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+".txt", strings.NewReader(string(body))) |
| 1614 | + request.Header.Set("Content-Type", "text/plain") |
| 1615 | + first := record(s.Handler(), request) |
| 1616 | + if first.Code != http.StatusCreated || !strings.Contains(first.Body.String(), sha) { |
| 1617 | + t.Fatalf("first path upload = %d %s", first.Code, first.Body.String()) |
| 1618 | + } |
| 1619 | + second := record(s.Handler(), httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha, strings.NewReader(string(body)))) |
| 1620 | + if second.Code != http.StatusOK { |
| 1621 | + t.Fatalf("duplicate path upload = %d %s", second.Code, second.Body.String()) |
| 1622 | + } |
| 1623 | +} |
| 1624 | + |
| 1625 | +func TestPathUploadRejectsHashMismatchWithoutPersisting(t *testing.T) { |
| 1626 | + s := testService(t) |
| 1627 | + wanted := strings.Repeat("a", 64) |
| 1628 | + response := record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, wanted) }, httptest.NewRequest(http.MethodPut, "https://relay.test/"+wanted, strings.NewReader("wrong"))) |
| 1629 | + if response.Code != http.StatusConflict { |
| 1630 | + t.Fatalf("mismatch status = %d, want 409", response.Code) |
| 1631 | + } |
| 1632 | + if _, _, err := s.Get(context.Background(), wanted); err == nil { |
| 1633 | + t.Fatal("mismatched path upload was persisted") |
| 1634 | + } |
| 1635 | +} |
| 1636 | + |
| 1637 | +func TestPathUploadAuthenticatesBeforeReadingBody(t *testing.T) { |
| 1638 | + s := testService(t) |
| 1639 | + read := false |
| 1640 | + s.config.Authorize = func(*http.Request, Action) (string, error) { return "", errors.New("bad auth") } |
| 1641 | + body := &trackingReader{Reader: strings.NewReader("should not be read"), read: &read} |
| 1642 | + sha := strings.Repeat("b", 64) |
| 1643 | + response := record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, sha) }, httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha, body)) |
| 1644 | + if response.Code != http.StatusUnauthorized || read { |
| 1645 | + t.Fatalf("auth status/read = %d/%v", response.Code, read) |
| 1646 | + } |
| 1647 | +} |
| 1648 | + |
| 1649 | +func TestPathUploadRejectsRemoteBodyAndMalformedURL(t *testing.T) { |
| 1650 | + s := testService(t) |
| 1651 | + sha := strings.Repeat("c", 64) |
| 1652 | + request := httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+"?url=https%3A%2F%2Fexample.com%2Fblob", strings.NewReader("body")) |
| 1653 | + response := record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, sha) }, request) |
| 1654 | + if response.Code != http.StatusBadRequest { |
| 1655 | + t.Fatalf("remote body status = %d, want 400", response.Code) |
| 1656 | + } |
| 1657 | + request = httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+"?url=file%3A%2F%2F%2Fetc%2Fpasswd", http.NoBody) |
| 1658 | + response = record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, sha) }, request) |
| 1659 | + if response.Code != http.StatusBadRequest { |
| 1660 | + t.Fatalf("malformed URL status = %d, want 400", response.Code) |
| 1661 | + } |
| 1662 | + request = httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+"?url=https%3A%2F%2Fexample.com%2Fx%23fragment", http.NoBody) |
| 1663 | + response = record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, sha) }, request) |
| 1664 | + if response.Code != http.StatusBadRequest { |
| 1665 | + t.Fatalf("fragment URL status = %d, want 400", response.Code) |
| 1666 | + } |
| 1667 | +} |
| 1668 | + |
| 1669 | +func TestPathUploadRemoteSourceVerifiesHashAndSupportsHTTP(t *testing.T) { |
| 1670 | + s := testService(t) |
| 1671 | + body := []byte("remote BUD-13 bytes") |
| 1672 | + digest := sha256.Sum256(body) |
| 1673 | + sha := hex.EncodeToString(digest[:]) |
| 1674 | + s.config.ResolveIP = func(context.Context, string) ([]net.IP, error) { return []net.IP{net.ParseIP("8.8.8.8")}, nil } |
| 1675 | + s.resolve = s.config.ResolveIP |
| 1676 | + s.config.HTTPClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { |
| 1677 | + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(string(body))), Header: http.Header{"Content-Type": []string{"text/plain"}}, Request: req}, nil |
| 1678 | + })} |
| 1679 | + request := httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+"?url=http%3A%2F%2Forigin.example%2Ffile.txt", http.NoBody) |
| 1680 | + response := record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, sha) }, request) |
| 1681 | + if response.Code != http.StatusCreated || !strings.Contains(response.Body.String(), sha) { |
| 1682 | + t.Fatalf("remote upload = %d %s", response.Code, response.Body.String()) |
| 1683 | + } |
| 1684 | +} |
| 1685 | + |
| 1686 | +func TestPathUploadRemoteRejectsPrivateOrigin(t *testing.T) { |
| 1687 | + s := testService(t) |
| 1688 | + s.config.ResolveIP = func(context.Context, string) ([]net.IP, error) { return []net.IP{net.ParseIP("127.0.0.1")}, nil } |
| 1689 | + s.resolve = s.config.ResolveIP |
| 1690 | + response := record(func(w http.ResponseWriter, r *http.Request) { s.pathUpload(w, r, strings.Repeat("d", 64)) }, httptest.NewRequest(http.MethodPut, "https://relay.test/"+strings.Repeat("d", 64)+"?url=http%3A%2F%2Forigin.example%2Fx", http.NoBody)) |
| 1691 | + if response.Code != http.StatusBadRequest { |
| 1692 | + t.Fatalf("private origin status = %d, want 400", response.Code) |
| 1693 | + } |
| 1694 | +} |
| 1695 | + |
| 1696 | +func TestPathUploadRemoteUsesRequestContext(t *testing.T) { |
| 1697 | + s := testService(t) |
| 1698 | + sha := strings.Repeat("e", 64) |
| 1699 | + s.config.ResolveIP = func(ctx context.Context, _ string) ([]net.IP, error) { |
| 1700 | + return []net.IP{net.ParseIP("8.8.8.8")}, nil |
| 1701 | + } |
| 1702 | + s.resolve = s.config.ResolveIP |
| 1703 | + s.config.HTTPClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { |
| 1704 | + return nil, req.Context().Err() |
| 1705 | + })} |
| 1706 | + ctx, cancel := context.WithCancel(context.Background()) |
| 1707 | + cancel() |
| 1708 | + request := httptest.NewRequest(http.MethodPut, "https://relay.test/"+sha+"?url=https%3A%2F%2Forigin.example%2Ffile", http.NoBody).WithContext(ctx) |
| 1709 | + response := record(s.Handler(), request) |
| 1710 | + if response.Code != http.StatusBadGateway { |
| 1711 | + t.Fatalf("cancelled remote upload = %d, want 502", response.Code) |
| 1712 | + } |
| 1713 | +} |
| 1714 | + |
| 1715 | +type trackingReader struct { |
| 1716 | + io.Reader |
| 1717 | + read *bool |
| 1718 | +} |
| 1719 | + |
| 1720 | +func (r *trackingReader) Read(p []byte) (int, error) { |
| 1721 | + *r.read = true |
| 1722 | + return r.Reader.Read(p) |
| 1723 | +} |
| 1724 | diff --git a/internal/daemon/blob_authorization_test.go b/internal/daemon/blob_authorization_test.go |
| 1725 | new file mode 100644 |
| 1726 | index 0000000..d9d4d4c |
| 1727 | --- /dev/null |
| 1728 | +++ b/internal/daemon/blob_authorization_test.go |
| 1729 | @@ -0,0 +1,103 @@ |
| 1730 | +package daemon |
| 1731 | + |
| 1732 | +import ( |
| 1733 | + "context" |
| 1734 | + "crypto/sha256" |
| 1735 | + "encoding/base64" |
| 1736 | + "encoding/hex" |
| 1737 | + "encoding/json" |
| 1738 | + "io" |
| 1739 | + "net" |
| 1740 | + "net/http" |
| 1741 | + "net/http/httptest" |
| 1742 | + "net/url" |
| 1743 | + "strconv" |
| 1744 | + "strings" |
| 1745 | + "testing" |
| 1746 | + "time" |
| 1747 | + |
| 1748 | + "github.com/FelineStateMachine/tinyrelay/internal/blob" |
| 1749 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 1750 | +) |
| 1751 | + |
| 1752 | +type blobOriginTransport func(*http.Request) (*http.Response, error) |
| 1753 | + |
| 1754 | +func (f blobOriginTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } |
| 1755 | + |
| 1756 | +func TestBlobMirrorAuthenticationBindsRequestAndFetchedContent(t *testing.T) { |
| 1757 | + const content = "fetched encrypted bytes" |
| 1758 | + sum := sha256.Sum256([]byte(content)) |
| 1759 | + hash := hex.EncodeToString(sum[:]) |
| 1760 | + for _, tc := range []struct { |
| 1761 | + name, mode, action, scope string |
| 1762 | + pathUpload, tamper bool |
| 1763 | + want int |
| 1764 | + }{ |
| 1765 | + {name: "mirror NIP98", mode: "nip98", want: 201}, |
| 1766 | + {name: "mirror NIP98 tampered JSON", mode: "nip98", tamper: true, want: 401}, |
| 1767 | + {name: "mirror Blossom upload verb", action: "upload", scope: hash, want: 201}, |
| 1768 | + {name: "mirror Blossom wrong verb", action: "mirror", scope: hash, want: 401}, |
| 1769 | + {name: "mirror Blossom wrong content", action: "upload", scope: strings.Repeat("a", 64), want: 401}, |
| 1770 | + {name: "remote NIP98", mode: "nip98", pathUpload: true, want: 201}, |
| 1771 | + {name: "remote Blossom", action: "upload", scope: hash, pathUpload: true, want: 201}, |
| 1772 | + } { |
| 1773 | + t.Run(tc.name, func(t *testing.T) { |
| 1774 | + _, tenant := testTenant(t) |
| 1775 | + fetches := 0 |
| 1776 | + service, err := blob.New(context.Background(), blob.Config{ |
| 1777 | + Root: t.TempDir(), Store: tenant.store, PublicURL: tenant.publicURL, |
| 1778 | + Authorize: tenant.authorizeBlob, ValidateUpload: tenant.validateBlobUpload, |
| 1779 | + ResolveIP: func(context.Context, string) ([]net.IP, error) { return []net.IP{net.ParseIP("8.8.8.8")}, nil }, |
| 1780 | + HTTPClient: &http.Client{Transport: blobOriginTransport(func(*http.Request) (*http.Response, error) { |
| 1781 | + fetches++ |
| 1782 | + return &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": []string{"application/octet-stream"}}, Body: io.NopCloser(strings.NewReader(content))}, nil |
| 1783 | + })}, |
| 1784 | + }) |
| 1785 | + if err != nil { |
| 1786 | + t.Fatal(err) |
| 1787 | + } |
| 1788 | + body := `{"url":"https://files.example/source"}` |
| 1789 | + target := "http://relay.test/mirror" |
| 1790 | + if tc.pathUpload { |
| 1791 | + body = "" |
| 1792 | + target = "http://relay.test/" + hash + "?url=" + url.QueryEscape("https://files.example/source") |
| 1793 | + } |
| 1794 | + r := httptest.NewRequest(http.MethodPut, target, strings.NewReader(body)) |
| 1795 | + if tc.mode == "nip98" { |
| 1796 | + signRequest(t, r, body) |
| 1797 | + } else { |
| 1798 | + token := event.Event{Kind: 24242, CreatedAt: time.Now().Unix(), Tags: [][]string{{"t", tc.action}, {"x", tc.scope}, {"expiration", strconv.FormatInt(time.Now().Unix()+60, 10)}}, Content: "Authorize file storage"} |
| 1799 | + if err := event.Sign(&token, strings.Repeat("0", 63)+"1"); err != nil { |
| 1800 | + t.Fatal(err) |
| 1801 | + } |
| 1802 | + raw, err := json.Marshal(token) |
| 1803 | + if err != nil { |
| 1804 | + t.Fatal(err) |
| 1805 | + } |
| 1806 | + r.Header.Set("Authorization", "Nostr "+base64.RawURLEncoding.EncodeToString(raw)) |
| 1807 | + } |
| 1808 | + if tc.tamper { |
| 1809 | + r.Body = io.NopCloser(strings.NewReader(`{"url":"https://files.example/altered"}`)) |
| 1810 | + } |
| 1811 | + w := httptest.NewRecorder() |
| 1812 | + service.Handler().ServeHTTP(w, r) |
| 1813 | + if w.Code != tc.want { |
| 1814 | + t.Fatalf("status=%d, want %d: %s", w.Code, tc.want, w.Body.String()) |
| 1815 | + } |
| 1816 | + if (tc.tamper || tc.action == "mirror") && fetches != 0 { |
| 1817 | + t.Fatal("unauthorized request fetched an origin") |
| 1818 | + } |
| 1819 | + }) |
| 1820 | + } |
| 1821 | +} |
| 1822 | + |
| 1823 | +func TestUploadURLQueryCannotBypassNIP98PayloadValidation(t *testing.T) { |
| 1824 | + _, tenant := testTenant(t) |
| 1825 | + r := httptest.NewRequest(http.MethodPut, "http://relay.test/upload?url=https://files.example/ignored", strings.NewReader("tampered body")) |
| 1826 | + signRequest(t, r, "signed body") |
| 1827 | + w := httptest.NewRecorder() |
| 1828 | + tenant.ServeHTTP(w, r) |
| 1829 | + if w.Code != http.StatusUnauthorized { |
| 1830 | + t.Fatalf("tampered upload status=%d: %s", w.Code, w.Body.String()) |
| 1831 | + } |
| 1832 | +} |
| 1833 | diff --git a/internal/daemon/blob_multipart_test.go b/internal/daemon/blob_multipart_test.go |
| 1834 | new file mode 100644 |
| 1835 | index 0000000..38ea2b3 |
| 1836 | --- /dev/null |
| 1837 | +++ b/internal/daemon/blob_multipart_test.go |
| 1838 | @@ -0,0 +1,125 @@ |
| 1839 | +package daemon |
| 1840 | + |
| 1841 | +import ( |
| 1842 | + "crypto/sha256" |
| 1843 | + "encoding/base64" |
| 1844 | + "encoding/hex" |
| 1845 | + "encoding/json" |
| 1846 | + "net/http" |
| 1847 | + "net/http/httptest" |
| 1848 | + "strconv" |
| 1849 | + "strings" |
| 1850 | + "testing" |
| 1851 | + "time" |
| 1852 | + |
| 1853 | + "github.com/FelineStateMachine/tinyrelay/internal/blob" |
| 1854 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 1855 | +) |
| 1856 | + |
| 1857 | +func multipartAuthorization(t *testing.T, hash string) string { |
| 1858 | + t.Helper() |
| 1859 | + token := event.Event{Kind: 24242, CreatedAt: time.Now().Unix(), Tags: [][]string{ |
| 1860 | + {"t", "upload"}, {"x", hash}, {"expiration", strconv.FormatInt(time.Now().Unix()+60, 10)}, |
| 1861 | + }, Content: "Authorize multipart upload"} |
| 1862 | + if err := event.Sign(&token, strings.Repeat("0", 63)+"1"); err != nil { |
| 1863 | + t.Fatal(err) |
| 1864 | + } |
| 1865 | + raw, err := json.Marshal(token) |
| 1866 | + if err != nil { |
| 1867 | + t.Fatal(err) |
| 1868 | + } |
| 1869 | + return "Nostr " + base64.RawURLEncoding.EncodeToString(raw) |
| 1870 | +} |
| 1871 | + |
| 1872 | +func TestMultipartAuthorizationBindsFinalHashBeforeReading(t *testing.T) { |
| 1873 | + _, tenant := testTenant(t) |
| 1874 | + finalHash := strings.Repeat("a", 64) |
| 1875 | + r := httptest.NewRequest(http.MethodPatch, "http://relay.test/"+finalHash+".txt", strings.NewReader("chunk")) |
| 1876 | + r.Header.Set("Authorization", multipartAuthorization(t, strings.Repeat("b", 64))) |
| 1877 | + if _, err := tenant.authorizeBlob(r, blob.ActionUpload); err == nil { |
| 1878 | + t.Fatal("multipart authorization accepted another final hash") |
| 1879 | + } |
| 1880 | + r.Header.Set("Authorization", multipartAuthorization(t, finalHash)) |
| 1881 | + if _, err := tenant.authorizeBlob(r, blob.ActionUpload); err != nil { |
| 1882 | + t.Fatal(err) |
| 1883 | + } |
| 1884 | + chunkHash := sha256.Sum256([]byte("chunk")) |
| 1885 | + if err := tenant.validateBlobUpload(r, hex.EncodeToString(chunkHash[:])); err != nil { |
| 1886 | + t.Fatalf("Blossom proof incorrectly required a chunk hash: %v", err) |
| 1887 | + } |
| 1888 | +} |
| 1889 | + |
| 1890 | +func TestMultipartNIP98BindsEachChunkBody(t *testing.T) { |
| 1891 | + _, tenant := testTenant(t) |
| 1892 | + r := httptest.NewRequest(http.MethodPatch, "http://relay.test/"+strings.Repeat("c", 64), strings.NewReader("chunk")) |
| 1893 | + signRequest(t, r, "chunk") |
| 1894 | + if _, err := tenant.authorizeBlob(r, blob.ActionUpload); err != nil { |
| 1895 | + t.Fatal(err) |
| 1896 | + } |
| 1897 | + chunkHash := sha256.Sum256([]byte("chunk")) |
| 1898 | + if err := tenant.validateBlobUpload(r, hex.EncodeToString(chunkHash[:])); err != nil { |
| 1899 | + t.Fatal(err) |
| 1900 | + } |
| 1901 | + if err := tenant.validateBlobUpload(r, strings.Repeat("c", 64)); err == nil { |
| 1902 | + t.Fatal("NIP-98 proof accepted final hash in place of chunk payload") |
| 1903 | + } |
| 1904 | +} |
| 1905 | + |
| 1906 | +func TestMultipartPreflightAdvertisesMountedFileDoor(t *testing.T) { |
| 1907 | + _, tenant := testTenant(t) |
| 1908 | + r := httptest.NewRequest(http.MethodOptions, "http://relay.test/"+strings.Repeat("a", 64), nil) |
| 1909 | + w := httptest.NewRecorder() |
| 1910 | + tenant.ServeHTTP(w, r) |
| 1911 | + if w.Code != http.StatusNoContent || !strings.Contains(w.Header().Get("Allow"), "PATCH") { |
| 1912 | + t.Fatalf("multipart preflight: %d, Allow=%q", w.Code, w.Header().Get("Allow")) |
| 1913 | + } |
| 1914 | + for _, header := range []string{"upload-type", "upload-length", "upload-offset"} { |
| 1915 | + if !strings.Contains(strings.ToLower(w.Header().Get("Access-Control-Allow-Headers")), header) { |
| 1916 | + t.Fatalf("multipart CORS does not allow %s", header) |
| 1917 | + } |
| 1918 | + } |
| 1919 | + if !strings.Contains(w.Header().Get("Access-Control-Expose-Headers"), "Allow") { |
| 1920 | + t.Fatal("browser cannot read PATCH support") |
| 1921 | + } |
| 1922 | + for _, capability := range tenant.Capabilities(nil) { |
| 1923 | + if capability.ID == "BUD-14" && capability.Status == "enabled" { |
| 1924 | + return |
| 1925 | + } |
| 1926 | + } |
| 1927 | + t.Fatal("BUD-14 capability is missing") |
| 1928 | +} |
| 1929 | + |
| 1930 | +func TestMultipartHTTPVerifiesEachNIP98ChunkBeforeAcceptingIt(t *testing.T) { |
| 1931 | + _, tenant := testTenant(t) |
| 1932 | + const body = "first and second" |
| 1933 | + digest := sha256.Sum256([]byte(body)) |
| 1934 | + hash := hex.EncodeToString(digest[:]) |
| 1935 | + patch := func(offset int, chunk, signed string) *httptest.ResponseRecorder { |
| 1936 | + r := httptest.NewRequest(http.MethodPatch, "http://relay.test/"+hash, strings.NewReader(chunk)) |
| 1937 | + r.Header.Set("Content-Type", "application/octet-stream") |
| 1938 | + r.Header.Set("Upload-Type", "text/plain") |
| 1939 | + r.Header.Set("Upload-Length", strconv.Itoa(len(body))) |
| 1940 | + r.Header.Set("Upload-Offset", strconv.Itoa(offset)) |
| 1941 | + signRequest(t, r, signed) |
| 1942 | + w := httptest.NewRecorder() |
| 1943 | + tenant.ServeHTTP(w, r) |
| 1944 | + return w |
| 1945 | + } |
| 1946 | + if w := patch(0, body[:6], "altered"); w.Code != http.StatusUnauthorized { |
| 1947 | + t.Fatalf("invalid chunk proof = %d: %s", w.Code, w.Body.String()) |
| 1948 | + } |
| 1949 | + if w := patch(6, body[6:], body[6:]); w.Code != http.StatusNoContent { |
| 1950 | + t.Fatalf("out-of-order chunk = %d: %s", w.Code, w.Body.String()) |
| 1951 | + } |
| 1952 | + if w := patch(0, body[:6], body[:6]); w.Code != http.StatusCreated { |
| 1953 | + t.Fatalf("complete upload = %d: %s", w.Code, w.Body.String()) |
| 1954 | + } |
| 1955 | + if w := patch(0, body[:6], "still altered"); w.Code != http.StatusUnauthorized { |
| 1956 | + t.Fatalf("existing blob bypassed payload validation = %d: %s", w.Code, w.Body.String()) |
| 1957 | + } |
| 1958 | + get := httptest.NewRecorder() |
| 1959 | + tenant.ServeHTTP(get, httptest.NewRequest(http.MethodGet, "http://relay.test/"+hash, nil)) |
| 1960 | + if get.Code != http.StatusOK || get.Body.String() != body { |
| 1961 | + t.Fatalf("retrieved upload = %d: %s", get.Code, get.Body.String()) |
| 1962 | + } |
| 1963 | +} |
| 1964 | diff --git a/internal/daemon/capabilities.go b/internal/daemon/capabilities.go |
| 1965 | index 494ba00..9097fe0 100644 |
| 1966 | --- a/internal/daemon/capabilities.go |
| 1967 | +++ b/internal/daemon/capabilities.go |
| 1968 | @@ -84,10 +84,13 @@ func CapabilityRegistryWithServices(p policy.Policy, gitAvailable, blobsAvailabl |
| 1969 | {ID: "BUD-01", Status: "enabled", Reason: "blob retrieval and basic server behavior"}, |
| 1970 | {ID: "BUD-02", Status: "enabled", Reason: "blob upload and management"}, |
| 1971 | {ID: "BUD-04", Status: "enabled", Reason: "blob mirroring"}, |
| 1972 | - {ID: "BUD-06", Status: "enabled", Reason: "blob existence and metadata checks"}, |
| 1973 | + {ID: "BUD-06", Status: "enabled", Reason: "upload preflight and storage allowances"}, |
| 1974 | + {ID: "BUD-08", Status: "enabled", Reason: "Nostr file metadata in blob descriptors"}, |
| 1975 | {ID: "BUD-09", Status: "enabled", Reason: "blob reports"}, |
| 1976 | {ID: "BUD-11", Status: "enabled", Reason: "Nostr authorization"}, |
| 1977 | {ID: "BUD-12", Status: "enabled", Reason: "cursor-paginated blob listing"}, |
| 1978 | + {ID: "BUD-13", Status: "enabled", Reason: "draft hash-addressed uploads and remote sources"}, |
| 1979 | + {ID: "BUD-14", Status: "enabled", Reason: "draft multipart uploads with bounded partial storage"}, |
| 1980 | {ID: "NIP-66", Status: "disabled", Reason: "operator peer liveness is not configured"}, |
| 1981 | } |
| 1982 | if p.Features.Count { |
| 1983 | diff --git a/internal/daemon/capabilities_test.go b/internal/daemon/capabilities_test.go |
| 1984 | index f7070ed..32b78a0 100644 |
| 1985 | --- a/internal/daemon/capabilities_test.go |
| 1986 | +++ b/internal/daemon/capabilities_test.go |
| 1987 | @@ -30,6 +30,9 @@ func TestCapabilityRegistryReportsHonestOptionalFeatures(t *testing.T) { |
| 1988 | if got["NIP-94"].Status != "enabled" || got["NIP-96"].Status != "enabled" { |
| 1989 | t.Fatalf("file capabilities = %#v", got) |
| 1990 | } |
| 1991 | + if got["BUD-08"].Status != "enabled" || got["BUD-13"].Status != "enabled" { |
| 1992 | + t.Fatalf("file metadata and upload capabilities = %#v", got) |
| 1993 | + } |
| 1994 | } |
| 1995 | |
| 1996 | func TestPublishPeerStatusStoresSignedDiscoveryEvent(t *testing.T) { |
| 1997 | diff --git a/internal/daemon/client_browse.go b/internal/daemon/client_browse.go |
| 1998 | index e3e2da6..87013ad 100644 |
| 1999 | --- a/internal/daemon/client_browse.go |
| 2000 | +++ b/internal/daemon/client_browse.go |
| 2001 | @@ -210,6 +210,29 @@ func (t *Tenant) browseFiles(ctx context.Context, actor string, q clientBrowseRe |
| 2002 | var next string |
| 2003 | if role == "owner" || role == "moderator" { |
| 2004 | entries, next, err = t.browseAllFiles(ctx, q) |
| 2005 | + } else if strings.TrimSpace(q.Query) != "" { |
| 2006 | + // ListPage is uploader scoped and cannot filter server-side. Walk its |
| 2007 | + // cursor until the requested number of matches is collected. |
| 2008 | + cursor := q.Cursor |
| 2009 | + for len(entries) < q.Limit { |
| 2010 | + var page []blob.Blob |
| 2011 | + page, next, err = t.blobs.ListPage(ctx, actor, q.Limit-len(entries), cursor) |
| 2012 | + if err != nil { |
| 2013 | + break |
| 2014 | + } |
| 2015 | + for _, entry := range page { |
| 2016 | + if blobMatchesQuery(entry, q.Query) { |
| 2017 | + entries = append(entries, entry) |
| 2018 | + if len(entries) == q.Limit { |
| 2019 | + break |
| 2020 | + } |
| 2021 | + } |
| 2022 | + } |
| 2023 | + if next == "" { |
| 2024 | + break |
| 2025 | + } |
| 2026 | + cursor = next |
| 2027 | + } |
| 2028 | } else { |
| 2029 | entries, next, err = t.blobs.ListPage(ctx, actor, q.Limit, q.Cursor) |
| 2030 | } |
| 2031 | @@ -218,14 +241,25 @@ func (t *Tenant) browseFiles(ctx context.Context, actor string, q clientBrowseRe |
| 2032 | } |
| 2033 | items := []map[string]any{} |
| 2034 | for _, entry := range entries { |
| 2035 | + if q.Query != "" && !blobMatchesQuery(entry, q.Query) { |
| 2036 | + continue |
| 2037 | + } |
| 2038 | items = append(items, t.browseBlobMetadata(entry)) |
| 2039 | } |
| 2040 | return map[string]any{"items": items, "next_cursor": next}, nil |
| 2041 | } |
| 2042 | |
| 2043 | func (t *Tenant) browseAllFiles(ctx context.Context, q clientBrowseRequest) ([]blob.Blob, string, error) { |
| 2044 | - query := "SELECT sha256,size,type,uploader,uploaded FROM blobs WHERE sha256>? ORDER BY sha256 LIMIT ?" |
| 2045 | - rows, err := t.store.DB().QueryContext(ctx, query, q.Cursor, q.Limit+1) |
| 2046 | + query := "SELECT sha256,size,type,uploader,uploaded FROM blobs WHERE sha256>?" |
| 2047 | + args := []any{q.Cursor} |
| 2048 | + if q.Query != "" { |
| 2049 | + query += " AND (instr(lower(sha256), ?) > 0 OR instr(lower(type), ?) > 0 OR instr(lower(uploader), ?) > 0)" |
| 2050 | + needle := strings.ToLower(strings.TrimSpace(q.Query)) |
| 2051 | + args = append(args, needle, needle, needle) |
| 2052 | + } |
| 2053 | + query += " ORDER BY sha256 LIMIT ?" |
| 2054 | + args = append(args, q.Limit+1) |
| 2055 | + rows, err := t.store.DB().QueryContext(ctx, query, args...) |
| 2056 | if err != nil { |
| 2057 | return nil, "", err |
| 2058 | } |
| 2059 | @@ -249,6 +283,16 @@ func (t *Tenant) browseAllFiles(ctx context.Context, q clientBrowseRequest) ([]b |
| 2060 | return entries, next, nil |
| 2061 | } |
| 2062 | |
| 2063 | +func blobMatchesQuery(entry blob.Blob, query string) bool { |
| 2064 | + needle := strings.ToLower(strings.TrimSpace(query)) |
| 2065 | + if needle == "" { |
| 2066 | + return true |
| 2067 | + } |
| 2068 | + return strings.Contains(strings.ToLower(entry.SHA256), needle) || |
| 2069 | + strings.Contains(strings.ToLower(entry.Type), needle) || |
| 2070 | + strings.Contains(strings.ToLower(entry.Uploader), needle) |
| 2071 | +} |
| 2072 | + |
| 2073 | func (t *Tenant) browseBlobMetadata(entry blob.Blob) map[string]any { |
| 2074 | portable := blob.BlossomURI{Hash: entry.SHA256, Extension: "bin", Author: entry.Uploader, Size: entry.Size} |
| 2075 | if origin, err := url.Parse(t.publicURL); err == nil && strings.Trim(origin.Path, "/") == "" { |
| 2076 | diff --git a/internal/daemon/client_browse_test.go b/internal/daemon/client_browse_test.go |
| 2077 | index 71e6813..a92b59e 100644 |
| 2078 | --- a/internal/daemon/client_browse_test.go |
| 2079 | +++ b/internal/daemon/client_browse_test.go |
| 2080 | @@ -3,6 +3,7 @@ package daemon |
| 2081 | import ( |
| 2082 | "context" |
| 2083 | "encoding/json" |
| 2084 | + "fmt" |
| 2085 | "net/http" |
| 2086 | "net/http/httptest" |
| 2087 | "strings" |
| 2088 | @@ -66,3 +67,67 @@ func TestClientStatusRequiresOwnerAndPreservesErrors(t *testing.T) { |
| 2089 | } |
| 2090 | } |
| 2091 | } |
| 2092 | + |
| 2093 | +func TestFileSearchPaginationPreservesEveryUploaderMatch(t *testing.T) { |
| 2094 | + _, tenant := testTenant(t) |
| 2095 | + ctx := context.Background() |
| 2096 | + uploader := strings.Repeat("b", 64) |
| 2097 | + var want []string |
| 2098 | + for i, typ := range []string{"text/plain", "image/png", "text/plain", "text/plain", "text/plain"} { |
| 2099 | + entry, err := tenant.blobs.Put(ctx, blob.PutOptions{Reader: strings.NewReader(fmt.Sprintf("file %d", i)), Type: typ, Uploader: uploader}) |
| 2100 | + if err != nil { |
| 2101 | + t.Fatal(err) |
| 2102 | + } |
| 2103 | + if _, err := tenant.store.DB().ExecContext(ctx, "UPDATE blobs SET uploaded=? WHERE sha256=?", 100-i, entry.SHA256); err != nil { |
| 2104 | + t.Fatal(err) |
| 2105 | + } |
| 2106 | + if typ == "text/plain" { |
| 2107 | + want = append(want, entry.SHA256) |
| 2108 | + } |
| 2109 | + } |
| 2110 | + if _, err := tenant.blobs.Put(ctx, blob.PutOptions{Reader: strings.NewReader("another user's matching file"), Type: "text/plain", Uploader: tenant.Policy().Owner}); err != nil { |
| 2111 | + t.Fatal(err) |
| 2112 | + } |
| 2113 | + var got []string |
| 2114 | + cursor := "" |
| 2115 | + for page := 0; page < 5; page++ { |
| 2116 | + params, err := json.Marshal(map[string]any{"q": "text/plain", "limit": 2, "cursor": cursor}) |
| 2117 | + if err != nil { |
| 2118 | + t.Fatal(err) |
| 2119 | + } |
| 2120 | + result, err := tenant.Execute(ctx, uploader, "browsefiles", []json.RawMessage{params}) |
| 2121 | + if err != nil { |
| 2122 | + t.Fatal(err) |
| 2123 | + } |
| 2124 | + row := result.(map[string]any) |
| 2125 | + for _, item := range row["items"].([]map[string]any) { |
| 2126 | + got = append(got, item["sha256"].(string)) |
| 2127 | + } |
| 2128 | + cursor = row["next_cursor"].(string) |
| 2129 | + if cursor == "" { |
| 2130 | + break |
| 2131 | + } |
| 2132 | + } |
| 2133 | + if fmt.Sprint(got) != fmt.Sprint(want) { |
| 2134 | + t.Fatalf("matching files = %v, want %v", got, want) |
| 2135 | + } |
| 2136 | +} |
| 2137 | + |
| 2138 | +func TestOwnerFileSearchUsesLiteralTextBeforePagination(t *testing.T) { |
| 2139 | + _, tenant := testTenant(t) |
| 2140 | + ctx := context.Background() |
| 2141 | + for i, typ := range []string{"text/plain", "application/x_foo", "image/png"} { |
| 2142 | + if _, err := tenant.blobs.Put(ctx, blob.PutOptions{Reader: strings.NewReader(fmt.Sprintf("owner file %d", i)), Type: typ, Uploader: tenant.Policy().Owner}); err != nil { |
| 2143 | + t.Fatal(err) |
| 2144 | + } |
| 2145 | + } |
| 2146 | + result, err := tenant.Execute(ctx, tenant.Policy().Owner, "browsefiles", []json.RawMessage{json.RawMessage(`{"q":"_","limit":1}`)}) |
| 2147 | + if err != nil { |
| 2148 | + t.Fatal(err) |
| 2149 | + } |
| 2150 | + row := result.(map[string]any) |
| 2151 | + items := row["items"].([]map[string]any) |
| 2152 | + if len(items) != 1 || items[0]["type"] != "application/x_foo" || row["next_cursor"] != "" { |
| 2153 | + t.Fatalf("literal search = %#v", row) |
| 2154 | + } |
| 2155 | +} |
| 2156 | diff --git a/internal/daemon/daemon_test.go b/internal/daemon/daemon_test.go |
| 2157 | index 97a97fa..a181d61 100644 |
| 2158 | --- a/internal/daemon/daemon_test.go |
| 2159 | +++ b/internal/daemon/daemon_test.go |
| 2160 | @@ -12,6 +12,7 @@ import ( |
| 2161 | "testing" |
| 2162 | "time" |
| 2163 | |
| 2164 | + "github.com/FelineStateMachine/tinyrelay/internal/blob" |
| 2165 | "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 2166 | ) |
| 2167 | |
| 2168 | @@ -82,6 +83,34 @@ func signRequest(t *testing.T, r *http.Request, body string) { |
| 2169 | r.Header.Set("Authorization", "Nostr "+base64.StdEncoding.EncodeToString(raw)) |
| 2170 | } |
| 2171 | |
| 2172 | +func TestBUD13RemoteNIP98AcceptsAbsentOrEmptyPayloadOnly(t *testing.T) { |
| 2173 | + _, tenant := testTenant(t) |
| 2174 | + sha := strings.Repeat("a", 64) |
| 2175 | + empty := sha256.Sum256(nil) |
| 2176 | + for name, payload := range map[string]string{"absent": "", "empty": hex.EncodeToString(empty[:]), "wrong": strings.Repeat("b", 64)} { |
| 2177 | + t.Run(name, func(t *testing.T) { |
| 2178 | + req := httptest.NewRequest(http.MethodPut, "http://relay.test/"+sha+"?url=https%3A%2F%2Forigin.example%2Fblob", http.NoBody) |
| 2179 | + tags := [][]string{{"u", req.URL.String()}, {"method", "PUT"}} |
| 2180 | + if payload != "" { |
| 2181 | + tags = append(tags, []string{"payload", payload}) |
| 2182 | + } |
| 2183 | + e := event.Event{Kind: 27235, CreatedAt: time.Now().Unix(), Tags: tags} |
| 2184 | + if err := event.Sign(&e, strings.Repeat("0", 63)+"1"); err != nil { |
| 2185 | + t.Fatal(err) |
| 2186 | + } |
| 2187 | + raw, _ := json.Marshal(e) |
| 2188 | + req.Header.Set("Authorization", "Nostr "+base64.StdEncoding.EncodeToString(raw)) |
| 2189 | + _, err := tenant.authorizeBlob(req, blob.ActionUpload) |
| 2190 | + if name == "wrong" && err == nil { |
| 2191 | + t.Fatal("accepted wrong empty-body payload") |
| 2192 | + } |
| 2193 | + if name != "wrong" && err != nil { |
| 2194 | + t.Fatalf("remote authorization: %v", err) |
| 2195 | + } |
| 2196 | + }) |
| 2197 | + } |
| 2198 | +} |
| 2199 | + |
| 2200 | func TestCreationRequiresOwnerAndInformationHasNoEconomy(t *testing.T) { |
| 2201 | ctx := context.Background() |
| 2202 | app, err := New(ctx, Config{DataDir: t.TempDir(), DefaultTenant: "main"}) |
| 2203 | diff --git a/internal/daemon/http.go b/internal/daemon/http.go |
| 2204 | index 7b6422d..4c21630 100644 |
| 2205 | --- a/internal/daemon/http.go |
| 2206 | +++ b/internal/daemon/http.go |
| 2207 | @@ -21,10 +21,15 @@ import ( |
| 2208 | |
| 2209 | func (t *Tenant) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 2210 | w.Header().Set("Access-Control-Allow-Origin", "*") |
| 2211 | - w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, X-SHA-256, X-Content-Length, X-Content-Type") |
| 2212 | - w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, DELETE, OPTIONS") |
| 2213 | + w.Header().Set("Access-Control-Allow-Headers", "Authorization, Content-Type, X-SHA-256, X-Content-Length, X-Content-Type, Upload-Type, Upload-Length, Upload-Offset") |
| 2214 | + w.Header().Set("Access-Control-Allow-Methods", "GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS") |
| 2215 | + w.Header().Set("Access-Control-Expose-Headers", "Allow, X-Reason") |
| 2216 | w.Header().Set("X-Content-Type-Options", "nosniff") |
| 2217 | if r.Method == http.MethodOptions { |
| 2218 | + if t.blobs != nil && t.Policy().Features.Files && blob.HandlesPath(r.URL.Path) { |
| 2219 | + t.blobs.Handler().ServeHTTP(w, r) |
| 2220 | + return |
| 2221 | + } |
| 2222 | w.WriteHeader(http.StatusNoContent) |
| 2223 | return |
| 2224 | } |
| 2225 | diff --git a/internal/daemon/scheduler.go b/internal/daemon/scheduler.go |
| 2226 | index fb4d7ef..6d9174d 100644 |
| 2227 | --- a/internal/daemon/scheduler.go |
| 2228 | +++ b/internal/daemon/scheduler.go |
| 2229 | @@ -45,6 +45,9 @@ func (t *Tenant) runScheduler() { |
| 2230 | if !wake { |
| 2231 | report("records", t.records.Tick(ctx, now.Unix())) |
| 2232 | if !now.Before(nextMaintenance) { |
| 2233 | + if t.blobs != nil { |
| 2234 | + report("partial uploads", t.blobs.CleanupMultipart(ctx)) |
| 2235 | + } |
| 2236 | report("retention", t.sweep(ctx, now.Unix())) |
| 2237 | report("restore-state", t.sweepReplicationState(ctx, now.Unix())) |
| 2238 | report("git-sync", t.git.GRASPService().Tick(ctx)) |
| 2239 | diff --git a/internal/daemon/services.go b/internal/daemon/services.go |
| 2240 | index a4ea205..19c124c 100644 |
| 2241 | --- a/internal/daemon/services.go |
| 2242 | +++ b/internal/daemon/services.go |
| 2243 | @@ -3,7 +3,9 @@ package daemon |
| 2244 | import ( |
| 2245 | "bytes" |
| 2246 | "context" |
| 2247 | + "crypto/sha256" |
| 2248 | "database/sql" |
| 2249 | + "encoding/hex" |
| 2250 | "encoding/json" |
| 2251 | "errors" |
| 2252 | "fmt" |
| 2253 | @@ -39,15 +41,11 @@ func (t *Tenant) initServices(ctx context.Context) error { |
| 2254 | } |
| 2255 | t.blobs, err = blob.New(ctx, blob.Config{ |
| 2256 | Root: t.meta.Paths.Root, PublicURL: t.publicURL, Store: t.store, Authorize: t.authorizeBlob, |
| 2257 | - ValidateUpload: func(r *http.Request, hash string) error { |
| 2258 | - // /mirror is authenticated against its complete JSON request body in |
| 2259 | - // authorizeBlob. Its resulting blob hash is not the NIP-98 payload |
| 2260 | - // hash; Blossom tokens still bind through their x tag. |
| 2261 | - if r.URL.Path == "/mirror" { |
| 2262 | - return t.auth.ValidateBlossomPayload(r.Header.Get("Authorization"), string(blob.ActionMirror), hash) |
| 2263 | - } |
| 2264 | - return t.auth.ValidateBlobPayload(r.Header.Get("Authorization"), hash) |
| 2265 | + Limits: func() blob.Limits { |
| 2266 | + limits := t.Policy().FileLimits |
| 2267 | + return blob.Limits{MaxFileBytes: limits.MaxFileBytes, UserStorageBytes: limits.UserStorageBytes} |
| 2268 | }, |
| 2269 | + ValidateUpload: t.validateBlobUpload, |
| 2270 | ResolveServers: func(ctx context.Context, pubkey string) ([]string, error) { |
| 2271 | result, err := t.store.Query(ctx, event.Filter{Authors: []string{pubkey}, Kinds: []int{10063}, Tags: map[string][]string{}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: 1}) |
| 2272 | if err != nil { |
| 2273 | @@ -224,10 +222,13 @@ func (t *Tenant) siteReadAccess(r *http.Request) bool { |
| 2274 | } |
| 2275 | func (t *Tenant) authorizeBlob(r *http.Request, action blob.Action) (string, error) { |
| 2276 | hash := "" |
| 2277 | - if action == blob.ActionGet || action == blob.ActionDelete { |
| 2278 | + if action == blob.ActionGet || action == blob.ActionDelete || (action == blob.ActionUpload && (r.Method == http.MethodPut || r.Method == http.MethodPatch)) { |
| 2279 | path := strings.TrimPrefix(r.URL.Path, "/") |
| 2280 | path = strings.TrimPrefix(path, "nip96/") |
| 2281 | hash = strings.Split(path, ".")[0] |
| 2282 | + if len(hash) != 64 || strings.Trim(hash, "0123456789abcdef") != "" { |
| 2283 | + hash = "" |
| 2284 | + } |
| 2285 | } |
| 2286 | body, bodyErr := blobAuthBody(r, action) |
| 2287 | if bodyErr != nil { |
| 2288 | @@ -241,17 +242,32 @@ func (t *Tenant) authorizeBlob(r *http.Request, action blob.Action) (string, err |
| 2289 | pubkey = token.PubKey |
| 2290 | } else { |
| 2291 | if auth.IsBlossomAuthorization(r.Header.Get("Authorization")) { |
| 2292 | + blossomAction := string(action) |
| 2293 | + if action == blob.ActionMirror { |
| 2294 | + // BUD-04 mirror uses the upload authorization verb; the fetched |
| 2295 | + // bytes are the resulting blob and are scoped by x below. |
| 2296 | + blossomAction = string(blob.ActionUpload) |
| 2297 | + } |
| 2298 | blobHash := hash |
| 2299 | if blobHash == "" && action == blob.ActionUpload { |
| 2300 | blobHash = strings.ToLower(strings.TrimSpace(r.Header.Get("x-sha-256"))) |
| 2301 | } |
| 2302 | var token event.Event |
| 2303 | - token, err = t.auth.VerifyBlossomRequest(r.Header.Get("Authorization"), string(action), t.siteDomain(), blobHash) |
| 2304 | + token, err = t.auth.VerifyBlossomRequest(r.Header.Get("Authorization"), blossomAction, t.siteDomain(), blobHash) |
| 2305 | pubkey = token.PubKey |
| 2306 | } else { |
| 2307 | - var keys []string |
| 2308 | - keys, err = t.auth.WhoAsks(r.Header.Get("Authorization"), t.requestURL(r), r.Method, body, string(action), hash) |
| 2309 | - pubkey = first(keys) |
| 2310 | + if isBUD13RemoteUpload(r) { |
| 2311 | + var token event.Event |
| 2312 | + token, err = t.auth.VerifyNIP98(r.Header.Get("Authorization"), t.requestURL(r), r.Method, "") |
| 2313 | + if err == nil && event.Tag(token, "payload") != "" && event.Tag(token, "payload") != emptyPayloadHash() { |
| 2314 | + err = errors.New("auth-required: remote upload token payload must hash an empty body") |
| 2315 | + } |
| 2316 | + pubkey = token.PubKey |
| 2317 | + } else { |
| 2318 | + var keys []string |
| 2319 | + keys, err = t.auth.WhoAsks(r.Header.Get("Authorization"), t.requestURL(r), r.Method, body, string(action), hash) |
| 2320 | + pubkey = first(keys) |
| 2321 | + } |
| 2322 | } |
| 2323 | } |
| 2324 | if err != nil { |
| 2325 | @@ -284,6 +300,24 @@ func (t *Tenant) authorizeBlob(r *http.Request, action blob.Action) (string, err |
| 2326 | return pubkey, nil |
| 2327 | } |
| 2328 | |
| 2329 | +func (t *Tenant) validateBlobUpload(r *http.Request, hash string) error { |
| 2330 | + // PATCH carries only one chunk. Blossom scopes the final path hash, while |
| 2331 | + // NIP-98 binds the bytes in this individual request body. |
| 2332 | + if r.Method == http.MethodPatch && auth.IsBlossomAuthorization(r.Header.Get("Authorization")) { |
| 2333 | + finalHash := strings.Split(strings.TrimPrefix(r.URL.Path, "/"), ".")[0] |
| 2334 | + return t.auth.ValidateBlossomPayload(r.Header.Get("Authorization"), string(blob.ActionUpload), finalHash) |
| 2335 | + } |
| 2336 | + // Mirror proofs bind the HTTP request body before fetching. Blossom |
| 2337 | + // authorization additionally names the fetched content through its x tag. |
| 2338 | + if r.URL.Path == "/mirror" || isBUD13RemoteUpload(r) { |
| 2339 | + if auth.IsBlossomAuthorization(r.Header.Get("Authorization")) { |
| 2340 | + return t.auth.ValidateBlossomPayload(r.Header.Get("Authorization"), string(blob.ActionUpload), hash) |
| 2341 | + } |
| 2342 | + return nil |
| 2343 | + } |
| 2344 | + return t.auth.ValidateBlobPayload(r.Header.Get("Authorization"), hash) |
| 2345 | +} |
| 2346 | + |
| 2347 | // NIP-98 binds its payload to the complete HTTP body. The streaming Blossom |
| 2348 | // upload remains unbuffered and is checked against its resulting SHA-256 by |
| 2349 | // ValidateUpload; the JSON and multipart doors are buffered only so their |
| 2350 | @@ -355,6 +389,21 @@ func first(v []string) string { |
| 2351 | } |
| 2352 | return v[0] |
| 2353 | } |
| 2354 | + |
| 2355 | +func isBUD13RemoteUpload(r *http.Request) bool { |
| 2356 | + if r == nil || r.Method != http.MethodPut || r.URL.Query().Get("url") == "" { |
| 2357 | + return false |
| 2358 | + } |
| 2359 | + path := strings.TrimPrefix(r.URL.Path, "/") |
| 2360 | + hash := strings.Split(path, ".")[0] |
| 2361 | + return len(hash) == 64 && strings.Trim(hash, "0123456789abcdefABCDEF") == "" |
| 2362 | +} |
| 2363 | + |
| 2364 | +func emptyPayloadHash() string { |
| 2365 | + sum := sha256.Sum256(nil) |
| 2366 | + return hex.EncodeToString(sum[:]) |
| 2367 | +} |
| 2368 | + |
| 2369 | func (t *Tenant) resolveUIActor(r *http.Request) (string, error) { |
| 2370 | if r.Header.Get("Authorization") == "" { |
| 2371 | if r.Method != http.MethodGet && r.Method != http.MethodHead { |
| 2372 | diff --git a/scripts/conformance/blossom.test.ts b/scripts/conformance/blossom.test.ts |
| 2373 | index b6e39c1..3ef8607 100644 |
| 2374 | --- a/scripts/conformance/blossom.test.ts |
| 2375 | +++ b/scripts/conformance/blossom.test.ts |
| 2376 | @@ -12,6 +12,181 @@ function token(sk: Uint8Array, verb: "upload" | "get" | "list" | "delete", sha?: |
| 2377 | } |
| 2378 | |
| 2379 | describe("Blossom media", () => { |
| 2380 | + it("advertises the BUD-06 browser and upload preflight contract", async () => { |
| 2381 | + const preflight = await fetch(`${HTTP_URL}/upload`, { |
| 2382 | + method: "OPTIONS", |
| 2383 | + headers: { |
| 2384 | + origin: "https://client.example", |
| 2385 | + "access-control-request-method": "PUT", |
| 2386 | + "access-control-request-headers": "authorization, content-type, x-content-length, x-content-type", |
| 2387 | + }, |
| 2388 | + }); |
| 2389 | + expect(preflight.status).toBe(204); |
| 2390 | + expect(preflight.headers.get("access-control-allow-origin")).toBe("*"); |
| 2391 | + expect(preflight.headers.get("access-control-allow-methods")).toContain("PUT"); |
| 2392 | + expect(preflight.headers.get("access-control-allow-headers")?.toLowerCase()).toContain("x-content-type"); |
| 2393 | + |
| 2394 | + const sk = newKey(); |
| 2395 | + const body = new TextEncoder().encode(`preflight ${rand()}`); |
| 2396 | + const sha = hex(sha256(body)); |
| 2397 | + const head = await fetch(`${HTTP_URL}/upload`, { |
| 2398 | + method: "HEAD", |
| 2399 | + headers: { |
| 2400 | + authorization: token(sk, "upload", sha), |
| 2401 | + "x-sha-256": sha, |
| 2402 | + "x-content-length": String(body.length), |
| 2403 | + "x-content-type": "text/plain", |
| 2404 | + }, |
| 2405 | + }); |
| 2406 | + expect(head.status).toBe(200); |
| 2407 | + expect(await head.text()).toBe(""); |
| 2408 | + }); |
| 2409 | + |
| 2410 | + it("accepts a BUD-13 hash-addressed upload and rejects a mismatched hash", async () => { |
| 2411 | + const sk = newKey(); |
| 2412 | + const body = new TextEncoder().encode(`path upload ${rand()}`); |
| 2413 | + const sha = hex(sha256(body)); |
| 2414 | + const upload = await fetch(`${HTTP_URL}/${sha}.txt`, { |
| 2415 | + method: "PUT", |
| 2416 | + headers: { authorization: token(sk, "upload", sha), "content-type": "text/plain" }, |
| 2417 | + body, |
| 2418 | + }); |
| 2419 | + expect(upload.status, await upload.clone().text()).toBe(201); |
| 2420 | + expect((await upload.json<any>()).sha256).toBe(sha); |
| 2421 | + |
| 2422 | + const mismatchHash = "0".repeat(64); |
| 2423 | + const mismatch = await fetch(`${HTTP_URL}/${mismatchHash}`, { |
| 2424 | + method: "PUT", |
| 2425 | + headers: { authorization: token(sk, "upload", mismatchHash), "content-type": "text/plain" }, |
| 2426 | + body, |
| 2427 | + }); |
| 2428 | + expect(mismatch.status).toBe(409); |
| 2429 | + |
| 2430 | + const del = await fetch(`${HTTP_URL}/${sha}`, { |
| 2431 | + method: "DELETE", |
| 2432 | + headers: { authorization: token(sk, "delete", sha) }, |
| 2433 | + }); |
| 2434 | + expect(del.status).toBe(204); |
| 2435 | + }); |
| 2436 | + |
| 2437 | + it("advertises and completes BUD-14 uploads out of order with overlap", async () => { |
| 2438 | + const sk = newKey(); |
| 2439 | + const body = new TextEncoder().encode(`bud14 ${rand()} multipart body`); |
| 2440 | + const sha = hex(sha256(body)); |
| 2441 | + const split = Math.floor(body.length / 2); |
| 2442 | + |
| 2443 | + const preflight = await fetch(`${HTTP_URL}/${sha}.bin`, { |
| 2444 | + method: "OPTIONS", |
| 2445 | + headers: { |
| 2446 | + origin: "https://client.example", |
| 2447 | + "access-control-request-method": "PATCH", |
| 2448 | + "access-control-request-headers": "authorization, content-type, upload-type, upload-length, upload-offset", |
| 2449 | + }, |
| 2450 | + }); |
| 2451 | + expect(preflight.status).toBe(204); |
| 2452 | + expect(preflight.headers.get("allow")).toContain("PATCH"); |
| 2453 | + expect(preflight.headers.get("access-control-allow-origin")).toBe("*"); |
| 2454 | + expect(preflight.headers.get("access-control-allow-methods")).toContain("PATCH"); |
| 2455 | + |
| 2456 | + // The authorization is scoped to the final hash. It is reused for both |
| 2457 | + // chunks; individual chunk hashes must not be required by BUD-14. |
| 2458 | + const auth = token(sk, "upload", sha); |
| 2459 | + const secondOffset = Math.max(1, split - 2); |
| 2460 | + const second = body.slice(secondOffset); |
| 2461 | + const first = body.slice(0, split + 2); |
| 2462 | + const send = (offset: number, chunk: Uint8Array) => fetch(`${HTTP_URL}/${sha}.bin`, { |
| 2463 | + method: "PATCH", |
| 2464 | + headers: { |
| 2465 | + authorization: auth, |
| 2466 | + "content-type": "application/octet-stream", |
| 2467 | + "upload-type": "text/plain", |
| 2468 | + "upload-length": String(body.length), |
| 2469 | + "upload-offset": String(offset), |
| 2470 | + "content-length": String(chunk.length), |
| 2471 | + }, |
| 2472 | + body: chunk, |
| 2473 | + }); |
| 2474 | + |
| 2475 | + const chunkScoped = await fetch(`${HTTP_URL}/${sha}.bin`, { |
| 2476 | + method: "PATCH", |
| 2477 | + headers: { |
| 2478 | + authorization: token(sk, "upload", hex(sha256(second))), |
| 2479 | + "content-type": "application/octet-stream", |
| 2480 | + "upload-type": "text/plain", |
| 2481 | + "upload-length": String(body.length), |
| 2482 | + "upload-offset": String(secondOffset), |
| 2483 | + "content-length": String(second.length), |
| 2484 | + }, |
| 2485 | + body: second, |
| 2486 | + }); |
| 2487 | + expect(chunkScoped.status).toBe(401); |
| 2488 | + |
| 2489 | + // Send the later range first, then an overlapping earlier range. |
| 2490 | + const later = await send(secondOffset, second); |
| 2491 | + expect(later.status).toBe(204); |
| 2492 | + expect((await fetch(`${HTTP_URL}/${sha}`)).status).toBe(404); |
| 2493 | + const complete = await send(0, first); |
| 2494 | + expect(complete.status, await complete.clone().text()).toBe(201); |
| 2495 | + expect((await fetch(`${HTTP_URL}/${sha}`)).status).toBe(200); |
| 2496 | + expect(new Uint8Array(await (await fetch(`${HTTP_URL}/${sha}`)).arrayBuffer())).toEqual(body); |
| 2497 | + |
| 2498 | + const del = await fetch(`${HTTP_URL}/${sha}`, { |
| 2499 | + method: "DELETE", |
| 2500 | + headers: { authorization: token(sk, "delete", sha) }, |
| 2501 | + }); |
| 2502 | + expect(del.status).toBe(204); |
| 2503 | + }); |
| 2504 | + |
| 2505 | + it("rejects a complete BUD-14 upload whose reconstructed bytes have the wrong hash", async () => { |
| 2506 | + const sk = newKey(); |
| 2507 | + const body = new TextEncoder().encode(`bud14 mismatch ${rand()}`); |
| 2508 | + const claimed = hex(sha256(new TextEncoder().encode(`${body} wrong`))); |
| 2509 | + const auth = token(sk, "upload", claimed); |
| 2510 | + const response = await fetch(`${HTTP_URL}/${claimed}`, { |
| 2511 | + method: "PATCH", |
| 2512 | + headers: { |
| 2513 | + authorization: auth, |
| 2514 | + "content-type": "application/octet-stream", |
| 2515 | + "upload-type": "text/plain", |
| 2516 | + "upload-length": String(body.length), |
| 2517 | + "upload-offset": "0", |
| 2518 | + "content-length": String(body.length), |
| 2519 | + }, |
| 2520 | + body, |
| 2521 | + }); |
| 2522 | + expect(response.status).toBe(409); |
| 2523 | + expect((await fetch(`${HTTP_URL}/${claimed}`)).status).toBe(404); |
| 2524 | + }); |
| 2525 | + |
| 2526 | + it("keeps repeated BUD-14 ranges idempotent while an upload is partial", async () => { |
| 2527 | + const sk = newKey(); |
| 2528 | + const body = new TextEncoder().encode(`bud14 duplicate ${rand()}`); |
| 2529 | + const sha = hex(sha256(body)); |
| 2530 | + const auth = token(sk, "upload", sha); |
| 2531 | + const send = (offset: number, chunk: Uint8Array) => fetch(`${HTTP_URL}/${sha}`, { |
| 2532 | + method: "PATCH", |
| 2533 | + headers: { |
| 2534 | + authorization: auth, |
| 2535 | + "content-type": "application/octet-stream", |
| 2536 | + "upload-type": "text/plain", |
| 2537 | + "upload-length": String(body.length), |
| 2538 | + "upload-offset": String(offset), |
| 2539 | + "content-length": String(chunk.length), |
| 2540 | + }, |
| 2541 | + body: chunk, |
| 2542 | + }); |
| 2543 | + const first = body.slice(0, Math.floor(body.length / 2)); |
| 2544 | + expect((await send(0, first)).status).toBe(204); |
| 2545 | + expect((await send(0, first)).status).toBe(204); |
| 2546 | + const finish = await send(first.length, body.slice(first.length)); |
| 2547 | + expect(finish.status, await finish.clone().text()).toBe(201); |
| 2548 | + expect((await fetch(`${HTTP_URL}/${sha}`)).status).toBe(200); |
| 2549 | + expect((await fetch(`${HTTP_URL}/${sha}`, { |
| 2550 | + method: "DELETE", |
| 2551 | + headers: { authorization: token(sk, "delete", sha) }, |
| 2552 | + })).status).toBe(204); |
| 2553 | + }); |
| 2554 | + |
| 2555 | it("uploads a file, serves it whole and by range, lists it, and deletes it", async () => { |
| 2556 | const sk = newKey(); |
| 2557 | const body = new TextEncoder().encode(`a small file ${rand()} with enough bytes for a range`); |