| 1 | diff --git a/Dockerfile b/Dockerfile |
| 2 | index 05fcae9..2c3455f 100644 |
| 3 | --- a/Dockerfile |
| 4 | +++ b/Dockerfile |
| 5 | @@ -8,17 +8,17 @@ COPY . . |
| 6 | |
| 7 | ARG TEST_PACKAGES=./... |
| 8 | FROM build AS test |
| 9 | -RUN go test -mod=mod -race ${TEST_PACKAGES} |
| 10 | +RUN go test -mod=readonly -race ${TEST_PACKAGES} |
| 11 | |
| 12 | ARG BENCH_PACKAGES=./internal/storage ./internal/relay |
| 13 | ARG BENCHTIME=5s |
| 14 | FROM build AS bench |
| 15 | ARG BENCH_PACKAGES |
| 16 | ARG BENCHTIME |
| 17 | -RUN go test -mod=mod -run '^$' -bench . -benchmem -benchtime=${BENCHTIME} ${BENCH_PACKAGES} |
| 18 | +RUN go test -mod=readonly -run '^$' -bench . -benchmem -benchtime=${BENCHTIME} ${BENCH_PACKAGES} |
| 19 | |
| 20 | FROM build AS binary |
| 21 | -RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /out/tiny ./cmd/tiny |
| 22 | +RUN CGO_ENABLED=0 go build -mod=readonly -trimpath -ldflags="-s -w" -o /out/tiny ./cmd/tiny |
| 23 | |
| 24 | FROM debian:bookworm-slim AS runtime |
| 25 | RUN apt-get update \ |
| 26 | @@ -28,7 +28,7 @@ RUN apt-get update \ |
| 27 | && mkdir -p /data \ |
| 28 | && chown relay:relay /data |
| 29 | COPY --from=binary /out/tiny /usr/local/bin/tiny |
| 30 | -COPY LICENSE THIRD_PARTY_NOTICES internal/webui/signer.js.license /usr/share/doc/tiny/ |
| 31 | +COPY LICENSE THIRD_PARTY_NOTICES internal/webui/signer.js.license internal/webui/nostr-name.js.license /usr/share/doc/tiny/ |
| 32 | USER relay |
| 33 | WORKDIR /home/relay |
| 34 | VOLUME ["/data"] |
| 35 | diff --git a/Makefile b/Makefile |
| 36 | index fd7f92e..38754f9 100644 |
| 37 | --- a/Makefile |
| 38 | +++ b/Makefile |
| 39 | @@ -1,18 +1,28 @@ |
| 40 | -.PHONY: test test-race test-internal-race benchmark docker-test docker-build linux-test |
| 41 | +.PHONY: test test-race test-internal-race benchmark verify web-test docker-test docker-build linux-test |
| 42 | |
| 43 | TEST_PACKAGES ?= ./... |
| 44 | |
| 45 | test: |
| 46 | - go test -mod=mod ./... |
| 47 | + go test -mod=readonly ./... |
| 48 | |
| 49 | test-race: |
| 50 | - go test -mod=mod -race ./... |
| 51 | + go test -mod=readonly -race ./... |
| 52 | |
| 53 | test-internal-race: |
| 54 | - go test -mod=mod -race $(TEST_PACKAGES) |
| 55 | + go test -mod=readonly -race $(TEST_PACKAGES) |
| 56 | |
| 57 | benchmark: |
| 58 | - go test -mod=mod ./internal/storage ./internal/relay -run '^$$' -bench . -benchmem -benchtime=$${BENCHTIME:-1s} |
| 59 | + go test -mod=readonly ./internal/storage ./internal/relay -run '^$$' -bench . -benchmem -benchtime=$${BENCHTIME:-1s} |
| 60 | + |
| 61 | +verify: |
| 62 | + go test -mod=readonly ./... |
| 63 | + go vet -mod=readonly ./... |
| 64 | + npm run verify:bundles |
| 65 | + npm run test:js |
| 66 | + |
| 67 | +web-test: |
| 68 | + npm run verify:bundles |
| 69 | + npm run test:js |
| 70 | |
| 71 | docker-test: |
| 72 | docker build --target test --build-arg TEST_PACKAGES="$(TEST_PACKAGES)" -t tinyrelay-test:local . |
| 73 | diff --git a/README.md b/README.md |
| 74 | index 536025d..7b5091c 100644 |
| 75 | --- a/README.md |
| 76 | +++ b/README.md |
| 77 | @@ -79,6 +79,8 @@ See [Wiki](docs/wiki.md) for pages, versions and forks, merge requests and redir |
| 78 | |
| 79 | See [Custom views](docs/views.md) for rendering fenced code blocks through a transform you run and keeping the result as signed artifacts. |
| 80 | |
| 81 | +See [Architecture](docs/architecture.md) for service responsibilities, event acceptance and dependency checks. |
| 82 | + |
| 83 | **Manage > Health** shows the relay and script versions and whether the browser's WebMCP tools are registered. See [WebMCP tools](docs/webmcp.md) for browser-agent integration and authorization behavior. |
| 84 | |
| 85 | Agents outside the browser connect to `/mcp`, a stateless Model Context Protocol endpoint authenticated with NIP-98, and read `/llms.txt` for a summary of the relay's machine surface. See [MCP](docs/mcp.md). An agent acts under a grant the owner signs, and the owner reviews what it proposes: |
| 86 | diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES |
| 87 | index b912b6e..bb4bc1e 100644 |
| 88 | --- a/THIRD_PARTY_NOTICES |
| 89 | +++ b/THIRD_PARTY_NOTICES |
| 90 | @@ -38,6 +38,17 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 91 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN |
| 92 | THE SOFTWARE. |
| 93 | |
| 94 | +nostr-name browser bundle |
| 95 | +------------------------- |
| 96 | +The generated internal/webui/nostr-name.js bundle is built from |
| 97 | +nostr-web-components 0.2.2, @jsr/nostr__gadgets 0.1.9 and |
| 98 | +@jsr/nostr__tools 2.23.0, as resolved by package-lock.json. The installed |
| 99 | +nostr-web-components package identifies its license as the Unlicense. The |
| 100 | +bundle also retains esbuild license comments for noble-curves and scure-base; |
| 101 | +their installed package metadata identifies them as MIT licensed. The |
| 102 | +corresponding installed license text is recorded in |
| 103 | +internal/webui/nostr-name.js.license. |
| 104 | + |
| 105 | Fixi BSD-0 notice |
| 106 | ----------------- |
| 107 | BSD Zero Clause License |
| 108 | diff --git a/docs/agents.md b/docs/agents.md |
| 109 | index 2598fe0..9e997a6 100644 |
| 110 | --- a/docs/agents.md |
| 111 | +++ b/docs/agents.md |
| 112 | @@ -323,6 +323,8 @@ Right before each POST the relay checks that the callback still exists and is no |
| 113 | |
| 114 | ### Retries and pauses |
| 115 | |
| 116 | +A valid event is accepted even when its optional callback work cannot be planned immediately. The relay makes up to three background planning attempts within 15 minutes, rechecks the source and callback, and skips work that is no longer relevant. Recovery fills missing work without restarting completed deliveries or sending earlier events to newly registered callbacks. |
| 117 | + |
| 118 | A response outside 2xx, a timeout or a connection failure counts as a failure. The relay tries the event again after one minute and once more after five, three attempts in all, then drops it. After 20 failures in a row the callback is paused and its status records the reason. A callback is also paused when its key is no longer a member. A successful delivery resets the failure count. Resume a paused callback with `resumecallback`. |
| 119 | |
| 120 | ### Manage callbacks |
| 121 | diff --git a/docs/architecture.md b/docs/architecture.md |
| 122 | new file mode 100644 |
| 123 | index 0000000..19a53b8 |
| 124 | --- /dev/null |
| 125 | +++ b/docs/architecture.md |
| 126 | @@ -0,0 +1,31 @@ |
| 127 | +# Architecture |
| 128 | + |
| 129 | +Each tenant has a durable store and a set of services. The daemon constructs those services, admits operations and manages their lifetimes. |
| 130 | + |
| 131 | +## Responsibilities |
| 132 | + |
| 133 | +| Area | Responsibility | |
| 134 | +| --- | --- | |
| 135 | +| `event` and `policy` | Event formats and shared policy decisions. | |
| 136 | +| `storage` | Event persistence, queries and transaction boundaries. | |
| 137 | +| `community` | Membership, moderation, rooms and their projections. | |
| 138 | +| `records` | Relay identity, signed records and notification records. Community data arrives through a required read interface. | |
| 139 | +| `blob` | File storage, upload rules and file HTTP endpoints. | |
| 140 | +| `gitrelay` | Repository admission, storage, synchronization, repair and Git HTTP endpoints. | |
| 141 | +| `replication` | Replication policy, jobs and delivery handlers. | |
| 142 | +| `work` | Durable queue claims, retries and worker lifetimes. The daemon combines handlers from the services. | |
| 143 | +| `webui` | Pages, typed room reads and browser interactions. | |
| 144 | + |
| 145 | +Keep schema changes and data access with the service that owns the data. Share a policy function or a small read contract when several services need the same rule or projection. Avoid introducing a general utility package for unrelated behavior. |
| 146 | + |
| 147 | +## Event acceptance |
| 148 | + |
| 149 | +Client publication and imported events share the persistence and projection path. Each entry point retains its own admission rules and protocol actions. Event projections and durable work records commit together. |
| 150 | + |
| 151 | +Callbacks and custom views are optional. Local planning can recover after acceptance, with bounded retries that recheck the source and captured registrations. Completed work keeps its identity and is not reset by planning recovery. Remote delivery and transforms retain their own retry limits. See [callbacks](agents.md#retries-and-pauses) and [custom views](views.md#failures). |
| 152 | + |
| 153 | +## Browser boundaries |
| 154 | + |
| 155 | +Pages render without JavaScript. The page template owns the shell; page templates own their content sections. Generic components live in `components.js`, room interactions live in `rooms.js`, and signed publication uses the shared `tiny.signing` contract. The verifier checks signed events before publication. |
| 156 | + |
| 157 | +Go module versions are declared in `go.mod`, with dependency checksums in `go.sum`. Browser dependencies are locked in `package-lock.json`. `make verify` checks the Go and JavaScript code and verifies generated browser bundles against their sources. See [Testing](testing.md) for local and Linux checks. |
| 158 | diff --git a/docs/testing.md b/docs/testing.md |
| 159 | index 2c67a1a..2656f74 100644 |
| 160 | --- a/docs/testing.md |
| 161 | +++ b/docs/testing.md |
| 162 | @@ -3,13 +3,12 @@ |
| 163 | The normal development loop is local: |
| 164 | |
| 165 | ```sh |
| 166 | -make test-internal-race |
| 167 | -make test |
| 168 | -npm run test:files |
| 169 | -npm run test:collaboration |
| 170 | -npm run test:approvals |
| 171 | +make verify |
| 172 | +make test-race |
| 173 | ``` |
| 174 | |
| 175 | +`make verify` runs the Go tests, Go vet, JavaScript tests and a check that the bundled browser dependencies match their locked sources. Install JavaScript dependencies with `npm ci` first. `make web-test` runs the browser checks alone. |
| 176 | + |
| 177 | The repository also contains a multistage Docker build. `make docker-test` runs `TEST_PACKAGES` (default `./...`) under Go 1.27.1 on Debian Bookworm with the race detector. `make docker-build` builds the self-hosted runtime image. |
| 178 | |
| 179 | ## Remote Linux host |
| 180 | diff --git a/docs/views.md b/docs/views.md |
| 181 | index 1bf0e24..9b7c0bc 100644 |
| 182 | --- a/docs/views.md |
| 183 | +++ b/docs/views.md |
| 184 | @@ -67,6 +67,8 @@ Each artifact is served at `/views/<name>/<hash>` with its stored media type, un |
| 185 | |
| 186 | ## Failures |
| 187 | |
| 188 | +A valid event is accepted even when its optional transform work cannot be planned immediately. The relay makes up to three background planning attempts within 15 minutes, rechecks the source and view, and skips work that is no longer relevant. Recovery fills missing work without restarting completed transforms or backfilling views registered after the event. |
| 189 | + |
| 190 | A response outside 2xx, a timeout or a connection failure counts as a failure. The relay tries the event again after one minute and once more after five, three attempts in all, then drops it. After 20 failures in a row the view is paused and its status records the reason. A successful run resets the failure count. |
| 191 | |
| 192 | ## Manage views |
| 193 | diff --git a/internal/blob/blob.go b/internal/blob/blob.go |
| 194 | deleted file mode 100644 |
| 195 | index baffbe4..0000000 |
| 196 | --- a/internal/blob/blob.go |
| 197 | +++ /dev/null |
| 198 | @@ -1,1615 +0,0 @@ |
| 199 | -// Package blob implements the Blossom and NIP-96 file doors for a local relay. |
| 200 | -package blob |
| 201 | - |
| 202 | -import ( |
| 203 | - "context" |
| 204 | - "crypto/sha256" |
| 205 | - "database/sql" |
| 206 | - "encoding/hex" |
| 207 | - "encoding/json" |
| 208 | - "errors" |
| 209 | - "fmt" |
| 210 | - "io" |
| 211 | - "net" |
| 212 | - "net/http" |
| 213 | - "net/url" |
| 214 | - "os" |
| 215 | - "path/filepath" |
| 216 | - "regexp" |
| 217 | - "sort" |
| 218 | - "strconv" |
| 219 | - "strings" |
| 220 | - "sync" |
| 221 | - "time" |
| 222 | - |
| 223 | - "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 224 | - "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 225 | -) |
| 226 | - |
| 227 | -type Action string |
| 228 | - |
| 229 | -const ( |
| 230 | - ActionUpload Action = "upload" |
| 231 | - ActionGet Action = "get" |
| 232 | - ActionDelete Action = "delete" |
| 233 | - ActionList Action = "list" |
| 234 | - ActionMirror Action = "mirror" |
| 235 | - ActionReport Action = "report" |
| 236 | -) |
| 237 | - |
| 238 | -type Authorize func(*http.Request, Action) (string, error) |
| 239 | -type CanRead func(context.Context, string, []string) bool |
| 240 | -type ResolveIP func(context.Context, string) ([]net.IP, error) |
| 241 | -type ReportRecorder func(context.Context, *sql.Tx, string, string, string, string, string) error |
| 242 | - |
| 243 | -type Config struct { |
| 244 | - Root string |
| 245 | - PublicURL string |
| 246 | - Store *storage.Store |
| 247 | - Authorize Authorize |
| 248 | - CanRead CanRead |
| 249 | - HTTPClient *http.Client |
| 250 | - ResolveIP ResolveIP |
| 251 | - IsOwner func(string) bool |
| 252 | - ValidateUpload func(*http.Request, string) error |
| 253 | - // ResolveServers returns the ordered BUD-03 servers for an authorized |
| 254 | - // caller. It is used only for Blossom URI mirror fallback. |
| 255 | - ResolveServers func(context.Context, string) ([]string, error) |
| 256 | - // RecordReport joins native blob reporting to the host moderation ledger. |
| 257 | - // It runs inside the blob report transaction. |
| 258 | - RecordReport ReportRecorder |
| 259 | - // Limits is read for each upload so policy changes apply without reopening |
| 260 | - // the blob service. Zero values mean unlimited. |
| 261 | - Limits func() Limits |
| 262 | - // UploadTerms returns what an uploader's standing imposes on its uploads: |
| 263 | - // an agent grant with a sites ttl gives its claims an expiry, and one with |
| 264 | - // the encrypted flag refuses plain uploads. Zero values mean no terms. |
| 265 | - UploadTerms func(context.Context, string) (UploadTerms, error) |
| 266 | -} |
| 267 | - |
| 268 | -type Limits struct { |
| 269 | - MaxFileBytes int64 |
| 270 | - UserStorageBytes int64 |
| 271 | -} |
| 272 | - |
| 273 | -// UploadTerms is what an uploader's grant imposes on the blobs it stores. |
| 274 | -// ExpiresAt is the unix time its claims lapse, 0 for never; Encrypted |
| 275 | -// requires the stored bytes to be an encrypted blob or manifest. |
| 276 | -type UploadTerms struct { |
| 277 | - ExpiresAt int64 |
| 278 | - Encrypted bool |
| 279 | -} |
| 280 | - |
| 281 | -// encryptedTypes are the content types the browser uploader gives encrypted |
| 282 | -// objects: AES-GCM ciphertext for files and chunks, and the BUD-16 directory |
| 283 | -// type for encrypted manifests. Anything the relay recognizes as a page, |
| 284 | -// image, video, audio or document is plain. |
| 285 | -var encryptedTypes = map[string]bool{"application/octet-stream": true, "application/vnd.blossom.directory+msgpack": true} |
| 286 | - |
| 287 | -type Blob struct { |
| 288 | - SHA256 string `json:"sha256"` |
| 289 | - Size int64 `json:"size"` |
| 290 | - Type string `json:"type"` |
| 291 | - Uploader string `json:"uploader"` |
| 292 | - Uploaded int64 `json:"uploaded"` |
| 293 | -} |
| 294 | - |
| 295 | -type PutOptions struct { |
| 296 | - Reader io.Reader |
| 297 | - Type string |
| 298 | - Uploader string |
| 299 | - Hash string |
| 300 | - // Commit records ownership metadata in the blob transaction. The final |
| 301 | - // argument reports whether the blob is new. An error rolls back all claims. |
| 302 | - // New bytes remain unavailable until their ownership metadata is durable. |
| 303 | - Commit func(context.Context, *sql.Tx, Blob, bool) error |
| 304 | -} |
| 305 | - |
| 306 | -type Service struct { |
| 307 | - config Config |
| 308 | - root string |
| 309 | - store *storage.Store |
| 310 | - resolve ResolveIP |
| 311 | - quotaMu sync.Mutex |
| 312 | - // multipartMu protects partial files and their range metadata. Lock it |
| 313 | - // before quotaMu; network bodies are never read while either is held. |
| 314 | - multipartMu sync.Mutex |
| 315 | - activeParts map[string]int |
| 316 | - // multipartFinalizing prevents another chunk from changing a complete |
| 317 | - // session while its assembled file is being hashed. Hashing is deliberately |
| 318 | - // outside multipartMu because the upload may be as large as 1 GiB. |
| 319 | - multipartFinalizing map[string]bool |
| 320 | - // verifyMultipartHook is used by package tests to pause finalization |
| 321 | - // without allocating a GiB fixture. It is nil in production. |
| 322 | - verifyMultipartHook func(context.Context, string, string, int64) error |
| 323 | -} |
| 324 | - |
| 325 | -var ErrBlocked = errors.New("blocked: this blob was removed by a moderator") |
| 326 | -var ErrHashMismatch = errors.New("conflict: mirrored blob hash does not match the requested hash") |
| 327 | -var ErrFileTooLarge = errors.New("invalid: blob exceeds the per-file size limit") |
| 328 | -var ErrQuotaExceeded = errors.New("restricted: uploader storage quota exceeded") |
| 329 | -var ErrMultipartConflict = errors.New("conflict: upload metadata differs from existing session") |
| 330 | -var ErrNotEncrypted = errors.New("restricted: agent grant requires encrypted uploads") |
| 331 | - |
| 332 | -var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) |
| 333 | -var blobPathPattern = regexp.MustCompile(`^/([0-9a-f]{64})(?:\.[a-z0-9]{1,8})?$`) |
| 334 | - |
| 335 | -// HandlesPath keeps the daemon mount in sync with every supported file door. |
| 336 | -func HandlesPath(path string) bool { |
| 337 | - return path == "/upload" || path == "/mirror" || path == "/report" || |
| 338 | - path == "/nip96" || path == "/.well-known/nostr/nip96.json" || |
| 339 | - strings.HasPrefix(path, "/nip96/") || strings.HasPrefix(path, "/list/") || |
| 340 | - blobPathPattern.MatchString(path) |
| 341 | -} |
| 342 | - |
| 343 | -var extensions = map[string]string{ |
| 344 | - "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp", "image/avif": "avif", "image/svg+xml": "svg", |
| 345 | - "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/wav": "wav", "audio/mp4": "m4a", |
| 346 | - "application/pdf": "pdf", "application/json": "json", "text/plain": "txt", "text/markdown": "md", |
| 347 | - "application/vnd.blossom.directory+msgpack": "bdir", |
| 348 | -} |
| 349 | - |
| 350 | -func New(ctx context.Context, config Config) (*Service, error) { |
| 351 | - if config.Root == "" || config.Store == nil { |
| 352 | - return nil, errors.New("blob root and store are required") |
| 353 | - } |
| 354 | - if err := os.MkdirAll(filepath.Join(config.Root, "blobs"), 0o700); err != nil { |
| 355 | - return nil, fmt.Errorf("create blob root: %w", err) |
| 356 | - } |
| 357 | - if _, err := config.Store.DB().ExecContext(ctx, `CREATE TABLE IF NOT EXISTS blobs ( |
| 358 | - sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL, type TEXT NOT NULL, |
| 359 | - uploader TEXT NOT NULL, uploaded INTEGER NOT NULL |
| 360 | - ); CREATE TABLE IF NOT EXISTS blob_claims ( |
| 361 | - sha256 TEXT NOT NULL, uploader TEXT NOT NULL, claimed_at INTEGER NOT NULL, |
| 362 | - PRIMARY KEY (sha256, uploader), FOREIGN KEY (sha256) REFERENCES blobs(sha256) ON DELETE CASCADE |
| 363 | - ); CREATE INDEX IF NOT EXISTS blobs_uploader ON blobs(uploader, uploaded DESC); |
| 364 | - CREATE INDEX IF NOT EXISTS blob_claims_uploader ON blob_claims(uploader, claimed_at DESC, sha256 DESC); |
| 365 | - CREATE TABLE IF NOT EXISTS blob_reports ( |
| 366 | - id TEXT PRIMARY KEY, reporter TEXT NOT NULL, target_pubkey TEXT NOT NULL DEFAULT '', |
| 367 | - target_blob TEXT NOT NULL, type TEXT NOT NULL, content TEXT NOT NULL, at INTEGER NOT NULL, |
| 368 | - status TEXT NOT NULL DEFAULT 'open', resolved_by TEXT NOT NULL DEFAULT '', |
| 369 | - resolved_at INTEGER NOT NULL DEFAULT 0, action TEXT NOT NULL DEFAULT '', |
| 370 | - UNIQUE(reporter,target_blob)); |
| 371 | - CREATE TRIGGER IF NOT EXISTS blob_claim_on_insert AFTER INSERT ON blobs |
| 372 | - WHEN NEW.uploader != '' BEGIN INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(NEW.sha256,NEW.uploader,NEW.uploaded); END; |
| 373 | - CREATE TABLE IF NOT EXISTS blob_blocks (sha256 TEXT PRIMARY KEY, reason TEXT NOT NULL DEFAULT '', blocked_at INTEGER NOT NULL DEFAULT 0); |
| 374 | - CREATE TABLE IF NOT EXISTS blob_tombstones (sha256 TEXT PRIMARY KEY, deleted_at INTEGER NOT NULL); |
| 375 | - CREATE TABLE IF NOT EXISTS multipart_uploads (id TEXT PRIMARY KEY, sha256 TEXT NOT NULL, uploader TEXT NOT NULL, length INTEGER NOT NULL, type TEXT NOT NULL, created INTEGER NOT NULL, last_seen INTEGER NOT NULL); |
| 376 | - CREATE TABLE IF NOT EXISTS multipart_parts (upload_id TEXT NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY(upload_id, offset), FOREIGN KEY(upload_id) REFERENCES multipart_uploads(id) ON DELETE CASCADE); |
| 377 | - INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) |
| 378 | - SELECT sha256,uploader,uploaded FROM blobs WHERE uploader != ''`); err != nil { |
| 379 | - return nil, fmt.Errorf("initialize blob metadata: %w", err) |
| 380 | - } |
| 381 | - // Claims made under an agent grant with a ttl carry an expiry; every |
| 382 | - // earlier claim keeps 0, which never lapses. |
| 383 | - if _, err := config.Store.DB().ExecContext(ctx, "ALTER TABLE blob_claims ADD COLUMN expires INTEGER NOT NULL DEFAULT 0"); err != nil && !strings.Contains(err.Error(), "duplicate column") { |
| 384 | - return nil, fmt.Errorf("add blob claim expiry: %w", err) |
| 385 | - } |
| 386 | - if _, err := config.Store.DB().ExecContext(ctx, "CREATE INDEX IF NOT EXISTS blob_claims_expires ON blob_claims(expires) WHERE expires>0"); err != nil { |
| 387 | - return nil, fmt.Errorf("index blob claim expiry: %w", err) |
| 388 | - } |
| 389 | - if err := reconcile(ctx, config.Store, filepath.Join(config.Root, "blobs")); err != nil { |
| 390 | - return nil, err |
| 391 | - } |
| 392 | - resolve := config.ResolveIP |
| 393 | - if resolve == nil { |
| 394 | - resolve = func(ctx context.Context, host string) ([]net.IP, error) { |
| 395 | - return net.DefaultResolver.LookupIP(ctx, "ip", host) |
| 396 | - } |
| 397 | - } |
| 398 | - s := &Service{config: config, root: filepath.Join(config.Root, "blobs"), store: config.Store, resolve: resolve, activeParts: make(map[string]int), multipartFinalizing: make(map[string]bool)} |
| 399 | - if err := s.reconcileMultipart(ctx); err != nil { |
| 400 | - return nil, err |
| 401 | - } |
| 402 | - return s, nil |
| 403 | -} |
| 404 | - |
| 405 | -func (s *Service) Handler() http.HandlerFunc { return s.serveHTTP } |
| 406 | - |
| 407 | -// Put stores bytes using their SHA-256 digest as the durable filename. |
| 408 | -func (s *Service) Put(ctx context.Context, options PutOptions) (Blob, error) { |
| 409 | - if options.Reader == nil { |
| 410 | - return Blob{}, errors.New("blob reader is required") |
| 411 | - } |
| 412 | - entry, _, err := s.putOptions(ctx, options) |
| 413 | - return entry, err |
| 414 | -} |
| 415 | - |
| 416 | -func (s *Service) putOptions(ctx context.Context, options PutOptions) (Blob, bool, error) { |
| 417 | - return s.putValidatedWithCommit(ctx, options.Reader, options.Type, options.Uploader, options.Hash, nil, options.Commit) |
| 418 | -} |
| 419 | - |
| 420 | -// Get opens a verified metadata-backed blob for streaming to another service. |
| 421 | -func (s *Service) Get(ctx context.Context, sha string) (Blob, io.ReadCloser, error) { |
| 422 | - if !shaPattern.MatchString(sha) { |
| 423 | - return Blob{}, nil, errors.New("invalid: bad blob sha256") |
| 424 | - } |
| 425 | - entry, err := s.lookup(ctx, sha) |
| 426 | - if err != nil { |
| 427 | - return Blob{}, nil, err |
| 428 | - } |
| 429 | - if s.blocked(ctx, sha) { |
| 430 | - return Blob{}, nil, ErrBlocked |
| 431 | - } |
| 432 | - file, err := os.Open(filepath.Join(s.root, sha)) |
| 433 | - if err != nil { |
| 434 | - return Blob{}, nil, err |
| 435 | - } |
| 436 | - return entry, file, nil |
| 437 | -} |
| 438 | - |
| 439 | -// ListMetadata returns all blobs uploaded by the supplied pubkey. |
| 440 | -func (s *Service) ListMetadata(ctx context.Context, uploader string) ([]Blob, error) { |
| 441 | - rows, err := s.store.DB().QueryContext(ctx, `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded |
| 442 | - FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=? |
| 443 | - ORDER BY b.uploaded DESC,b.sha256 DESC`, uploader) |
| 444 | - if err != nil { |
| 445 | - return nil, fmt.Errorf("list blob metadata: %w", err) |
| 446 | - } |
| 447 | - defer rows.Close() |
| 448 | - entries := make([]Blob, 0) |
| 449 | - for rows.Next() { |
| 450 | - var entry Blob |
| 451 | - if err := rows.Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded); err != nil { |
| 452 | - return nil, fmt.Errorf("scan blob metadata: %w", err) |
| 453 | - } |
| 454 | - entries = append(entries, entry) |
| 455 | - } |
| 456 | - return entries, rows.Err() |
| 457 | -} |
| 458 | - |
| 459 | -// ListPage returns a BUD-12 cursor page. The cursor is the last blob hash |
| 460 | -// returned by the previous page and is intentionally opaque to callers. |
| 461 | -func (s *Service) ListPage(ctx context.Context, uploader string, limit int, cursor string) ([]Blob, string, error) { |
| 462 | - if limit < 1 { |
| 463 | - limit = 20 |
| 464 | - } |
| 465 | - if limit > 100 { |
| 466 | - limit = 100 |
| 467 | - } |
| 468 | - query := `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded FROM blobs b |
| 469 | - JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=?` |
| 470 | - args := []any{uploader} |
| 471 | - if cursor != "" { |
| 472 | - var uploaded int64 |
| 473 | - if err := s.store.DB().QueryRowContext(ctx, `SELECT b.uploaded FROM blobs b |
| 474 | - JOIN blob_claims c ON c.sha256=b.sha256 WHERE b.sha256=? AND c.uploader=?`, cursor, uploader).Scan(&uploaded); err != nil { |
| 475 | - return nil, "", fmt.Errorf("resolve blob cursor: %w", err) |
| 476 | - } |
| 477 | - query += " AND (b.uploaded < ? OR (b.uploaded = ? AND b.sha256 < ?))" |
| 478 | - args = append(args, uploaded, uploaded, cursor) |
| 479 | - } |
| 480 | - query += " ORDER BY b.uploaded DESC,b.sha256 DESC LIMIT ?" |
| 481 | - args = append(args, limit+1) |
| 482 | - rows, err := s.store.DB().QueryContext(ctx, query, args...) |
| 483 | - if err != nil { |
| 484 | - return nil, "", fmt.Errorf("list blob page: %w", err) |
| 485 | - } |
| 486 | - defer rows.Close() |
| 487 | - entries := make([]Blob, 0, limit) |
| 488 | - for rows.Next() { |
| 489 | - var entry Blob |
| 490 | - if err := rows.Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded); err != nil { |
| 491 | - return nil, "", fmt.Errorf("scan blob page: %w", err) |
| 492 | - } |
| 493 | - entries = append(entries, entry) |
| 494 | - } |
| 495 | - if err := rows.Err(); err != nil { |
| 496 | - return nil, "", fmt.Errorf("read blob page: %w", err) |
| 497 | - } |
| 498 | - next := "" |
| 499 | - if len(entries) > limit { |
| 500 | - next = entries[limit-1].SHA256 |
| 501 | - entries = entries[:limit] |
| 502 | - } |
| 503 | - return entries, next, nil |
| 504 | -} |
| 505 | - |
| 506 | -// Delete removes a blob's metadata and content. Authorization is the caller's responsibility. |
| 507 | -func (s *Service) Delete(ctx context.Context, sha string) error { |
| 508 | - if !shaPattern.MatchString(sha) { |
| 509 | - return errors.New("invalid: bad blob sha256") |
| 510 | - } |
| 511 | - if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 512 | - if _, err := tx.ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 513 | - return fmt.Errorf("record blob deletion: %w", err) |
| 514 | - } |
| 515 | - if _, err := tx.ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 516 | - return fmt.Errorf("delete blob metadata: %w", err) |
| 517 | - } |
| 518 | - return nil |
| 519 | - }); err != nil { |
| 520 | - return err |
| 521 | - } |
| 522 | - if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 523 | - return fmt.Errorf("delete blob content: %w", err) |
| 524 | - } |
| 525 | - return nil |
| 526 | -} |
| 527 | - |
| 528 | -// DeleteForUploader removes one owner's claim. The physical object remains |
| 529 | -// while another owner still references it, which makes deduplicated uploads |
| 530 | -// safe to manage independently. |
| 531 | -func (s *Service) DeleteForUploader(ctx context.Context, sha, uploader string) error { |
| 532 | - if !shaPattern.MatchString(sha) { |
| 533 | - return errors.New("invalid: bad blob sha256") |
| 534 | - } |
| 535 | - removed := false |
| 536 | - if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 537 | - result, err := tx.ExecContext(ctx, "DELETE FROM blob_claims WHERE sha256=? AND uploader=?", sha, uploader) |
| 538 | - if err != nil { |
| 539 | - return fmt.Errorf("delete blob claim: %w", err) |
| 540 | - } |
| 541 | - changed, err := result.RowsAffected() |
| 542 | - if err != nil { |
| 543 | - return fmt.Errorf("check blob claim deletion: %w", err) |
| 544 | - } |
| 545 | - if changed == 0 { |
| 546 | - return sql.ErrNoRows |
| 547 | - } |
| 548 | - var remaining int |
| 549 | - if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM blob_claims WHERE sha256=?", sha).Scan(&remaining); err != nil { |
| 550 | - return fmt.Errorf("count blob claims: %w", err) |
| 551 | - } |
| 552 | - if remaining == 0 { |
| 553 | - removed = true |
| 554 | - if _, err := tx.ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 555 | - return fmt.Errorf("record blob deletion: %w", err) |
| 556 | - } |
| 557 | - if _, err := tx.ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 558 | - return fmt.Errorf("delete blob metadata: %w", err) |
| 559 | - } |
| 560 | - } |
| 561 | - return nil |
| 562 | - }); err != nil { |
| 563 | - return err |
| 564 | - } |
| 565 | - if removed { |
| 566 | - if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 567 | - return fmt.Errorf("delete blob content: %w", err) |
| 568 | - } |
| 569 | - } |
| 570 | - return nil |
| 571 | -} |
| 572 | - |
| 573 | -// Descriptor returns the Blossom descriptor and NIP-94 tags for a blob. |
| 574 | -func (s *Service) Descriptor(baseURL string, entry Blob) map[string]any { |
| 575 | - return map[string]any{ |
| 576 | - "url": baseURL + "/" + entry.SHA256 + extensionSuffix(entry.Type), |
| 577 | - "sha256": entry.SHA256, |
| 578 | - "size": entry.Size, |
| 579 | - "type": entry.Type, |
| 580 | - "uploaded": entry.Uploaded, |
| 581 | - "nip94": [][]string{{"url", baseURL + "/" + entry.SHA256 + extensionSuffix(entry.Type)}, {"m", entry.Type}, {"x", entry.SHA256}, {"ox", entry.SHA256}, {"size", strconv.FormatInt(entry.Size, 10)}}, |
| 582 | - } |
| 583 | -} |
| 584 | - |
| 585 | -func (s *Service) serveHTTP(w http.ResponseWriter, r *http.Request) { |
| 586 | - w.Header().Set("access-control-allow-origin", "*") |
| 587 | - w.Header().Set("access-control-allow-headers", "authorization, content-type, x-sha-256, x-content-length, x-content-type, upload-type, upload-length, upload-offset") |
| 588 | - w.Header().Set("access-control-allow-methods", "GET, HEAD, PUT, PATCH, POST, DELETE, OPTIONS") |
| 589 | - w.Header().Set("Access-Control-Expose-Headers", "Allow, X-Reason") |
| 590 | - if r.Method == http.MethodOptions { |
| 591 | - if blobPathPattern.MatchString(r.URL.Path) { |
| 592 | - w.Header().Set("Allow", "GET, HEAD, PUT, PATCH, DELETE, OPTIONS") |
| 593 | - } |
| 594 | - w.WriteHeader(http.StatusNoContent) |
| 595 | - return |
| 596 | - } |
| 597 | - if r.URL.Path == "/.well-known/nostr/nip96.json" { |
| 598 | - s.writeJSON(w, http.StatusOK, s.nip96Info(r)) |
| 599 | - return |
| 600 | - } |
| 601 | - if r.URL.Path == "/upload" && r.Method == http.MethodHead { |
| 602 | - s.uploadHead(w, r) |
| 603 | - return |
| 604 | - } |
| 605 | - if r.URL.Path == "/upload" && r.Method == http.MethodPut { |
| 606 | - s.upload(w, r) |
| 607 | - return |
| 608 | - } |
| 609 | - if r.URL.Path == "/mirror" && r.Method == http.MethodPut { |
| 610 | - s.mirror(w, r) |
| 611 | - return |
| 612 | - } |
| 613 | - if match := blobPathPattern.FindStringSubmatch(r.URL.Path); match != nil && (r.Method == http.MethodPut || r.Method == http.MethodPatch || r.Method == http.MethodOptions) { |
| 614 | - if r.Method == http.MethodPatch || r.Method == http.MethodOptions { |
| 615 | - s.multipart(w, r, match[1]) |
| 616 | - return |
| 617 | - } |
| 618 | - s.pathUpload(w, r, match[1]) |
| 619 | - return |
| 620 | - } |
| 621 | - if r.URL.Path == "/report" && r.Method == http.MethodPut { |
| 622 | - s.report(w, r) |
| 623 | - return |
| 624 | - } |
| 625 | - if strings.HasPrefix(r.URL.Path, "/list/") && r.Method == http.MethodGet { |
| 626 | - s.list(w, r, strings.TrimPrefix(r.URL.Path, "/list/")) |
| 627 | - return |
| 628 | - } |
| 629 | - if r.URL.Path == "/nip96" { |
| 630 | - s.nip96(w, r) |
| 631 | - return |
| 632 | - } |
| 633 | - s.blob(w, r) |
| 634 | -} |
| 635 | - |
| 636 | -func (s *Service) upload(w http.ResponseWriter, r *http.Request) { |
| 637 | - pubkey, err := s.authorize(r, ActionUpload) |
| 638 | - if err != nil { |
| 639 | - s.fail(w, http.StatusUnauthorized, err) |
| 640 | - return |
| 641 | - } |
| 642 | - claimed := strings.ToLower(strings.TrimSpace(r.Header.Get("x-sha-256"))) |
| 643 | - if claimed != "" && !shaPattern.MatchString(claimed) { |
| 644 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: X-SHA-256 must be lowercase hex")) |
| 645 | - return |
| 646 | - } |
| 647 | - blob, created, err := s.putValidated(r.Context(), r.Body, contentType(r.Header.Get("content-type")), pubkey, claimed, func(actual string) error { |
| 648 | - if s.config.ValidateUpload != nil { |
| 649 | - return s.config.ValidateUpload(r, actual) |
| 650 | - } |
| 651 | - return nil |
| 652 | - }) |
| 653 | - if err != nil { |
| 654 | - s.fail(w, statusFor(err), err) |
| 655 | - return |
| 656 | - } |
| 657 | - _ = created |
| 658 | - s.writeJSON(w, http.StatusOK, s.descriptor(r, blob)) |
| 659 | -} |
| 660 | - |
| 661 | -func (s *Service) uploadHead(w http.ResponseWriter, r *http.Request) { |
| 662 | - sha := strings.ToLower(strings.TrimSpace(r.Header.Get("x-sha-256"))) |
| 663 | - if !shaPattern.MatchString(sha) { |
| 664 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: X-SHA-256 must be lowercase hex")) |
| 665 | - return |
| 666 | - } |
| 667 | - rawLength := strings.TrimSpace(r.Header.Get("x-content-length")) |
| 668 | - if rawLength == "" { |
| 669 | - s.fail(w, http.StatusLengthRequired, errors.New("invalid: X-Content-Length is required")) |
| 670 | - return |
| 671 | - } |
| 672 | - length, err := strconv.ParseInt(rawLength, 10, 64) |
| 673 | - if err != nil || length < 0 { |
| 674 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Length must be a nonnegative integer")) |
| 675 | - return |
| 676 | - } |
| 677 | - pubkey, err := s.authorize(r, ActionUpload) |
| 678 | - if err != nil { |
| 679 | - s.fail(w, http.StatusUnauthorized, err) |
| 680 | - return |
| 681 | - } |
| 682 | - typ := r.Header.Get("x-content-type") |
| 683 | - if typ == "" { |
| 684 | - typ = r.Header.Get("content-type") |
| 685 | - } |
| 686 | - if strings.TrimSpace(typ) == "" { |
| 687 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Type is required")) |
| 688 | - return |
| 689 | - } |
| 690 | - if existing, lookupErr := s.lookup(r.Context(), sha); lookupErr == nil && existing.SHA256 != "" && s.hasClaim(r.Context(), sha, pubkey) { |
| 691 | - w.WriteHeader(http.StatusOK) |
| 692 | - return |
| 693 | - } |
| 694 | - limits := s.currentLimits() |
| 695 | - if limits.MaxFileBytes > 0 && length > limits.MaxFileBytes { |
| 696 | - s.fail(w, http.StatusRequestEntityTooLarge, ErrFileTooLarge) |
| 697 | - return |
| 698 | - } |
| 699 | - // HEAD is advisory for deduplicated uploads, whose final physical size is |
| 700 | - // already present, but it should still reflect the current quota budget. |
| 701 | - if limits.UserStorageBytes > 0 { |
| 702 | - if used, quotaErr := s.quotaUsage(r.Context(), pubkey); quotaErr != nil { |
| 703 | - s.fail(w, http.StatusInternalServerError, quotaErr) |
| 704 | - return |
| 705 | - } else if length > limits.UserStorageBytes-used { |
| 706 | - s.fail(w, http.StatusForbidden, ErrQuotaExceeded) |
| 707 | - return |
| 708 | - } |
| 709 | - } |
| 710 | - if existing, err := s.lookup(r.Context(), sha); err == nil && existing.SHA256 != "" { |
| 711 | - w.WriteHeader(http.StatusOK) |
| 712 | - return |
| 713 | - } |
| 714 | - w.WriteHeader(http.StatusOK) |
| 715 | -} |
| 716 | - |
| 717 | -func (s *Service) blob(w http.ResponseWriter, r *http.Request) { |
| 718 | - path := r.URL.Path |
| 719 | - if strings.HasPrefix(path, "/nip96/") { |
| 720 | - path = strings.TrimPrefix(path, "/nip96") |
| 721 | - } |
| 722 | - match := blobPathPattern.FindStringSubmatch(path) |
| 723 | - if match == nil { |
| 724 | - s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 725 | - return |
| 726 | - } |
| 727 | - sha := match[1] |
| 728 | - if r.Method == http.MethodDelete { |
| 729 | - s.delete(w, r, sha) |
| 730 | - return |
| 731 | - } |
| 732 | - if r.Method != http.MethodGet && r.Method != http.MethodHead { |
| 733 | - s.fail(w, http.StatusMethodNotAllowed, errors.New("method not allowed")) |
| 734 | - return |
| 735 | - } |
| 736 | - if s.config.CanRead != nil { |
| 737 | - pubkey, err := s.authorize(r, ActionGet) |
| 738 | - if err != nil { |
| 739 | - s.fail(w, http.StatusUnauthorized, err) |
| 740 | - return |
| 741 | - } |
| 742 | - if !s.config.CanRead(r.Context(), sha, []string{pubkey}) { |
| 743 | - s.fail(w, http.StatusForbidden, errors.New("restricted: read access denied")) |
| 744 | - return |
| 745 | - } |
| 746 | - } |
| 747 | - entry, err := s.lookup(r.Context(), sha) |
| 748 | - if errors.Is(err, sql.ErrNoRows) { |
| 749 | - s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 750 | - return |
| 751 | - } |
| 752 | - if err != nil { |
| 753 | - s.fail(w, http.StatusInternalServerError, err) |
| 754 | - return |
| 755 | - } |
| 756 | - if s.blocked(r.Context(), sha) { |
| 757 | - s.fail(w, http.StatusForbidden, ErrBlocked) |
| 758 | - return |
| 759 | - } |
| 760 | - file, err := os.Open(filepath.Join(s.root, sha)) |
| 761 | - if err != nil { |
| 762 | - s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 763 | - return |
| 764 | - } |
| 765 | - defer file.Close() |
| 766 | - w.Header().Set("etag", `"`+sha+`"`) |
| 767 | - w.Header().Set("Content-Type", entry.Type) |
| 768 | - w.Header().Set("x-content-type-options", "nosniff") |
| 769 | - w.Header().Set("accept-ranges", "bytes") |
| 770 | - w.Header().Set("content-security-policy", "default-src 'none'") |
| 771 | - if s.config.CanRead == nil { |
| 772 | - w.Header().Set("cache-control", "public, max-age=31536000, immutable") |
| 773 | - } else { |
| 774 | - w.Header().Set("cache-control", "private, no-store") |
| 775 | - } |
| 776 | - http.ServeContent(w, r, sha, time.Unix(entry.Uploaded, 0), file) |
| 777 | -} |
| 778 | - |
| 779 | -func (s *Service) delete(w http.ResponseWriter, r *http.Request, sha string) { |
| 780 | - pubkey, err := s.authorize(r, ActionDelete) |
| 781 | - if err != nil { |
| 782 | - s.fail(w, http.StatusUnauthorized, err) |
| 783 | - return |
| 784 | - } |
| 785 | - _, err = s.lookup(r.Context(), sha) |
| 786 | - if errors.Is(err, sql.ErrNoRows) { |
| 787 | - s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 788 | - return |
| 789 | - } |
| 790 | - if err != nil { |
| 791 | - s.fail(w, http.StatusInternalServerError, err) |
| 792 | - return |
| 793 | - } |
| 794 | - if !s.hasClaim(r.Context(), sha, pubkey) && !s.isOwner(pubkey) { |
| 795 | - s.fail(w, http.StatusForbidden, errors.New("restricted: not the uploader")) |
| 796 | - return |
| 797 | - } |
| 798 | - if s.isOwner(pubkey) { |
| 799 | - if err := s.Delete(r.Context(), sha); err != nil { |
| 800 | - s.fail(w, http.StatusInternalServerError, err) |
| 801 | - return |
| 802 | - } |
| 803 | - } else if err := s.DeleteForUploader(r.Context(), sha, pubkey); err != nil { |
| 804 | - if errors.Is(err, sql.ErrNoRows) { |
| 805 | - s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 806 | - return |
| 807 | - } |
| 808 | - s.fail(w, http.StatusInternalServerError, err) |
| 809 | - return |
| 810 | - } |
| 811 | - // Owners and uploaders may remove their claim; a shared physical object is |
| 812 | - // retained until its final claim disappears. |
| 813 | - w.WriteHeader(http.StatusNoContent) |
| 814 | -} |
| 815 | - |
| 816 | -func (s *Service) put(ctx context.Context, source io.Reader, typ, uploader, claimed string) (Blob, bool, error) { |
| 817 | - return s.putValidatedWithCommit(ctx, source, typ, uploader, claimed, nil, nil) |
| 818 | -} |
| 819 | - |
| 820 | -func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploader, claimed string, validate func(string) error) (Blob, bool, error) { |
| 821 | - return s.putValidatedWithCommit(ctx, source, typ, uploader, claimed, validate, nil) |
| 822 | -} |
| 823 | - |
| 824 | -func (s *Service) putValidatedWithCommit(ctx context.Context, source io.Reader, typ, uploader, claimed string, validate func(string) error, commit func(context.Context, *sql.Tx, Blob, bool) error) (Blob, bool, error) { |
| 825 | - limits := s.currentLimits() |
| 826 | - streamLimit := limits.MaxFileBytes |
| 827 | - quotaStream := false |
| 828 | - if streamLimit == 0 && limits.UserStorageBytes > 0 && uploader != "" { |
| 829 | - streamLimit = limits.UserStorageBytes |
| 830 | - quotaStream = true |
| 831 | - } |
| 832 | - if streamLimit > 0 && streamLimit < int64(^uint64(0)>>1) { |
| 833 | - source = io.LimitReader(source, streamLimit+1) |
| 834 | - } |
| 835 | - tmp, err := os.CreateTemp(s.root, ".upload-") |
| 836 | - if err != nil { |
| 837 | - return Blob{}, false, fmt.Errorf("create upload temporary file: %w", err) |
| 838 | - } |
| 839 | - tmpName := tmp.Name() |
| 840 | - defer os.Remove(tmpName) |
| 841 | - hasher := sha256.New() |
| 842 | - count, err := io.Copy(io.MultiWriter(tmp, hasher), source) |
| 843 | - if err != nil { |
| 844 | - _ = tmp.Close() |
| 845 | - return Blob{}, false, fmt.Errorf("read upload: %w", err) |
| 846 | - } |
| 847 | - if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 848 | - _ = tmp.Close() |
| 849 | - return Blob{}, false, ErrFileTooLarge |
| 850 | - } |
| 851 | - if quotaStream && count > streamLimit { |
| 852 | - _ = tmp.Close() |
| 853 | - return Blob{}, false, ErrQuotaExceeded |
| 854 | - } |
| 855 | - if err := tmp.Sync(); err != nil { |
| 856 | - _ = tmp.Close() |
| 857 | - return Blob{}, false, fmt.Errorf("sync upload: %w", err) |
| 858 | - } |
| 859 | - if err := tmp.Close(); err != nil { |
| 860 | - return Blob{}, false, fmt.Errorf("close upload: %w", err) |
| 861 | - } |
| 862 | - sha := hex.EncodeToString(hasher.Sum(nil)) |
| 863 | - if claimed != "" && claimed != sha { |
| 864 | - return Blob{}, false, ErrHashMismatch |
| 865 | - } |
| 866 | - if validate != nil { |
| 867 | - if err := validate(sha); err != nil { |
| 868 | - return Blob{}, false, err |
| 869 | - } |
| 870 | - } |
| 871 | - return s.installUpload(ctx, uploadCandidate{path: tmpName, hash: sha, size: count, typ: typ, uploader: uploader, commit: commit}) |
| 872 | -} |
| 873 | - |
| 874 | -type uploadCandidate struct { |
| 875 | - path, hash, typ, uploader, reservation string |
| 876 | - size int64 |
| 877 | - commit func(context.Context, *sql.Tx, Blob, bool) error |
| 878 | -} |
| 879 | - |
| 880 | -func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate) (Blob, bool, error) { |
| 881 | - sha, count, typ, uploader := candidate.hash, candidate.size, candidate.typ, candidate.uploader |
| 882 | - // Serialize the lookup, quota check, installation and metadata transaction. |
| 883 | - // This closes the race where concurrent uploads could each observe the same |
| 884 | - // remaining quota before one of them installs a new physical object. |
| 885 | - s.quotaMu.Lock() |
| 886 | - defer s.quotaMu.Unlock() |
| 887 | - if s.blocked(ctx, sha) { |
| 888 | - return Blob{}, false, ErrBlocked |
| 889 | - } |
| 890 | - terms, err := s.uploadTerms(ctx, uploader) |
| 891 | - if err != nil { |
| 892 | - return Blob{}, false, err |
| 893 | - } |
| 894 | - limits := s.currentLimits() |
| 895 | - if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 896 | - return Blob{}, false, ErrFileTooLarge |
| 897 | - } |
| 898 | - if limits.UserStorageBytes > 0 && uploader != "" && count > limits.UserStorageBytes { |
| 899 | - return Blob{}, false, ErrQuotaExceeded |
| 900 | - } |
| 901 | - if existing, err := s.lookup(ctx, sha); err == nil { |
| 902 | - if terms.Encrypted && !encryptedTypes[existing.Type] { |
| 903 | - return Blob{}, false, ErrNotEncrypted |
| 904 | - } |
| 905 | - if uploader != "" { |
| 906 | - if limits.UserStorageBytes > 0 && !s.hasClaim(ctx, sha, uploader) { |
| 907 | - used, usageErr := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 908 | - if usageErr != nil { |
| 909 | - return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", usageErr) |
| 910 | - } |
| 911 | - if existing.Size > limits.UserStorageBytes-used { |
| 912 | - return Blob{}, false, ErrQuotaExceeded |
| 913 | - } |
| 914 | - } |
| 915 | - } |
| 916 | - if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 917 | - if candidate.commit != nil { |
| 918 | - if err := candidate.commit(ctx, tx, existing, false); err != nil { |
| 919 | - return err |
| 920 | - } |
| 921 | - } |
| 922 | - if uploader != "" { |
| 923 | - if _, err := tx.ExecContext(ctx, claimSQL, sha, uploader, time.Now().Unix(), terms.ExpiresAt); err != nil { |
| 924 | - return err |
| 925 | - } |
| 926 | - } |
| 927 | - if candidate.reservation != "" { |
| 928 | - if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 929 | - return err |
| 930 | - } |
| 931 | - } |
| 932 | - return nil |
| 933 | - }); err != nil { |
| 934 | - return Blob{}, false, err |
| 935 | - } |
| 936 | - // A crash after a scoped metadata commit may leave the bytes missing. |
| 937 | - // A retry supplies those same verified bytes without losing the scope. |
| 938 | - if _, err := os.Stat(filepath.Join(s.root, sha)); errors.Is(err, os.ErrNotExist) { |
| 939 | - if err := s.installBlobFile(candidate.path, sha); err != nil { |
| 940 | - return Blob{}, false, err |
| 941 | - } |
| 942 | - } else if err != nil { |
| 943 | - return Blob{}, false, err |
| 944 | - } |
| 945 | - return existing, false, nil |
| 946 | - } else if !errors.Is(err, sql.ErrNoRows) { |
| 947 | - return Blob{}, false, err |
| 948 | - } |
| 949 | - if limits.UserStorageBytes > 0 && uploader != "" { |
| 950 | - used, err := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 951 | - if err != nil { |
| 952 | - return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", err) |
| 953 | - } |
| 954 | - if count > limits.UserStorageBytes-used { |
| 955 | - return Blob{}, false, ErrQuotaExceeded |
| 956 | - } |
| 957 | - } |
| 958 | - // The type is settled before the bytes are installed so a plain upload |
| 959 | - // under an encrypted-only grant leaves nothing behind. A declared octet |
| 960 | - // stream is sniffed: encrypted bytes stay one, a page or an image does not. |
| 961 | - typ = contentType(typ) |
| 962 | - if typ == "application/octet-stream" { |
| 963 | - typ = detectFileType(candidate.path) |
| 964 | - } |
| 965 | - if terms.Encrypted && !encryptedTypes[typ] { |
| 966 | - return Blob{}, false, ErrNotEncrypted |
| 967 | - } |
| 968 | - stage := "" |
| 969 | - if candidate.commit != nil { |
| 970 | - stage = filepath.Join(s.root, ".scoped-"+sha) |
| 971 | - if err := os.Rename(candidate.path, stage); err != nil { |
| 972 | - return Blob{}, false, fmt.Errorf("stage scoped blob: %w", err) |
| 973 | - } |
| 974 | - if err := syncDirectory(s.root); err != nil { |
| 975 | - return Blob{}, false, fmt.Errorf("sync staged blob directory: %w", err) |
| 976 | - } |
| 977 | - } |
| 978 | - if candidate.commit == nil { |
| 979 | - if err := s.installBlobFile(candidate.path, sha); err != nil { |
| 980 | - return Blob{}, false, err |
| 981 | - } |
| 982 | - } |
| 983 | - now := time.Now().UTC().Unix() |
| 984 | - entry := Blob{SHA256: sha, Size: count, Type: typ, Uploader: uploader, Uploaded: now} |
| 985 | - err = s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 986 | - if _, err := tx.ExecContext(ctx, "DELETE FROM blob_tombstones WHERE sha256=?", entry.SHA256); err != nil { |
| 987 | - return fmt.Errorf("clear blob deletion tombstone: %w", err) |
| 988 | - } |
| 989 | - if _, err := tx.ExecContext(ctx, "INSERT OR IGNORE INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)", entry.SHA256, entry.Size, entry.Type, entry.Uploader, entry.Uploaded); err != nil { |
| 990 | - return fmt.Errorf("record blob: %w", err) |
| 991 | - } |
| 992 | - if entry.Uploader != "" { |
| 993 | - if _, err := tx.ExecContext(ctx, claimSQL, entry.SHA256, entry.Uploader, entry.Uploaded, terms.ExpiresAt); err != nil { |
| 994 | - return fmt.Errorf("record blob claim: %w", err) |
| 995 | - } |
| 996 | - } |
| 997 | - if candidate.reservation != "" { |
| 998 | - if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 999 | - return fmt.Errorf("release upload reservation: %w", err) |
| 1000 | - } |
| 1001 | - } |
| 1002 | - if candidate.commit != nil { |
| 1003 | - if err := candidate.commit(ctx, tx, entry, true); err != nil { |
| 1004 | - return err |
| 1005 | - } |
| 1006 | - } |
| 1007 | - return nil |
| 1008 | - }) |
| 1009 | - if err != nil { |
| 1010 | - if stage != "" { |
| 1011 | - _ = os.Remove(stage) |
| 1012 | - } |
| 1013 | - return Blob{}, false, err |
| 1014 | - } |
| 1015 | - if candidate.commit != nil { |
| 1016 | - if err := s.installBlobFile(stage, sha); err != nil { |
| 1017 | - return Blob{}, false, err |
| 1018 | - } |
| 1019 | - } |
| 1020 | - if saved, err := s.lookup(ctx, sha); err == nil { |
| 1021 | - return saved, saved.Uploader == uploader && saved.Uploaded == now, nil |
| 1022 | - } |
| 1023 | - return entry, true, nil |
| 1024 | -} |
| 1025 | - |
| 1026 | -func (s *Service) installBlobFile(source, hash string) error { |
| 1027 | - if err := os.Rename(source, filepath.Join(s.root, hash)); err != nil { |
| 1028 | - return fmt.Errorf("install blob: %w", err) |
| 1029 | - } |
| 1030 | - if err := syncDirectory(s.root); err != nil { |
| 1031 | - return fmt.Errorf("sync blob directory: %w", err) |
| 1032 | - } |
| 1033 | - return nil |
| 1034 | -} |
| 1035 | - |
| 1036 | -func (s *Service) quotaUsage(ctx context.Context, uploader string) (int64, error) { |
| 1037 | - return s.quotaUsageExcept(ctx, uploader, "") |
| 1038 | -} |
| 1039 | - |
| 1040 | -func (s *Service) quotaUsageExcept(ctx context.Context, uploader, reservation string) (int64, error) { |
| 1041 | - var used sql.NullInt64 |
| 1042 | - err := s.store.DB().QueryRowContext(ctx, `SELECT COALESCE((SELECT SUM(b.size) |
| 1043 | - FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=?),0) + |
| 1044 | - COALESCE((SELECT SUM(length) FROM multipart_uploads WHERE uploader=? AND id != ?),0)`, uploader, uploader, reservation).Scan(&used) |
| 1045 | - if err != nil { |
| 1046 | - return 0, err |
| 1047 | - } |
| 1048 | - return used.Int64, nil |
| 1049 | -} |
| 1050 | - |
| 1051 | -// claimSQL records a claim. The insert trigger on blobs has usually made the |
| 1052 | -// row already, without an expiry, so the newest upload's terms replace it. |
| 1053 | -const claimSQL = "INSERT INTO blob_claims(sha256,uploader,claimed_at,expires) VALUES(?,?,?,?) ON CONFLICT(sha256,uploader) DO UPDATE SET expires=excluded.expires" |
| 1054 | - |
| 1055 | -func (s *Service) claim(ctx context.Context, sha, uploader string, expires int64) error { |
| 1056 | - _, err := s.store.DB().ExecContext(ctx, claimSQL, sha, uploader, time.Now().UTC().Unix(), expires) |
| 1057 | - if err != nil { |
| 1058 | - return fmt.Errorf("record blob claim: %w", err) |
| 1059 | - } |
| 1060 | - return nil |
| 1061 | -} |
| 1062 | - |
| 1063 | -func (s *Service) uploadTerms(ctx context.Context, uploader string) (UploadTerms, error) { |
| 1064 | - if s.config.UploadTerms == nil || uploader == "" { |
| 1065 | - return UploadTerms{}, nil |
| 1066 | - } |
| 1067 | - // A refusal keeps its reason so the door answers with the right status. |
| 1068 | - return s.config.UploadTerms(ctx, uploader) |
| 1069 | -} |
| 1070 | - |
| 1071 | -// SweepExpired removes the blobs whose every claim has lapsed and releases |
| 1072 | -// lapsed claims on blobs someone else still holds, so an agent's files leave |
| 1073 | -// with its site while a person's claim keeps them. It returns how many blobs |
| 1074 | -// were removed. |
| 1075 | -func (s *Service) SweepExpired(ctx context.Context, now int64) (int, error) { |
| 1076 | - s.quotaMu.Lock() |
| 1077 | - defer s.quotaMu.Unlock() |
| 1078 | - rows, err := s.store.DB().QueryContext(ctx, `SELECT DISTINCT sha256 FROM blob_claims c WHERE c.expires>0 AND c.expires<=? |
| 1079 | - AND NOT EXISTS(SELECT 1 FROM blob_claims live WHERE live.sha256=c.sha256 AND (live.expires=0 OR live.expires>?))`, now, now) |
| 1080 | - if err != nil { |
| 1081 | - return 0, fmt.Errorf("list expired blobs: %w", err) |
| 1082 | - } |
| 1083 | - var expired []string |
| 1084 | - for rows.Next() { |
| 1085 | - var sha string |
| 1086 | - if err := rows.Scan(&sha); err != nil { |
| 1087 | - _ = rows.Close() |
| 1088 | - return 0, err |
| 1089 | - } |
| 1090 | - expired = append(expired, sha) |
| 1091 | - } |
| 1092 | - if err := errors.Join(rows.Err(), rows.Close()); err != nil { |
| 1093 | - return 0, err |
| 1094 | - } |
| 1095 | - for _, sha := range expired { |
| 1096 | - if err := s.Delete(ctx, sha); err != nil { |
| 1097 | - return 0, err |
| 1098 | - } |
| 1099 | - } |
| 1100 | - if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM blob_claims WHERE expires>0 AND expires<=?", now); err != nil { |
| 1101 | - return 0, fmt.Errorf("release expired blob claims: %w", err) |
| 1102 | - } |
| 1103 | - return len(expired), nil |
| 1104 | -} |
| 1105 | - |
| 1106 | -func (s *Service) currentLimits() Limits { |
| 1107 | - if s.config.Limits == nil { |
| 1108 | - return Limits{} |
| 1109 | - } |
| 1110 | - limits := s.config.Limits() |
| 1111 | - if limits.MaxFileBytes < 0 { |
| 1112 | - limits.MaxFileBytes = 0 |
| 1113 | - } |
| 1114 | - if limits.UserStorageBytes < 0 { |
| 1115 | - limits.UserStorageBytes = 0 |
| 1116 | - } |
| 1117 | - return limits |
| 1118 | -} |
| 1119 | - |
| 1120 | -func (s *Service) hasClaim(ctx context.Context, sha, uploader string) bool { |
| 1121 | - var found int |
| 1122 | - err := s.store.DB().QueryRowContext(ctx, "SELECT 1 FROM blob_claims WHERE sha256=? AND uploader=?", sha, uploader).Scan(&found) |
| 1123 | - return err == nil && found == 1 |
| 1124 | -} |
| 1125 | - |
| 1126 | -func (s *Service) lookup(ctx context.Context, sha string) (Blob, error) { |
| 1127 | - var entry Blob |
| 1128 | - err := s.store.DB().QueryRowContext(ctx, "SELECT sha256,size,type,uploader,uploaded FROM blobs WHERE sha256=?", sha).Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded) |
| 1129 | - return entry, err |
| 1130 | -} |
| 1131 | - |
| 1132 | -func (s *Service) blocked(ctx context.Context, sha string) bool { |
| 1133 | - var found int |
| 1134 | - err := s.store.DB().QueryRowContext(ctx, `SELECT 1 FROM hidden_events WHERE id=? |
| 1135 | - UNION ALL SELECT 1 FROM blob_blocks WHERE sha256=? LIMIT 1`, sha, sha).Scan(&found) |
| 1136 | - return err == nil && found == 1 |
| 1137 | -} |
| 1138 | - |
| 1139 | -func reconcile(ctx context.Context, store *storage.Store, root string) error { |
| 1140 | - entries, err := os.ReadDir(root) |
| 1141 | - if err != nil { |
| 1142 | - return fmt.Errorf("scan blob directory: %w", err) |
| 1143 | - } |
| 1144 | - for _, entry := range entries { |
| 1145 | - if entry.IsDir() { |
| 1146 | - continue |
| 1147 | - } |
| 1148 | - name := entry.Name() |
| 1149 | - if strings.HasPrefix(name, ".scoped-") { |
| 1150 | - sha := strings.TrimPrefix(name, ".scoped-") |
| 1151 | - if !shaPattern.MatchString(sha) { |
| 1152 | - if err := os.Remove(filepath.Join(root, name)); err != nil { |
| 1153 | - return fmt.Errorf("remove invalid staged blob: %w", err) |
| 1154 | - } |
| 1155 | - continue |
| 1156 | - } |
| 1157 | - if err := reconcileStaged(ctx, store, root, name, sha); err != nil { |
| 1158 | - return err |
| 1159 | - } |
| 1160 | - continue |
| 1161 | - } |
| 1162 | - if strings.HasPrefix(name, ".upload-") || strings.HasPrefix(name, ".multipart-chunk-") { |
| 1163 | - if err := os.Remove(filepath.Join(root, name)); err != nil { |
| 1164 | - return fmt.Errorf("remove abandoned upload: %w", err) |
| 1165 | - } |
| 1166 | - continue |
| 1167 | - } |
| 1168 | - if !shaPattern.MatchString(name) { |
| 1169 | - continue |
| 1170 | - } |
| 1171 | - var deleted int |
| 1172 | - if err := store.DB().QueryRowContext(ctx, "SELECT 1 FROM blob_tombstones WHERE sha256=? UNION ALL SELECT 1 FROM blob_blocks WHERE sha256=? LIMIT 1", name, name).Scan(&deleted); err == nil { |
| 1173 | - continue |
| 1174 | - } else if !errors.Is(err, sql.ErrNoRows) { |
| 1175 | - return fmt.Errorf("check deleted blob %q: %w", name, err) |
| 1176 | - } |
| 1177 | - var known int |
| 1178 | - if err := store.DB().QueryRowContext(ctx, "SELECT 1 FROM blobs WHERE sha256=?", name).Scan(&known); err == nil { |
| 1179 | - continue |
| 1180 | - } else if !errors.Is(err, sql.ErrNoRows) { |
| 1181 | - return fmt.Errorf("check orphan blob %q: %w", name, err) |
| 1182 | - } |
| 1183 | - if err := recoverBlob(ctx, store, root, name); err != nil { |
| 1184 | - return err |
| 1185 | - } |
| 1186 | - } |
| 1187 | - return nil |
| 1188 | -} |
| 1189 | - |
| 1190 | -func reconcileStaged(ctx context.Context, store *storage.Store, root, name, sha string) error { |
| 1191 | - var size int64 |
| 1192 | - if err := store.DB().QueryRowContext(ctx, "SELECT size FROM blobs WHERE sha256=?", sha).Scan(&size); errors.Is(err, sql.ErrNoRows) { |
| 1193 | - if removeErr := os.Remove(filepath.Join(root, name)); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { |
| 1194 | - return removeErr |
| 1195 | - } |
| 1196 | - return nil |
| 1197 | - } else if err != nil { |
| 1198 | - return fmt.Errorf("check staged blob %q: %w", sha, err) |
| 1199 | - } |
| 1200 | - file, err := os.Open(filepath.Join(root, name)) |
| 1201 | - if err != nil { |
| 1202 | - return err |
| 1203 | - } |
| 1204 | - info, err := file.Stat() |
| 1205 | - if err != nil { |
| 1206 | - _ = file.Close() |
| 1207 | - return err |
| 1208 | - } |
| 1209 | - if info.Size() != size { |
| 1210 | - _ = file.Close() |
| 1211 | - return fmt.Errorf("staged blob %q has size %d, want %d", sha, info.Size(), size) |
| 1212 | - } |
| 1213 | - hasher := sha256.New() |
| 1214 | - if _, err := io.Copy(hasher, file); err != nil { |
| 1215 | - _ = file.Close() |
| 1216 | - return err |
| 1217 | - } |
| 1218 | - if err := file.Close(); err != nil { |
| 1219 | - return err |
| 1220 | - } |
| 1221 | - if hex.EncodeToString(hasher.Sum(nil)) != sha { |
| 1222 | - return fmt.Errorf("staged blob %q failed content hash", sha) |
| 1223 | - } |
| 1224 | - if _, err := os.Stat(filepath.Join(root, sha)); errors.Is(err, os.ErrNotExist) { |
| 1225 | - if err := os.Rename(filepath.Join(root, name), filepath.Join(root, sha)); err != nil { |
| 1226 | - return err |
| 1227 | - } |
| 1228 | - } else if err != nil { |
| 1229 | - return err |
| 1230 | - } else { |
| 1231 | - _ = os.Remove(filepath.Join(root, name)) |
| 1232 | - } |
| 1233 | - return syncDirectory(root) |
| 1234 | -} |
| 1235 | - |
| 1236 | -func recoverBlob(ctx context.Context, store *storage.Store, root, name string) error { |
| 1237 | - file, err := os.Open(filepath.Join(root, name)) |
| 1238 | - if err != nil { |
| 1239 | - return fmt.Errorf("open orphan blob %q: %w", name, err) |
| 1240 | - } |
| 1241 | - hash := sha256.New() |
| 1242 | - size, copyErr := io.Copy(hash, file) |
| 1243 | - closeErr := file.Close() |
| 1244 | - if copyErr != nil || closeErr != nil { |
| 1245 | - return fmt.Errorf("read orphan blob %q: %w", name, errors.Join(copyErr, closeErr)) |
| 1246 | - } |
| 1247 | - if hex.EncodeToString(hash.Sum(nil)) != name { |
| 1248 | - return fmt.Errorf("orphan blob %q failed content hash", name) |
| 1249 | - } |
| 1250 | - info, err := os.Stat(filepath.Join(root, name)) |
| 1251 | - if err != nil { |
| 1252 | - return fmt.Errorf("stat orphan blob %q: %w", name, err) |
| 1253 | - } |
| 1254 | - _, err = store.DB().ExecContext(ctx, "INSERT OR IGNORE INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)", name, size, "application/octet-stream", "", info.ModTime().Unix()) |
| 1255 | - if err != nil { |
| 1256 | - return fmt.Errorf("recover orphan blob %q: %w", name, err) |
| 1257 | - } |
| 1258 | - return nil |
| 1259 | -} |
| 1260 | - |
| 1261 | -func syncDirectory(path string) error { |
| 1262 | - directory, err := os.Open(path) |
| 1263 | - if err != nil { |
| 1264 | - return err |
| 1265 | - } |
| 1266 | - defer directory.Close() |
| 1267 | - return directory.Sync() |
| 1268 | -} |
| 1269 | - |
| 1270 | -func (s *Service) list(w http.ResponseWriter, r *http.Request, pubkey string) { |
| 1271 | - if !shaPattern.MatchString(pubkey) { |
| 1272 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: bad pubkey")) |
| 1273 | - return |
| 1274 | - } |
| 1275 | - if _, err := s.authorize(r, ActionList); err != nil { |
| 1276 | - s.fail(w, http.StatusUnauthorized, err) |
| 1277 | - return |
| 1278 | - } |
| 1279 | - limit, err := parseListLimit(r.URL.Query().Get("limit")) |
| 1280 | - if err != nil { |
| 1281 | - s.fail(w, http.StatusBadRequest, err) |
| 1282 | - return |
| 1283 | - } |
| 1284 | - entries, _, err := s.ListPage(r.Context(), pubkey, limit, strings.TrimSpace(r.URL.Query().Get("cursor"))) |
| 1285 | - if err != nil { |
| 1286 | - if errors.Is(err, sql.ErrNoRows) { |
| 1287 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: cursor does not identify a blob")) |
| 1288 | - return |
| 1289 | - } |
| 1290 | - s.fail(w, http.StatusInternalServerError, err) |
| 1291 | - return |
| 1292 | - } |
| 1293 | - descriptors := make([]map[string]any, 0, len(entries)) |
| 1294 | - for _, b := range entries { |
| 1295 | - descriptors = append(descriptors, s.descriptor(r, b)) |
| 1296 | - } |
| 1297 | - s.writeJSON(w, http.StatusOK, descriptors) |
| 1298 | -} |
| 1299 | - |
| 1300 | -func (s *Service) report(w http.ResponseWriter, r *http.Request) { |
| 1301 | - reporter, err := s.authorize(r, ActionReport) |
| 1302 | - if err != nil { |
| 1303 | - s.fail(w, http.StatusUnauthorized, err) |
| 1304 | - return |
| 1305 | - } |
| 1306 | - var input event.Event |
| 1307 | - if err := json.NewDecoder(r.Body).Decode(&input); err != nil || input.Kind != 1984 || input.PubKey != reporter { |
| 1308 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: report must be a signed kind 1984 event")) |
| 1309 | - return |
| 1310 | - } |
| 1311 | - if err := event.Validate(input); err != nil || time.Since(time.Unix(input.CreatedAt, 0)) > time.Hour || time.Until(time.Unix(input.CreatedAt, 0)) > time.Hour { |
| 1312 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: report is not a current signed kind 1984 event")) |
| 1313 | - return |
| 1314 | - } |
| 1315 | - filed := 0 |
| 1316 | - type reportTarget struct { |
| 1317 | - hash string |
| 1318 | - uploader string |
| 1319 | - } |
| 1320 | - targets := make([]reportTarget, 0, len(input.Tags)) |
| 1321 | - for _, tag := range input.Tags { |
| 1322 | - if len(tag) < 2 || tag[0] != "x" || !shaPattern.MatchString(tag[1]) { |
| 1323 | - continue |
| 1324 | - } |
| 1325 | - entry, lookupErr := s.lookup(r.Context(), tag[1]) |
| 1326 | - if lookupErr != nil { |
| 1327 | - continue |
| 1328 | - } |
| 1329 | - id := input.ID |
| 1330 | - if filed > 0 { |
| 1331 | - id += "." + strconv.Itoa(filed) |
| 1332 | - } |
| 1333 | - targets = append(targets, reportTarget{hash: tag[1], uploader: entry.Uploader}) |
| 1334 | - } |
| 1335 | - if len(targets) == 0 { |
| 1336 | - s.fail(w, http.StatusNotFound, errors.New("not found: relay holds none of the reported blobs")) |
| 1337 | - return |
| 1338 | - } |
| 1339 | - err = s.store.WithTx(r.Context(), func(tx *sql.Tx) error { |
| 1340 | - for i, target := range targets { |
| 1341 | - id := input.ID |
| 1342 | - if i > 0 { |
| 1343 | - id += "." + strconv.Itoa(i) |
| 1344 | - } |
| 1345 | - result, insertErr := tx.ExecContext(r.Context(), `INSERT OR IGNORE INTO blob_reports(id,reporter,target_pubkey,target_blob,type,content,at) VALUES(?,?,?,?,?,?,?)`, id, reporter, target.uploader, target.hash, "blob", input.Content, input.CreatedAt) |
| 1346 | - if insertErr != nil { |
| 1347 | - return insertErr |
| 1348 | - } |
| 1349 | - changed, rowsErr := result.RowsAffected() |
| 1350 | - if rowsErr != nil { |
| 1351 | - return rowsErr |
| 1352 | - } |
| 1353 | - if changed == 0 { |
| 1354 | - continue |
| 1355 | - } |
| 1356 | - if s.config.RecordReport != nil { |
| 1357 | - if err := s.config.RecordReport(r.Context(), tx, reporter, target.hash, "blob", "blob", input.Content); err != nil { |
| 1358 | - return err |
| 1359 | - } |
| 1360 | - } |
| 1361 | - filed++ |
| 1362 | - } |
| 1363 | - return nil |
| 1364 | - }) |
| 1365 | - if err != nil { |
| 1366 | - s.fail(w, http.StatusInternalServerError, err) |
| 1367 | - return |
| 1368 | - } |
| 1369 | - if filed == 0 { |
| 1370 | - // Repeating an identical report is idempotent and still successful. |
| 1371 | - filed = len(targets) |
| 1372 | - } |
| 1373 | - s.writeJSON(w, http.StatusOK, map[string]any{"ok": true, "filed": filed}) |
| 1374 | -} |
| 1375 | - |
| 1376 | -// Block permanently prevents a hash from returning through any upload door. |
| 1377 | -func (s *Service) Block(ctx context.Context, sha, reason string) error { |
| 1378 | - if !shaPattern.MatchString(sha) { |
| 1379 | - return errors.New("invalid: bad blob sha256") |
| 1380 | - } |
| 1381 | - _, err := s.store.DB().ExecContext(ctx, "INSERT INTO blob_blocks(sha256,reason,blocked_at) VALUES(?,?,?) ON CONFLICT(sha256) DO UPDATE SET reason=excluded.reason,blocked_at=excluded.blocked_at", sha, reason, time.Now().UTC().Unix()) |
| 1382 | - if err != nil { |
| 1383 | - return fmt.Errorf("block blob: %w", err) |
| 1384 | - } |
| 1385 | - if _, err := s.store.DB().ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 1386 | - return fmt.Errorf("record blocked blob deletion: %w", err) |
| 1387 | - } |
| 1388 | - if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 1389 | - return fmt.Errorf("remove blocked blob metadata: %w", err) |
| 1390 | - } |
| 1391 | - if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 1392 | - return fmt.Errorf("remove blocked blob: %w", err) |
| 1393 | - } |
| 1394 | - return nil |
| 1395 | -} |
| 1396 | - |
| 1397 | -func (s *Service) mirror(w http.ResponseWriter, r *http.Request) { |
| 1398 | - pubkey, err := s.authorize(r, ActionMirror) |
| 1399 | - if err != nil { |
| 1400 | - s.fail(w, http.StatusUnauthorized, err) |
| 1401 | - return |
| 1402 | - } |
| 1403 | - var input struct { |
| 1404 | - URL string `json:"url"` |
| 1405 | - } |
| 1406 | - if err := json.NewDecoder(r.Body).Decode(&input); err != nil || input.URL == "" { |
| 1407 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: body needs a url")) |
| 1408 | - return |
| 1409 | - } |
| 1410 | - requestedURL := input.URL |
| 1411 | - expectedHash := "" |
| 1412 | - extension := "" |
| 1413 | - uriServers := []string{} |
| 1414 | - authors := []string{} |
| 1415 | - if strings.HasPrefix(strings.ToLower(requestedURL), "blossom:") { |
| 1416 | - uri, err := ParseBlossomURI(requestedURL) |
| 1417 | - if err != nil { |
| 1418 | - s.fail(w, http.StatusBadRequest, err) |
| 1419 | - return |
| 1420 | - } |
| 1421 | - expectedHash, extension = uri.Hash, uri.Extension |
| 1422 | - uriServers = append(uriServers, uri.Servers...) |
| 1423 | - authors = append(authors, uri.Authors...) |
| 1424 | - requestedURL = "" |
| 1425 | - } else if hash := BlobHashFromURL(requestedURL); hash != "" { |
| 1426 | - expectedHash = hash |
| 1427 | - } |
| 1428 | - client := s.config.HTTPClient |
| 1429 | - if client == nil { |
| 1430 | - client = s.safeClient() |
| 1431 | - } |
| 1432 | - candidates := []string{} |
| 1433 | - if requestedURL != "" { |
| 1434 | - candidates = append(candidates, requestedURL) |
| 1435 | - } |
| 1436 | - candidates = append(candidates, FallbackURLs(uriServers, expectedHash, extension)...) |
| 1437 | - if s.config.ResolveServers != nil && expectedHash != "" { |
| 1438 | - if len(authors) == 0 && pubkey != "" { |
| 1439 | - authors = []string{pubkey} |
| 1440 | - } |
| 1441 | - for _, author := range uniqueStrings(authors) { |
| 1442 | - servers, resolveErr := s.config.ResolveServers(r.Context(), author) |
| 1443 | - if resolveErr != nil { |
| 1444 | - continue |
| 1445 | - } |
| 1446 | - candidates = append(candidates, FallbackURLs(servers, expectedHash, extension)...) |
| 1447 | - } |
| 1448 | - } |
| 1449 | - var lastErr error |
| 1450 | - for _, candidate := range uniqueStrings(candidates) { |
| 1451 | - if err := validateMirrorURL(candidate, s.resolve); err != nil { |
| 1452 | - lastErr = err |
| 1453 | - continue |
| 1454 | - } |
| 1455 | - response, fetchErr := client.Get(candidate) |
| 1456 | - if fetchErr != nil { |
| 1457 | - lastErr = fetchErr |
| 1458 | - continue |
| 1459 | - } |
| 1460 | - if response.StatusCode < 200 || response.StatusCode >= 300 { |
| 1461 | - lastErr = fmt.Errorf("error: origin answered %d", response.StatusCode) |
| 1462 | - _ = response.Body.Close() |
| 1463 | - continue |
| 1464 | - } |
| 1465 | - typ := contentType(response.Header.Get("content-type")) |
| 1466 | - if typ == "application/octet-stream" { |
| 1467 | - typ = typeForExtension(candidate) |
| 1468 | - } |
| 1469 | - entry, created, putErr := s.putValidated(r.Context(), response.Body, typ, pubkey, expectedHash, func(actual string) error { |
| 1470 | - if s.config.ValidateUpload != nil { |
| 1471 | - return s.config.ValidateUpload(r, actual) |
| 1472 | - } |
| 1473 | - return nil |
| 1474 | - }) |
| 1475 | - _ = response.Body.Close() |
| 1476 | - if putErr == nil { |
| 1477 | - s.writeJSON(w, chooseStatus(created, http.StatusCreated), s.descriptor(r, entry)) |
| 1478 | - return |
| 1479 | - } |
| 1480 | - lastErr = putErr |
| 1481 | - if !errors.Is(putErr, ErrHashMismatch) { |
| 1482 | - s.fail(w, statusFor(putErr), putErr) |
| 1483 | - return |
| 1484 | - } |
| 1485 | - } |
| 1486 | - if lastErr == nil { |
| 1487 | - lastErr = errors.New("error: no mirror origin available") |
| 1488 | - } |
| 1489 | - s.fail(w, statusFor(lastErr), lastErr) |
| 1490 | -} |
| 1491 | - |
| 1492 | -func (s *Service) nip96(w http.ResponseWriter, r *http.Request) { |
| 1493 | - if r.Method == http.MethodGet { |
| 1494 | - pubkey, err := s.authorize(r, ActionList) |
| 1495 | - if err != nil { |
| 1496 | - s.fail(w, http.StatusUnauthorized, err) |
| 1497 | - return |
| 1498 | - } |
| 1499 | - s.nip96List(w, r, pubkey) |
| 1500 | - return |
| 1501 | - } |
| 1502 | - if r.Method != http.MethodPost { |
| 1503 | - s.fail(w, http.StatusMethodNotAllowed, errors.New("method not allowed")) |
| 1504 | - return |
| 1505 | - } |
| 1506 | - pubkey, err := s.authorize(r, ActionUpload) |
| 1507 | - if err != nil { |
| 1508 | - s.fail(w, http.StatusUnauthorized, err) |
| 1509 | - return |
| 1510 | - } |
| 1511 | - if err := r.ParseMultipartForm(0); err != nil { |
| 1512 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: body is not multipart form data")) |
| 1513 | - return |
| 1514 | - } |
| 1515 | - defer r.MultipartForm.RemoveAll() |
| 1516 | - file, header, err := r.FormFile("file") |
| 1517 | - if err != nil { |
| 1518 | - s.fail(w, http.StatusBadRequest, errors.New("invalid: form needs a file field")) |
| 1519 | - return |
| 1520 | - } |
| 1521 | - defer file.Close() |
| 1522 | - typ := contentType(header.Header.Get("Content-Type")) |
| 1523 | - if typ == "application/octet-stream" { |
| 1524 | - typ = contentType(r.FormValue("content_type")) |
| 1525 | - } |
| 1526 | - entry, created, err := s.put(r.Context(), file, typ, pubkey, "") |
| 1527 | - if err != nil { |
| 1528 | - s.fail(w, statusFor(err), err) |
| 1529 | - return |
| 1530 | - } |
| 1531 | - s.writeJSON(w, chooseStatus(created, http.StatusCreated), map[string]any{"status": "success", "message": "Upload successful.", "url": s.blobURL(r, entry), "nip94_event": map[string]any{"tags": s.nip94Tags(r, entry), "content": r.FormValue("caption")}}) |
| 1532 | -} |
| 1533 | - |
| 1534 | -func (s *Service) nip96List(w http.ResponseWriter, r *http.Request, pubkey string) { |
| 1535 | - count := parsePageCount(r.URL.Query().Get("count")) |
| 1536 | - page := parsePage(r.URL.Query().Get("page")) |
| 1537 | - var total int |
| 1538 | - if err := s.store.DB().QueryRowContext(r.Context(), "SELECT count(*) FROM blob_claims WHERE uploader=?", pubkey).Scan(&total); err != nil { |
| 1539 | - s.fail(w, http.StatusInternalServerError, err) |
| 1540 | - return |
| 1541 | - } |
| 1542 | - rows, err := s.store.DB().QueryContext(r.Context(), `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded |
| 1543 | - FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=? |
| 1544 | - ORDER BY b.uploaded DESC,b.sha256 DESC LIMIT ? OFFSET ?`, pubkey, count, page*count) |
| 1545 | - if err != nil { |
| 1546 | - s.fail(w, http.StatusInternalServerError, err) |
| 1547 | - return |
| 1548 | - } |
| 1549 | - defer rows.Close() |
| 1550 | - files := make([]map[string]any, 0) |
| 1551 | - for rows.Next() { |
| 1552 | - var b Blob |
| 1553 | - if err := rows.Scan(&b.SHA256, &b.Size, &b.Type, &b.Uploader, &b.Uploaded); err != nil { |
| 1554 | - s.fail(w, http.StatusInternalServerError, err) |
| 1555 | - return |
| 1556 | - } |
| 1557 | - files = append(files, map[string]any{"tags": s.nip94Tags(r, b), "content": "", "created_at": b.Uploaded}) |
| 1558 | - } |
| 1559 | - s.writeJSON(w, http.StatusOK, map[string]any{"count": count, "total": total, "page": page, "files": files}) |
| 1560 | -} |
| 1561 | - |
| 1562 | -func (s *Service) authorize(r *http.Request, action Action) (string, error) { |
| 1563 | - if s.config.Authorize == nil { |
| 1564 | - return "", errors.New("auth-required: authorization is required") |
| 1565 | - } |
| 1566 | - return s.config.Authorize(r, action) |
| 1567 | -} |
| 1568 | -func (s *Service) isOwner(pubkey string) bool { |
| 1569 | - return s.config.IsOwner != nil && s.config.IsOwner(pubkey) |
| 1570 | -} |
| 1571 | -func (s *Service) descriptor(r *http.Request, b Blob) map[string]any { |
| 1572 | - return map[string]any{"url": s.blobURL(r, b), "sha256": b.SHA256, "size": b.Size, "type": b.Type, "uploaded": b.Uploaded, "nip94": s.nip94Tags(r, b)} |
| 1573 | -} |
| 1574 | -func (s *Service) nip94Tags(r *http.Request, b Blob) [][]string { |
| 1575 | - return [][]string{{"url", s.blobURL(r, b)}, {"m", b.Type}, {"x", b.SHA256}, {"ox", b.SHA256}, {"size", strconv.FormatInt(b.Size, 10)}} |
| 1576 | -} |
| 1577 | -func (s *Service) blobURL(r *http.Request, b Blob) string { |
| 1578 | - return s.origin(r) + "/" + b.SHA256 + extensionSuffix(b.Type) |
| 1579 | -} |
| 1580 | -func (s *Service) nip96Info(r *http.Request) map[string]any { |
| 1581 | - return map[string]any{"api_url": s.origin(r) + "/nip96", "download_url": s.origin(r), "supported_nips": []int{94, 98}, "content_types": contentTypes(), "plans": map[string]any{"self-hosted": map[string]any{"name": "Self-hosted", "is_nip98_required": true, "max_byte_size": 0, "file_expiration": []int{0, 0}, "media_transformations": map[string]any{}}}} |
| 1582 | -} |
| 1583 | -func (s *Service) writeJSON(w http.ResponseWriter, status int, value any) { |
| 1584 | - w.Header().Set("content-type", "application/json") |
| 1585 | - w.WriteHeader(status) |
| 1586 | - _ = json.NewEncoder(w).Encode(value) |
| 1587 | -} |
| 1588 | -func (s *Service) fail(w http.ResponseWriter, status int, err error) { |
| 1589 | - w.Header().Set("x-reason", err.Error()) |
| 1590 | - s.writeJSON(w, status, map[string]string{"error": err.Error()}) |
| 1591 | -} |
| 1592 | -func origin(r *http.Request) string { |
| 1593 | - scheme := r.URL.Scheme |
| 1594 | - if scheme == "" { |
| 1595 | - scheme = "http" |
| 1596 | - if r.TLS != nil { |
| 1597 | - scheme = "https" |
| 1598 | - } |
| 1599 | - } |
| 1600 | - return scheme + "://" + r.Host |
| 1601 | -} |
| 1602 | - |
| 1603 | -func (s *Service) origin(r *http.Request) string { |
| 1604 | - if s.config.PublicURL != "" { |
| 1605 | - return strings.TrimRight(s.config.PublicURL, "/") |
| 1606 | - } |
| 1607 | - return origin(r) |
| 1608 | -} |
| 1609 | - |
| 1610 | -// DetectMediaType recognises images, video, audio and PDF from the first |
| 1611 | -// bytes of a blob. Anything else stays an octet stream so uploads can never |
| 1612 | -// be reinterpreted as a page or a script. |
| 1613 | -func DetectMediaType(head []byte) string { |
| 1614 | - detected := contentType(http.DetectContentType(head)) |
| 1615 | - for _, prefix := range []string{"image/", "video/", "audio/"} { |
| 1616 | - if strings.HasPrefix(detected, prefix) { |
| 1617 | - return detected |
| 1618 | - } |
| 1619 | - } |
| 1620 | - if detected == "application/pdf" { |
| 1621 | - return detected |
| 1622 | - } |
| 1623 | - return "application/octet-stream" |
| 1624 | -} |
| 1625 | - |
| 1626 | -func detectFileType(name string) string { |
| 1627 | - file, err := os.Open(name) |
| 1628 | - if err != nil { |
| 1629 | - return "application/octet-stream" |
| 1630 | - } |
| 1631 | - defer file.Close() |
| 1632 | - head := make([]byte, 512) |
| 1633 | - n, _ := io.ReadFull(file, head) |
| 1634 | - return DetectMediaType(head[:n]) |
| 1635 | -} |
| 1636 | - |
| 1637 | -func contentType(raw string) string { |
| 1638 | - raw = strings.ToLower(strings.TrimSpace(strings.Split(raw, ";")[0])) |
| 1639 | - if raw == "" { |
| 1640 | - return "application/octet-stream" |
| 1641 | - } |
| 1642 | - return raw |
| 1643 | -} |
| 1644 | -func extensionSuffix(typ string) string { |
| 1645 | - if ext := extensions[typ]; ext != "" { |
| 1646 | - return "." + ext |
| 1647 | - } |
| 1648 | - return "" |
| 1649 | -} |
| 1650 | - |
| 1651 | -func typeForExtension(raw string) string { |
| 1652 | - u, err := url.Parse(raw) |
| 1653 | - if err != nil { |
| 1654 | - return "application/octet-stream" |
| 1655 | - } |
| 1656 | - name := strings.ToLower(filepath.Ext(u.Path)) |
| 1657 | - for typ, ext := range extensions { |
| 1658 | - if "."+ext == name { |
| 1659 | - return typ |
| 1660 | - } |
| 1661 | - } |
| 1662 | - return "application/octet-stream" |
| 1663 | -} |
| 1664 | -func contentTypes() []string { |
| 1665 | - out := make([]string, 0, len(extensions)) |
| 1666 | - for typ := range extensions { |
| 1667 | - out = append(out, typ) |
| 1668 | - } |
| 1669 | - sort.Strings(out) |
| 1670 | - return out |
| 1671 | -} |
| 1672 | - |
| 1673 | -func parsePageCount(raw string) int { |
| 1674 | - value, err := strconv.Atoi(raw) |
| 1675 | - if err != nil || value < 1 { |
| 1676 | - return 20 |
| 1677 | - } |
| 1678 | - if value > 100 { |
| 1679 | - return 100 |
| 1680 | - } |
| 1681 | - return value |
| 1682 | -} |
| 1683 | - |
| 1684 | -func parsePage(raw string) int { |
| 1685 | - value, err := strconv.Atoi(raw) |
| 1686 | - if err != nil || value < 0 { |
| 1687 | - return 0 |
| 1688 | - } |
| 1689 | - return value |
| 1690 | -} |
| 1691 | - |
| 1692 | -func parseListLimit(raw string) (int, error) { |
| 1693 | - if raw == "" { |
| 1694 | - return 20, nil |
| 1695 | - } |
| 1696 | - value, err := strconv.Atoi(raw) |
| 1697 | - if err != nil || value < 1 || value > 100 { |
| 1698 | - return 0, errors.New("invalid: limit must be between 1 and 100") |
| 1699 | - } |
| 1700 | - return value, nil |
| 1701 | -} |
| 1702 | -func chooseStatus(created bool, status int) int { |
| 1703 | - if created { |
| 1704 | - return status |
| 1705 | - } |
| 1706 | - return http.StatusOK |
| 1707 | -} |
| 1708 | -func statusFor(err error) int { |
| 1709 | - if errors.Is(err, ErrHashMismatch) { |
| 1710 | - return http.StatusConflict |
| 1711 | - } |
| 1712 | - if errors.Is(err, ErrMultipartConflict) { |
| 1713 | - return http.StatusConflict |
| 1714 | - } |
| 1715 | - if errors.Is(err, ErrFileTooLarge) { |
| 1716 | - return http.StatusRequestEntityTooLarge |
| 1717 | - } |
| 1718 | - if errors.Is(err, ErrQuotaExceeded) { |
| 1719 | - return http.StatusForbidden |
| 1720 | - } |
| 1721 | - if errors.Is(err, ErrBlocked) || errors.Is(err, ErrNotEncrypted) { |
| 1722 | - return http.StatusForbidden |
| 1723 | - } |
| 1724 | - if strings.HasPrefix(err.Error(), "auth-required:") { |
| 1725 | - return http.StatusUnauthorized |
| 1726 | - } |
| 1727 | - if strings.HasPrefix(err.Error(), "invalid:") { |
| 1728 | - return http.StatusBadRequest |
| 1729 | - } |
| 1730 | - if strings.HasPrefix(err.Error(), "restricted:") { |
| 1731 | - return http.StatusForbidden |
| 1732 | - } |
| 1733 | - return http.StatusInternalServerError |
| 1734 | -} |
| 1735 | - |
| 1736 | -func uniqueStrings(values []string) []string { |
| 1737 | - seen := make(map[string]struct{}, len(values)) |
| 1738 | - result := make([]string, 0, len(values)) |
| 1739 | - for _, value := range values { |
| 1740 | - if _, ok := seen[value]; ok || value == "" { |
| 1741 | - continue |
| 1742 | - } |
| 1743 | - seen[value] = struct{}{} |
| 1744 | - result = append(result, value) |
| 1745 | - } |
| 1746 | - return result |
| 1747 | -} |
| 1748 | - |
| 1749 | -func validateMirrorURL(raw string, resolve ResolveIP) error { |
| 1750 | - u, err := url.Parse(raw) |
| 1751 | - if err != nil || u.Scheme != "https" || u.Hostname() == "" { |
| 1752 | - return errors.New("invalid: only https urls can be mirrored") |
| 1753 | - } |
| 1754 | - ips, err := resolve(context.Background(), u.Hostname()) |
| 1755 | - if err != nil || len(ips) == 0 { |
| 1756 | - return errors.New("invalid: origin hostname did not resolve") |
| 1757 | - } |
| 1758 | - for _, ip := range ips { |
| 1759 | - if !publicIP(ip) { |
| 1760 | - return errors.New("invalid: origin resolves to a private address") |
| 1761 | - } |
| 1762 | - } |
| 1763 | - return nil |
| 1764 | -} |
| 1765 | - |
| 1766 | -func publicIP(ip net.IP) bool { |
| 1767 | - if ip == nil || ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified() { |
| 1768 | - return false |
| 1769 | - } |
| 1770 | - if ip4 := ip.To4(); ip4 != nil { |
| 1771 | - if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 { |
| 1772 | - return false |
| 1773 | - } |
| 1774 | - if ip4[0] == 192 && ip4[1] == 0 && ip4[2] == 0 { |
| 1775 | - return false |
| 1776 | - } |
| 1777 | - if ip4[0] == 198 && (ip4[1] == 18 || ip4[1] == 19) { |
| 1778 | - return false |
| 1779 | - } |
| 1780 | - } |
| 1781 | - return true |
| 1782 | -} |
| 1783 | -func (s *Service) safeClient() *http.Client { |
| 1784 | - base, ok := http.DefaultTransport.(*http.Transport) |
| 1785 | - if !ok { |
| 1786 | - base = &http.Transport{} |
| 1787 | - } |
| 1788 | - transport := base.Clone() |
| 1789 | - dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} |
| 1790 | - transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { |
| 1791 | - host, port, err := net.SplitHostPort(address) |
| 1792 | - if err != nil { |
| 1793 | - return nil, err |
| 1794 | - } |
| 1795 | - ips, err := s.resolve(ctx, host) |
| 1796 | - if err != nil { |
| 1797 | - return nil, err |
| 1798 | - } |
| 1799 | - for _, ip := range ips { |
| 1800 | - if !publicIP(ip) { |
| 1801 | - continue |
| 1802 | - } |
| 1803 | - conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) |
| 1804 | - if err == nil { |
| 1805 | - return conn, nil |
| 1806 | - } |
| 1807 | - } |
| 1808 | - return nil, errors.New("origin has no reachable public address") |
| 1809 | - } |
| 1810 | - return &http.Client{Timeout: 5 * time.Minute, Transport: transport, CheckRedirect: func(req *http.Request, _ []*http.Request) error { |
| 1811 | - return validateMirrorURL(req.URL.String(), s.resolve) |
| 1812 | - }} |
| 1813 | -} |
| 1814 | diff --git a/internal/blob/doc.go b/internal/blob/doc.go |
| 1815 | new file mode 100644 |
| 1816 | index 0000000..c0ee9d9 |
| 1817 | --- /dev/null |
| 1818 | +++ b/internal/blob/doc.go |
| 1819 | @@ -0,0 +1,2 @@ |
| 1820 | +// Package blob implements the Blossom and NIP-96 file doors for a local relay. |
| 1821 | +package blob |
| 1822 | diff --git a/internal/blob/format.go b/internal/blob/format.go |
| 1823 | new file mode 100644 |
| 1824 | index 0000000..1d1c05a |
| 1825 | --- /dev/null |
| 1826 | +++ b/internal/blob/format.go |
| 1827 | @@ -0,0 +1,221 @@ |
| 1828 | +package blob |
| 1829 | + |
| 1830 | +import ( |
| 1831 | + "context" |
| 1832 | + "errors" |
| 1833 | + "io" |
| 1834 | + "net" |
| 1835 | + "net/http" |
| 1836 | + "net/url" |
| 1837 | + "os" |
| 1838 | + "path/filepath" |
| 1839 | + "sort" |
| 1840 | + "strconv" |
| 1841 | + "strings" |
| 1842 | + "time" |
| 1843 | +) |
| 1844 | + |
| 1845 | +// DetectMediaType recognises images, video, audio and PDF from the first |
| 1846 | +// bytes of a blob. Anything else stays an octet stream so uploads can never |
| 1847 | +// be reinterpreted as a page or a script. |
| 1848 | +func DetectMediaType(head []byte) string { |
| 1849 | + detected := contentType(http.DetectContentType(head)) |
| 1850 | + for _, prefix := range []string{"image/", "video/", "audio/"} { |
| 1851 | + if strings.HasPrefix(detected, prefix) { |
| 1852 | + return detected |
| 1853 | + } |
| 1854 | + } |
| 1855 | + if detected == "application/pdf" { |
| 1856 | + return detected |
| 1857 | + } |
| 1858 | + return "application/octet-stream" |
| 1859 | +} |
| 1860 | + |
| 1861 | +func detectFileType(name string) string { |
| 1862 | + file, err := os.Open(name) |
| 1863 | + if err != nil { |
| 1864 | + return "application/octet-stream" |
| 1865 | + } |
| 1866 | + defer file.Close() |
| 1867 | + head := make([]byte, 512) |
| 1868 | + n, _ := io.ReadFull(file, head) |
| 1869 | + return DetectMediaType(head[:n]) |
| 1870 | +} |
| 1871 | + |
| 1872 | +func contentType(raw string) string { |
| 1873 | + raw = strings.ToLower(strings.TrimSpace(strings.Split(raw, ";")[0])) |
| 1874 | + if raw == "" { |
| 1875 | + return "application/octet-stream" |
| 1876 | + } |
| 1877 | + return raw |
| 1878 | +} |
| 1879 | +func extensionSuffix(typ string) string { |
| 1880 | + if ext := extensions[typ]; ext != "" { |
| 1881 | + return "." + ext |
| 1882 | + } |
| 1883 | + return "" |
| 1884 | +} |
| 1885 | + |
| 1886 | +func typeForExtension(raw string) string { |
| 1887 | + u, err := url.Parse(raw) |
| 1888 | + if err != nil { |
| 1889 | + return "application/octet-stream" |
| 1890 | + } |
| 1891 | + name := strings.ToLower(filepath.Ext(u.Path)) |
| 1892 | + for typ, ext := range extensions { |
| 1893 | + if "."+ext == name { |
| 1894 | + return typ |
| 1895 | + } |
| 1896 | + } |
| 1897 | + return "application/octet-stream" |
| 1898 | +} |
| 1899 | +func contentTypes() []string { |
| 1900 | + out := make([]string, 0, len(extensions)) |
| 1901 | + for typ := range extensions { |
| 1902 | + out = append(out, typ) |
| 1903 | + } |
| 1904 | + sort.Strings(out) |
| 1905 | + return out |
| 1906 | +} |
| 1907 | + |
| 1908 | +func parsePageCount(raw string) int { |
| 1909 | + value, err := strconv.Atoi(raw) |
| 1910 | + if err != nil || value < 1 { |
| 1911 | + return 20 |
| 1912 | + } |
| 1913 | + if value > 100 { |
| 1914 | + return 100 |
| 1915 | + } |
| 1916 | + return value |
| 1917 | +} |
| 1918 | + |
| 1919 | +func parsePage(raw string) int { |
| 1920 | + value, err := strconv.Atoi(raw) |
| 1921 | + if err != nil || value < 0 { |
| 1922 | + return 0 |
| 1923 | + } |
| 1924 | + return value |
| 1925 | +} |
| 1926 | + |
| 1927 | +func parseListLimit(raw string) (int, error) { |
| 1928 | + if raw == "" { |
| 1929 | + return 20, nil |
| 1930 | + } |
| 1931 | + value, err := strconv.Atoi(raw) |
| 1932 | + if err != nil || value < 1 || value > 100 { |
| 1933 | + return 0, errors.New("invalid: limit must be between 1 and 100") |
| 1934 | + } |
| 1935 | + return value, nil |
| 1936 | +} |
| 1937 | +func chooseStatus(created bool, status int) int { |
| 1938 | + if created { |
| 1939 | + return status |
| 1940 | + } |
| 1941 | + return http.StatusOK |
| 1942 | +} |
| 1943 | +func statusFor(err error) int { |
| 1944 | + if errors.Is(err, ErrHashMismatch) { |
| 1945 | + return http.StatusConflict |
| 1946 | + } |
| 1947 | + if errors.Is(err, ErrMultipartConflict) { |
| 1948 | + return http.StatusConflict |
| 1949 | + } |
| 1950 | + if errors.Is(err, ErrFileTooLarge) { |
| 1951 | + return http.StatusRequestEntityTooLarge |
| 1952 | + } |
| 1953 | + if errors.Is(err, ErrQuotaExceeded) { |
| 1954 | + return http.StatusForbidden |
| 1955 | + } |
| 1956 | + if errors.Is(err, ErrBlocked) || errors.Is(err, ErrNotEncrypted) { |
| 1957 | + return http.StatusForbidden |
| 1958 | + } |
| 1959 | + if strings.HasPrefix(err.Error(), "auth-required:") { |
| 1960 | + return http.StatusUnauthorized |
| 1961 | + } |
| 1962 | + if strings.HasPrefix(err.Error(), "invalid:") { |
| 1963 | + return http.StatusBadRequest |
| 1964 | + } |
| 1965 | + if strings.HasPrefix(err.Error(), "restricted:") { |
| 1966 | + return http.StatusForbidden |
| 1967 | + } |
| 1968 | + return http.StatusInternalServerError |
| 1969 | +} |
| 1970 | + |
| 1971 | +func uniqueStrings(values []string) []string { |
| 1972 | + seen := make(map[string]struct{}, len(values)) |
| 1973 | + result := make([]string, 0, len(values)) |
| 1974 | + for _, value := range values { |
| 1975 | + if _, ok := seen[value]; ok || value == "" { |
| 1976 | + continue |
| 1977 | + } |
| 1978 | + seen[value] = struct{}{} |
| 1979 | + result = append(result, value) |
| 1980 | + } |
| 1981 | + return result |
| 1982 | +} |
| 1983 | + |
| 1984 | +func validateMirrorURL(raw string, resolve ResolveIP) error { |
| 1985 | + u, err := url.Parse(raw) |
| 1986 | + if err != nil || u.Scheme != "https" || u.Hostname() == "" { |
| 1987 | + return errors.New("invalid: only https urls can be mirrored") |
| 1988 | + } |
| 1989 | + ips, err := resolve(context.Background(), u.Hostname()) |
| 1990 | + if err != nil || len(ips) == 0 { |
| 1991 | + return errors.New("invalid: origin hostname did not resolve") |
| 1992 | + } |
| 1993 | + for _, ip := range ips { |
| 1994 | + if !publicIP(ip) { |
| 1995 | + return errors.New("invalid: origin resolves to a private address") |
| 1996 | + } |
| 1997 | + } |
| 1998 | + return nil |
| 1999 | +} |
| 2000 | + |
| 2001 | +func publicIP(ip net.IP) bool { |
| 2002 | + if ip == nil || ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsUnspecified() { |
| 2003 | + return false |
| 2004 | + } |
| 2005 | + if ip4 := ip.To4(); ip4 != nil { |
| 2006 | + if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 { |
| 2007 | + return false |
| 2008 | + } |
| 2009 | + if ip4[0] == 192 && ip4[1] == 0 && ip4[2] == 0 { |
| 2010 | + return false |
| 2011 | + } |
| 2012 | + if ip4[0] == 198 && (ip4[1] == 18 || ip4[1] == 19) { |
| 2013 | + return false |
| 2014 | + } |
| 2015 | + } |
| 2016 | + return true |
| 2017 | +} |
| 2018 | +func (s *Service) safeClient() *http.Client { |
| 2019 | + base, ok := http.DefaultTransport.(*http.Transport) |
| 2020 | + if !ok { |
| 2021 | + base = &http.Transport{} |
| 2022 | + } |
| 2023 | + transport := base.Clone() |
| 2024 | + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} |
| 2025 | + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { |
| 2026 | + host, port, err := net.SplitHostPort(address) |
| 2027 | + if err != nil { |
| 2028 | + return nil, err |
| 2029 | + } |
| 2030 | + ips, err := s.resolve(ctx, host) |
| 2031 | + if err != nil { |
| 2032 | + return nil, err |
| 2033 | + } |
| 2034 | + for _, ip := range ips { |
| 2035 | + if !publicIP(ip) { |
| 2036 | + continue |
| 2037 | + } |
| 2038 | + conn, err := dialer.DialContext(ctx, network, net.JoinHostPort(ip.String(), port)) |
| 2039 | + if err == nil { |
| 2040 | + return conn, nil |
| 2041 | + } |
| 2042 | + } |
| 2043 | + return nil, errors.New("origin has no reachable public address") |
| 2044 | + } |
| 2045 | + return &http.Client{Timeout: 5 * time.Minute, Transport: transport, CheckRedirect: func(req *http.Request, _ []*http.Request) error { |
| 2046 | + return validateMirrorURL(req.URL.String(), s.resolve) |
| 2047 | + }} |
| 2048 | +} |
| 2049 | diff --git a/internal/blob/http.go b/internal/blob/http.go |
| 2050 | new file mode 100644 |
| 2051 | index 0000000..0ad5772 |
| 2052 | --- /dev/null |
| 2053 | +++ b/internal/blob/http.go |
| 2054 | @@ -0,0 +1,588 @@ |
| 2055 | +package blob |
| 2056 | + |
| 2057 | +import ( |
| 2058 | + "context" |
| 2059 | + "database/sql" |
| 2060 | + "encoding/json" |
| 2061 | + "errors" |
| 2062 | + "fmt" |
| 2063 | + "net/http" |
| 2064 | + "os" |
| 2065 | + "path/filepath" |
| 2066 | + "strconv" |
| 2067 | + "strings" |
| 2068 | + "time" |
| 2069 | + |
| 2070 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 2071 | +) |
| 2072 | + |
| 2073 | +func (s *Service) serveHTTP(w http.ResponseWriter, r *http.Request) { |
| 2074 | + w.Header().Set("access-control-allow-origin", "*") |
| 2075 | + w.Header().Set("access-control-allow-headers", "authorization, content-type, x-sha-256, x-content-length, x-content-type, upload-type, upload-length, upload-offset") |
| 2076 | + w.Header().Set("access-control-allow-methods", "GET, HEAD, PUT, PATCH, POST, DELETE, OPTIONS") |
| 2077 | + w.Header().Set("Access-Control-Expose-Headers", "Allow, X-Reason") |
| 2078 | + if r.Method == http.MethodOptions { |
| 2079 | + if blobPathPattern.MatchString(r.URL.Path) { |
| 2080 | + w.Header().Set("Allow", "GET, HEAD, PUT, PATCH, DELETE, OPTIONS") |
| 2081 | + } |
| 2082 | + w.WriteHeader(http.StatusNoContent) |
| 2083 | + return |
| 2084 | + } |
| 2085 | + if r.URL.Path == "/.well-known/nostr/nip96.json" { |
| 2086 | + s.writeJSON(w, http.StatusOK, s.nip96Info(r)) |
| 2087 | + return |
| 2088 | + } |
| 2089 | + if r.URL.Path == "/upload" && r.Method == http.MethodHead { |
| 2090 | + s.uploadHead(w, r) |
| 2091 | + return |
| 2092 | + } |
| 2093 | + if r.URL.Path == "/upload" && r.Method == http.MethodPut { |
| 2094 | + s.upload(w, r) |
| 2095 | + return |
| 2096 | + } |
| 2097 | + if r.URL.Path == "/mirror" && r.Method == http.MethodPut { |
| 2098 | + s.mirror(w, r) |
| 2099 | + return |
| 2100 | + } |
| 2101 | + if match := blobPathPattern.FindStringSubmatch(r.URL.Path); match != nil && (r.Method == http.MethodPut || r.Method == http.MethodPatch || r.Method == http.MethodOptions) { |
| 2102 | + if r.Method == http.MethodPatch || r.Method == http.MethodOptions { |
| 2103 | + s.multipart(w, r, match[1]) |
| 2104 | + return |
| 2105 | + } |
| 2106 | + s.pathUpload(w, r, match[1]) |
| 2107 | + return |
| 2108 | + } |
| 2109 | + if r.URL.Path == "/report" && r.Method == http.MethodPut { |
| 2110 | + s.report(w, r) |
| 2111 | + return |
| 2112 | + } |
| 2113 | + if strings.HasPrefix(r.URL.Path, "/list/") && r.Method == http.MethodGet { |
| 2114 | + s.list(w, r, strings.TrimPrefix(r.URL.Path, "/list/")) |
| 2115 | + return |
| 2116 | + } |
| 2117 | + if r.URL.Path == "/nip96" { |
| 2118 | + s.nip96(w, r) |
| 2119 | + return |
| 2120 | + } |
| 2121 | + s.blob(w, r) |
| 2122 | +} |
| 2123 | + |
| 2124 | +func (s *Service) upload(w http.ResponseWriter, r *http.Request) { |
| 2125 | + pubkey, err := s.authorize(r, ActionUpload) |
| 2126 | + if err != nil { |
| 2127 | + s.fail(w, http.StatusUnauthorized, err) |
| 2128 | + return |
| 2129 | + } |
| 2130 | + claimed := strings.ToLower(strings.TrimSpace(r.Header.Get("x-sha-256"))) |
| 2131 | + if claimed != "" && !shaPattern.MatchString(claimed) { |
| 2132 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: X-SHA-256 must be lowercase hex")) |
| 2133 | + return |
| 2134 | + } |
| 2135 | + blob, created, err := s.putValidated(r.Context(), r.Body, contentType(r.Header.Get("content-type")), pubkey, claimed, func(actual string) error { |
| 2136 | + if s.config.ValidateUpload != nil { |
| 2137 | + return s.config.ValidateUpload(r, actual) |
| 2138 | + } |
| 2139 | + return nil |
| 2140 | + }) |
| 2141 | + if err != nil { |
| 2142 | + s.fail(w, statusFor(err), err) |
| 2143 | + return |
| 2144 | + } |
| 2145 | + _ = created |
| 2146 | + s.writeJSON(w, http.StatusOK, s.descriptor(r, blob)) |
| 2147 | +} |
| 2148 | + |
| 2149 | +func (s *Service) uploadHead(w http.ResponseWriter, r *http.Request) { |
| 2150 | + sha := strings.ToLower(strings.TrimSpace(r.Header.Get("x-sha-256"))) |
| 2151 | + if !shaPattern.MatchString(sha) { |
| 2152 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: X-SHA-256 must be lowercase hex")) |
| 2153 | + return |
| 2154 | + } |
| 2155 | + rawLength := strings.TrimSpace(r.Header.Get("x-content-length")) |
| 2156 | + if rawLength == "" { |
| 2157 | + s.fail(w, http.StatusLengthRequired, errors.New("invalid: X-Content-Length is required")) |
| 2158 | + return |
| 2159 | + } |
| 2160 | + length, err := strconv.ParseInt(rawLength, 10, 64) |
| 2161 | + if err != nil || length < 0 { |
| 2162 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Length must be a nonnegative integer")) |
| 2163 | + return |
| 2164 | + } |
| 2165 | + pubkey, err := s.authorize(r, ActionUpload) |
| 2166 | + if err != nil { |
| 2167 | + s.fail(w, http.StatusUnauthorized, err) |
| 2168 | + return |
| 2169 | + } |
| 2170 | + typ := r.Header.Get("x-content-type") |
| 2171 | + if typ == "" { |
| 2172 | + typ = r.Header.Get("content-type") |
| 2173 | + } |
| 2174 | + if strings.TrimSpace(typ) == "" { |
| 2175 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: X-Content-Type is required")) |
| 2176 | + return |
| 2177 | + } |
| 2178 | + if existing, lookupErr := s.lookup(r.Context(), sha); lookupErr == nil && existing.SHA256 != "" && s.hasClaim(r.Context(), sha, pubkey) { |
| 2179 | + w.WriteHeader(http.StatusOK) |
| 2180 | + return |
| 2181 | + } |
| 2182 | + limits := s.currentLimits() |
| 2183 | + if limits.MaxFileBytes > 0 && length > limits.MaxFileBytes { |
| 2184 | + s.fail(w, http.StatusRequestEntityTooLarge, ErrFileTooLarge) |
| 2185 | + return |
| 2186 | + } |
| 2187 | + // HEAD is advisory for deduplicated uploads, whose final physical size is |
| 2188 | + // already present, but it should still reflect the current quota budget. |
| 2189 | + if limits.UserStorageBytes > 0 { |
| 2190 | + if used, quotaErr := s.quotaUsage(r.Context(), pubkey); quotaErr != nil { |
| 2191 | + s.fail(w, http.StatusInternalServerError, quotaErr) |
| 2192 | + return |
| 2193 | + } else if length > limits.UserStorageBytes-used { |
| 2194 | + s.fail(w, http.StatusForbidden, ErrQuotaExceeded) |
| 2195 | + return |
| 2196 | + } |
| 2197 | + } |
| 2198 | + if existing, err := s.lookup(r.Context(), sha); err == nil && existing.SHA256 != "" { |
| 2199 | + w.WriteHeader(http.StatusOK) |
| 2200 | + return |
| 2201 | + } |
| 2202 | + w.WriteHeader(http.StatusOK) |
| 2203 | +} |
| 2204 | + |
| 2205 | +func (s *Service) blob(w http.ResponseWriter, r *http.Request) { |
| 2206 | + path := r.URL.Path |
| 2207 | + if strings.HasPrefix(path, "/nip96/") { |
| 2208 | + path = strings.TrimPrefix(path, "/nip96") |
| 2209 | + } |
| 2210 | + match := blobPathPattern.FindStringSubmatch(path) |
| 2211 | + if match == nil { |
| 2212 | + s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 2213 | + return |
| 2214 | + } |
| 2215 | + sha := match[1] |
| 2216 | + if r.Method == http.MethodDelete { |
| 2217 | + s.delete(w, r, sha) |
| 2218 | + return |
| 2219 | + } |
| 2220 | + if r.Method != http.MethodGet && r.Method != http.MethodHead { |
| 2221 | + s.fail(w, http.StatusMethodNotAllowed, errors.New("method not allowed")) |
| 2222 | + return |
| 2223 | + } |
| 2224 | + if s.config.CanRead != nil { |
| 2225 | + pubkey, err := s.authorize(r, ActionGet) |
| 2226 | + if err != nil { |
| 2227 | + s.fail(w, http.StatusUnauthorized, err) |
| 2228 | + return |
| 2229 | + } |
| 2230 | + if !s.config.CanRead(r.Context(), sha, []string{pubkey}) { |
| 2231 | + s.fail(w, http.StatusForbidden, errors.New("restricted: read access denied")) |
| 2232 | + return |
| 2233 | + } |
| 2234 | + } |
| 2235 | + entry, err := s.lookup(r.Context(), sha) |
| 2236 | + if errors.Is(err, sql.ErrNoRows) { |
| 2237 | + s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 2238 | + return |
| 2239 | + } |
| 2240 | + if err != nil { |
| 2241 | + s.fail(w, http.StatusInternalServerError, err) |
| 2242 | + return |
| 2243 | + } |
| 2244 | + if s.blocked(r.Context(), sha) { |
| 2245 | + s.fail(w, http.StatusForbidden, ErrBlocked) |
| 2246 | + return |
| 2247 | + } |
| 2248 | + file, err := os.Open(filepath.Join(s.root, sha)) |
| 2249 | + if err != nil { |
| 2250 | + s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 2251 | + return |
| 2252 | + } |
| 2253 | + defer file.Close() |
| 2254 | + w.Header().Set("etag", `"`+sha+`"`) |
| 2255 | + w.Header().Set("Content-Type", entry.Type) |
| 2256 | + w.Header().Set("x-content-type-options", "nosniff") |
| 2257 | + w.Header().Set("accept-ranges", "bytes") |
| 2258 | + w.Header().Set("content-security-policy", "default-src 'none'") |
| 2259 | + if s.config.CanRead == nil { |
| 2260 | + w.Header().Set("cache-control", "public, max-age=31536000, immutable") |
| 2261 | + } else { |
| 2262 | + w.Header().Set("cache-control", "private, no-store") |
| 2263 | + } |
| 2264 | + http.ServeContent(w, r, sha, time.Unix(entry.Uploaded, 0), file) |
| 2265 | +} |
| 2266 | + |
| 2267 | +func (s *Service) delete(w http.ResponseWriter, r *http.Request, sha string) { |
| 2268 | + pubkey, err := s.authorize(r, ActionDelete) |
| 2269 | + if err != nil { |
| 2270 | + s.fail(w, http.StatusUnauthorized, err) |
| 2271 | + return |
| 2272 | + } |
| 2273 | + _, err = s.lookup(r.Context(), sha) |
| 2274 | + if errors.Is(err, sql.ErrNoRows) { |
| 2275 | + s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 2276 | + return |
| 2277 | + } |
| 2278 | + if err != nil { |
| 2279 | + s.fail(w, http.StatusInternalServerError, err) |
| 2280 | + return |
| 2281 | + } |
| 2282 | + if !s.hasClaim(r.Context(), sha, pubkey) && !s.isOwner(pubkey) { |
| 2283 | + s.fail(w, http.StatusForbidden, errors.New("restricted: not the uploader")) |
| 2284 | + return |
| 2285 | + } |
| 2286 | + if s.isOwner(pubkey) { |
| 2287 | + if err := s.Delete(r.Context(), sha); err != nil { |
| 2288 | + s.fail(w, http.StatusInternalServerError, err) |
| 2289 | + return |
| 2290 | + } |
| 2291 | + } else if err := s.DeleteForUploader(r.Context(), sha, pubkey); err != nil { |
| 2292 | + if errors.Is(err, sql.ErrNoRows) { |
| 2293 | + s.fail(w, http.StatusNotFound, errors.New("not found")) |
| 2294 | + return |
| 2295 | + } |
| 2296 | + s.fail(w, http.StatusInternalServerError, err) |
| 2297 | + return |
| 2298 | + } |
| 2299 | + // Owners and uploaders may remove their claim; a shared physical object is |
| 2300 | + // retained until its final claim disappears. |
| 2301 | + w.WriteHeader(http.StatusNoContent) |
| 2302 | +} |
| 2303 | + |
| 2304 | +func (s *Service) list(w http.ResponseWriter, r *http.Request, pubkey string) { |
| 2305 | + if !shaPattern.MatchString(pubkey) { |
| 2306 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: bad pubkey")) |
| 2307 | + return |
| 2308 | + } |
| 2309 | + if _, err := s.authorize(r, ActionList); err != nil { |
| 2310 | + s.fail(w, http.StatusUnauthorized, err) |
| 2311 | + return |
| 2312 | + } |
| 2313 | + limit, err := parseListLimit(r.URL.Query().Get("limit")) |
| 2314 | + if err != nil { |
| 2315 | + s.fail(w, http.StatusBadRequest, err) |
| 2316 | + return |
| 2317 | + } |
| 2318 | + entries, _, err := s.ListPage(r.Context(), pubkey, limit, strings.TrimSpace(r.URL.Query().Get("cursor"))) |
| 2319 | + if err != nil { |
| 2320 | + if errors.Is(err, sql.ErrNoRows) { |
| 2321 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: cursor does not identify a blob")) |
| 2322 | + return |
| 2323 | + } |
| 2324 | + s.fail(w, http.StatusInternalServerError, err) |
| 2325 | + return |
| 2326 | + } |
| 2327 | + descriptors := make([]map[string]any, 0, len(entries)) |
| 2328 | + for _, b := range entries { |
| 2329 | + descriptors = append(descriptors, s.descriptor(r, b)) |
| 2330 | + } |
| 2331 | + s.writeJSON(w, http.StatusOK, descriptors) |
| 2332 | +} |
| 2333 | + |
| 2334 | +func (s *Service) report(w http.ResponseWriter, r *http.Request) { |
| 2335 | + reporter, err := s.authorize(r, ActionReport) |
| 2336 | + if err != nil { |
| 2337 | + s.fail(w, http.StatusUnauthorized, err) |
| 2338 | + return |
| 2339 | + } |
| 2340 | + var input event.Event |
| 2341 | + if err := json.NewDecoder(r.Body).Decode(&input); err != nil || input.Kind != 1984 || input.PubKey != reporter { |
| 2342 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: report must be a signed kind 1984 event")) |
| 2343 | + return |
| 2344 | + } |
| 2345 | + if err := event.Validate(input); err != nil || time.Since(time.Unix(input.CreatedAt, 0)) > time.Hour || time.Until(time.Unix(input.CreatedAt, 0)) > time.Hour { |
| 2346 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: report is not a current signed kind 1984 event")) |
| 2347 | + return |
| 2348 | + } |
| 2349 | + filed := 0 |
| 2350 | + type reportTarget struct { |
| 2351 | + hash string |
| 2352 | + uploader string |
| 2353 | + } |
| 2354 | + targets := make([]reportTarget, 0, len(input.Tags)) |
| 2355 | + for _, tag := range input.Tags { |
| 2356 | + if len(tag) < 2 || tag[0] != "x" || !shaPattern.MatchString(tag[1]) { |
| 2357 | + continue |
| 2358 | + } |
| 2359 | + entry, lookupErr := s.lookup(r.Context(), tag[1]) |
| 2360 | + if lookupErr != nil { |
| 2361 | + continue |
| 2362 | + } |
| 2363 | + id := input.ID |
| 2364 | + if filed > 0 { |
| 2365 | + id += "." + strconv.Itoa(filed) |
| 2366 | + } |
| 2367 | + targets = append(targets, reportTarget{hash: tag[1], uploader: entry.Uploader}) |
| 2368 | + } |
| 2369 | + if len(targets) == 0 { |
| 2370 | + s.fail(w, http.StatusNotFound, errors.New("not found: relay holds none of the reported blobs")) |
| 2371 | + return |
| 2372 | + } |
| 2373 | + err = s.store.WithTx(r.Context(), func(tx *sql.Tx) error { |
| 2374 | + for i, target := range targets { |
| 2375 | + id := input.ID |
| 2376 | + if i > 0 { |
| 2377 | + id += "." + strconv.Itoa(i) |
| 2378 | + } |
| 2379 | + result, insertErr := tx.ExecContext(r.Context(), `INSERT OR IGNORE INTO blob_reports(id,reporter,target_pubkey,target_blob,type,content,at) VALUES(?,?,?,?,?,?,?)`, id, reporter, target.uploader, target.hash, "blob", input.Content, input.CreatedAt) |
| 2380 | + if insertErr != nil { |
| 2381 | + return insertErr |
| 2382 | + } |
| 2383 | + changed, rowsErr := result.RowsAffected() |
| 2384 | + if rowsErr != nil { |
| 2385 | + return rowsErr |
| 2386 | + } |
| 2387 | + if changed == 0 { |
| 2388 | + continue |
| 2389 | + } |
| 2390 | + if s.config.RecordReport != nil { |
| 2391 | + if err := s.config.RecordReport(r.Context(), tx, reporter, target.hash, "blob", "blob", input.Content); err != nil { |
| 2392 | + return err |
| 2393 | + } |
| 2394 | + } |
| 2395 | + filed++ |
| 2396 | + } |
| 2397 | + return nil |
| 2398 | + }) |
| 2399 | + if err != nil { |
| 2400 | + s.fail(w, http.StatusInternalServerError, err) |
| 2401 | + return |
| 2402 | + } |
| 2403 | + if filed == 0 { |
| 2404 | + // Repeating an identical report is idempotent and still successful. |
| 2405 | + filed = len(targets) |
| 2406 | + } |
| 2407 | + s.writeJSON(w, http.StatusOK, map[string]any{"ok": true, "filed": filed}) |
| 2408 | +} |
| 2409 | + |
| 2410 | +// Block permanently prevents a hash from returning through any upload door. |
| 2411 | +func (s *Service) Block(ctx context.Context, sha, reason string) error { |
| 2412 | + if !shaPattern.MatchString(sha) { |
| 2413 | + return errors.New("invalid: bad blob sha256") |
| 2414 | + } |
| 2415 | + _, err := s.store.DB().ExecContext(ctx, "INSERT INTO blob_blocks(sha256,reason,blocked_at) VALUES(?,?,?) ON CONFLICT(sha256) DO UPDATE SET reason=excluded.reason,blocked_at=excluded.blocked_at", sha, reason, time.Now().UTC().Unix()) |
| 2416 | + if err != nil { |
| 2417 | + return fmt.Errorf("block blob: %w", err) |
| 2418 | + } |
| 2419 | + if _, err := s.store.DB().ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 2420 | + return fmt.Errorf("record blocked blob deletion: %w", err) |
| 2421 | + } |
| 2422 | + if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 2423 | + return fmt.Errorf("remove blocked blob metadata: %w", err) |
| 2424 | + } |
| 2425 | + if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 2426 | + return fmt.Errorf("remove blocked blob: %w", err) |
| 2427 | + } |
| 2428 | + return nil |
| 2429 | +} |
| 2430 | + |
| 2431 | +func (s *Service) mirror(w http.ResponseWriter, r *http.Request) { |
| 2432 | + pubkey, err := s.authorize(r, ActionMirror) |
| 2433 | + if err != nil { |
| 2434 | + s.fail(w, http.StatusUnauthorized, err) |
| 2435 | + return |
| 2436 | + } |
| 2437 | + var input struct { |
| 2438 | + URL string `json:"url"` |
| 2439 | + } |
| 2440 | + if err := json.NewDecoder(r.Body).Decode(&input); err != nil || input.URL == "" { |
| 2441 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: body needs a url")) |
| 2442 | + return |
| 2443 | + } |
| 2444 | + requestedURL := input.URL |
| 2445 | + expectedHash := "" |
| 2446 | + extension := "" |
| 2447 | + uriServers := []string{} |
| 2448 | + authors := []string{} |
| 2449 | + if strings.HasPrefix(strings.ToLower(requestedURL), "blossom:") { |
| 2450 | + uri, err := ParseBlossomURI(requestedURL) |
| 2451 | + if err != nil { |
| 2452 | + s.fail(w, http.StatusBadRequest, err) |
| 2453 | + return |
| 2454 | + } |
| 2455 | + expectedHash, extension = uri.Hash, uri.Extension |
| 2456 | + uriServers = append(uriServers, uri.Servers...) |
| 2457 | + authors = append(authors, uri.Authors...) |
| 2458 | + requestedURL = "" |
| 2459 | + } else if hash := BlobHashFromURL(requestedURL); hash != "" { |
| 2460 | + expectedHash = hash |
| 2461 | + } |
| 2462 | + client := s.config.HTTPClient |
| 2463 | + if client == nil { |
| 2464 | + client = s.safeClient() |
| 2465 | + } |
| 2466 | + candidates := []string{} |
| 2467 | + if requestedURL != "" { |
| 2468 | + candidates = append(candidates, requestedURL) |
| 2469 | + } |
| 2470 | + candidates = append(candidates, FallbackURLs(uriServers, expectedHash, extension)...) |
| 2471 | + if s.config.ResolveServers != nil && expectedHash != "" { |
| 2472 | + if len(authors) == 0 && pubkey != "" { |
| 2473 | + authors = []string{pubkey} |
| 2474 | + } |
| 2475 | + for _, author := range uniqueStrings(authors) { |
| 2476 | + servers, resolveErr := s.config.ResolveServers(r.Context(), author) |
| 2477 | + if resolveErr != nil { |
| 2478 | + continue |
| 2479 | + } |
| 2480 | + candidates = append(candidates, FallbackURLs(servers, expectedHash, extension)...) |
| 2481 | + } |
| 2482 | + } |
| 2483 | + var lastErr error |
| 2484 | + for _, candidate := range uniqueStrings(candidates) { |
| 2485 | + if err := validateMirrorURL(candidate, s.resolve); err != nil { |
| 2486 | + lastErr = err |
| 2487 | + continue |
| 2488 | + } |
| 2489 | + response, fetchErr := client.Get(candidate) |
| 2490 | + if fetchErr != nil { |
| 2491 | + lastErr = fetchErr |
| 2492 | + continue |
| 2493 | + } |
| 2494 | + if response.StatusCode < 200 || response.StatusCode >= 300 { |
| 2495 | + lastErr = fmt.Errorf("error: origin answered %d", response.StatusCode) |
| 2496 | + _ = response.Body.Close() |
| 2497 | + continue |
| 2498 | + } |
| 2499 | + typ := contentType(response.Header.Get("content-type")) |
| 2500 | + if typ == "application/octet-stream" { |
| 2501 | + typ = typeForExtension(candidate) |
| 2502 | + } |
| 2503 | + entry, created, putErr := s.putValidated(r.Context(), response.Body, typ, pubkey, expectedHash, func(actual string) error { |
| 2504 | + if s.config.ValidateUpload != nil { |
| 2505 | + return s.config.ValidateUpload(r, actual) |
| 2506 | + } |
| 2507 | + return nil |
| 2508 | + }) |
| 2509 | + _ = response.Body.Close() |
| 2510 | + if putErr == nil { |
| 2511 | + s.writeJSON(w, chooseStatus(created, http.StatusCreated), s.descriptor(r, entry)) |
| 2512 | + return |
| 2513 | + } |
| 2514 | + lastErr = putErr |
| 2515 | + if !errors.Is(putErr, ErrHashMismatch) { |
| 2516 | + s.fail(w, statusFor(putErr), putErr) |
| 2517 | + return |
| 2518 | + } |
| 2519 | + } |
| 2520 | + if lastErr == nil { |
| 2521 | + lastErr = errors.New("error: no mirror origin available") |
| 2522 | + } |
| 2523 | + s.fail(w, statusFor(lastErr), lastErr) |
| 2524 | +} |
| 2525 | + |
| 2526 | +func (s *Service) nip96(w http.ResponseWriter, r *http.Request) { |
| 2527 | + if r.Method == http.MethodGet { |
| 2528 | + pubkey, err := s.authorize(r, ActionList) |
| 2529 | + if err != nil { |
| 2530 | + s.fail(w, http.StatusUnauthorized, err) |
| 2531 | + return |
| 2532 | + } |
| 2533 | + s.nip96List(w, r, pubkey) |
| 2534 | + return |
| 2535 | + } |
| 2536 | + if r.Method != http.MethodPost { |
| 2537 | + s.fail(w, http.StatusMethodNotAllowed, errors.New("method not allowed")) |
| 2538 | + return |
| 2539 | + } |
| 2540 | + pubkey, err := s.authorize(r, ActionUpload) |
| 2541 | + if err != nil { |
| 2542 | + s.fail(w, http.StatusUnauthorized, err) |
| 2543 | + return |
| 2544 | + } |
| 2545 | + if err := r.ParseMultipartForm(0); err != nil { |
| 2546 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: body is not multipart form data")) |
| 2547 | + return |
| 2548 | + } |
| 2549 | + defer r.MultipartForm.RemoveAll() |
| 2550 | + file, header, err := r.FormFile("file") |
| 2551 | + if err != nil { |
| 2552 | + s.fail(w, http.StatusBadRequest, errors.New("invalid: form needs a file field")) |
| 2553 | + return |
| 2554 | + } |
| 2555 | + defer file.Close() |
| 2556 | + typ := contentType(header.Header.Get("Content-Type")) |
| 2557 | + if typ == "application/octet-stream" { |
| 2558 | + typ = contentType(r.FormValue("content_type")) |
| 2559 | + } |
| 2560 | + entry, created, err := s.put(r.Context(), file, typ, pubkey, "") |
| 2561 | + if err != nil { |
| 2562 | + s.fail(w, statusFor(err), err) |
| 2563 | + return |
| 2564 | + } |
| 2565 | + s.writeJSON(w, chooseStatus(created, http.StatusCreated), map[string]any{"status": "success", "message": "Upload successful.", "url": s.blobURL(r, entry), "nip94_event": map[string]any{"tags": s.nip94Tags(r, entry), "content": r.FormValue("caption")}}) |
| 2566 | +} |
| 2567 | + |
| 2568 | +func (s *Service) nip96List(w http.ResponseWriter, r *http.Request, pubkey string) { |
| 2569 | + count := parsePageCount(r.URL.Query().Get("count")) |
| 2570 | + page := parsePage(r.URL.Query().Get("page")) |
| 2571 | + var total int |
| 2572 | + if err := s.store.DB().QueryRowContext(r.Context(), "SELECT count(*) FROM blob_claims WHERE uploader=?", pubkey).Scan(&total); err != nil { |
| 2573 | + s.fail(w, http.StatusInternalServerError, err) |
| 2574 | + return |
| 2575 | + } |
| 2576 | + rows, err := s.store.DB().QueryContext(r.Context(), `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded |
| 2577 | + FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=? |
| 2578 | + ORDER BY b.uploaded DESC,b.sha256 DESC LIMIT ? OFFSET ?`, pubkey, count, page*count) |
| 2579 | + if err != nil { |
| 2580 | + s.fail(w, http.StatusInternalServerError, err) |
| 2581 | + return |
| 2582 | + } |
| 2583 | + defer rows.Close() |
| 2584 | + files := make([]map[string]any, 0) |
| 2585 | + for rows.Next() { |
| 2586 | + var b Blob |
| 2587 | + if err := rows.Scan(&b.SHA256, &b.Size, &b.Type, &b.Uploader, &b.Uploaded); err != nil { |
| 2588 | + s.fail(w, http.StatusInternalServerError, err) |
| 2589 | + return |
| 2590 | + } |
| 2591 | + files = append(files, map[string]any{"tags": s.nip94Tags(r, b), "content": "", "created_at": b.Uploaded}) |
| 2592 | + } |
| 2593 | + s.writeJSON(w, http.StatusOK, map[string]any{"count": count, "total": total, "page": page, "files": files}) |
| 2594 | +} |
| 2595 | + |
| 2596 | +func (s *Service) authorize(r *http.Request, action Action) (string, error) { |
| 2597 | + if s.config.Authorize == nil { |
| 2598 | + return "", errors.New("auth-required: authorization is required") |
| 2599 | + } |
| 2600 | + return s.config.Authorize(r, action) |
| 2601 | +} |
| 2602 | +func (s *Service) isOwner(pubkey string) bool { |
| 2603 | + return s.config.IsOwner != nil && s.config.IsOwner(pubkey) |
| 2604 | +} |
| 2605 | +func (s *Service) descriptor(r *http.Request, b Blob) map[string]any { |
| 2606 | + return map[string]any{"url": s.blobURL(r, b), "sha256": b.SHA256, "size": b.Size, "type": b.Type, "uploaded": b.Uploaded, "nip94": s.nip94Tags(r, b)} |
| 2607 | +} |
| 2608 | +func (s *Service) nip94Tags(r *http.Request, b Blob) [][]string { |
| 2609 | + return [][]string{{"url", s.blobURL(r, b)}, {"m", b.Type}, {"x", b.SHA256}, {"ox", b.SHA256}, {"size", strconv.FormatInt(b.Size, 10)}} |
| 2610 | +} |
| 2611 | +func (s *Service) blobURL(r *http.Request, b Blob) string { |
| 2612 | + return s.origin(r) + "/" + b.SHA256 + extensionSuffix(b.Type) |
| 2613 | +} |
| 2614 | +func (s *Service) nip96Info(r *http.Request) map[string]any { |
| 2615 | + return map[string]any{"api_url": s.origin(r) + "/nip96", "download_url": s.origin(r), "supported_nips": []int{94, 98}, "content_types": contentTypes(), "plans": map[string]any{"self-hosted": map[string]any{"name": "Self-hosted", "is_nip98_required": true, "max_byte_size": 0, "file_expiration": []int{0, 0}, "media_transformations": map[string]any{}}}} |
| 2616 | +} |
| 2617 | +func (s *Service) writeJSON(w http.ResponseWriter, status int, value any) { |
| 2618 | + w.Header().Set("content-type", "application/json") |
| 2619 | + w.WriteHeader(status) |
| 2620 | + _ = json.NewEncoder(w).Encode(value) |
| 2621 | +} |
| 2622 | +func (s *Service) fail(w http.ResponseWriter, status int, err error) { |
| 2623 | + w.Header().Set("x-reason", err.Error()) |
| 2624 | + s.writeJSON(w, status, map[string]string{"error": err.Error()}) |
| 2625 | +} |
| 2626 | +func origin(r *http.Request) string { |
| 2627 | + scheme := r.URL.Scheme |
| 2628 | + if scheme == "" { |
| 2629 | + scheme = "http" |
| 2630 | + if r.TLS != nil { |
| 2631 | + scheme = "https" |
| 2632 | + } |
| 2633 | + } |
| 2634 | + return scheme + "://" + r.Host |
| 2635 | +} |
| 2636 | + |
| 2637 | +func (s *Service) origin(r *http.Request) string { |
| 2638 | + if s.config.PublicURL != "" { |
| 2639 | + return strings.TrimRight(s.config.PublicURL, "/") |
| 2640 | + } |
| 2641 | + return origin(r) |
| 2642 | +} |
| 2643 | diff --git a/internal/blob/service.go b/internal/blob/service.go |
| 2644 | new file mode 100644 |
| 2645 | index 0000000..871a16a |
| 2646 | --- /dev/null |
| 2647 | +++ b/internal/blob/service.go |
| 2648 | @@ -0,0 +1,197 @@ |
| 2649 | +package blob |
| 2650 | + |
| 2651 | +import ( |
| 2652 | + "context" |
| 2653 | + "database/sql" |
| 2654 | + "errors" |
| 2655 | + "fmt" |
| 2656 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 2657 | + "io" |
| 2658 | + "net" |
| 2659 | + "net/http" |
| 2660 | + "os" |
| 2661 | + "path/filepath" |
| 2662 | + "regexp" |
| 2663 | + "strings" |
| 2664 | + "sync" |
| 2665 | +) |
| 2666 | + |
| 2667 | +type Action string |
| 2668 | + |
| 2669 | +const ( |
| 2670 | + ActionUpload Action = "upload" |
| 2671 | + ActionGet Action = "get" |
| 2672 | + ActionDelete Action = "delete" |
| 2673 | + ActionList Action = "list" |
| 2674 | + ActionMirror Action = "mirror" |
| 2675 | + ActionReport Action = "report" |
| 2676 | +) |
| 2677 | + |
| 2678 | +type Authorize func(*http.Request, Action) (string, error) |
| 2679 | +type CanRead func(context.Context, string, []string) bool |
| 2680 | +type ResolveIP func(context.Context, string) ([]net.IP, error) |
| 2681 | +type ReportRecorder func(context.Context, *sql.Tx, string, string, string, string, string) error |
| 2682 | + |
| 2683 | +type Config struct { |
| 2684 | + Root string |
| 2685 | + PublicURL string |
| 2686 | + Store *storage.Store |
| 2687 | + Authorize Authorize |
| 2688 | + CanRead CanRead |
| 2689 | + HTTPClient *http.Client |
| 2690 | + ResolveIP ResolveIP |
| 2691 | + IsOwner func(string) bool |
| 2692 | + ValidateUpload func(*http.Request, string) error |
| 2693 | + // ResolveServers returns the ordered BUD-03 servers for an authorized |
| 2694 | + // caller. It is used only for Blossom URI mirror fallback. |
| 2695 | + ResolveServers func(context.Context, string) ([]string, error) |
| 2696 | + // RecordReport joins native blob reporting to the host moderation ledger. |
| 2697 | + // It runs inside the blob report transaction. |
| 2698 | + RecordReport ReportRecorder |
| 2699 | + // Limits is read for each upload so policy changes apply without reopening |
| 2700 | + // the blob service. Zero values mean unlimited. |
| 2701 | + Limits func() Limits |
| 2702 | + // UploadTerms returns what an uploader's standing imposes on its uploads: |
| 2703 | + // an agent grant with a sites ttl gives its claims an expiry, and one with |
| 2704 | + // the encrypted flag refuses plain uploads. Zero values mean no terms. |
| 2705 | + UploadTerms func(context.Context, string) (UploadTerms, error) |
| 2706 | +} |
| 2707 | + |
| 2708 | +type Limits struct { |
| 2709 | + MaxFileBytes int64 |
| 2710 | + UserStorageBytes int64 |
| 2711 | +} |
| 2712 | + |
| 2713 | +// UploadTerms is what an uploader's grant imposes on the blobs it stores. |
| 2714 | +// ExpiresAt is the unix time its claims lapse, 0 for never; Encrypted |
| 2715 | +// requires the stored bytes to be an encrypted blob or manifest. |
| 2716 | +type UploadTerms struct { |
| 2717 | + ExpiresAt int64 |
| 2718 | + Encrypted bool |
| 2719 | +} |
| 2720 | + |
| 2721 | +// encryptedTypes are the content types the browser uploader gives encrypted |
| 2722 | +// objects: AES-GCM ciphertext for files and chunks, and the BUD-16 directory |
| 2723 | +// type for encrypted manifests. Anything the relay recognizes as a page, |
| 2724 | +// image, video, audio or document is plain. |
| 2725 | +var encryptedTypes = map[string]bool{"application/octet-stream": true, "application/vnd.blossom.directory+msgpack": true} |
| 2726 | + |
| 2727 | +type Blob struct { |
| 2728 | + SHA256 string `json:"sha256"` |
| 2729 | + Size int64 `json:"size"` |
| 2730 | + Type string `json:"type"` |
| 2731 | + Uploader string `json:"uploader"` |
| 2732 | + Uploaded int64 `json:"uploaded"` |
| 2733 | +} |
| 2734 | + |
| 2735 | +type PutOptions struct { |
| 2736 | + Reader io.Reader |
| 2737 | + Type string |
| 2738 | + Uploader string |
| 2739 | + Hash string |
| 2740 | + // Commit records ownership metadata in the blob transaction. The final |
| 2741 | + // argument reports whether the blob is new. An error rolls back all claims. |
| 2742 | + // New bytes remain unavailable until their ownership metadata is durable. |
| 2743 | + Commit func(context.Context, *sql.Tx, Blob, bool) error |
| 2744 | +} |
| 2745 | + |
| 2746 | +type Service struct { |
| 2747 | + config Config |
| 2748 | + root string |
| 2749 | + store *storage.Store |
| 2750 | + resolve ResolveIP |
| 2751 | + quotaMu sync.Mutex |
| 2752 | + // multipartMu protects partial files and their range metadata. Lock it |
| 2753 | + // before quotaMu; network bodies are never read while either is held. |
| 2754 | + multipartMu sync.Mutex |
| 2755 | + activeParts map[string]int |
| 2756 | + // multipartFinalizing prevents another chunk from changing a complete |
| 2757 | + // session while its assembled file is being hashed. Hashing is deliberately |
| 2758 | + // outside multipartMu because the upload may be as large as 1 GiB. |
| 2759 | + multipartFinalizing map[string]bool |
| 2760 | + // verifyMultipartHook is used by package tests to pause finalization |
| 2761 | + // without allocating a GiB fixture. It is nil in production. |
| 2762 | + verifyMultipartHook func(context.Context, string, string, int64) error |
| 2763 | +} |
| 2764 | + |
| 2765 | +var ErrBlocked = errors.New("blocked: this blob was removed by a moderator") |
| 2766 | +var ErrHashMismatch = errors.New("conflict: mirrored blob hash does not match the requested hash") |
| 2767 | +var ErrFileTooLarge = errors.New("invalid: blob exceeds the per-file size limit") |
| 2768 | +var ErrQuotaExceeded = errors.New("restricted: uploader storage quota exceeded") |
| 2769 | +var ErrMultipartConflict = errors.New("conflict: upload metadata differs from existing session") |
| 2770 | +var ErrNotEncrypted = errors.New("restricted: agent grant requires encrypted uploads") |
| 2771 | + |
| 2772 | +var shaPattern = regexp.MustCompile(`^[0-9a-f]{64}$`) |
| 2773 | +var blobPathPattern = regexp.MustCompile(`^/([0-9a-f]{64})(?:\.[a-z0-9]{1,8})?$`) |
| 2774 | + |
| 2775 | +// HandlesPath keeps the daemon mount in sync with every supported file door. |
| 2776 | +func HandlesPath(path string) bool { |
| 2777 | + return path == "/upload" || path == "/mirror" || path == "/report" || |
| 2778 | + path == "/nip96" || path == "/.well-known/nostr/nip96.json" || |
| 2779 | + strings.HasPrefix(path, "/nip96/") || strings.HasPrefix(path, "/list/") || |
| 2780 | + blobPathPattern.MatchString(path) |
| 2781 | +} |
| 2782 | + |
| 2783 | +var extensions = map[string]string{ |
| 2784 | + "image/png": "png", "image/jpeg": "jpg", "image/gif": "gif", "image/webp": "webp", "image/avif": "avif", "image/svg+xml": "svg", |
| 2785 | + "video/mp4": "mp4", "video/webm": "webm", "video/quicktime": "mov", "audio/mpeg": "mp3", "audio/ogg": "ogg", "audio/wav": "wav", "audio/mp4": "m4a", |
| 2786 | + "application/pdf": "pdf", "application/json": "json", "text/plain": "txt", "text/markdown": "md", |
| 2787 | + "application/vnd.blossom.directory+msgpack": "bdir", |
| 2788 | +} |
| 2789 | + |
| 2790 | +func New(ctx context.Context, config Config) (*Service, error) { |
| 2791 | + if config.Root == "" || config.Store == nil { |
| 2792 | + return nil, errors.New("blob root and store are required") |
| 2793 | + } |
| 2794 | + if err := os.MkdirAll(filepath.Join(config.Root, "blobs"), 0o700); err != nil { |
| 2795 | + return nil, fmt.Errorf("create blob root: %w", err) |
| 2796 | + } |
| 2797 | + if _, err := config.Store.DB().ExecContext(ctx, `CREATE TABLE IF NOT EXISTS blobs ( |
| 2798 | + sha256 TEXT PRIMARY KEY, size INTEGER NOT NULL, type TEXT NOT NULL, |
| 2799 | + uploader TEXT NOT NULL, uploaded INTEGER NOT NULL |
| 2800 | + ); CREATE TABLE IF NOT EXISTS blob_claims ( |
| 2801 | + sha256 TEXT NOT NULL, uploader TEXT NOT NULL, claimed_at INTEGER NOT NULL, |
| 2802 | + PRIMARY KEY (sha256, uploader), FOREIGN KEY (sha256) REFERENCES blobs(sha256) ON DELETE CASCADE |
| 2803 | + ); CREATE INDEX IF NOT EXISTS blobs_uploader ON blobs(uploader, uploaded DESC); |
| 2804 | + CREATE INDEX IF NOT EXISTS blob_claims_uploader ON blob_claims(uploader, claimed_at DESC, sha256 DESC); |
| 2805 | + CREATE TABLE IF NOT EXISTS blob_reports ( |
| 2806 | + id TEXT PRIMARY KEY, reporter TEXT NOT NULL, target_pubkey TEXT NOT NULL DEFAULT '', |
| 2807 | + target_blob TEXT NOT NULL, type TEXT NOT NULL, content TEXT NOT NULL, at INTEGER NOT NULL, |
| 2808 | + status TEXT NOT NULL DEFAULT 'open', resolved_by TEXT NOT NULL DEFAULT '', |
| 2809 | + resolved_at INTEGER NOT NULL DEFAULT 0, action TEXT NOT NULL DEFAULT '', |
| 2810 | + UNIQUE(reporter,target_blob)); |
| 2811 | + CREATE TRIGGER IF NOT EXISTS blob_claim_on_insert AFTER INSERT ON blobs |
| 2812 | + WHEN NEW.uploader != '' BEGIN INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) VALUES(NEW.sha256,NEW.uploader,NEW.uploaded); END; |
| 2813 | + CREATE TABLE IF NOT EXISTS blob_blocks (sha256 TEXT PRIMARY KEY, reason TEXT NOT NULL DEFAULT '', blocked_at INTEGER NOT NULL DEFAULT 0); |
| 2814 | + CREATE TABLE IF NOT EXISTS blob_tombstones (sha256 TEXT PRIMARY KEY, deleted_at INTEGER NOT NULL); |
| 2815 | + CREATE TABLE IF NOT EXISTS multipart_uploads (id TEXT PRIMARY KEY, sha256 TEXT NOT NULL, uploader TEXT NOT NULL, length INTEGER NOT NULL, type TEXT NOT NULL, created INTEGER NOT NULL, last_seen INTEGER NOT NULL); |
| 2816 | + CREATE TABLE IF NOT EXISTS multipart_parts (upload_id TEXT NOT NULL, offset INTEGER NOT NULL, length INTEGER NOT NULL, PRIMARY KEY(upload_id, offset), FOREIGN KEY(upload_id) REFERENCES multipart_uploads(id) ON DELETE CASCADE); |
| 2817 | + INSERT OR IGNORE INTO blob_claims(sha256,uploader,claimed_at) |
| 2818 | + SELECT sha256,uploader,uploaded FROM blobs WHERE uploader != ''`); err != nil { |
| 2819 | + return nil, fmt.Errorf("initialize blob metadata: %w", err) |
| 2820 | + } |
| 2821 | + // Claims made under an agent grant with a ttl carry an expiry; every |
| 2822 | + // earlier claim keeps 0, which never lapses. |
| 2823 | + if _, err := config.Store.DB().ExecContext(ctx, "ALTER TABLE blob_claims ADD COLUMN expires INTEGER NOT NULL DEFAULT 0"); err != nil && !strings.Contains(err.Error(), "duplicate column") { |
| 2824 | + return nil, fmt.Errorf("add blob claim expiry: %w", err) |
| 2825 | + } |
| 2826 | + if _, err := config.Store.DB().ExecContext(ctx, "CREATE INDEX IF NOT EXISTS blob_claims_expires ON blob_claims(expires) WHERE expires>0"); err != nil { |
| 2827 | + return nil, fmt.Errorf("index blob claim expiry: %w", err) |
| 2828 | + } |
| 2829 | + if err := reconcile(ctx, config.Store, filepath.Join(config.Root, "blobs")); err != nil { |
| 2830 | + return nil, err |
| 2831 | + } |
| 2832 | + resolve := config.ResolveIP |
| 2833 | + if resolve == nil { |
| 2834 | + resolve = func(ctx context.Context, host string) ([]net.IP, error) { |
| 2835 | + return net.DefaultResolver.LookupIP(ctx, "ip", host) |
| 2836 | + } |
| 2837 | + } |
| 2838 | + s := &Service{config: config, root: filepath.Join(config.Root, "blobs"), store: config.Store, resolve: resolve, activeParts: make(map[string]int), multipartFinalizing: make(map[string]bool)} |
| 2839 | + if err := s.reconcileMultipart(ctx); err != nil { |
| 2840 | + return nil, err |
| 2841 | + } |
| 2842 | + return s, nil |
| 2843 | +} |
| 2844 | + |
| 2845 | +func (s *Service) Handler() http.HandlerFunc { return s.serveHTTP } |
| 2846 | diff --git a/internal/blob/storage.go b/internal/blob/storage.go |
| 2847 | new file mode 100644 |
| 2848 | index 0000000..811e54b |
| 2849 | --- /dev/null |
| 2850 | +++ b/internal/blob/storage.go |
| 2851 | @@ -0,0 +1,649 @@ |
| 2852 | +package blob |
| 2853 | + |
| 2854 | +import ( |
| 2855 | + "context" |
| 2856 | + "crypto/sha256" |
| 2857 | + "database/sql" |
| 2858 | + "encoding/hex" |
| 2859 | + "errors" |
| 2860 | + "fmt" |
| 2861 | + "io" |
| 2862 | + "os" |
| 2863 | + "path/filepath" |
| 2864 | + "strconv" |
| 2865 | + "strings" |
| 2866 | + "time" |
| 2867 | + |
| 2868 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 2869 | +) |
| 2870 | + |
| 2871 | +// Put stores bytes using their SHA-256 digest as the durable filename. |
| 2872 | +func (s *Service) Put(ctx context.Context, options PutOptions) (Blob, error) { |
| 2873 | + if options.Reader == nil { |
| 2874 | + return Blob{}, errors.New("blob reader is required") |
| 2875 | + } |
| 2876 | + entry, _, err := s.putOptions(ctx, options) |
| 2877 | + return entry, err |
| 2878 | +} |
| 2879 | + |
| 2880 | +func (s *Service) putOptions(ctx context.Context, options PutOptions) (Blob, bool, error) { |
| 2881 | + return s.putValidatedWithCommit(ctx, options.Reader, options.Type, options.Uploader, options.Hash, nil, options.Commit) |
| 2882 | +} |
| 2883 | + |
| 2884 | +// Get opens a verified metadata-backed blob for streaming to another service. |
| 2885 | +func (s *Service) Get(ctx context.Context, sha string) (Blob, io.ReadCloser, error) { |
| 2886 | + if !shaPattern.MatchString(sha) { |
| 2887 | + return Blob{}, nil, errors.New("invalid: bad blob sha256") |
| 2888 | + } |
| 2889 | + entry, err := s.lookup(ctx, sha) |
| 2890 | + if err != nil { |
| 2891 | + return Blob{}, nil, err |
| 2892 | + } |
| 2893 | + if s.blocked(ctx, sha) { |
| 2894 | + return Blob{}, nil, ErrBlocked |
| 2895 | + } |
| 2896 | + file, err := os.Open(filepath.Join(s.root, sha)) |
| 2897 | + if err != nil { |
| 2898 | + return Blob{}, nil, err |
| 2899 | + } |
| 2900 | + return entry, file, nil |
| 2901 | +} |
| 2902 | + |
| 2903 | +// ListMetadata returns all blobs uploaded by the supplied pubkey. |
| 2904 | +func (s *Service) ListMetadata(ctx context.Context, uploader string) ([]Blob, error) { |
| 2905 | + rows, err := s.store.DB().QueryContext(ctx, `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded |
| 2906 | + FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=? |
| 2907 | + ORDER BY b.uploaded DESC,b.sha256 DESC`, uploader) |
| 2908 | + if err != nil { |
| 2909 | + return nil, fmt.Errorf("list blob metadata: %w", err) |
| 2910 | + } |
| 2911 | + defer rows.Close() |
| 2912 | + entries := make([]Blob, 0) |
| 2913 | + for rows.Next() { |
| 2914 | + var entry Blob |
| 2915 | + if err := rows.Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded); err != nil { |
| 2916 | + return nil, fmt.Errorf("scan blob metadata: %w", err) |
| 2917 | + } |
| 2918 | + entries = append(entries, entry) |
| 2919 | + } |
| 2920 | + return entries, rows.Err() |
| 2921 | +} |
| 2922 | + |
| 2923 | +// ListPage returns a BUD-12 cursor page. The cursor is the last blob hash |
| 2924 | +// returned by the previous page and is intentionally opaque to callers. |
| 2925 | +func (s *Service) ListPage(ctx context.Context, uploader string, limit int, cursor string) ([]Blob, string, error) { |
| 2926 | + if limit < 1 { |
| 2927 | + limit = 20 |
| 2928 | + } |
| 2929 | + if limit > 100 { |
| 2930 | + limit = 100 |
| 2931 | + } |
| 2932 | + query := `SELECT b.sha256,b.size,b.type,c.uploader,b.uploaded FROM blobs b |
| 2933 | + JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=?` |
| 2934 | + args := []any{uploader} |
| 2935 | + if cursor != "" { |
| 2936 | + var uploaded int64 |
| 2937 | + if err := s.store.DB().QueryRowContext(ctx, `SELECT b.uploaded FROM blobs b |
| 2938 | + JOIN blob_claims c ON c.sha256=b.sha256 WHERE b.sha256=? AND c.uploader=?`, cursor, uploader).Scan(&uploaded); err != nil { |
| 2939 | + return nil, "", fmt.Errorf("resolve blob cursor: %w", err) |
| 2940 | + } |
| 2941 | + query += " AND (b.uploaded < ? OR (b.uploaded = ? AND b.sha256 < ?))" |
| 2942 | + args = append(args, uploaded, uploaded, cursor) |
| 2943 | + } |
| 2944 | + query += " ORDER BY b.uploaded DESC,b.sha256 DESC LIMIT ?" |
| 2945 | + args = append(args, limit+1) |
| 2946 | + rows, err := s.store.DB().QueryContext(ctx, query, args...) |
| 2947 | + if err != nil { |
| 2948 | + return nil, "", fmt.Errorf("list blob page: %w", err) |
| 2949 | + } |
| 2950 | + defer rows.Close() |
| 2951 | + entries := make([]Blob, 0, limit) |
| 2952 | + for rows.Next() { |
| 2953 | + var entry Blob |
| 2954 | + if err := rows.Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded); err != nil { |
| 2955 | + return nil, "", fmt.Errorf("scan blob page: %w", err) |
| 2956 | + } |
| 2957 | + entries = append(entries, entry) |
| 2958 | + } |
| 2959 | + if err := rows.Err(); err != nil { |
| 2960 | + return nil, "", fmt.Errorf("read blob page: %w", err) |
| 2961 | + } |
| 2962 | + next := "" |
| 2963 | + if len(entries) > limit { |
| 2964 | + next = entries[limit-1].SHA256 |
| 2965 | + entries = entries[:limit] |
| 2966 | + } |
| 2967 | + return entries, next, nil |
| 2968 | +} |
| 2969 | + |
| 2970 | +// Delete removes a blob's metadata and content. Authorization is the caller's responsibility. |
| 2971 | +func (s *Service) Delete(ctx context.Context, sha string) error { |
| 2972 | + if !shaPattern.MatchString(sha) { |
| 2973 | + return errors.New("invalid: bad blob sha256") |
| 2974 | + } |
| 2975 | + if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 2976 | + if _, err := tx.ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 2977 | + return fmt.Errorf("record blob deletion: %w", err) |
| 2978 | + } |
| 2979 | + if _, err := tx.ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 2980 | + return fmt.Errorf("delete blob metadata: %w", err) |
| 2981 | + } |
| 2982 | + return nil |
| 2983 | + }); err != nil { |
| 2984 | + return err |
| 2985 | + } |
| 2986 | + if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 2987 | + return fmt.Errorf("delete blob content: %w", err) |
| 2988 | + } |
| 2989 | + return nil |
| 2990 | +} |
| 2991 | + |
| 2992 | +// DeleteForUploader removes one owner's claim. The physical object remains |
| 2993 | +// while another owner still references it, which makes deduplicated uploads |
| 2994 | +// safe to manage independently. |
| 2995 | +func (s *Service) DeleteForUploader(ctx context.Context, sha, uploader string) error { |
| 2996 | + if !shaPattern.MatchString(sha) { |
| 2997 | + return errors.New("invalid: bad blob sha256") |
| 2998 | + } |
| 2999 | + removed := false |
| 3000 | + if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 3001 | + result, err := tx.ExecContext(ctx, "DELETE FROM blob_claims WHERE sha256=? AND uploader=?", sha, uploader) |
| 3002 | + if err != nil { |
| 3003 | + return fmt.Errorf("delete blob claim: %w", err) |
| 3004 | + } |
| 3005 | + changed, err := result.RowsAffected() |
| 3006 | + if err != nil { |
| 3007 | + return fmt.Errorf("check blob claim deletion: %w", err) |
| 3008 | + } |
| 3009 | + if changed == 0 { |
| 3010 | + return sql.ErrNoRows |
| 3011 | + } |
| 3012 | + var remaining int |
| 3013 | + if err := tx.QueryRowContext(ctx, "SELECT count(*) FROM blob_claims WHERE sha256=?", sha).Scan(&remaining); err != nil { |
| 3014 | + return fmt.Errorf("count blob claims: %w", err) |
| 3015 | + } |
| 3016 | + if remaining == 0 { |
| 3017 | + removed = true |
| 3018 | + if _, err := tx.ExecContext(ctx, "INSERT INTO blob_tombstones(sha256,deleted_at) VALUES(?,?) ON CONFLICT(sha256) DO UPDATE SET deleted_at=excluded.deleted_at", sha, time.Now().UTC().Unix()); err != nil { |
| 3019 | + return fmt.Errorf("record blob deletion: %w", err) |
| 3020 | + } |
| 3021 | + if _, err := tx.ExecContext(ctx, "DELETE FROM blobs WHERE sha256=?", sha); err != nil { |
| 3022 | + return fmt.Errorf("delete blob metadata: %w", err) |
| 3023 | + } |
| 3024 | + } |
| 3025 | + return nil |
| 3026 | + }); err != nil { |
| 3027 | + return err |
| 3028 | + } |
| 3029 | + if removed { |
| 3030 | + if err := os.Remove(filepath.Join(s.root, sha)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 3031 | + return fmt.Errorf("delete blob content: %w", err) |
| 3032 | + } |
| 3033 | + } |
| 3034 | + return nil |
| 3035 | +} |
| 3036 | + |
| 3037 | +// Descriptor returns the Blossom descriptor and NIP-94 tags for a blob. |
| 3038 | +func (s *Service) Descriptor(baseURL string, entry Blob) map[string]any { |
| 3039 | + return map[string]any{ |
| 3040 | + "url": baseURL + "/" + entry.SHA256 + extensionSuffix(entry.Type), |
| 3041 | + "sha256": entry.SHA256, |
| 3042 | + "size": entry.Size, |
| 3043 | + "type": entry.Type, |
| 3044 | + "uploaded": entry.Uploaded, |
| 3045 | + "nip94": [][]string{{"url", baseURL + "/" + entry.SHA256 + extensionSuffix(entry.Type)}, {"m", entry.Type}, {"x", entry.SHA256}, {"ox", entry.SHA256}, {"size", strconv.FormatInt(entry.Size, 10)}}, |
| 3046 | + } |
| 3047 | +} |
| 3048 | +func (s *Service) put(ctx context.Context, source io.Reader, typ, uploader, claimed string) (Blob, bool, error) { |
| 3049 | + return s.putValidatedWithCommit(ctx, source, typ, uploader, claimed, nil, nil) |
| 3050 | +} |
| 3051 | + |
| 3052 | +func (s *Service) putValidated(ctx context.Context, source io.Reader, typ, uploader, claimed string, validate func(string) error) (Blob, bool, error) { |
| 3053 | + return s.putValidatedWithCommit(ctx, source, typ, uploader, claimed, validate, nil) |
| 3054 | +} |
| 3055 | + |
| 3056 | +func (s *Service) putValidatedWithCommit(ctx context.Context, source io.Reader, typ, uploader, claimed string, validate func(string) error, commit func(context.Context, *sql.Tx, Blob, bool) error) (Blob, bool, error) { |
| 3057 | + limits := s.currentLimits() |
| 3058 | + streamLimit := limits.MaxFileBytes |
| 3059 | + quotaStream := false |
| 3060 | + if streamLimit == 0 && limits.UserStorageBytes > 0 && uploader != "" { |
| 3061 | + streamLimit = limits.UserStorageBytes |
| 3062 | + quotaStream = true |
| 3063 | + } |
| 3064 | + if streamLimit > 0 && streamLimit < int64(^uint64(0)>>1) { |
| 3065 | + source = io.LimitReader(source, streamLimit+1) |
| 3066 | + } |
| 3067 | + tmp, err := os.CreateTemp(s.root, ".upload-") |
| 3068 | + if err != nil { |
| 3069 | + return Blob{}, false, fmt.Errorf("create upload temporary file: %w", err) |
| 3070 | + } |
| 3071 | + tmpName := tmp.Name() |
| 3072 | + defer os.Remove(tmpName) |
| 3073 | + hasher := sha256.New() |
| 3074 | + count, err := io.Copy(io.MultiWriter(tmp, hasher), source) |
| 3075 | + if err != nil { |
| 3076 | + _ = tmp.Close() |
| 3077 | + return Blob{}, false, fmt.Errorf("read upload: %w", err) |
| 3078 | + } |
| 3079 | + if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 3080 | + _ = tmp.Close() |
| 3081 | + return Blob{}, false, ErrFileTooLarge |
| 3082 | + } |
| 3083 | + if quotaStream && count > streamLimit { |
| 3084 | + _ = tmp.Close() |
| 3085 | + return Blob{}, false, ErrQuotaExceeded |
| 3086 | + } |
| 3087 | + if err := tmp.Sync(); err != nil { |
| 3088 | + _ = tmp.Close() |
| 3089 | + return Blob{}, false, fmt.Errorf("sync upload: %w", err) |
| 3090 | + } |
| 3091 | + if err := tmp.Close(); err != nil { |
| 3092 | + return Blob{}, false, fmt.Errorf("close upload: %w", err) |
| 3093 | + } |
| 3094 | + sha := hex.EncodeToString(hasher.Sum(nil)) |
| 3095 | + if claimed != "" && claimed != sha { |
| 3096 | + return Blob{}, false, ErrHashMismatch |
| 3097 | + } |
| 3098 | + if validate != nil { |
| 3099 | + if err := validate(sha); err != nil { |
| 3100 | + return Blob{}, false, err |
| 3101 | + } |
| 3102 | + } |
| 3103 | + return s.installUpload(ctx, uploadCandidate{path: tmpName, hash: sha, size: count, typ: typ, uploader: uploader, commit: commit}) |
| 3104 | +} |
| 3105 | + |
| 3106 | +type uploadCandidate struct { |
| 3107 | + path, hash, typ, uploader, reservation string |
| 3108 | + size int64 |
| 3109 | + commit func(context.Context, *sql.Tx, Blob, bool) error |
| 3110 | +} |
| 3111 | + |
| 3112 | +func (s *Service) installUpload(ctx context.Context, candidate uploadCandidate) (Blob, bool, error) { |
| 3113 | + sha, count, typ, uploader := candidate.hash, candidate.size, candidate.typ, candidate.uploader |
| 3114 | + // Serialize the lookup, quota check, installation and metadata transaction. |
| 3115 | + // This closes the race where concurrent uploads could each observe the same |
| 3116 | + // remaining quota before one of them installs a new physical object. |
| 3117 | + s.quotaMu.Lock() |
| 3118 | + defer s.quotaMu.Unlock() |
| 3119 | + if s.blocked(ctx, sha) { |
| 3120 | + return Blob{}, false, ErrBlocked |
| 3121 | + } |
| 3122 | + terms, err := s.uploadTerms(ctx, uploader) |
| 3123 | + if err != nil { |
| 3124 | + return Blob{}, false, err |
| 3125 | + } |
| 3126 | + limits := s.currentLimits() |
| 3127 | + if limits.MaxFileBytes > 0 && count > limits.MaxFileBytes { |
| 3128 | + return Blob{}, false, ErrFileTooLarge |
| 3129 | + } |
| 3130 | + if limits.UserStorageBytes > 0 && uploader != "" && count > limits.UserStorageBytes { |
| 3131 | + return Blob{}, false, ErrQuotaExceeded |
| 3132 | + } |
| 3133 | + if existing, err := s.lookup(ctx, sha); err == nil { |
| 3134 | + if terms.Encrypted && !encryptedTypes[existing.Type] { |
| 3135 | + return Blob{}, false, ErrNotEncrypted |
| 3136 | + } |
| 3137 | + if uploader != "" { |
| 3138 | + if limits.UserStorageBytes > 0 && !s.hasClaim(ctx, sha, uploader) { |
| 3139 | + used, usageErr := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 3140 | + if usageErr != nil { |
| 3141 | + return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", usageErr) |
| 3142 | + } |
| 3143 | + if existing.Size > limits.UserStorageBytes-used { |
| 3144 | + return Blob{}, false, ErrQuotaExceeded |
| 3145 | + } |
| 3146 | + } |
| 3147 | + } |
| 3148 | + if err := s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 3149 | + if candidate.commit != nil { |
| 3150 | + if err := candidate.commit(ctx, tx, existing, false); err != nil { |
| 3151 | + return err |
| 3152 | + } |
| 3153 | + } |
| 3154 | + if uploader != "" { |
| 3155 | + if _, err := tx.ExecContext(ctx, claimSQL, sha, uploader, time.Now().Unix(), terms.ExpiresAt); err != nil { |
| 3156 | + return err |
| 3157 | + } |
| 3158 | + } |
| 3159 | + if candidate.reservation != "" { |
| 3160 | + if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 3161 | + return err |
| 3162 | + } |
| 3163 | + } |
| 3164 | + return nil |
| 3165 | + }); err != nil { |
| 3166 | + return Blob{}, false, err |
| 3167 | + } |
| 3168 | + // A crash after a scoped metadata commit may leave the bytes missing. |
| 3169 | + // A retry supplies those same verified bytes without losing the scope. |
| 3170 | + if _, err := os.Stat(filepath.Join(s.root, sha)); errors.Is(err, os.ErrNotExist) { |
| 3171 | + if err := s.installBlobFile(candidate.path, sha); err != nil { |
| 3172 | + return Blob{}, false, err |
| 3173 | + } |
| 3174 | + } else if err != nil { |
| 3175 | + return Blob{}, false, err |
| 3176 | + } |
| 3177 | + return existing, false, nil |
| 3178 | + } else if !errors.Is(err, sql.ErrNoRows) { |
| 3179 | + return Blob{}, false, err |
| 3180 | + } |
| 3181 | + if limits.UserStorageBytes > 0 && uploader != "" { |
| 3182 | + used, err := s.quotaUsageExcept(ctx, uploader, candidate.reservation) |
| 3183 | + if err != nil { |
| 3184 | + return Blob{}, false, fmt.Errorf("check uploader storage quota: %w", err) |
| 3185 | + } |
| 3186 | + if count > limits.UserStorageBytes-used { |
| 3187 | + return Blob{}, false, ErrQuotaExceeded |
| 3188 | + } |
| 3189 | + } |
| 3190 | + // The type is settled before the bytes are installed so a plain upload |
| 3191 | + // under an encrypted-only grant leaves nothing behind. A declared octet |
| 3192 | + // stream is sniffed: encrypted bytes stay one, a page or an image does not. |
| 3193 | + typ = contentType(typ) |
| 3194 | + if typ == "application/octet-stream" { |
| 3195 | + typ = detectFileType(candidate.path) |
| 3196 | + } |
| 3197 | + if terms.Encrypted && !encryptedTypes[typ] { |
| 3198 | + return Blob{}, false, ErrNotEncrypted |
| 3199 | + } |
| 3200 | + stage := "" |
| 3201 | + if candidate.commit != nil { |
| 3202 | + stage = filepath.Join(s.root, ".scoped-"+sha) |
| 3203 | + if err := os.Rename(candidate.path, stage); err != nil { |
| 3204 | + return Blob{}, false, fmt.Errorf("stage scoped blob: %w", err) |
| 3205 | + } |
| 3206 | + if err := syncDirectory(s.root); err != nil { |
| 3207 | + return Blob{}, false, fmt.Errorf("sync staged blob directory: %w", err) |
| 3208 | + } |
| 3209 | + } |
| 3210 | + if candidate.commit == nil { |
| 3211 | + if err := s.installBlobFile(candidate.path, sha); err != nil { |
| 3212 | + return Blob{}, false, err |
| 3213 | + } |
| 3214 | + } |
| 3215 | + now := time.Now().UTC().Unix() |
| 3216 | + entry := Blob{SHA256: sha, Size: count, Type: typ, Uploader: uploader, Uploaded: now} |
| 3217 | + err = s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 3218 | + if _, err := tx.ExecContext(ctx, "DELETE FROM blob_tombstones WHERE sha256=?", entry.SHA256); err != nil { |
| 3219 | + return fmt.Errorf("clear blob deletion tombstone: %w", err) |
| 3220 | + } |
| 3221 | + if _, err := tx.ExecContext(ctx, "INSERT OR IGNORE INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)", entry.SHA256, entry.Size, entry.Type, entry.Uploader, entry.Uploaded); err != nil { |
| 3222 | + return fmt.Errorf("record blob: %w", err) |
| 3223 | + } |
| 3224 | + if entry.Uploader != "" { |
| 3225 | + if _, err := tx.ExecContext(ctx, claimSQL, entry.SHA256, entry.Uploader, entry.Uploaded, terms.ExpiresAt); err != nil { |
| 3226 | + return fmt.Errorf("record blob claim: %w", err) |
| 3227 | + } |
| 3228 | + } |
| 3229 | + if candidate.reservation != "" { |
| 3230 | + if _, err := tx.ExecContext(ctx, "DELETE FROM multipart_uploads WHERE id=?", candidate.reservation); err != nil { |
| 3231 | + return fmt.Errorf("release upload reservation: %w", err) |
| 3232 | + } |
| 3233 | + } |
| 3234 | + if candidate.commit != nil { |
| 3235 | + if err := candidate.commit(ctx, tx, entry, true); err != nil { |
| 3236 | + return err |
| 3237 | + } |
| 3238 | + } |
| 3239 | + return nil |
| 3240 | + }) |
| 3241 | + if err != nil { |
| 3242 | + if stage != "" { |
| 3243 | + _ = os.Remove(stage) |
| 3244 | + } |
| 3245 | + return Blob{}, false, err |
| 3246 | + } |
| 3247 | + if candidate.commit != nil { |
| 3248 | + if err := s.installBlobFile(stage, sha); err != nil { |
| 3249 | + return Blob{}, false, err |
| 3250 | + } |
| 3251 | + } |
| 3252 | + if saved, err := s.lookup(ctx, sha); err == nil { |
| 3253 | + return saved, saved.Uploader == uploader && saved.Uploaded == now, nil |
| 3254 | + } |
| 3255 | + return entry, true, nil |
| 3256 | +} |
| 3257 | + |
| 3258 | +func (s *Service) installBlobFile(source, hash string) error { |
| 3259 | + if err := os.Rename(source, filepath.Join(s.root, hash)); err != nil { |
| 3260 | + return fmt.Errorf("install blob: %w", err) |
| 3261 | + } |
| 3262 | + if err := syncDirectory(s.root); err != nil { |
| 3263 | + return fmt.Errorf("sync blob directory: %w", err) |
| 3264 | + } |
| 3265 | + return nil |
| 3266 | +} |
| 3267 | + |
| 3268 | +func (s *Service) quotaUsage(ctx context.Context, uploader string) (int64, error) { |
| 3269 | + return s.quotaUsageExcept(ctx, uploader, "") |
| 3270 | +} |
| 3271 | + |
| 3272 | +func (s *Service) quotaUsageExcept(ctx context.Context, uploader, reservation string) (int64, error) { |
| 3273 | + var used sql.NullInt64 |
| 3274 | + err := s.store.DB().QueryRowContext(ctx, `SELECT COALESCE((SELECT SUM(b.size) |
| 3275 | + FROM blobs b JOIN blob_claims c ON c.sha256=b.sha256 WHERE c.uploader=?),0) + |
| 3276 | + COALESCE((SELECT SUM(length) FROM multipart_uploads WHERE uploader=? AND id != ?),0)`, uploader, uploader, reservation).Scan(&used) |
| 3277 | + if err != nil { |
| 3278 | + return 0, err |
| 3279 | + } |
| 3280 | + return used.Int64, nil |
| 3281 | +} |
| 3282 | + |
| 3283 | +// claimSQL records a claim. The insert trigger on blobs has usually made the |
| 3284 | +// row already, without an expiry, so the newest upload's terms replace it. |
| 3285 | +const claimSQL = "INSERT INTO blob_claims(sha256,uploader,claimed_at,expires) VALUES(?,?,?,?) ON CONFLICT(sha256,uploader) DO UPDATE SET expires=excluded.expires" |
| 3286 | + |
| 3287 | +func (s *Service) claim(ctx context.Context, sha, uploader string, expires int64) error { |
| 3288 | + _, err := s.store.DB().ExecContext(ctx, claimSQL, sha, uploader, time.Now().UTC().Unix(), expires) |
| 3289 | + if err != nil { |
| 3290 | + return fmt.Errorf("record blob claim: %w", err) |
| 3291 | + } |
| 3292 | + return nil |
| 3293 | +} |
| 3294 | + |
| 3295 | +func (s *Service) uploadTerms(ctx context.Context, uploader string) (UploadTerms, error) { |
| 3296 | + if s.config.UploadTerms == nil || uploader == "" { |
| 3297 | + return UploadTerms{}, nil |
| 3298 | + } |
| 3299 | + // A refusal keeps its reason so the door answers with the right status. |
| 3300 | + return s.config.UploadTerms(ctx, uploader) |
| 3301 | +} |
| 3302 | + |
| 3303 | +// SweepExpired removes the blobs whose every claim has lapsed and releases |
| 3304 | +// lapsed claims on blobs someone else still holds, so an agent's files leave |
| 3305 | +// with its site while a person's claim keeps them. It returns how many blobs |
| 3306 | +// were removed. |
| 3307 | +func (s *Service) SweepExpired(ctx context.Context, now int64) (int, error) { |
| 3308 | + s.quotaMu.Lock() |
| 3309 | + defer s.quotaMu.Unlock() |
| 3310 | + rows, err := s.store.DB().QueryContext(ctx, `SELECT DISTINCT sha256 FROM blob_claims c WHERE c.expires>0 AND c.expires<=? |
| 3311 | + AND NOT EXISTS(SELECT 1 FROM blob_claims live WHERE live.sha256=c.sha256 AND (live.expires=0 OR live.expires>?))`, now, now) |
| 3312 | + if err != nil { |
| 3313 | + return 0, fmt.Errorf("list expired blobs: %w", err) |
| 3314 | + } |
| 3315 | + var expired []string |
| 3316 | + for rows.Next() { |
| 3317 | + var sha string |
| 3318 | + if err := rows.Scan(&sha); err != nil { |
| 3319 | + _ = rows.Close() |
| 3320 | + return 0, err |
| 3321 | + } |
| 3322 | + expired = append(expired, sha) |
| 3323 | + } |
| 3324 | + if err := errors.Join(rows.Err(), rows.Close()); err != nil { |
| 3325 | + return 0, err |
| 3326 | + } |
| 3327 | + for _, sha := range expired { |
| 3328 | + if err := s.Delete(ctx, sha); err != nil { |
| 3329 | + return 0, err |
| 3330 | + } |
| 3331 | + } |
| 3332 | + if _, err := s.store.DB().ExecContext(ctx, "DELETE FROM blob_claims WHERE expires>0 AND expires<=?", now); err != nil { |
| 3333 | + return 0, fmt.Errorf("release expired blob claims: %w", err) |
| 3334 | + } |
| 3335 | + return len(expired), nil |
| 3336 | +} |
| 3337 | + |
| 3338 | +func (s *Service) currentLimits() Limits { |
| 3339 | + if s.config.Limits == nil { |
| 3340 | + return Limits{} |
| 3341 | + } |
| 3342 | + limits := s.config.Limits() |
| 3343 | + if limits.MaxFileBytes < 0 { |
| 3344 | + limits.MaxFileBytes = 0 |
| 3345 | + } |
| 3346 | + if limits.UserStorageBytes < 0 { |
| 3347 | + limits.UserStorageBytes = 0 |
| 3348 | + } |
| 3349 | + return limits |
| 3350 | +} |
| 3351 | + |
| 3352 | +func (s *Service) hasClaim(ctx context.Context, sha, uploader string) bool { |
| 3353 | + var found int |
| 3354 | + err := s.store.DB().QueryRowContext(ctx, "SELECT 1 FROM blob_claims WHERE sha256=? AND uploader=?", sha, uploader).Scan(&found) |
| 3355 | + return err == nil && found == 1 |
| 3356 | +} |
| 3357 | + |
| 3358 | +func (s *Service) lookup(ctx context.Context, sha string) (Blob, error) { |
| 3359 | + var entry Blob |
| 3360 | + err := s.store.DB().QueryRowContext(ctx, "SELECT sha256,size,type,uploader,uploaded FROM blobs WHERE sha256=?", sha).Scan(&entry.SHA256, &entry.Size, &entry.Type, &entry.Uploader, &entry.Uploaded) |
| 3361 | + return entry, err |
| 3362 | +} |
| 3363 | + |
| 3364 | +func (s *Service) blocked(ctx context.Context, sha string) bool { |
| 3365 | + var found int |
| 3366 | + err := s.store.DB().QueryRowContext(ctx, `SELECT 1 FROM hidden_events WHERE id=? |
| 3367 | + UNION ALL SELECT 1 FROM blob_blocks WHERE sha256=? LIMIT 1`, sha, sha).Scan(&found) |
| 3368 | + return err == nil && found == 1 |
| 3369 | +} |
| 3370 | + |
| 3371 | +func reconcile(ctx context.Context, store *storage.Store, root string) error { |
| 3372 | + entries, err := os.ReadDir(root) |
| 3373 | + if err != nil { |
| 3374 | + return fmt.Errorf("scan blob directory: %w", err) |
| 3375 | + } |
| 3376 | + for _, entry := range entries { |
| 3377 | + if entry.IsDir() { |
| 3378 | + continue |
| 3379 | + } |
| 3380 | + name := entry.Name() |
| 3381 | + if strings.HasPrefix(name, ".scoped-") { |
| 3382 | + sha := strings.TrimPrefix(name, ".scoped-") |
| 3383 | + if !shaPattern.MatchString(sha) { |
| 3384 | + if err := os.Remove(filepath.Join(root, name)); err != nil { |
| 3385 | + return fmt.Errorf("remove invalid staged blob: %w", err) |
| 3386 | + } |
| 3387 | + continue |
| 3388 | + } |
| 3389 | + if err := reconcileStaged(ctx, store, root, name, sha); err != nil { |
| 3390 | + return err |
| 3391 | + } |
| 3392 | + continue |
| 3393 | + } |
| 3394 | + if strings.HasPrefix(name, ".upload-") || strings.HasPrefix(name, ".multipart-chunk-") { |
| 3395 | + if err := os.Remove(filepath.Join(root, name)); err != nil { |
| 3396 | + return fmt.Errorf("remove abandoned upload: %w", err) |
| 3397 | + } |
| 3398 | + continue |
| 3399 | + } |
| 3400 | + if !shaPattern.MatchString(name) { |
| 3401 | + continue |
| 3402 | + } |
| 3403 | + var deleted int |
| 3404 | + if err := store.DB().QueryRowContext(ctx, "SELECT 1 FROM blob_tombstones WHERE sha256=? UNION ALL SELECT 1 FROM blob_blocks WHERE sha256=? LIMIT 1", name, name).Scan(&deleted); err == nil { |
| 3405 | + continue |
| 3406 | + } else if !errors.Is(err, sql.ErrNoRows) { |
| 3407 | + return fmt.Errorf("check deleted blob %q: %w", name, err) |
| 3408 | + } |
| 3409 | + var known int |
| 3410 | + if err := store.DB().QueryRowContext(ctx, "SELECT 1 FROM blobs WHERE sha256=?", name).Scan(&known); err == nil { |
| 3411 | + continue |
| 3412 | + } else if !errors.Is(err, sql.ErrNoRows) { |
| 3413 | + return fmt.Errorf("check orphan blob %q: %w", name, err) |
| 3414 | + } |
| 3415 | + if err := recoverBlob(ctx, store, root, name); err != nil { |
| 3416 | + return err |
| 3417 | + } |
| 3418 | + } |
| 3419 | + return nil |
| 3420 | +} |
| 3421 | + |
| 3422 | +func reconcileStaged(ctx context.Context, store *storage.Store, root, name, sha string) error { |
| 3423 | + var size int64 |
| 3424 | + if err := store.DB().QueryRowContext(ctx, "SELECT size FROM blobs WHERE sha256=?", sha).Scan(&size); errors.Is(err, sql.ErrNoRows) { |
| 3425 | + if removeErr := os.Remove(filepath.Join(root, name)); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { |
| 3426 | + return removeErr |
| 3427 | + } |
| 3428 | + return nil |
| 3429 | + } else if err != nil { |
| 3430 | + return fmt.Errorf("check staged blob %q: %w", sha, err) |
| 3431 | + } |
| 3432 | + file, err := os.Open(filepath.Join(root, name)) |
| 3433 | + if err != nil { |
| 3434 | + return err |
| 3435 | + } |
| 3436 | + info, err := file.Stat() |
| 3437 | + if err != nil { |
| 3438 | + _ = file.Close() |
| 3439 | + return err |
| 3440 | + } |
| 3441 | + if info.Size() != size { |
| 3442 | + _ = file.Close() |
| 3443 | + return fmt.Errorf("staged blob %q has size %d, want %d", sha, info.Size(), size) |
| 3444 | + } |
| 3445 | + hasher := sha256.New() |
| 3446 | + if _, err := io.Copy(hasher, file); err != nil { |
| 3447 | + _ = file.Close() |
| 3448 | + return err |
| 3449 | + } |
| 3450 | + if err := file.Close(); err != nil { |
| 3451 | + return err |
| 3452 | + } |
| 3453 | + if hex.EncodeToString(hasher.Sum(nil)) != sha { |
| 3454 | + return fmt.Errorf("staged blob %q failed content hash", sha) |
| 3455 | + } |
| 3456 | + if _, err := os.Stat(filepath.Join(root, sha)); errors.Is(err, os.ErrNotExist) { |
| 3457 | + if err := os.Rename(filepath.Join(root, name), filepath.Join(root, sha)); err != nil { |
| 3458 | + return err |
| 3459 | + } |
| 3460 | + } else if err != nil { |
| 3461 | + return err |
| 3462 | + } else { |
| 3463 | + _ = os.Remove(filepath.Join(root, name)) |
| 3464 | + } |
| 3465 | + return syncDirectory(root) |
| 3466 | +} |
| 3467 | + |
| 3468 | +func recoverBlob(ctx context.Context, store *storage.Store, root, name string) error { |
| 3469 | + file, err := os.Open(filepath.Join(root, name)) |
| 3470 | + if err != nil { |
| 3471 | + return fmt.Errorf("open orphan blob %q: %w", name, err) |
| 3472 | + } |
| 3473 | + hash := sha256.New() |
| 3474 | + size, copyErr := io.Copy(hash, file) |
| 3475 | + closeErr := file.Close() |
| 3476 | + if copyErr != nil || closeErr != nil { |
| 3477 | + return fmt.Errorf("read orphan blob %q: %w", name, errors.Join(copyErr, closeErr)) |
| 3478 | + } |
| 3479 | + if hex.EncodeToString(hash.Sum(nil)) != name { |
| 3480 | + return fmt.Errorf("orphan blob %q failed content hash", name) |
| 3481 | + } |
| 3482 | + info, err := os.Stat(filepath.Join(root, name)) |
| 3483 | + if err != nil { |
| 3484 | + return fmt.Errorf("stat orphan blob %q: %w", name, err) |
| 3485 | + } |
| 3486 | + _, err = store.DB().ExecContext(ctx, "INSERT OR IGNORE INTO blobs(sha256,size,type,uploader,uploaded) VALUES(?,?,?,?,?)", name, size, "application/octet-stream", "", info.ModTime().Unix()) |
| 3487 | + if err != nil { |
| 3488 | + return fmt.Errorf("recover orphan blob %q: %w", name, err) |
| 3489 | + } |
| 3490 | + return nil |
| 3491 | +} |
| 3492 | + |
| 3493 | +func syncDirectory(path string) error { |
| 3494 | + directory, err := os.Open(path) |
| 3495 | + if err != nil { |
| 3496 | + return err |
| 3497 | + } |
| 3498 | + defer directory.Close() |
| 3499 | + return directory.Sync() |
| 3500 | +} |
| 3501 | diff --git a/internal/community/config.go b/internal/community/config.go |
| 3502 | new file mode 100644 |
| 3503 | index 0000000..919f646 |
| 3504 | --- /dev/null |
| 3505 | +++ b/internal/community/config.go |
| 3506 | @@ -0,0 +1,253 @@ |
| 3507 | +package community |
| 3508 | + |
| 3509 | +import ( |
| 3510 | + "context" |
| 3511 | + "database/sql" |
| 3512 | + "time" |
| 3513 | +) |
| 3514 | + |
| 3515 | +// ConfigSnapshot is the community-owned portion of relay configuration. |
| 3516 | +// The type deliberately contains no configport types so the domain owns its |
| 3517 | +// storage contract. |
| 3518 | +type ConfigSnapshot struct { |
| 3519 | + Members []Member |
| 3520 | + Bans []ConfigBan |
| 3521 | + IPBlocks []ConfigIPBlock |
| 3522 | + EventBans []ConfigEventBan |
| 3523 | + KindRules []ConfigKindRule |
| 3524 | + Retention []ConfigRetention |
| 3525 | +} |
| 3526 | + |
| 3527 | +type ConfigBan struct { |
| 3528 | + PubKey string |
| 3529 | + Reason string |
| 3530 | +} |
| 3531 | + |
| 3532 | +type ConfigIPBlock struct { |
| 3533 | + IP string |
| 3534 | + Reason string |
| 3535 | +} |
| 3536 | + |
| 3537 | +type ConfigEventBan struct { |
| 3538 | + ID string |
| 3539 | + Reason string |
| 3540 | +} |
| 3541 | + |
| 3542 | +type ConfigKindRule struct { |
| 3543 | + Kind int |
| 3544 | + Rule string |
| 3545 | +} |
| 3546 | + |
| 3547 | +type ConfigRetention struct { |
| 3548 | + Kind *int |
| 3549 | + Days int |
| 3550 | +} |
| 3551 | + |
| 3552 | +type ConfigMemberRetention struct { |
| 3553 | + PubKey string |
| 3554 | + Days int |
| 3555 | +} |
| 3556 | + |
| 3557 | +// RetentionRules returns the community-owned retention settings. |
| 3558 | +func (s *Service) RetentionRules(ctx context.Context) ([]ConfigRetention, error) { |
| 3559 | + rows, err := s.store.DB().QueryContext(ctx, `SELECT kind,days FROM community_retention ORDER BY kind`) |
| 3560 | + if err != nil { |
| 3561 | + return nil, err |
| 3562 | + } |
| 3563 | + defer rows.Close() |
| 3564 | + var out []ConfigRetention |
| 3565 | + for rows.Next() { |
| 3566 | + var kind, days int |
| 3567 | + if err := rows.Scan(&kind, &days); err != nil { |
| 3568 | + return nil, err |
| 3569 | + } |
| 3570 | + out = append(out, ConfigRetention{Kind: &kind, Days: days}) |
| 3571 | + } |
| 3572 | + return out, rows.Err() |
| 3573 | +} |
| 3574 | + |
| 3575 | +// MembersForRetention returns members with an explicit retention override. |
| 3576 | +func (s *Service) MembersForRetention(ctx context.Context) ([]ConfigMemberRetention, error) { |
| 3577 | + rows, err := s.store.DB().QueryContext(ctx, `SELECT pubkey,keep_days FROM community_members WHERE keep_days>0 ORDER BY pubkey`) |
| 3578 | + if err != nil { |
| 3579 | + return nil, err |
| 3580 | + } |
| 3581 | + defer rows.Close() |
| 3582 | + var out []ConfigMemberRetention |
| 3583 | + for rows.Next() { |
| 3584 | + var member ConfigMemberRetention |
| 3585 | + if err := rows.Scan(&member.PubKey, &member.Days); err != nil { |
| 3586 | + return nil, err |
| 3587 | + } |
| 3588 | + out = append(out, member) |
| 3589 | + } |
| 3590 | + return out, rows.Err() |
| 3591 | +} |
| 3592 | + |
| 3593 | +// ExportConfig reads the community-owned configuration tables. |
| 3594 | +func ExportConfig(ctx context.Context, store interface { |
| 3595 | + DB() *sql.DB |
| 3596 | +}) (ConfigSnapshot, error) { |
| 3597 | + var out ConfigSnapshot |
| 3598 | + rows, err := store.DB().QueryContext(ctx, `SELECT pubkey,name,note,role,invited_by,via,created_at,keep_days FROM community_members WHERE role<>'owner' ORDER BY pubkey`) |
| 3599 | + if err != nil { |
| 3600 | + return out, err |
| 3601 | + } |
| 3602 | + defer rows.Close() |
| 3603 | + for rows.Next() { |
| 3604 | + var member Member |
| 3605 | + if err := rows.Scan(&member.PubKey, &member.Name, &member.Note, &member.Role, &member.InvitedBy, &member.Via, &member.CreatedAt, &member.KeepDays); err != nil { |
| 3606 | + return out, err |
| 3607 | + } |
| 3608 | + out.Members = append(out.Members, member) |
| 3609 | + } |
| 3610 | + if err := rows.Err(); err != nil { |
| 3611 | + return out, err |
| 3612 | + } |
| 3613 | + if err := readConfigRows(ctx, store.DB(), `SELECT pubkey,reason FROM community_bans ORDER BY pubkey`, func(rows *sql.Rows) error { |
| 3614 | + var item ConfigBan |
| 3615 | + if err := rows.Scan(&item.PubKey, &item.Reason); err != nil { |
| 3616 | + return err |
| 3617 | + } |
| 3618 | + out.Bans = append(out.Bans, item) |
| 3619 | + return nil |
| 3620 | + }); err != nil { |
| 3621 | + return out, err |
| 3622 | + } |
| 3623 | + if err := readConfigRows(ctx, store.DB(), `SELECT ip,reason FROM community_ip_blocks ORDER BY ip`, func(rows *sql.Rows) error { |
| 3624 | + var item ConfigIPBlock |
| 3625 | + if err := rows.Scan(&item.IP, &item.Reason); err != nil { |
| 3626 | + return err |
| 3627 | + } |
| 3628 | + out.IPBlocks = append(out.IPBlocks, item) |
| 3629 | + return nil |
| 3630 | + }); err != nil { |
| 3631 | + return out, err |
| 3632 | + } |
| 3633 | + if err := readConfigRows(ctx, store.DB(), `SELECT id,reason FROM community_event_bans ORDER BY id`, func(rows *sql.Rows) error { |
| 3634 | + var item ConfigEventBan |
| 3635 | + if err := rows.Scan(&item.ID, &item.Reason); err != nil { |
| 3636 | + return err |
| 3637 | + } |
| 3638 | + out.EventBans = append(out.EventBans, item) |
| 3639 | + return nil |
| 3640 | + }); err != nil { |
| 3641 | + return out, err |
| 3642 | + } |
| 3643 | + if err := readConfigRows(ctx, store.DB(), `SELECT kind,rule FROM community_kind_rules ORDER BY kind`, func(rows *sql.Rows) error { |
| 3644 | + var item ConfigKindRule |
| 3645 | + if err := rows.Scan(&item.Kind, &item.Rule); err != nil { |
| 3646 | + return err |
| 3647 | + } |
| 3648 | + out.KindRules = append(out.KindRules, item) |
| 3649 | + return nil |
| 3650 | + }); err != nil { |
| 3651 | + return out, err |
| 3652 | + } |
| 3653 | + if err := readConfigRows(ctx, store.DB(), `SELECT kind,days FROM community_retention ORDER BY kind`, func(rows *sql.Rows) error { |
| 3654 | + var kind, days int |
| 3655 | + if err := rows.Scan(&kind, &days); err != nil { |
| 3656 | + return err |
| 3657 | + } |
| 3658 | + if kind < 0 { |
| 3659 | + out.Retention = append(out.Retention, ConfigRetention{Days: days}) |
| 3660 | + return nil |
| 3661 | + } |
| 3662 | + out.Retention = append(out.Retention, ConfigRetention{Kind: &kind, Days: days}) |
| 3663 | + return nil |
| 3664 | + }); err != nil { |
| 3665 | + return out, err |
| 3666 | + } |
| 3667 | + return out, nil |
| 3668 | +} |
| 3669 | + |
| 3670 | +func readConfigRows(ctx context.Context, db *sql.DB, query string, scan func(*sql.Rows) error) error { |
| 3671 | + rows, err := db.QueryContext(ctx, query) |
| 3672 | + if err != nil { |
| 3673 | + return err |
| 3674 | + } |
| 3675 | + defer rows.Close() |
| 3676 | + for rows.Next() { |
| 3677 | + if err := scan(rows); err != nil { |
| 3678 | + return err |
| 3679 | + } |
| 3680 | + } |
| 3681 | + return rows.Err() |
| 3682 | +} |
| 3683 | + |
| 3684 | +// ApplyConfigTx replaces selected community configuration sections inside a |
| 3685 | +// caller-owned transaction. Empty sections are meaningful replacements. |
| 3686 | +func ApplyConfigTx(ctx context.Context, tx *sql.Tx, c ConfigSnapshot, members, bans, addresses, eventBans, kinds, retention bool) error { |
| 3687 | + now := time.Now().Unix() |
| 3688 | + if members { |
| 3689 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_members WHERE role<>'owner'`); err != nil { |
| 3690 | + return err |
| 3691 | + } |
| 3692 | + for _, member := range c.Members { |
| 3693 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_members(pubkey,name,note,role,invited_by,via,created_at,keep_days) VALUES(?,?,?,?,?,?,?,?)`, member.PubKey, member.Name, member.Note, member.Role, member.InvitedBy, member.Via, now, member.KeepDays); err != nil { |
| 3694 | + return err |
| 3695 | + } |
| 3696 | + } |
| 3697 | + } |
| 3698 | + if bans { |
| 3699 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_bans`); err != nil { |
| 3700 | + return err |
| 3701 | + } |
| 3702 | + for _, item := range c.Bans { |
| 3703 | + if item.PubKey != "" { |
| 3704 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_bans(pubkey,reason,at) VALUES(?,?,?)`, item.PubKey, item.Reason, now); err != nil { |
| 3705 | + return err |
| 3706 | + } |
| 3707 | + } |
| 3708 | + } |
| 3709 | + } |
| 3710 | + if addresses { |
| 3711 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_ip_blocks`); err != nil { |
| 3712 | + return err |
| 3713 | + } |
| 3714 | + for _, item := range c.IPBlocks { |
| 3715 | + if item.IP != "" { |
| 3716 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_ip_blocks(ip,reason,at) VALUES(?,?,?)`, item.IP, item.Reason, now); err != nil { |
| 3717 | + return err |
| 3718 | + } |
| 3719 | + } |
| 3720 | + } |
| 3721 | + } |
| 3722 | + if eventBans { |
| 3723 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_event_bans`); err != nil { |
| 3724 | + return err |
| 3725 | + } |
| 3726 | + for _, item := range c.EventBans { |
| 3727 | + if item.ID != "" { |
| 3728 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_event_bans(id,reason,at) VALUES(?,?,?)`, item.ID, item.Reason, now); err != nil { |
| 3729 | + return err |
| 3730 | + } |
| 3731 | + } |
| 3732 | + } |
| 3733 | + } |
| 3734 | + if kinds { |
| 3735 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_kind_rules`); err != nil { |
| 3736 | + return err |
| 3737 | + } |
| 3738 | + for _, item := range c.KindRules { |
| 3739 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_kind_rules(kind,rule) VALUES(?,?)`, item.Kind, item.Rule); err != nil { |
| 3740 | + return err |
| 3741 | + } |
| 3742 | + } |
| 3743 | + } |
| 3744 | + if retention { |
| 3745 | + if _, err := tx.ExecContext(ctx, `DELETE FROM community_retention`); err != nil { |
| 3746 | + return err |
| 3747 | + } |
| 3748 | + for _, item := range c.Retention { |
| 3749 | + kind := -1 |
| 3750 | + if item.Kind != nil { |
| 3751 | + kind = *item.Kind |
| 3752 | + } |
| 3753 | + if _, err := tx.ExecContext(ctx, `INSERT INTO community_retention(kind,days) VALUES(?,?)`, kind, item.Days); err != nil { |
| 3754 | + return err |
| 3755 | + } |
| 3756 | + } |
| 3757 | + } |
| 3758 | + return nil |
| 3759 | +} |
| 3760 | diff --git a/internal/community/records_reader.go b/internal/community/records_reader.go |
| 3761 | new file mode 100644 |
| 3762 | index 0000000..6edda93 |
| 3763 | --- /dev/null |
| 3764 | +++ b/internal/community/records_reader.go |
| 3765 | @@ -0,0 +1,72 @@ |
| 3766 | +package community |
| 3767 | + |
| 3768 | +import ( |
| 3769 | + "context" |
| 3770 | + "errors" |
| 3771 | + "strings" |
| 3772 | + |
| 3773 | + "github.com/FelineStateMachine/tinyrelay/internal/communityread" |
| 3774 | +) |
| 3775 | + |
| 3776 | +// Members exposes the community-owned membership projection to records. The |
| 3777 | +// records package consumes this narrow read model and never relies on table |
| 3778 | +// names or columns. |
| 3779 | +func (s *Service) Members(ctx context.Context) ([]communityread.Member, error) { |
| 3780 | + rows, err := s.store.DB().QueryContext(ctx, `SELECT pubkey,name,role FROM community_members ORDER BY pubkey`) |
| 3781 | + if err != nil { |
| 3782 | + if strings.Contains(err.Error(), "no such table") { |
| 3783 | + return nil, nil |
| 3784 | + } |
| 3785 | + return nil, err |
| 3786 | + } |
| 3787 | + defer rows.Close() |
| 3788 | + var out []communityread.Member |
| 3789 | + for rows.Next() { |
| 3790 | + var member communityread.Member |
| 3791 | + if err := rows.Scan(&member.PubKey, &member.Name, &member.Role); err != nil { |
| 3792 | + return nil, err |
| 3793 | + } |
| 3794 | + out = append(out, member) |
| 3795 | + } |
| 3796 | + return out, rows.Err() |
| 3797 | +} |
| 3798 | + |
| 3799 | +// MemberStatus reports heir membership and the current member count in one |
| 3800 | +// call so succession checks use a consistent community snapshot. |
| 3801 | +func (s *Service) MemberStatus(ctx context.Context, pubkey string) (bool, int, error) { |
| 3802 | + var count int |
| 3803 | + var member int |
| 3804 | + if err := s.store.DB().QueryRowContext(ctx, `SELECT count(*), coalesce(max(pubkey=?),0) FROM community_members`, pubkey).Scan(&count, &member); err != nil { |
| 3805 | + if strings.Contains(err.Error(), "no such table") { |
| 3806 | + return false, 0, nil |
| 3807 | + } |
| 3808 | + return false, 0, err |
| 3809 | + } |
| 3810 | + return member != 0, count, nil |
| 3811 | +} |
| 3812 | + |
| 3813 | +// ModerationCounts returns the counters needed by the moderation view. |
| 3814 | +func (s *Service) ModerationCounts(ctx context.Context, start int64) (communityread.ModerationCounts, error) { |
| 3815 | + var out communityread.ModerationCounts |
| 3816 | + queries := []struct { |
| 3817 | + table string |
| 3818 | + to *int |
| 3819 | + where string |
| 3820 | + }{ |
| 3821 | + {"community_bans", &out.Bans, "at>=?"}, |
| 3822 | + {"community_reports", &out.Reports, "at>=?"}, |
| 3823 | + {"community_reports", &out.Resolved, "resolved_at>=?"}, |
| 3824 | + {"community_event_bans", &out.Hidden, "at>=?"}, |
| 3825 | + {"community_ip_blocks", &out.BlockedAddresses, "at>=?"}, |
| 3826 | + } |
| 3827 | + var failures error |
| 3828 | + for _, query := range queries { |
| 3829 | + if err := s.store.DB().QueryRowContext(ctx, "SELECT count(*) FROM "+query.table+" WHERE "+query.where, start).Scan(query.to); err != nil { |
| 3830 | + if strings.Contains(err.Error(), "no such table") { |
| 3831 | + continue |
| 3832 | + } |
| 3833 | + failures = errors.Join(failures, err) |
| 3834 | + } |
| 3835 | + } |
| 3836 | + return out, failures |
| 3837 | +} |
| 3838 | diff --git a/internal/communityread/communityread.go b/internal/communityread/communityread.go |
| 3839 | new file mode 100644 |
| 3840 | index 0000000..22c18d1 |
| 3841 | --- /dev/null |
| 3842 | +++ b/internal/communityread/communityread.go |
| 3843 | @@ -0,0 +1,13 @@ |
| 3844 | +// Package communityread contains the small read models shared by community |
| 3845 | +// and records. It has no storage or service dependencies. |
| 3846 | +package communityread |
| 3847 | + |
| 3848 | +type Member struct { |
| 3849 | + PubKey string |
| 3850 | + Name string |
| 3851 | + Role string |
| 3852 | +} |
| 3853 | + |
| 3854 | +type ModerationCounts struct { |
| 3855 | + Bans, Reports, Resolved, Hidden, BlockedAddresses int |
| 3856 | +} |
| 3857 | diff --git a/internal/configport/configport.go b/internal/configport/configport.go |
| 3858 | index e0a578d..4fa9eb4 100644 |
| 3859 | --- a/internal/configport/configport.go |
| 3860 | +++ b/internal/configport/configport.go |
| 3861 | @@ -14,6 +14,7 @@ import ( |
| 3862 | |
| 3863 | "github.com/FelineStateMachine/tinyrelay/internal/community" |
| 3864 | "github.com/FelineStateMachine/tinyrelay/internal/policy" |
| 3865 | + "github.com/FelineStateMachine/tinyrelay/internal/replication" |
| 3866 | "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 3867 | "github.com/FelineStateMachine/tinyrelay/internal/templates" |
| 3868 | ) |
| 3869 | @@ -78,7 +79,7 @@ func New(c ConfigStore) *ConfigStore { |
| 3870 | if c.Store != nil { |
| 3871 | _, _ = c.Store.DB().Exec(`CREATE TABLE IF NOT EXISTS config_connections(position INTEGER PRIMARY KEY, value TEXT NOT NULL)`) |
| 3872 | _, _ = c.Store.DB().Exec(`CREATE TABLE IF NOT EXISTS config_jobs(id TEXT PRIMARY KEY, recipe TEXT NOT NULL)`) |
| 3873 | - _, _ = c.Store.DB().Exec(`CREATE TABLE IF NOT EXISTS replication_jobs (id TEXT PRIMARY KEY, payload TEXT NOT NULL, state TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, phase TEXT NOT NULL DEFAULT 'pending', error TEXT NOT NULL DEFAULT '', stored INTEGER NOT NULL DEFAULT 0, sent INTEGER NOT NULL DEFAULT 0, skipped INTEGER NOT NULL DEFAULT 0, refused INTEGER NOT NULL DEFAULT 0, cursor INTEGER NOT NULL DEFAULT 0, rounds INTEGER NOT NULL DEFAULT 0, started_at INTEGER NOT NULL DEFAULT 0, finished_at INTEGER NOT NULL DEFAULT 0)`) |
| 3874 | + _ = replication.EnsureJobSchema(c.Store.DB()) |
| 3875 | } |
| 3876 | return &c |
| 3877 | } |
| 3878 | @@ -179,55 +180,33 @@ func (s *ConfigStore) Export(ctx context.Context) (Config, error) { |
| 3879 | if s.Store == nil { |
| 3880 | return c, errors.New("config: missing store") |
| 3881 | } |
| 3882 | - rows, err := s.Store.DB().QueryContext(ctx, `SELECT pubkey,name,note,role FROM community_members WHERE role<>'owner' ORDER BY pubkey`) |
| 3883 | + communityConfig, err := community.ExportConfig(ctx, s.Store) |
| 3884 | if err != nil { |
| 3885 | return c, err |
| 3886 | } |
| 3887 | - defer rows.Close() |
| 3888 | - for rows.Next() { |
| 3889 | - var m Member |
| 3890 | - if err := rows.Scan(&m.PubKey, &m.Name, &m.Note, &m.Role); err != nil { |
| 3891 | - return c, err |
| 3892 | - } |
| 3893 | - c.Members = append(c.Members, m) |
| 3894 | + for _, m := range communityConfig.Members { |
| 3895 | + c.Members = append(c.Members, Member{PubKey: m.PubKey, Name: m.Name, Note: m.Note, Role: m.Role}) |
| 3896 | } |
| 3897 | - if err := rows.Err(); err != nil { |
| 3898 | - return c, err |
| 3899 | + for _, item := range communityConfig.Bans { |
| 3900 | + c.Bans = append(c.Bans, map[string]any{"pubkey": item.PubKey, "reason": item.Reason}) |
| 3901 | } |
| 3902 | - for _, q := range []struct { |
| 3903 | - query string |
| 3904 | - dst *[]map[string]any |
| 3905 | - fields []any |
| 3906 | - }{{`SELECT pubkey,reason FROM community_bans ORDER BY pubkey`, &c.Bans, nil}, {`SELECT ip,reason FROM community_ip_blocks ORDER BY ip`, &c.Addresses, nil}, {`SELECT id,reason FROM community_event_bans ORDER BY id`, &c.BannedEvents, nil}} { |
| 3907 | - rs, e := s.Store.DB().QueryContext(ctx, q.query) |
| 3908 | - if e != nil { |
| 3909 | - return c, e |
| 3910 | - } |
| 3911 | - for rs.Next() { |
| 3912 | - if q.dst == &c.Bans { |
| 3913 | - var k, v string |
| 3914 | - if e = rs.Scan(&k, &v); e == nil { |
| 3915 | - *q.dst = append(*q.dst, map[string]any{"pubkey": k, "reason": v}) |
| 3916 | - } |
| 3917 | - } else if q.dst == &c.Addresses { |
| 3918 | - var k, v string |
| 3919 | - if e = rs.Scan(&k, &v); e == nil { |
| 3920 | - *q.dst = append(*q.dst, map[string]any{"ip": k, "reason": v}) |
| 3921 | - } |
| 3922 | - } else { |
| 3923 | - var k, v string |
| 3924 | - if e = rs.Scan(&k, &v); e == nil { |
| 3925 | - *q.dst = append(*q.dst, map[string]any{"id": k, "reason": v}) |
| 3926 | - } |
| 3927 | - } |
| 3928 | - if e != nil { |
| 3929 | - rs.Close() |
| 3930 | - return c, e |
| 3931 | - } |
| 3932 | + for _, item := range communityConfig.IPBlocks { |
| 3933 | + c.Addresses = append(c.Addresses, map[string]any{"ip": item.IP, "reason": item.Reason}) |
| 3934 | + } |
| 3935 | + for _, item := range communityConfig.EventBans { |
| 3936 | + c.BannedEvents = append(c.BannedEvents, map[string]any{"id": item.ID, "reason": item.Reason}) |
| 3937 | + } |
| 3938 | + for _, item := range communityConfig.KindRules { |
| 3939 | + if item.Rule == "allow" { |
| 3940 | + c.Kinds.Allow = append(c.Kinds.Allow, item.Kind) |
| 3941 | + } else if item.Rule == "block" { |
| 3942 | + c.Kinds.Block = append(c.Kinds.Block, item.Kind) |
| 3943 | } |
| 3944 | - rs.Close() |
| 3945 | } |
| 3946 | - rows, err = s.Store.DB().QueryContext(ctx, `SELECT value FROM config_connections ORDER BY position`) |
| 3947 | + for _, item := range communityConfig.Retention { |
| 3948 | + c.Retention = append(c.Retention, Rule{Kind: item.Kind, Days: item.Days}) |
| 3949 | + } |
| 3950 | + rows, err := s.Store.DB().QueryContext(ctx, `SELECT value FROM config_connections ORDER BY position`) |
| 3951 | if err == nil { |
| 3952 | defer rows.Close() |
| 3953 | for rows.Next() { |
| 3954 | @@ -362,91 +341,41 @@ func (s *ConfigStore) ApplyWithOptions(ctx context.Context, c Config, opts Apply |
| 3955 | return err |
| 3956 | } |
| 3957 | } |
| 3958 | - if has(c, "members") { |
| 3959 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_members WHERE role<>'owner'`); err != nil { |
| 3960 | - return err |
| 3961 | - } |
| 3962 | - for _, m := range c.Members { |
| 3963 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_members(pubkey,name,note,role,created_at) VALUES(?,?,?,?,?)`, m.PubKey, m.Name, m.Note, m.Role, time.Now().Unix()); err != nil { |
| 3964 | - return err |
| 3965 | - } |
| 3966 | - } |
| 3967 | + communityConfig := community.ConfigSnapshot{} |
| 3968 | + for _, m := range c.Members { |
| 3969 | + communityConfig.Members = append(communityConfig.Members, community.Member{PubKey: m.PubKey, Name: m.Name, Note: m.Note, Role: m.Role}) |
| 3970 | } |
| 3971 | - if s.Community != nil && ch.FinalPolicy.Owner != "" && ch.FinalPolicy.Owner != cur.Owner { |
| 3972 | - if err := s.Community.ApplyOwnerTx(ctx, tx, cur.Owner, ch.FinalPolicy.Owner); err != nil { |
| 3973 | - return err |
| 3974 | - } |
| 3975 | + for _, b := range c.Bans { |
| 3976 | + pk, _ := b["pubkey"].(string) |
| 3977 | + reason, _ := b["reason"].(string) |
| 3978 | + communityConfig.Bans = append(communityConfig.Bans, community.ConfigBan{PubKey: pk, Reason: reason}) |
| 3979 | } |
| 3980 | - if has(c, "bans") { |
| 3981 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_bans`); err != nil { |
| 3982 | - return err |
| 3983 | - } |
| 3984 | - for _, b := range c.Bans { |
| 3985 | - pk, _ := b["pubkey"].(string) |
| 3986 | - reason, _ := b["reason"].(string) |
| 3987 | - if pk != "" { |
| 3988 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_bans(pubkey,reason,at) VALUES(?,?,?)`, pk, reason, time.Now().Unix()); err != nil { |
| 3989 | - return err |
| 3990 | - } |
| 3991 | - } |
| 3992 | - } |
| 3993 | + for _, b := range c.Addresses { |
| 3994 | + ip, _ := b["ip"].(string) |
| 3995 | + reason, _ := b["reason"].(string) |
| 3996 | + communityConfig.IPBlocks = append(communityConfig.IPBlocks, community.ConfigIPBlock{IP: ip, Reason: reason}) |
| 3997 | } |
| 3998 | - if has(c, "addresses") { |
| 3999 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_ip_blocks`); err != nil { |
| 4000 | - return err |
| 4001 | - } |
| 4002 | - for _, b := range c.Addresses { |
| 4003 | - ip, _ := b["ip"].(string) |
| 4004 | - reason, _ := b["reason"].(string) |
| 4005 | - if ip != "" { |
| 4006 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_ip_blocks(ip,reason,at) VALUES(?,?,?)`, ip, reason, time.Now().Unix()); err != nil { |
| 4007 | - return err |
| 4008 | - } |
| 4009 | - } |
| 4010 | - } |
| 4011 | + for _, b := range c.BannedEvents { |
| 4012 | + id, _ := b["id"].(string) |
| 4013 | + reason, _ := b["reason"].(string) |
| 4014 | + communityConfig.EventBans = append(communityConfig.EventBans, community.ConfigEventBan{ID: id, Reason: reason}) |
| 4015 | } |
| 4016 | - if has(c, "banned_events") { |
| 4017 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_event_bans`); err != nil { |
| 4018 | - return err |
| 4019 | - } |
| 4020 | - for _, b := range c.BannedEvents { |
| 4021 | - id, _ := b["id"].(string) |
| 4022 | - reason, _ := b["reason"].(string) |
| 4023 | - if id != "" { |
| 4024 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_event_bans(id,reason,at) VALUES(?,?,?)`, id, reason, time.Now().Unix()); err != nil { |
| 4025 | - return err |
| 4026 | - } |
| 4027 | - } |
| 4028 | - } |
| 4029 | + for _, k := range c.Kinds.Allow { |
| 4030 | + communityConfig.KindRules = append(communityConfig.KindRules, community.ConfigKindRule{Kind: k, Rule: "allow"}) |
| 4031 | } |
| 4032 | - if has(c, "kinds") { |
| 4033 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_kind_rules`); err != nil { |
| 4034 | - return err |
| 4035 | - } |
| 4036 | - for _, k := range c.Kinds.Allow { |
| 4037 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_kind_rules(kind,rule) VALUES(?,?)`, k, "allow"); err != nil { |
| 4038 | - return err |
| 4039 | - } |
| 4040 | - } |
| 4041 | - for _, k := range c.Kinds.Block { |
| 4042 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_kind_rules(kind,rule) VALUES(?,?)`, k, "block"); err != nil { |
| 4043 | - return err |
| 4044 | - } |
| 4045 | - } |
| 4046 | + for _, k := range c.Kinds.Block { |
| 4047 | + communityConfig.KindRules = append(communityConfig.KindRules, community.ConfigKindRule{Kind: k, Rule: "block"}) |
| 4048 | } |
| 4049 | - if has(c, "retention") { |
| 4050 | - if _, err := tx.ExecContext(ctx, `DELETE FROM community_retention`); err != nil { |
| 4051 | + for _, r := range c.Retention { |
| 4052 | + communityConfig.Retention = append(communityConfig.Retention, community.ConfigRetention{Kind: r.Kind, Days: r.Days}) |
| 4053 | + } |
| 4054 | + if err := community.ApplyConfigTx(ctx, tx, communityConfig, has(c, "members"), has(c, "bans"), has(c, "addresses"), has(c, "banned_events"), has(c, "kinds"), has(c, "retention")); err != nil { |
| 4055 | + return err |
| 4056 | + } |
| 4057 | + if s.Community != nil && ch.FinalPolicy.Owner != "" && ch.FinalPolicy.Owner != cur.Owner { |
| 4058 | + if err := s.Community.ApplyOwnerTx(ctx, tx, cur.Owner, ch.FinalPolicy.Owner); err != nil { |
| 4059 | return err |
| 4060 | } |
| 4061 | - for _, r := range c.Retention { |
| 4062 | - kind := -1 |
| 4063 | - if r.Kind != nil { |
| 4064 | - kind = *r.Kind |
| 4065 | - } |
| 4066 | - if _, err := tx.ExecContext(ctx, `INSERT INTO community_retention(kind,days) VALUES(?,?)`, kind, r.Days); err != nil { |
| 4067 | - return err |
| 4068 | - } |
| 4069 | - } |
| 4070 | } |
| 4071 | if has(c, "connections") { |
| 4072 | if _, err := tx.ExecContext(ctx, `DELETE FROM config_connections`); err != nil { |
| 4073 | @@ -463,9 +392,7 @@ func (s *ConfigStore) ApplyWithOptions(ctx context.Context, c Config, opts Apply |
| 4074 | if _, err := tx.ExecContext(ctx, `DELETE FROM config_jobs`); err != nil { |
| 4075 | return err |
| 4076 | } |
| 4077 | - if _, err := tx.ExecContext(ctx, `DELETE FROM replication_jobs`); err != nil { |
| 4078 | - return err |
| 4079 | - } |
| 4080 | + var replicationJobs []replication.ConfigJob |
| 4081 | for _, job := range c.Jobs { |
| 4082 | id, _ := job["id"].(string) |
| 4083 | if id == "" { |
| 4084 | @@ -489,9 +416,10 @@ func (s *ConfigStore) ApplyWithOptions(ctx context.Context, c Config, opts Apply |
| 4085 | if err != nil { |
| 4086 | return err |
| 4087 | } |
| 4088 | - if _, err := tx.ExecContext(ctx, `INSERT INTO replication_jobs(id,payload,state,created_at,updated_at) VALUES(?,?, 'pending',strftime('%s','now'),strftime('%s','now')) ON CONFLICT(id) DO UPDATE SET payload=excluded.payload,updated_at=excluded.updated_at,state='pending'`, id, string(payload)); err != nil { |
| 4089 | - return err |
| 4090 | - } |
| 4091 | + replicationJobs = append(replicationJobs, replication.ConfigJob{ID: id, Payload: payload}) |
| 4092 | + } |
| 4093 | + if err := replication.ReplaceJobsTx(ctx, tx, replicationJobs); err != nil { |
| 4094 | + return err |
| 4095 | } |
| 4096 | } |
| 4097 | return nil |
| 4098 | diff --git a/internal/daemon/callback_followups_test.go b/internal/daemon/callback_followups_test.go |
| 4099 | new file mode 100644 |
| 4100 | index 0000000..cd5fa8d |
| 4101 | --- /dev/null |
| 4102 | +++ b/internal/daemon/callback_followups_test.go |
| 4103 | @@ -0,0 +1,213 @@ |
| 4104 | +package daemon |
| 4105 | + |
| 4106 | +import ( |
| 4107 | + "context" |
| 4108 | + "database/sql" |
| 4109 | + "encoding/json" |
| 4110 | + "errors" |
| 4111 | + "strconv" |
| 4112 | + "strings" |
| 4113 | + "testing" |
| 4114 | + "time" |
| 4115 | + |
| 4116 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 4117 | + "github.com/FelineStateMachine/tinyrelay/internal/relay" |
| 4118 | + "github.com/FelineStateMachine/tinyrelay/internal/replication" |
| 4119 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 4120 | +) |
| 4121 | + |
| 4122 | +func TestCallbackFollowupPlanningFailureDoesNotRejectEvents(t *testing.T) { |
| 4123 | + for _, source := range []string{"client", "import"} { |
| 4124 | + t.Run(source, func(t *testing.T) { |
| 4125 | + _, tenant := loopbackTenant(t) |
| 4126 | + ctx := context.Background() |
| 4127 | + member, _ := event.PublicKey(testMemberSecret) |
| 4128 | + setRole(t, tenant, member, "member") |
| 4129 | + addCallback(t, tenant, member, "https://127.0.0.1:1/wake", `{"kinds":[1]}`) |
| 4130 | + if _, err := tenant.store.DB().ExecContext(ctx, `CREATE TRIGGER reject_callback_work BEFORE INSERT ON work_intents WHEN NEW.kind='callback-delivery' BEGIN SELECT RAISE(ABORT,'callback work unavailable'); END`); err != nil { |
| 4131 | + t.Fatal(err) |
| 4132 | + } |
| 4133 | + e := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), [][]string{{"p", member}}, "callback planning") |
| 4134 | + var err error |
| 4135 | + if source == "import" { |
| 4136 | + err = tenant.commitImported(ctx, e, replication.OriginImport) |
| 4137 | + } else { |
| 4138 | + _, err = tenant.Publish(ctx, e, relay.Session{PubKeys: []string{e.PubKey}}) |
| 4139 | + } |
| 4140 | + if err != nil { |
| 4141 | + t.Fatalf("%s event rejected after optional planning failure: %v", source, err) |
| 4142 | + } |
| 4143 | + var stored int |
| 4144 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM events WHERE id=?`, e.ID).Scan(&stored); err != nil { |
| 4145 | + t.Fatal(err) |
| 4146 | + } |
| 4147 | + if stored != 1 { |
| 4148 | + t.Fatalf("stored events=%d, want 1", stored) |
| 4149 | + } |
| 4150 | + var pending int |
| 4151 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM work_intents WHERE kind=? AND event_id=? AND state='pending'`, eventFollowupKind, e.ID).Scan(&pending); err != nil { |
| 4152 | + t.Fatal(err) |
| 4153 | + } |
| 4154 | + if pending != 1 { |
| 4155 | + t.Fatalf("pending follow-up jobs=%d, want 1", pending) |
| 4156 | + } |
| 4157 | + }) |
| 4158 | + } |
| 4159 | +} |
| 4160 | + |
| 4161 | +func TestCallbackFollowupRecoveryPreservesCompletedDelivery(t *testing.T) { |
| 4162 | + _, tenant := loopbackTenant(t) |
| 4163 | + ctx := context.Background() |
| 4164 | + member, _ := event.PublicKey(testMemberSecret) |
| 4165 | + setRole(t, tenant, member, "member") |
| 4166 | + first := addCallback(t, tenant, member, "https://127.0.0.1:1/first", `{"kinds":[1]}`) |
| 4167 | + second := addCallback(t, tenant, member, "https://127.0.0.1:1/second", `{"kinds":[1]}`) |
| 4168 | + e := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), [][]string{{"p", member}}, "recover callback work") |
| 4169 | + followup := tenant.followups.prepare(ctx, e, time.Now().Unix(), false) |
| 4170 | + if followup == nil { |
| 4171 | + t.Fatal("missing callback follow-up") |
| 4172 | + } |
| 4173 | + if _, err := tenant.store.Save(ctx, e, storage.SaveOptions{Now: time.Now().Unix(), Intents: []storage.Intent{*followup}}); err != nil { |
| 4174 | + t.Fatal(err) |
| 4175 | + } |
| 4176 | + job := followupJob(t, tenant, e.ID) |
| 4177 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 4178 | + t.Fatal(err) |
| 4179 | + } |
| 4180 | + firstID := first["id"].(string) |
| 4181 | + secondID := second["id"].(string) |
| 4182 | + markCallbackDeliveryCompleted(t, tenant, firstID, e.ID) |
| 4183 | + deleteCallbackDelivery(t, tenant, secondID, e.ID) |
| 4184 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 4185 | + t.Fatal(err) |
| 4186 | + } |
| 4187 | + if state := callbackDeliveryState(t, tenant, firstID, e.ID); state != "completed" { |
| 4188 | + t.Fatalf("completed delivery state=%q, want completed", state) |
| 4189 | + } |
| 4190 | + if state := callbackDeliveryState(t, tenant, secondID, e.ID); state != "pending" { |
| 4191 | + t.Fatalf("recovered delivery state=%q, want pending", state) |
| 4192 | + } |
| 4193 | +} |
| 4194 | + |
| 4195 | +func TestCallbackFollowupSkipsPausedRemovedAndLaterRegistrations(t *testing.T) { |
| 4196 | + for _, action := range []string{"pausecallback", "removecallback"} { |
| 4197 | + t.Run(action, func(t *testing.T) { |
| 4198 | + _, tenant := loopbackTenant(t) |
| 4199 | + ctx := context.Background() |
| 4200 | + member, _ := event.PublicKey(testMemberSecret) |
| 4201 | + setRole(t, tenant, member, "member") |
| 4202 | + captured := addCallback(t, tenant, member, "https://127.0.0.1:1/captured", `{"kinds":[1]}`) |
| 4203 | + e := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), [][]string{{"p", member}}, "captured callback") |
| 4204 | + followup := tenant.followups.prepare(ctx, e, time.Now().Unix(), false) |
| 4205 | + if followup == nil { |
| 4206 | + t.Fatal("missing callback follow-up") |
| 4207 | + } |
| 4208 | + id := captured["id"].(string) |
| 4209 | + if _, err := tenant.Execute(ctx, member, action, []json.RawMessage{json.RawMessage(strconv.Quote(id))}); err != nil { |
| 4210 | + t.Fatal(err) |
| 4211 | + } |
| 4212 | + later := addCallback(t, tenant, member, "https://127.0.0.1:1/later", `{"kinds":[1]}`) |
| 4213 | + if _, err := tenant.store.Save(ctx, e, storage.SaveOptions{Now: time.Now().Unix(), Intents: []storage.Intent{*followup}}); err != nil { |
| 4214 | + t.Fatal(err) |
| 4215 | + } |
| 4216 | + if err := tenant.followups.handle(ctx, followupJob(t, tenant, e.ID)); err != nil { |
| 4217 | + t.Fatal(err) |
| 4218 | + } |
| 4219 | + if got := callbackDeliveryState(t, tenant, id, e.ID); got != "" { |
| 4220 | + t.Fatalf("%s callback state=%q, want absent", action, got) |
| 4221 | + } |
| 4222 | + if got := callbackDeliveryState(t, tenant, later["id"].(string), e.ID); got != "" { |
| 4223 | + t.Fatalf("later callback state=%q, want absent", got) |
| 4224 | + } |
| 4225 | + }) |
| 4226 | + } |
| 4227 | +} |
| 4228 | + |
| 4229 | +func TestCallbackFollowupPlanningExhaustionStops(t *testing.T) { |
| 4230 | + _, tenant := loopbackTenant(t) |
| 4231 | + ctx := context.Background() |
| 4232 | + member, _ := event.PublicKey(testMemberSecret) |
| 4233 | + setRole(t, tenant, member, "member") |
| 4234 | + addCallback(t, tenant, member, "https://127.0.0.1:1/wake", `{"kinds":[1]}`) |
| 4235 | + e := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), [][]string{{"p", member}}, "exhaust planning") |
| 4236 | + followup := tenant.followups.prepare(ctx, e, time.Now().Unix(), false) |
| 4237 | + if followup == nil { |
| 4238 | + t.Fatal("missing callback follow-up") |
| 4239 | + } |
| 4240 | + if _, err := tenant.store.Save(ctx, e, storage.SaveOptions{Now: time.Now().Unix(), Intents: []storage.Intent{*followup}}); err != nil { |
| 4241 | + t.Fatal(err) |
| 4242 | + } |
| 4243 | + if _, err := tenant.store.DB().ExecContext(ctx, `CREATE TRIGGER reject_callback_work BEFORE INSERT ON work_intents WHEN NEW.kind='callback-delivery' BEGIN SELECT RAISE(ABORT,'callback work unavailable'); END`); err != nil { |
| 4244 | + t.Fatal(err) |
| 4245 | + } |
| 4246 | + job := followupJob(t, tenant, e.ID) |
| 4247 | + job.Attempts = followupMaxAttempts |
| 4248 | + err := tenant.followups.handle(ctx, job) |
| 4249 | + if err == nil || !strings.Contains(err.Error(), "callback work unavailable") || errors.Unwrap(err) == nil { |
| 4250 | + t.Fatalf("exhausted planning error=%v, want terminal wrapped error", err) |
| 4251 | + } |
| 4252 | +} |
| 4253 | + |
| 4254 | +func TestCallbackFollowupRecoversWhenViewLookupIsUnavailable(t *testing.T) { |
| 4255 | + tenant, _, _ := viewTenant(t, []int{1}, nil) |
| 4256 | + ctx := context.Background() |
| 4257 | + member, _ := event.PublicKey(testMemberSecret) |
| 4258 | + setRole(t, tenant, member, "member") |
| 4259 | + callback := addCallback(t, tenant, member, "https://127.0.0.1:1/wake", `{"kinds":[1]}`) |
| 4260 | + e := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), [][]string{{"p", member}}, "```mermaid\ngraph LR; A-->B\n```") |
| 4261 | + if _, err := tenant.store.DB().ExecContext(ctx, `ALTER TABLE custom_views RENAME TO custom_views_unavailable`); err != nil { |
| 4262 | + t.Fatal(err) |
| 4263 | + } |
| 4264 | + if _, err := tenant.Publish(ctx, e, relay.Session{PubKeys: []string{e.PubKey}}); err != nil { |
| 4265 | + t.Fatalf("publish with unavailable optional view lookup: %v", err) |
| 4266 | + } |
| 4267 | + job := followupJob(t, tenant, e.ID) |
| 4268 | + callbackID := callback["id"].(string) |
| 4269 | + if state := callbackDeliveryState(t, tenant, callbackID, e.ID); state != "pending" { |
| 4270 | + t.Fatalf("callback state after partial planning=%q, want pending", state) |
| 4271 | + } |
| 4272 | + markCallbackDeliveryCompleted(t, tenant, callbackID, e.ID) |
| 4273 | + if _, err := tenant.store.DB().ExecContext(ctx, `ALTER TABLE custom_views_unavailable RENAME TO custom_views`); err != nil { |
| 4274 | + t.Fatal(err) |
| 4275 | + } |
| 4276 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 4277 | + t.Fatalf("recover follow-up planning: %v", err) |
| 4278 | + } |
| 4279 | + if state := callbackDeliveryState(t, tenant, callbackID, e.ID); state != "completed" { |
| 4280 | + t.Fatalf("callback state after recovery=%q, want completed", state) |
| 4281 | + } |
| 4282 | + var callbackCount int |
| 4283 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM work_intents WHERE kind=? AND target=? AND event_id=?`, callbackDelivery, callbackID, e.ID).Scan(&callbackCount); err != nil { |
| 4284 | + t.Fatal(err) |
| 4285 | + } |
| 4286 | + if callbackCount != 1 { |
| 4287 | + t.Fatalf("callback intent count=%d, want 1", callbackCount) |
| 4288 | + } |
| 4289 | +} |
| 4290 | + |
| 4291 | +func callbackDeliveryState(t *testing.T, tenant *Tenant, target, eventID string) string { |
| 4292 | + t.Helper() |
| 4293 | + var state string |
| 4294 | + err := tenant.store.DB().QueryRow(`SELECT state FROM work_intents WHERE kind=? AND target=? AND event_id=?`, callbackDelivery, target, eventID).Scan(&state) |
| 4295 | + if errors.Is(err, sql.ErrNoRows) { |
| 4296 | + return "" |
| 4297 | + } |
| 4298 | + if err != nil { |
| 4299 | + t.Fatal(err) |
| 4300 | + } |
| 4301 | + return state |
| 4302 | +} |
| 4303 | + |
| 4304 | +func markCallbackDeliveryCompleted(t *testing.T, tenant *Tenant, target, eventID string) { |
| 4305 | + t.Helper() |
| 4306 | + if _, err := tenant.store.DB().Exec(`UPDATE work_intents SET state='completed' WHERE kind=? AND target=? AND event_id=?`, callbackDelivery, target, eventID); err != nil { |
| 4307 | + t.Fatal(err) |
| 4308 | + } |
| 4309 | +} |
| 4310 | + |
| 4311 | +func deleteCallbackDelivery(t *testing.T, tenant *Tenant, target, eventID string) { |
| 4312 | + t.Helper() |
| 4313 | + if _, err := tenant.store.DB().Exec(`DELETE FROM work_intents WHERE kind=? AND target=? AND event_id=?`, callbackDelivery, target, eventID); err != nil { |
| 4314 | + t.Fatal(err) |
| 4315 | + } |
| 4316 | +} |
| 4317 | diff --git a/internal/daemon/callback_restart_acceptance_test.go b/internal/daemon/callback_restart_acceptance_test.go |
| 4318 | index fe4e0a4..863efea 100644 |
| 4319 | --- a/internal/daemon/callback_restart_acceptance_test.go |
| 4320 | +++ b/internal/daemon/callback_restart_acceptance_test.go |
| 4321 | @@ -117,7 +117,7 @@ func TestCallbackIntentSurvivesRestartAndHonorsRevocation(t *testing.T) { |
| 4322 | } |
| 4323 | runCtx, cancel := context.WithCancel(ctx) |
| 4324 | done := make(chan error, 1) |
| 4325 | - go func() { done <- reloaded.replication.Run(runCtx) }() |
| 4326 | + go func() { done <- reloaded.runWork(runCtx) }() |
| 4327 | if !waitFor(t, 3*time.Second, func() bool { |
| 4328 | items, listErr := reloaded.replication.Queue().Pending(ctx, "callback") |
| 4329 | return listErr == nil && len(items) == 0 |
| 4330 | diff --git a/internal/daemon/callbacks.go b/internal/daemon/callbacks.go |
| 4331 | index f18e355..cfc5580 100644 |
| 4332 | --- a/internal/daemon/callbacks.go |
| 4333 | +++ b/internal/daemon/callbacks.go |
| 4334 | @@ -185,15 +185,15 @@ func callbackHost(raw string) string { |
| 4335 | return parsed.Host |
| 4336 | } |
| 4337 | |
| 4338 | -func (t *Tenant) initCallbacks(ctx context.Context) error { |
| 4339 | - _, err := t.store.DB().ExecContext(ctx, callbackSchema) |
| 4340 | +func (s *callbackService) initCallbacks(ctx context.Context) error { |
| 4341 | + _, err := s.store.DB().ExecContext(ctx, callbackSchema) |
| 4342 | return err |
| 4343 | } |
| 4344 | |
| 4345 | -// checkCallbackURL accepts https URLs to public hosts. When the relay itself |
| 4346 | -// runs on a loopback address, as it does in tests, private targets are |
| 4347 | +// validateWebhookURL accepts https URLs to public hosts. When the relay |
| 4348 | +// itself runs on a loopback address, as it does in tests, private targets are |
| 4349 | // allowed so a local receiver can be exercised. |
| 4350 | -func (t *Tenant) checkCallbackURL(raw string) error { |
| 4351 | +func validateWebhookURL(raw, publicURL string) error { |
| 4352 | if len(raw) > callbackURLMax { |
| 4353 | return errors.New("invalid: url is too long") |
| 4354 | } |
| 4355 | @@ -201,7 +201,7 @@ func (t *Tenant) checkCallbackURL(raw string) error { |
| 4356 | if err != nil || parsed.Scheme != "https" || parsed.Hostname() == "" || parsed.User != nil || parsed.Fragment != "" { |
| 4357 | return errors.New("invalid: url must be https without credentials or a fragment") |
| 4358 | } |
| 4359 | - if t.loopbackRelay() { |
| 4360 | + if loopbackPublicURL(publicURL) { |
| 4361 | return nil |
| 4362 | } |
| 4363 | host := strings.ToLower(parsed.Hostname()) |
| 4364 | @@ -225,10 +225,10 @@ func privateAddress(ip net.IP) bool { |
| 4365 | return shared.Contains(ip) |
| 4366 | } |
| 4367 | |
| 4368 | -// loopbackRelay reports whether the relay's own public URL is a loopback or |
| 4369 | -// private address, which only happens in tests and local trials. |
| 4370 | -func (t *Tenant) loopbackRelay() bool { |
| 4371 | - parsed, err := url.Parse(t.publicURL) |
| 4372 | +// loopbackPublicURL reports whether the relay's own public URL is a loopback |
| 4373 | +// or private address, which only happens in tests and local trials. |
| 4374 | +func loopbackPublicURL(publicURL string) bool { |
| 4375 | + parsed, err := url.Parse(publicURL) |
| 4376 | if err != nil { |
| 4377 | return false |
| 4378 | } |
| 4379 | @@ -247,8 +247,8 @@ func callbackMethod(method string) bool { |
| 4380 | // callbackExecute serves the callback management methods. Members and agents |
| 4381 | // manage callbacks for their own key; the owner and moderators see and |
| 4382 | // control every callback. Each call runs under the callback operation. |
| 4383 | -func (t *Tenant) callbackExecute(ctx context.Context, actor, method string, params []json.RawMessage) (result any, err error) { |
| 4384 | - ctx, finish := t.app.telemetry.Start(ctx, "callback") |
| 4385 | +func (s *callbackService) callbackExecute(ctx context.Context, actor, method string, params []json.RawMessage) (result any, err error) { |
| 4386 | + ctx, finish := s.telemetry.Start(ctx, "callback") |
| 4387 | outcome := "error" |
| 4388 | defer func() { |
| 4389 | if err != nil { |
| 4390 | @@ -256,7 +256,7 @@ func (t *Tenant) callbackExecute(ctx context.Context, actor, method string, para |
| 4391 | } |
| 4392 | finish(outcome) |
| 4393 | }() |
| 4394 | - role, err := t.community.Role(ctx, actor) |
| 4395 | + role, err := s.community.Role(ctx, actor) |
| 4396 | if err != nil { |
| 4397 | return nil, err |
| 4398 | } |
| 4399 | @@ -268,9 +268,9 @@ func (t *Tenant) callbackExecute(ctx context.Context, actor, method string, para |
| 4400 | case "listcallbacks": |
| 4401 | var rows []callbackRecord |
| 4402 | if operator { |
| 4403 | - rows, err = t.callbackRows(ctx, "") |
| 4404 | + rows, err = s.callbackRows(ctx, "") |
| 4405 | } else { |
| 4406 | - rows, err = t.callbackRows(ctx, actor) |
| 4407 | + rows, err = s.callbackRows(ctx, actor) |
| 4408 | } |
| 4409 | if err != nil { |
| 4410 | return nil, err |
| 4411 | @@ -288,9 +288,9 @@ func (t *Tenant) callbackExecute(ctx context.Context, actor, method string, para |
| 4412 | outcome = "ok" |
| 4413 | return out, nil |
| 4414 | case "addcallback": |
| 4415 | - result, err = t.addCallback(ctx, actor, role, params) |
| 4416 | + result, err = s.addCallback(ctx, actor, role, params) |
| 4417 | default: |
| 4418 | - result, err = t.changeCallback(ctx, actor, operator, method, params) |
| 4419 | + result, err = s.changeCallback(ctx, actor, operator, method, params) |
| 4420 | } |
| 4421 | if err == nil { |
| 4422 | outcome = "ok" |
| 4423 | @@ -309,7 +309,7 @@ func callbackOutcome(err error) string { |
| 4424 | } |
| 4425 | } |
| 4426 | |
| 4427 | -func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []json.RawMessage) (any, error) { |
| 4428 | +func (s *callbackService) addCallback(ctx context.Context, actor, role string, params []json.RawMessage) (any, error) { |
| 4429 | if len(params) != 1 { |
| 4430 | return nil, errors.New("invalid: addcallback expects one object with url, filter and an optional secret") |
| 4431 | } |
| 4432 | @@ -322,7 +322,7 @@ func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []j |
| 4433 | return nil, errors.New("invalid: addcallback expects one object with url, filter and an optional secret") |
| 4434 | } |
| 4435 | options.URL = strings.TrimSpace(options.URL) |
| 4436 | - if err := t.checkCallbackURL(options.URL); err != nil { |
| 4437 | + if err := validateWebhookURL(options.URL, s.publicURL); err != nil { |
| 4438 | return nil, err |
| 4439 | } |
| 4440 | filter, err := parseCallbackFilter(options.Filter) |
| 4441 | @@ -330,7 +330,7 @@ func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []j |
| 4442 | return nil, err |
| 4443 | } |
| 4444 | if role == "agent" { |
| 4445 | - if err := t.checkCallbackScope(ctx, actor, filter); err != nil { |
| 4446 | + if err := s.checkCallbackScope(ctx, actor, filter); err != nil { |
| 4447 | return nil, err |
| 4448 | } |
| 4449 | } |
| 4450 | @@ -353,9 +353,9 @@ func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []j |
| 4451 | if err != nil { |
| 4452 | return nil, err |
| 4453 | } |
| 4454 | - limit := t.Policy().Callbacks |
| 4455 | + limit := s.policy().Callbacks |
| 4456 | now := time.Now().Unix() |
| 4457 | - err = t.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4458 | + err = s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4459 | if role != "owner" { |
| 4460 | var count int |
| 4461 | if err := tx.QueryRowContext(ctx, `SELECT count(*) FROM callbacks WHERE owner=?`, actor).Scan(&count); err != nil { |
| 4462 | @@ -371,12 +371,12 @@ func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []j |
| 4463 | if err != nil { |
| 4464 | return nil, err |
| 4465 | } |
| 4466 | - t.invalidateCallbacks() |
| 4467 | - if err := t.community.Record(ctx, actor, "addcallback", id, ""); err != nil { |
| 4468 | + s.invalidateCallbacks() |
| 4469 | + if err := s.community.Record(ctx, actor, "addcallback", id, ""); err != nil { |
| 4470 | return nil, err |
| 4471 | } |
| 4472 | // Counts only: the host, path and secret stay out of the log. |
| 4473 | - t.app.telemetry.Logger().Info("callback registered", "tenant", t.meta.Name, "kinds", len(filter.Kinds), "generated", generated) |
| 4474 | + s.telemetry.Logger().Info("callback registered", "tenant", s.tenant.Name, "kinds", len(filter.Kinds), "generated", generated) |
| 4475 | record := callbackRecord{ID: id, Owner: actor, URL: options.URL, Filter: filter, CreatedAt: now} |
| 4476 | out := record.summary() |
| 4477 | out["secret"] = secret |
| 4478 | @@ -387,8 +387,8 @@ func (t *Tenant) addCallback(ctx context.Context, actor, role string, params []j |
| 4479 | // names must be one it may post in and a repository it names must be one |
| 4480 | // the grant covers. Kinds are not restricted, so an agent can listen for |
| 4481 | // answers it may not publish itself. |
| 4482 | -func (t *Tenant) checkCallbackScope(ctx context.Context, actor string, filter callbackFilter) error { |
| 4483 | - grant, ok, err := t.community.AgentGrant(ctx, actor) |
| 4484 | +func (s *callbackService) checkCallbackScope(ctx context.Context, actor string, filter callbackFilter) error { |
| 4485 | + grant, ok, err := s.community.AgentGrant(ctx, actor) |
| 4486 | if err != nil { |
| 4487 | return err |
| 4488 | } |
| 4489 | @@ -412,12 +412,12 @@ func (t *Tenant) checkCallbackScope(ctx context.Context, actor string, filter ca |
| 4490 | return nil |
| 4491 | } |
| 4492 | |
| 4493 | -func (t *Tenant) changeCallback(ctx context.Context, actor string, operator bool, method string, params []json.RawMessage) (any, error) { |
| 4494 | +func (s *callbackService) changeCallback(ctx context.Context, actor string, operator bool, method string, params []json.RawMessage) (any, error) { |
| 4495 | id := callbackIDParam(params) |
| 4496 | if id == "" { |
| 4497 | return nil, errors.New("invalid: callback id required") |
| 4498 | } |
| 4499 | - record, err := t.callback(ctx, id) |
| 4500 | + record, err := s.callback(ctx, id) |
| 4501 | if err != nil { |
| 4502 | return nil, err |
| 4503 | } |
| 4504 | @@ -426,12 +426,12 @@ func (t *Tenant) changeCallback(ctx context.Context, actor string, operator bool |
| 4505 | } |
| 4506 | switch method { |
| 4507 | case "removecallback": |
| 4508 | - _, err = t.store.DB().ExecContext(ctx, `DELETE FROM callbacks WHERE id=?`, id) |
| 4509 | + _, err = s.store.DB().ExecContext(ctx, `DELETE FROM callbacks WHERE id=?`, id) |
| 4510 | case "pausecallback": |
| 4511 | - _, err = t.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=1, last_status='paused' WHERE id=?`, id) |
| 4512 | + _, err = s.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=1, last_status='paused' WHERE id=?`, id) |
| 4513 | record.Paused, record.LastStatus = true, "paused" |
| 4514 | case "resumecallback": |
| 4515 | - _, err = t.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=0, failures=0, last_status='' WHERE id=?`, id) |
| 4516 | + _, err = s.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=0, failures=0, last_status='' WHERE id=?`, id) |
| 4517 | record.Paused, record.Failures, record.LastStatus = false, 0, "" |
| 4518 | default: |
| 4519 | return nil, fmt.Errorf("unsupported: management method %q", method) |
| 4520 | @@ -439,8 +439,8 @@ func (t *Tenant) changeCallback(ctx context.Context, actor string, operator bool |
| 4521 | if err != nil { |
| 4522 | return nil, err |
| 4523 | } |
| 4524 | - t.invalidateCallbacks() |
| 4525 | - if err := t.community.Record(ctx, actor, method, id, ""); err != nil { |
| 4526 | + s.invalidateCallbacks() |
| 4527 | + if err := s.community.Record(ctx, actor, method, id, ""); err != nil { |
| 4528 | return nil, err |
| 4529 | } |
| 4530 | if method == "removecallback" { |
| 4531 | @@ -464,8 +464,8 @@ func callbackIDParam(params []json.RawMessage) string { |
| 4532 | return strings.TrimSpace(options.ID) |
| 4533 | } |
| 4534 | |
| 4535 | -func (t *Tenant) callback(ctx context.Context, id string) (callbackRecord, error) { |
| 4536 | - rows, err := t.callbackRows(ctx, "", id) |
| 4537 | +func (s *callbackService) callback(ctx context.Context, id string) (callbackRecord, error) { |
| 4538 | + rows, err := s.callbackRows(ctx, "", id) |
| 4539 | if err != nil { |
| 4540 | return callbackRecord{}, err |
| 4541 | } |
| 4542 | @@ -476,7 +476,7 @@ func (t *Tenant) callback(ctx context.Context, id string) (callbackRecord, error |
| 4543 | } |
| 4544 | |
| 4545 | // callbackRows lists callbacks, narrowed to an owner or to one id. |
| 4546 | -func (t *Tenant) callbackRows(ctx context.Context, owner string, ids ...string) ([]callbackRecord, error) { |
| 4547 | +func (s *callbackService) callbackRows(ctx context.Context, owner string, ids ...string) ([]callbackRecord, error) { |
| 4548 | query := `SELECT id,owner,url,filter,secret,created_at,last_delivery_at,last_status,failures,paused FROM callbacks` |
| 4549 | args := []any{} |
| 4550 | switch { |
| 4551 | @@ -488,7 +488,7 @@ func (t *Tenant) callbackRows(ctx context.Context, owner string, ids ...string) |
| 4552 | args = append(args, owner) |
| 4553 | } |
| 4554 | query += ` ORDER BY created_at,id` |
| 4555 | - rows, err := t.store.DB().QueryContext(ctx, query, args...) |
| 4556 | + rows, err := s.store.DB().QueryContext(ctx, query, args...) |
| 4557 | if err != nil { |
| 4558 | return nil, fmt.Errorf("list callbacks: %w", err) |
| 4559 | } |
| 4560 | @@ -513,16 +513,16 @@ func (t *Tenant) callbackRows(ctx context.Context, owner string, ids ...string) |
| 4561 | // callbackRules returns the active callbacks indexed by kind. The index is |
| 4562 | // built from the table on first use and dropped whenever a callback |
| 4563 | // changes, so matching an event costs one map lookup. |
| 4564 | -func (t *Tenant) callbackRules(ctx context.Context) map[int][]callbackRecord { |
| 4565 | - t.callbackMu.Lock() |
| 4566 | - defer t.callbackMu.Unlock() |
| 4567 | - if t.callbackIndex != nil { |
| 4568 | - return t.callbackIndex |
| 4569 | +func (s *callbackService) callbackRules(ctx context.Context) map[int][]callbackRecord { |
| 4570 | + s.callbackMu.Lock() |
| 4571 | + defer s.callbackMu.Unlock() |
| 4572 | + if s.callbackIndex != nil { |
| 4573 | + return s.callbackIndex |
| 4574 | } |
| 4575 | index := map[int][]callbackRecord{} |
| 4576 | - rows, err := t.callbackRows(ctx, "") |
| 4577 | + rows, err := s.callbackRows(ctx, "") |
| 4578 | if err != nil { |
| 4579 | - t.app.telemetry.Logger().Error("load callbacks", "tenant", t.meta.Name, "error", err) |
| 4580 | + s.telemetry.Logger().Error("load callbacks", "tenant", s.tenant.Name, "error", err) |
| 4581 | return index |
| 4582 | } |
| 4583 | for _, row := range rows { |
| 4584 | @@ -533,52 +533,106 @@ func (t *Tenant) callbackRules(ctx context.Context) map[int][]callbackRecord { |
| 4585 | index[kind] = append(index[kind], row) |
| 4586 | } |
| 4587 | } |
| 4588 | - t.callbackIndex = index |
| 4589 | + s.callbackIndex = index |
| 4590 | return index |
| 4591 | } |
| 4592 | |
| 4593 | -func (t *Tenant) invalidateCallbacks() { |
| 4594 | - t.callbackMu.Lock() |
| 4595 | - t.callbackIndex = nil |
| 4596 | - t.callbackMu.Unlock() |
| 4597 | +func (s *callbackService) invalidateCallbacks() { |
| 4598 | + s.callbackMu.Lock() |
| 4599 | + s.callbackIndex = nil |
| 4600 | + s.callbackMu.Unlock() |
| 4601 | } |
| 4602 | |
| 4603 | -// notifyCallbacks runs after an event is stored. Each active callback whose |
| 4604 | -// filter matches and whose key may see the event gets one delivery intent. |
| 4605 | -// It never fails the publish; a callback is best effort on top of the |
| 4606 | -// durable event. |
| 4607 | -func (t *Tenant) notifyCallbacks(ctx context.Context, e event.Event) { |
| 4608 | - candidates := t.callbackRules(ctx)[e.Kind] |
| 4609 | - if len(candidates) == 0 { |
| 4610 | - return |
| 4611 | +// CandidateIDs captures the active callback registrations that match an |
| 4612 | +// event. The caller must persist these IDs with the event so later planning |
| 4613 | +// cannot include callbacks registered after the event was accepted. |
| 4614 | +func (s *callbackService) CandidateIDs(ctx context.Context, e event.Event) ([]string, error) { |
| 4615 | + rows, err := s.callbackRows(ctx, "") |
| 4616 | + if err != nil { |
| 4617 | + return nil, err |
| 4618 | } |
| 4619 | - var intents []storage.Intent |
| 4620 | - unauthorized := 0 |
| 4621 | - for _, candidate := range candidates { |
| 4622 | - if !event.Matches(candidate.Filter.eventFilter(), e) { |
| 4623 | + ids := make([]string, 0, len(rows)) |
| 4624 | + for _, candidate := range rows { |
| 4625 | + if candidate.Paused || !event.Matches(candidate.Filter.eventFilter(), e) { |
| 4626 | continue |
| 4627 | } |
| 4628 | - if !t.gate.CanSee(ctx, e, relay.Session{PubKeys: []string{candidate.Owner}, RelayURL: t.RelayURL()}, nil) { |
| 4629 | - unauthorized++ |
| 4630 | + ids = append(ids, candidate.ID) |
| 4631 | + } |
| 4632 | + return ids, nil |
| 4633 | +} |
| 4634 | + |
| 4635 | +// PrepareFor creates delivery intents for captured registration IDs. Each |
| 4636 | +// registration is re-read so deletion, pausing, membership changes and |
| 4637 | +// visibility changes take effect before an intent is created. |
| 4638 | +func (s *callbackService) PrepareFor(ctx context.Context, e event.Event, ids []string) ([]storage.Intent, error) { |
| 4639 | + if len(ids) == 0 { |
| 4640 | + return nil, nil |
| 4641 | + } |
| 4642 | + intents := make([]storage.Intent, 0, len(ids)) |
| 4643 | + seen := make(map[string]struct{}, len(ids)) |
| 4644 | + for _, id := range ids { |
| 4645 | + if id == "" { |
| 4646 | continue |
| 4647 | } |
| 4648 | - payload, err := json.Marshal(callbackPayload{Event: e, Attempt: 1}) |
| 4649 | + if _, ok := seen[id]; ok { |
| 4650 | + continue |
| 4651 | + } |
| 4652 | + seen[id] = struct{}{} |
| 4653 | + candidate, err := s.callback(ctx, id) |
| 4654 | if err != nil { |
| 4655 | + if strings.HasPrefix(err.Error(), "not found:") { |
| 4656 | + continue |
| 4657 | + } |
| 4658 | + return nil, err |
| 4659 | + } |
| 4660 | + if candidate.Paused || !event.Matches(candidate.Filter.eventFilter(), e) { |
| 4661 | + continue |
| 4662 | + } |
| 4663 | + if !s.callbackOwnerCanSee(ctx, candidate.Owner, e) { |
| 4664 | continue |
| 4665 | } |
| 4666 | + payload, err := json.Marshal(callbackPayload{Event: e, Attempt: 1}) |
| 4667 | + if err != nil { |
| 4668 | + return nil, fmt.Errorf("encode callback delivery: %w", err) |
| 4669 | + } |
| 4670 | intents = append(intents, storage.Intent{Kind: callbackDelivery, EventID: e.ID, Target: candidate.ID, Payload: string(payload)}) |
| 4671 | } |
| 4672 | + return intents, nil |
| 4673 | +} |
| 4674 | + |
| 4675 | +func (s *callbackService) callbackOwnerCanSee(ctx context.Context, owner string, e event.Event) bool { |
| 4676 | + return s.gate.CanSee(ctx, e, relay.Session{PubKeys: []string{owner}, RelayURL: s.relayURL}, nil) |
| 4677 | +} |
| 4678 | + |
| 4679 | +// Prepare preserves the original planning API for callers that do not yet |
| 4680 | +// persist captured candidate IDs alongside the event. |
| 4681 | +func (s *callbackService) Prepare(ctx context.Context, e event.Event) ([]storage.Intent, error) { |
| 4682 | + ids, err := s.CandidateIDs(ctx, e) |
| 4683 | + if err != nil { |
| 4684 | + return nil, err |
| 4685 | + } |
| 4686 | + return s.PrepareFor(ctx, e, ids) |
| 4687 | +} |
| 4688 | + |
| 4689 | +// notifyCallbacks runs after an event is stored. It preserves the legacy |
| 4690 | +// best-effort queueing path for callers that do not share the event tx. |
| 4691 | +func (s *callbackService) notifyCallbacks(ctx context.Context, e event.Event) { |
| 4692 | + intents, err := s.Prepare(ctx, e) |
| 4693 | + if err != nil { |
| 4694 | + s.telemetry.Logger().Error("callback deliveries not prepared", "tenant", s.tenant.Name, "error", err) |
| 4695 | + return |
| 4696 | + } |
| 4697 | if len(intents) == 0 { |
| 4698 | return |
| 4699 | } |
| 4700 | - err := t.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4701 | + err = s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4702 | return storage.AddIntents(ctx, tx, intents, time.Now().Unix()) |
| 4703 | }) |
| 4704 | if err != nil { |
| 4705 | - t.app.telemetry.Logger().Error("callback deliveries not queued", "tenant", t.meta.Name, "error", err) |
| 4706 | + s.telemetry.Logger().Error("callback deliveries not queued", "tenant", s.tenant.Name, "error", err) |
| 4707 | return |
| 4708 | } |
| 4709 | - t.app.telemetry.Logger().Debug("callback deliveries queued", "tenant", t.meta.Name, "queued", len(intents), "unauthorized", unauthorized) |
| 4710 | + s.telemetry.Logger().Debug("callback deliveries queued", "tenant", s.tenant.Name, "queued", len(intents)) |
| 4711 | } |
| 4712 | |
| 4713 | func randomHex(size int) (string, error) { |
| 4714 | @@ -588,3 +642,17 @@ func randomHex(size int) (string, error) { |
| 4715 | } |
| 4716 | return hex.EncodeToString(buffer), nil |
| 4717 | } |
| 4718 | + |
| 4719 | +// Tenant wrappers keep management and work dispatch independent of the |
| 4720 | +// callback service's storage and delivery state. |
| 4721 | +func (t *Tenant) initCallbacks(ctx context.Context) error { |
| 4722 | + return t.callbacks.initCallbacks(ctx) |
| 4723 | +} |
| 4724 | + |
| 4725 | +func (t *Tenant) callbackExecute(ctx context.Context, actor, method string, params []json.RawMessage) (any, error) { |
| 4726 | + return t.callbacks.callbackExecute(ctx, actor, method, params) |
| 4727 | +} |
| 4728 | + |
| 4729 | +func (t *Tenant) notifyCallbacks(ctx context.Context, e event.Event) { |
| 4730 | + t.callbacks.notifyCallbacks(ctx, e) |
| 4731 | +} |
| 4732 | diff --git a/internal/daemon/callbacks_delivery.go b/internal/daemon/callbacks_delivery.go |
| 4733 | index 7cbcfaa..95aed6b 100644 |
| 4734 | --- a/internal/daemon/callbacks_delivery.go |
| 4735 | +++ b/internal/daemon/callbacks_delivery.go |
| 4736 | @@ -44,27 +44,27 @@ func callbackSignature(secret string, body []byte) string { |
| 4737 | |
| 4738 | // callbackHTTPClient refuses private addresses and redirects unless the |
| 4739 | // relay itself runs on a loopback address. |
| 4740 | -func (t *Tenant) callbackHTTPClient() *http.Client { |
| 4741 | - if t.callbackClient != nil { |
| 4742 | - return t.callbackClient |
| 4743 | +func (s *callbackService) callbackHTTPClient() *http.Client { |
| 4744 | + if s.callbackClient != nil { |
| 4745 | + return s.callbackClient |
| 4746 | } |
| 4747 | - if t.loopbackRelay() { |
| 4748 | + if loopbackPublicURL(s.publicURL) { |
| 4749 | return &http.Client{Timeout: callbackTimeout, Transport: &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| 4750 | } |
| 4751 | return replication.NewPinnedClient(callbackTimeout) |
| 4752 | } |
| 4753 | |
| 4754 | // callbackLock bounds delivery to one POST per callback at a time. |
| 4755 | -func (t *Tenant) callbackLock(id string) *sync.Mutex { |
| 4756 | - t.callbackMu.Lock() |
| 4757 | - defer t.callbackMu.Unlock() |
| 4758 | - if t.callbackLocks == nil { |
| 4759 | - t.callbackLocks = map[string]*sync.Mutex{} |
| 4760 | +func (s *callbackService) callbackLock(id string) *sync.Mutex { |
| 4761 | + s.callbackMu.Lock() |
| 4762 | + defer s.callbackMu.Unlock() |
| 4763 | + if s.callbackLocks == nil { |
| 4764 | + s.callbackLocks = map[string]*sync.Mutex{} |
| 4765 | } |
| 4766 | - lock := t.callbackLocks[id] |
| 4767 | + lock := s.callbackLocks[id] |
| 4768 | if lock == nil { |
| 4769 | lock = &sync.Mutex{} |
| 4770 | - t.callbackLocks[id] = lock |
| 4771 | + s.callbackLocks[id] = lock |
| 4772 | } |
| 4773 | return lock |
| 4774 | } |
| 4775 | @@ -72,8 +72,8 @@ func (t *Tenant) callbackLock(id string) *sync.Mutex { |
| 4776 | // handleCallbackDelivery serves one callback-delivery intent. It rechecks |
| 4777 | // the callback and the key's access right before the POST, so a callback |
| 4778 | // removed or paused after the event arrived delivers nothing. |
| 4779 | -func (t *Tenant) handleCallbackDelivery(ctx context.Context, intent work.Intent) (err error) { |
| 4780 | - ctx, finish := t.app.telemetry.Start(ctx, "callback") |
| 4781 | +func (s *callbackService) handleCallbackDelivery(ctx context.Context, intent work.Intent) (err error) { |
| 4782 | + ctx, finish := s.telemetry.Start(ctx, "callback") |
| 4783 | outcome := "error" |
| 4784 | defer func() { finish(outcome) }() |
| 4785 | var payload callbackPayload |
| 4786 | @@ -84,10 +84,10 @@ func (t *Tenant) handleCallbackDelivery(ctx context.Context, intent work.Intent) |
| 4787 | if payload.Attempt < 1 { |
| 4788 | payload.Attempt = 1 |
| 4789 | } |
| 4790 | - lock := t.callbackLock(intent.Target) |
| 4791 | + lock := s.callbackLock(intent.Target) |
| 4792 | lock.Lock() |
| 4793 | defer lock.Unlock() |
| 4794 | - record, err := t.callback(ctx, intent.Target) |
| 4795 | + record, err := s.callback(ctx, intent.Target) |
| 4796 | if err != nil { |
| 4797 | if strings.HasPrefix(err.Error(), "not found:") { |
| 4798 | outcome = "paused" |
| 4799 | @@ -99,26 +99,26 @@ func (t *Tenant) handleCallbackDelivery(ctx context.Context, intent work.Intent) |
| 4800 | outcome = "paused" |
| 4801 | return nil |
| 4802 | } |
| 4803 | - role, err := t.community.Role(ctx, record.Owner) |
| 4804 | + role, err := s.community.Role(ctx, record.Owner) |
| 4805 | if err != nil { |
| 4806 | return err |
| 4807 | } |
| 4808 | - if role == "" && record.Owner != t.Policy().Owner { |
| 4809 | + if role == "" && record.Owner != s.policy().Owner { |
| 4810 | // The key lost its standing since it registered; stop until a |
| 4811 | // person resumes the callback. |
| 4812 | outcome = "unauthorized" |
| 4813 | - return t.pauseCallback(ctx, record.ID, record.Failures, "paused: the key is no longer a member") |
| 4814 | + return s.pauseCallback(ctx, record.ID, record.Failures, "paused: the key is no longer a member") |
| 4815 | } |
| 4816 | - if !t.gate.CanSee(ctx, payload.Event, relay.Session{PubKeys: []string{record.Owner}, RelayURL: t.RelayURL()}, nil) { |
| 4817 | + if !s.gate.CanSee(ctx, payload.Event, relay.Session{PubKeys: []string{record.Owner}, RelayURL: s.relayURL}, nil) { |
| 4818 | outcome = "unauthorized" |
| 4819 | return nil |
| 4820 | } |
| 4821 | - status, postErr := t.postCallback(ctx, record, payload.Event) |
| 4822 | + status, postErr := s.postCallback(ctx, record, payload.Event) |
| 4823 | now := time.Now().Unix() |
| 4824 | if postErr == nil { |
| 4825 | outcome = "ok" |
| 4826 | - _, err = t.store.DB().ExecContext(ctx, `UPDATE callbacks SET last_delivery_at=?, last_status='ok', failures=0 WHERE id=?`, now, record.ID) |
| 4827 | - t.app.telemetry.Logger().Info("callback delivered", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status) |
| 4828 | + _, err = s.store.DB().ExecContext(ctx, `UPDATE callbacks SET last_delivery_at=?, last_status='ok', failures=0 WHERE id=?`, now, record.ID) |
| 4829 | + s.telemetry.Logger().Info("callback delivered", "tenant", s.tenant.Name, "attempt", payload.Attempt, "status", status) |
| 4830 | return err |
| 4831 | } |
| 4832 | // Counts only: the reason names the HTTP status or the failure class, |
| 4833 | @@ -128,32 +128,32 @@ func (t *Tenant) handleCallbackDelivery(ctx context.Context, intent work.Intent) |
| 4834 | paused := failures >= callbackPauseFailures |
| 4835 | if paused { |
| 4836 | outcome = "paused" |
| 4837 | - if err := t.pauseCallback(ctx, record.ID, failures, fmt.Sprintf("paused after %d failures: %s", failures, reason)); err != nil { |
| 4838 | + if err := s.pauseCallback(ctx, record.ID, failures, fmt.Sprintf("paused after %d failures: %s", failures, reason)); err != nil { |
| 4839 | return err |
| 4840 | } |
| 4841 | - } else if _, err := t.store.DB().ExecContext(ctx, `UPDATE callbacks SET failures=?, last_status=? WHERE id=?`, failures, reason, record.ID); err != nil { |
| 4842 | + } else if _, err := s.store.DB().ExecContext(ctx, `UPDATE callbacks SET failures=?, last_status=? WHERE id=?`, failures, reason, record.ID); err != nil { |
| 4843 | return err |
| 4844 | } |
| 4845 | - t.app.telemetry.Logger().Info("callback delivery failed", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status, "failures", failures, "paused", paused) |
| 4846 | + s.telemetry.Logger().Info("callback delivery failed", "tenant", s.tenant.Name, "attempt", payload.Attempt, "status", status, "failures", failures, "paused", paused) |
| 4847 | if paused || payload.Attempt >= callbackMaxAttempts { |
| 4848 | return nil |
| 4849 | } |
| 4850 | - return t.retryCallback(ctx, intent, payload) |
| 4851 | + return s.retryCallback(ctx, intent, payload) |
| 4852 | } |
| 4853 | |
| 4854 | // pauseCallback stops deliveries and records the count and reason, then |
| 4855 | // drops the index so no new intents are queued for the callback. |
| 4856 | -func (t *Tenant) pauseCallback(ctx context.Context, id string, failures int, status string) error { |
| 4857 | - if _, err := t.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=1, failures=?, last_status=? WHERE id=?`, failures, status, id); err != nil { |
| 4858 | +func (s *callbackService) pauseCallback(ctx context.Context, id string, failures int, status string) error { |
| 4859 | + if _, err := s.store.DB().ExecContext(ctx, `UPDATE callbacks SET paused=1, failures=?, last_status=? WHERE id=?`, failures, status, id); err != nil { |
| 4860 | return err |
| 4861 | } |
| 4862 | - t.invalidateCallbacks() |
| 4863 | + s.invalidateCallbacks() |
| 4864 | return nil |
| 4865 | } |
| 4866 | |
| 4867 | // retryCallback queues the next attempt after its backoff. Each attempt is |
| 4868 | // its own intent so the durable queue's ordering and fencing still apply. |
| 4869 | -func (t *Tenant) retryCallback(ctx context.Context, intent work.Intent, payload callbackPayload) error { |
| 4870 | +func (s *callbackService) retryCallback(ctx context.Context, intent work.Intent, payload callbackPayload) error { |
| 4871 | delay := callbackBackoff[len(callbackBackoff)-1] |
| 4872 | if payload.Attempt-1 < len(callbackBackoff) { |
| 4873 | delay = callbackBackoff[payload.Attempt-1] |
| 4874 | @@ -165,7 +165,7 @@ func (t *Tenant) retryCallback(ctx context.Context, intent work.Intent, payload |
| 4875 | } |
| 4876 | now := time.Now().Unix() |
| 4877 | key := sha256.Sum256([]byte(callbackDelivery + "\x00" + payload.Event.ID + "\x00" + intent.Target + "\x00" + fmt.Sprint(payload.Attempt))) |
| 4878 | - return t.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4879 | + return s.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 4880 | _, err := tx.ExecContext(ctx, `INSERT OR IGNORE INTO work_intents(id,kind,event_id,target,payload,next_at,created_at,updated_at) VALUES(?,?,?,?,?,?,?,?)`, hex.EncodeToString(key[:]), callbackDelivery, payload.Event.ID+"#"+fmt.Sprint(payload.Attempt), intent.Target, string(encoded), now+int64(delay/time.Second), now, now) |
| 4881 | return err |
| 4882 | }) |
| 4883 | @@ -173,7 +173,7 @@ func (t *Tenant) retryCallback(ctx context.Context, intent work.Intent, payload |
| 4884 | |
| 4885 | // postCallback sends the event and returns the HTTP status with a bounded |
| 4886 | // reason on failure. |
| 4887 | -func (t *Tenant) postCallback(ctx context.Context, record callbackRecord, e event.Event) (int, error) { |
| 4888 | +func (s *callbackService) postCallback(ctx context.Context, record callbackRecord, e event.Event) (int, error) { |
| 4889 | body, err := json.Marshal(e) |
| 4890 | if err != nil { |
| 4891 | return 0, errors.New("encode event") |
| 4892 | @@ -187,9 +187,9 @@ func (t *Tenant) postCallback(ctx context.Context, record callbackRecord, e even |
| 4893 | request.Header.Set("Content-Type", "application/json") |
| 4894 | request.Header.Set("X-Tiny-Callback", record.ID) |
| 4895 | request.Header.Set("X-Tiny-Signature", callbackSignature(record.secret, body)) |
| 4896 | - request.Header.Set("X-Tiny-Relay", t.publicURL) |
| 4897 | + request.Header.Set("X-Tiny-Relay", s.publicURL) |
| 4898 | request.Header.Set("User-Agent", "tinyrelay") |
| 4899 | - response, err := t.callbackHTTPClient().Do(request) |
| 4900 | + response, err := s.callbackHTTPClient().Do(request) |
| 4901 | if err != nil { |
| 4902 | if errors.Is(err, context.DeadlineExceeded) || ctx.Err() != nil { |
| 4903 | return 0, errors.New("timeout") |
| 4904 | @@ -203,3 +203,7 @@ func (t *Tenant) postCallback(ctx context.Context, record callbackRecord, e even |
| 4905 | } |
| 4906 | return response.StatusCode, nil |
| 4907 | } |
| 4908 | + |
| 4909 | +func (t *Tenant) handleCallbackDelivery(ctx context.Context, intent work.Intent) error { |
| 4910 | + return t.callbacks.handleCallbackDelivery(ctx, intent) |
| 4911 | +} |
| 4912 | diff --git a/internal/daemon/callbacks_service.go b/internal/daemon/callbacks_service.go |
| 4913 | new file mode 100644 |
| 4914 | index 0000000..041de27 |
| 4915 | --- /dev/null |
| 4916 | +++ b/internal/daemon/callbacks_service.go |
| 4917 | @@ -0,0 +1,50 @@ |
| 4918 | +package daemon |
| 4919 | + |
| 4920 | +import ( |
| 4921 | + "net/http" |
| 4922 | + "sync" |
| 4923 | + |
| 4924 | + "github.com/FelineStateMachine/tinyrelay/internal/catalog" |
| 4925 | + "github.com/FelineStateMachine/tinyrelay/internal/community" |
| 4926 | + "github.com/FelineStateMachine/tinyrelay/internal/gates" |
| 4927 | + "github.com/FelineStateMachine/tinyrelay/internal/policy" |
| 4928 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 4929 | + "github.com/FelineStateMachine/tinyrelay/internal/telemetry" |
| 4930 | +) |
| 4931 | + |
| 4932 | +// callbackService owns callback registration, matching and durable delivery. |
| 4933 | +// Its dependencies are explicit so callback state does not enlarge Tenant. |
| 4934 | +type callbackService struct { |
| 4935 | + store *storage.Store |
| 4936 | + community *community.Service |
| 4937 | + gate *gates.Gate |
| 4938 | + policy func() policy.Policy |
| 4939 | + publicURL string |
| 4940 | + relayURL string |
| 4941 | + tenant catalog.Tenant |
| 4942 | + telemetry *telemetry.Telemetry |
| 4943 | + |
| 4944 | + callbackMu sync.Mutex |
| 4945 | + callbackIndex map[int][]callbackRecord |
| 4946 | + callbackLocks map[string]*sync.Mutex |
| 4947 | + callbackClient *http.Client |
| 4948 | +} |
| 4949 | + |
| 4950 | +type callbackServiceConfig struct { |
| 4951 | + Store *storage.Store |
| 4952 | + Community *community.Service |
| 4953 | + Gate *gates.Gate |
| 4954 | + Policy func() policy.Policy |
| 4955 | + PublicURL string |
| 4956 | + RelayURL string |
| 4957 | + Tenant catalog.Tenant |
| 4958 | + Telemetry *telemetry.Telemetry |
| 4959 | +} |
| 4960 | + |
| 4961 | +func newCallbackService(cfg callbackServiceConfig) *callbackService { |
| 4962 | + return &callbackService{ |
| 4963 | + store: cfg.Store, community: cfg.Community, gate: cfg.Gate, |
| 4964 | + policy: cfg.Policy, publicURL: cfg.PublicURL, relayURL: cfg.RelayURL, |
| 4965 | + tenant: cfg.Tenant, telemetry: cfg.Telemetry, |
| 4966 | + } |
| 4967 | +} |
| 4968 | diff --git a/internal/daemon/callbacks_test.go b/internal/daemon/callbacks_test.go |
| 4969 | index 4ab3713..c1ecd51 100644 |
| 4970 | --- a/internal/daemon/callbacks_test.go |
| 4971 | +++ b/internal/daemon/callbacks_test.go |
| 4972 | @@ -412,7 +412,7 @@ func TestCallbackDeliverySignsBodyRetriesAndPauses(t *testing.T) { |
| 4973 | receiver := &callbackReceiver{status: http.StatusAccepted} |
| 4974 | service := httptest.NewTLSServer(receiver) |
| 4975 | defer service.Close() |
| 4976 | - tenant.callbackClient = service.Client() |
| 4977 | + tenant.callbacks.callbackClient = service.Client() |
| 4978 | registered := addCallback(t, tenant, member, service.URL+"/wake?token=t0k3n", `{"kinds":[1621]}`) |
| 4979 | id, secret := registered["id"].(string), registered["secret"].(string) |
| 4980 | now := time.Now().Unix() |
| 4981 | @@ -561,7 +561,7 @@ func TestCallbackDeliveryRechecksTheGateAndMembership(t *testing.T) { |
| 4982 | receiver := &callbackReceiver{status: http.StatusOK} |
| 4983 | service := httptest.NewTLSServer(receiver) |
| 4984 | defer service.Close() |
| 4985 | - tenant.callbackClient = service.Client() |
| 4986 | + tenant.callbacks.callbackClient = service.Client() |
| 4987 | registered := addCallback(t, tenant, member, service.URL+"/wake", `{"kinds":[1]}`) |
| 4988 | id := registered["id"].(string) |
| 4989 | now := time.Now().Unix() |
| 4990 | diff --git a/internal/daemon/client_events.go b/internal/daemon/client_events.go |
| 4991 | new file mode 100644 |
| 4992 | index 0000000..a90f886 |
| 4993 | --- /dev/null |
| 4994 | +++ b/internal/daemon/client_events.go |
| 4995 | @@ -0,0 +1,146 @@ |
| 4996 | +package daemon |
| 4997 | + |
| 4998 | +import ( |
| 4999 | + "context" |
| 5000 | + "database/sql" |
| 5001 | + "errors" |
| 5002 | + "strings" |
| 5003 | + |
| 5004 | + "github.com/FelineStateMachine/tinyrelay/internal/community" |
| 5005 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 5006 | + "github.com/FelineStateMachine/tinyrelay/internal/policy" |
| 5007 | + "github.com/FelineStateMachine/tinyrelay/internal/relay" |
| 5008 | + "github.com/FelineStateMachine/tinyrelay/internal/replication" |
| 5009 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 5010 | +) |
| 5011 | + |
| 5012 | +type metadataProjection struct { |
| 5013 | + next policy.Policy |
| 5014 | + changed bool |
| 5015 | +} |
| 5016 | + |
| 5017 | +func (t *Tenant) publishReport(ctx context.Context, e event.Event) (string, error) { |
| 5018 | + target := "" |
| 5019 | + targetType := "" |
| 5020 | + reportType := "" |
| 5021 | + for _, tag := range e.Tags { |
| 5022 | + if len(tag) > 1 && tag[0] == "e" && target == "" { |
| 5023 | + target = tag[1] |
| 5024 | + targetType = "event" |
| 5025 | + if len(tag) > 2 { |
| 5026 | + reportType = tag[2] |
| 5027 | + } |
| 5028 | + } |
| 5029 | + if len(tag) > 1 && tag[0] == "p" && target == "" { |
| 5030 | + target = tag[1] |
| 5031 | + targetType = "pubkey" |
| 5032 | + if len(tag) > 2 { |
| 5033 | + reportType = tag[2] |
| 5034 | + } |
| 5035 | + } |
| 5036 | + if len(tag) > 1 && tag[0] == "x" && target == "" { |
| 5037 | + target = tag[1] |
| 5038 | + targetType = "blob" |
| 5039 | + if len(tag) > 2 { |
| 5040 | + reportType = tag[2] |
| 5041 | + } |
| 5042 | + } |
| 5043 | + } |
| 5044 | + if target == "" { |
| 5045 | + return "", errors.New("invalid: report needs an e or p tag") |
| 5046 | + } |
| 5047 | + if err := t.community.SubmitReportTarget(ctx, e.PubKey, target, targetType, reportType, e.Content, t.Policy().ReportThreshold); err != nil { |
| 5048 | + return "", err |
| 5049 | + } |
| 5050 | + return "info: report received", nil |
| 5051 | +} |
| 5052 | + |
| 5053 | +func (t *Tenant) prepareClientEvent(ctx context.Context, e event.Event, s relay.Session, now int64) (eventCommitPlan, *metadataProjection, error) { |
| 5054 | + metadata := &metadataProjection{} |
| 5055 | + plan, err := t.prepareStoredEvent(ctx, e, replication.OriginClient, now) |
| 5056 | + if err != nil { |
| 5057 | + return plan, metadata, err |
| 5058 | + } |
| 5059 | + opts := plan.options |
| 5060 | + if (e.Kind == event.KIND_EDIT_METADATA || e.Kind == event.KIND_PINS) && t.roomScope(e) == "" { |
| 5061 | + role, roleErr := t.community.Role(ctx, e.PubKey) |
| 5062 | + if roleErr != nil { |
| 5063 | + return plan, metadata, roleErr |
| 5064 | + } |
| 5065 | + if role != "owner" && role != "moderator" { |
| 5066 | + return plan, metadata, errors.New("restricted: not a group admin") |
| 5067 | + } |
| 5068 | + opts.AddBeforeCommit(func(txCtx context.Context, tx *sql.Tx) error { |
| 5069 | + return t.community.HandleProjectionEventTx(txCtx, tx, e, func(sideTx *sql.Tx) error { |
| 5070 | + if e.Kind == event.KIND_EDIT_METADATA { |
| 5071 | + metadata.next = t.Policy() |
| 5072 | + for _, tag := range e.Tags { |
| 5073 | + if len(tag) < 2 { |
| 5074 | + continue |
| 5075 | + } |
| 5076 | + switch tag[0] { |
| 5077 | + case "name": |
| 5078 | + metadata.next.Name = tag[1][:min(200, len(tag[1]))] |
| 5079 | + case "about": |
| 5080 | + metadata.next.Description = tag[1][:min(2000, len(tag[1]))] |
| 5081 | + case "picture": |
| 5082 | + metadata.next.Icon = tag[1][:min(2000, len(tag[1]))] |
| 5083 | + } |
| 5084 | + } |
| 5085 | + metadata.changed = true |
| 5086 | + return storage.PutSetting(txCtx, sideTx, "policy", metadata.next) |
| 5087 | + } |
| 5088 | + pins, err := parsePinTags(e.Tags) |
| 5089 | + if err != nil { |
| 5090 | + return err |
| 5091 | + } |
| 5092 | + return t.records.ReplacePinsTx(txCtx, sideTx, pins) |
| 5093 | + }) |
| 5094 | + }) |
| 5095 | + } |
| 5096 | + opts.Intents = append(opts.Intents, t.replication.Prepare(e, replication.OriginClient)...) |
| 5097 | + if e.Kind == event.KIND_MARMOT_GROUP { |
| 5098 | + principal, principalErr := t.gate.Principal(ctx, e, s) |
| 5099 | + if principalErr != nil { |
| 5100 | + return plan, metadata, principalErr |
| 5101 | + } |
| 5102 | + opts.AddBeforeCommit(func(txCtx context.Context, tx *sql.Tx) error { |
| 5103 | + _, err := tx.ExecContext(txCtx, `INSERT OR REPLACE INTO marmot_principals(event_id,pubkey) VALUES(?,?)`, e.ID, principal) |
| 5104 | + return err |
| 5105 | + }) |
| 5106 | + } |
| 5107 | + plan.options = opts |
| 5108 | + return plan, metadata, nil |
| 5109 | +} |
| 5110 | + |
| 5111 | +func (t *Tenant) persistClientEvent(ctx context.Context, e event.Event, opts storage.SaveOptions) (string, error) { |
| 5112 | + var err error |
| 5113 | + persist := func(tx *sql.Tx) error { |
| 5114 | + _, saveErr := storage.SaveTx(ctx, tx, e, opts) |
| 5115 | + return saveErr |
| 5116 | + } |
| 5117 | + // reason is the OK message; an access request tells the asker it waits, |
| 5118 | + // and a member asking again is told so. |
| 5119 | + reason := "" |
| 5120 | + room := t.roomScope(e) |
| 5121 | + switch { |
| 5122 | + case room != "" && community.RoomAdminKind(e.Kind): |
| 5123 | + _, err = t.community.HandleRoomEventTx(ctx, e, persist) |
| 5124 | + case room != "" && !event.IsEphemeral(e.Kind): |
| 5125 | + err = t.community.HandleRoomMessageTx(ctx, e, persist) |
| 5126 | + case e.Kind == event.KIND_JOIN, e.Kind == event.KIND_LEAVE, e.Kind == event.KIND_NIP43_JOIN, e.Kind == event.KIND_NIP43_LEAVE: |
| 5127 | + var membership community.MembershipResult |
| 5128 | + membership, err = t.community.HandleMembershipEventTx(ctx, e, persist) |
| 5129 | + if err == nil && (membership.AccessRequest || strings.HasPrefix(membership.Message, "duplicate:")) { |
| 5130 | + reason = membership.Message |
| 5131 | + } |
| 5132 | + if err == nil && membership.NewRequest { |
| 5133 | + t.notifyJoinRequest(ctx, e) |
| 5134 | + } |
| 5135 | + case e.Kind == event.KIND_PUT_USER, e.Kind == event.KIND_REMOVE_USER, e.Kind == event.KIND_DELETE_EVENT, e.Kind == event.KIND_CREATE_INVITE: |
| 5136 | + _, err = t.community.HandleModerationEventTx(ctx, e, persist) |
| 5137 | + default: |
| 5138 | + _, err = t.store.Save(ctx, e, opts) |
| 5139 | + } |
| 5140 | + return reason, err |
| 5141 | +} |
| 5142 | diff --git a/internal/daemon/custom_views.go b/internal/daemon/custom_views.go |
| 5143 | index 2d3e248..b6c63ea 100644 |
| 5144 | --- a/internal/daemon/custom_views.go |
| 5145 | +++ b/internal/daemon/custom_views.go |
| 5146 | @@ -8,20 +8,76 @@ package daemon |
| 5147 | |
| 5148 | import ( |
| 5149 | "context" |
| 5150 | + "crypto/sha256" |
| 5151 | "database/sql" |
| 5152 | + "encoding/hex" |
| 5153 | "encoding/json" |
| 5154 | "errors" |
| 5155 | "fmt" |
| 5156 | + "net/http" |
| 5157 | "sort" |
| 5158 | "strconv" |
| 5159 | "strings" |
| 5160 | + "sync" |
| 5161 | "time" |
| 5162 | |
| 5163 | + "github.com/FelineStateMachine/tinyrelay/internal/community" |
| 5164 | "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 5165 | + "github.com/FelineStateMachine/tinyrelay/internal/gitrelay" |
| 5166 | + "github.com/FelineStateMachine/tinyrelay/internal/records" |
| 5167 | "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 5168 | + "github.com/FelineStateMachine/tinyrelay/internal/telemetry" |
| 5169 | "github.com/FelineStateMachine/tinyrelay/internal/views" |
| 5170 | ) |
| 5171 | |
| 5172 | +// customViewService owns custom view definitions, transform execution and |
| 5173 | +// artifact storage. Tenant supplies the narrow lifecycle callbacks when it |
| 5174 | +// constructs the service. |
| 5175 | +type customViewService struct { |
| 5176 | + store *storage.Store |
| 5177 | + community *community.Service |
| 5178 | + records *records.Service |
| 5179 | + git *gitrelay.GitRelay |
| 5180 | + telemetry *telemetry.Telemetry |
| 5181 | + tenantName string |
| 5182 | + publicURL string |
| 5183 | + loopback func() bool |
| 5184 | + resolveActor func(*http.Request) (string, error) |
| 5185 | + browseRead func(context.Context, string) error |
| 5186 | + viewClient *http.Client |
| 5187 | + viewMu sync.Mutex |
| 5188 | + viewIndex map[int][]customView |
| 5189 | + viewLocks map[string]*sync.Mutex |
| 5190 | +} |
| 5191 | + |
| 5192 | +type customViewServiceConfig struct { |
| 5193 | + Store *storage.Store |
| 5194 | + Community *community.Service |
| 5195 | + Records *records.Service |
| 5196 | + Git *gitrelay.GitRelay |
| 5197 | + Telemetry *telemetry.Telemetry |
| 5198 | + TenantName string |
| 5199 | + PublicURL string |
| 5200 | + Loopback func() bool |
| 5201 | + ResolveActor func(*http.Request) (string, error) |
| 5202 | + BrowseRead func(context.Context, string) error |
| 5203 | +} |
| 5204 | + |
| 5205 | +func newCustomViewService(cfg customViewServiceConfig) *customViewService { |
| 5206 | + return &customViewService{ |
| 5207 | + store: cfg.Store, |
| 5208 | + community: cfg.Community, |
| 5209 | + records: cfg.Records, |
| 5210 | + git: cfg.Git, |
| 5211 | + telemetry: cfg.Telemetry, |
| 5212 | + tenantName: cfg.TenantName, |
| 5213 | + publicURL: cfg.PublicURL, |
| 5214 | + loopback: cfg.Loopback, |
| 5215 | + resolveActor: cfg.ResolveActor, |
| 5216 | + browseRead: cfg.BrowseRead, |
| 5217 | + } |
| 5218 | +} |
| 5219 | + |
| 5220 | const ( |
| 5221 | viewTransform = "view-transform" |
| 5222 | viewKindMax = 16 |
| 5223 | @@ -40,7 +96,7 @@ const ( |
| 5224 | // viewBackoff is the wait before the second and third attempt. |
| 5225 | var viewBackoff = []time.Duration{time.Minute, 5 * time.Minute} |
| 5226 | |
| 5227 | -const customViewSchema = `CREATE TABLE IF NOT EXISTS custom_views(name TEXT PRIMARY KEY, kinds TEXT NOT NULL, transform TEXT NOT NULL, trigger TEXT NOT NULL, audience TEXT NOT NULL, languages TEXT NOT NULL, max_bytes INTEGER NOT NULL, secret TEXT NOT NULL, enabled INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, last_run_at INTEGER NOT NULL DEFAULT 0, last_status TEXT NOT NULL DEFAULT '', failures INTEGER NOT NULL DEFAULT 0); |
| 5228 | +const customViewSchema = `CREATE TABLE IF NOT EXISTS custom_views(name TEXT PRIMARY KEY, kinds TEXT NOT NULL, transform TEXT NOT NULL, trigger TEXT NOT NULL, audience TEXT NOT NULL, languages TEXT NOT NULL, max_bytes INTEGER NOT NULL, secret TEXT NOT NULL, generation TEXT NOT NULL DEFAULT '', enabled INTEGER NOT NULL DEFAULT 1, created_at INTEGER NOT NULL, last_run_at INTEGER NOT NULL DEFAULT 0, last_status TEXT NOT NULL DEFAULT '', failures INTEGER NOT NULL DEFAULT 0); |
| 5229 | CREATE TABLE IF NOT EXISTS custom_view_artifacts(view TEXT NOT NULL, hash TEXT NOT NULL, event_id TEXT NOT NULL, type TEXT NOT NULL, engine TEXT NOT NULL DEFAULT '', audience TEXT NOT NULL, content TEXT NOT NULL, raw TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY(view,hash)); |
| 5230 | CREATE TABLE IF NOT EXISTS custom_view_sources(view TEXT NOT NULL, hash TEXT NOT NULL, source TEXT NOT NULL, block INTEGER NOT NULL, expires INTEGER NOT NULL DEFAULT 0, PRIMARY KEY(view,hash,source)); |
| 5231 | CREATE INDEX IF NOT EXISTS custom_view_sources_source ON custom_view_sources(view,source);` |
| 5232 | @@ -61,6 +117,30 @@ type customView struct { |
| 5233 | LastStatus string `json:"lastStatus"` |
| 5234 | Failures int `json:"failures"` |
| 5235 | secret string |
| 5236 | + generation string |
| 5237 | +} |
| 5238 | + |
| 5239 | +type customViewTarget struct { |
| 5240 | + Name string |
| 5241 | + Fingerprint string |
| 5242 | +} |
| 5243 | + |
| 5244 | +func customViewFingerprint(view customView) string { |
| 5245 | + canonical := struct { |
| 5246 | + Name string |
| 5247 | + Kinds []int |
| 5248 | + Transform string |
| 5249 | + Trigger string |
| 5250 | + Audience string |
| 5251 | + Languages []string |
| 5252 | + MaxBytes int |
| 5253 | + Secret string |
| 5254 | + CreatedAt int64 |
| 5255 | + Generation string |
| 5256 | + }{view.Name, view.Kinds, view.Transform, view.Trigger, view.Audience, view.Languages, view.MaxBytes, view.secret, view.CreatedAt, view.generation} |
| 5257 | + raw, _ := json.Marshal(canonical) |
| 5258 | + sum := sha256.Sum256(raw) |
| 5259 | + return hex.EncodeToString(sum[:]) |
| 5260 | } |
| 5261 | |
| 5262 | func (v customView) summary() map[string]any { |
| 5263 | @@ -80,9 +160,40 @@ func (v customView) watches(kind int) bool { |
| 5264 | return false |
| 5265 | } |
| 5266 | |
| 5267 | -func (t *Tenant) initCustomViews(ctx context.Context) error { |
| 5268 | - _, err := t.store.DB().ExecContext(ctx, customViewSchema) |
| 5269 | - return err |
| 5270 | +func (t *customViewService) initCustomViews(ctx context.Context) error { |
| 5271 | + if _, err := t.store.DB().ExecContext(ctx, customViewSchema); err != nil { |
| 5272 | + return err |
| 5273 | + } |
| 5274 | + _, err := t.store.DB().ExecContext(ctx, `ALTER TABLE custom_views ADD COLUMN generation TEXT NOT NULL DEFAULT ''`) |
| 5275 | + if err != nil && !strings.Contains(strings.ToLower(err.Error()), "duplicate column") { |
| 5276 | + return err |
| 5277 | + } |
| 5278 | + rows, err := t.store.DB().QueryContext(ctx, `SELECT name FROM custom_views WHERE generation=''`) |
| 5279 | + if err != nil { |
| 5280 | + return err |
| 5281 | + } |
| 5282 | + defer rows.Close() |
| 5283 | + var names []string |
| 5284 | + for rows.Next() { |
| 5285 | + var name string |
| 5286 | + if err := rows.Scan(&name); err != nil { |
| 5287 | + return err |
| 5288 | + } |
| 5289 | + names = append(names, name) |
| 5290 | + } |
| 5291 | + if err := rows.Err(); err != nil { |
| 5292 | + return err |
| 5293 | + } |
| 5294 | + for _, name := range names { |
| 5295 | + generation, err := randomHex(16) |
| 5296 | + if err != nil { |
| 5297 | + return err |
| 5298 | + } |
| 5299 | + if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET generation=? WHERE name=? AND generation=''`, generation, name); err != nil { |
| 5300 | + return err |
| 5301 | + } |
| 5302 | + } |
| 5303 | + return nil |
| 5304 | } |
| 5305 | |
| 5306 | func customViewMethod(method string) bool { |
| 5307 | @@ -91,8 +202,8 @@ func customViewMethod(method string) bool { |
| 5308 | |
| 5309 | // customViewExecute serves the owner's custom view methods under the view |
| 5310 | // operation. Every change is recorded in the audit log. |
| 5311 | -func (t *Tenant) customViewExecute(ctx context.Context, actor, method string, params []json.RawMessage) (result any, err error) { |
| 5312 | - ctx, finish := t.app.telemetry.Start(ctx, "view") |
| 5313 | +func (t *customViewService) customViewExecute(ctx context.Context, actor, method string, params []json.RawMessage) (result any, err error) { |
| 5314 | + ctx, finish := t.telemetry.Start(ctx, "view") |
| 5315 | outcome := "error" |
| 5316 | defer func() { |
| 5317 | if err != nil { |
| 5318 | @@ -143,7 +254,7 @@ type customViewOptions struct { |
| 5319 | Secret string `json:"secret"` |
| 5320 | } |
| 5321 | |
| 5322 | -func (t *Tenant) addCustomView(ctx context.Context, actor string, params []json.RawMessage) (any, error) { |
| 5323 | +func (t *customViewService) addCustomView(ctx context.Context, actor string, params []json.RawMessage) (any, error) { |
| 5324 | if len(params) != 1 { |
| 5325 | return nil, errors.New("invalid: addcustomview expects one object with name, kinds, transform, trigger, audience, languages and optional max_bytes and secret") |
| 5326 | } |
| 5327 | @@ -167,11 +278,15 @@ func (t *Tenant) addCustomView(ctx context.Context, actor string, params []json. |
| 5328 | return nil, fmt.Errorf("invalid: secret must be %d to %d printable ASCII characters", callbackSecretMin, callbackSecretMax) |
| 5329 | } |
| 5330 | view.secret = secret |
| 5331 | + view.generation, err = randomHex(16) |
| 5332 | + if err != nil { |
| 5333 | + return nil, err |
| 5334 | + } |
| 5335 | view.Enabled = true |
| 5336 | view.CreatedAt = time.Now().Unix() |
| 5337 | kinds, _ := json.Marshal(view.Kinds) |
| 5338 | languages, _ := json.Marshal(view.Languages) |
| 5339 | - _, err = t.store.DB().ExecContext(ctx, `INSERT INTO custom_views(name,kinds,transform,trigger,audience,languages,max_bytes,secret,enabled,created_at) VALUES(?,?,?,?,?,?,?,?,1,?)`, view.Name, string(kinds), view.Transform, view.Trigger, view.Audience, string(languages), view.MaxBytes, secret, view.CreatedAt) |
| 5340 | + _, err = t.store.DB().ExecContext(ctx, `INSERT INTO custom_views(name,kinds,transform,trigger,audience,languages,max_bytes,secret,generation,enabled,created_at) VALUES(?,?,?,?,?,?,?,?,?,1,?)`, view.Name, string(kinds), view.Transform, view.Trigger, view.Audience, string(languages), view.MaxBytes, secret, view.generation, view.CreatedAt) |
| 5341 | if err != nil { |
| 5342 | if strings.Contains(err.Error(), "UNIQUE") { |
| 5343 | return nil, errors.New("invalid: a view with that name exists") |
| 5344 | @@ -183,7 +298,7 @@ func (t *Tenant) addCustomView(ctx context.Context, actor string, params []json. |
| 5345 | return nil, err |
| 5346 | } |
| 5347 | // Counts only: the transform host, path and secret stay out of the log. |
| 5348 | - t.app.telemetry.Logger().Info("custom view added", "tenant", t.meta.Name, "kinds", len(view.Kinds), "languages", len(view.Languages), "generated", generated) |
| 5349 | + t.telemetry.Logger().Info("custom view added", "tenant", t.tenantName, "kinds", len(view.Kinds), "languages", len(view.Languages), "generated", generated) |
| 5350 | out := view.summary() |
| 5351 | out["secret"] = secret |
| 5352 | return out, nil |
| 5353 | @@ -192,7 +307,7 @@ func (t *Tenant) addCustomView(ctx context.Context, actor string, params []json. |
| 5354 | // parseCustomView validates a definition: the name, the kinds it watches, |
| 5355 | // the https transform, the trigger, the audience, the languages and the |
| 5356 | // artifact size limit. |
| 5357 | -func (t *Tenant) parseCustomView(options customViewOptions) (customView, error) { |
| 5358 | +func (t *customViewService) parseCustomView(options customViewOptions) (customView, error) { |
| 5359 | view := customView{Name: strings.TrimSpace(options.Name), Transform: strings.TrimSpace(options.Transform), Trigger: strings.TrimSpace(options.Trigger), Audience: strings.TrimSpace(options.Audience), MaxBytes: viewDefaultMaxBytes} |
| 5360 | if !views.NamePattern.MatchString(view.Name) { |
| 5361 | return customView{}, errors.New("invalid: name must be 1 to 32 lowercase letters, digits or hyphens") |
| 5362 | @@ -208,7 +323,7 @@ func (t *Tenant) parseCustomView(options customViewOptions) (customView, error) |
| 5363 | } |
| 5364 | sort.Ints(kinds) |
| 5365 | view.Kinds = dedupeInts(kinds) |
| 5366 | - if err := t.checkCallbackURL(view.Transform); err != nil { |
| 5367 | + if err := validateWebhookURL(view.Transform, t.publicURL); err != nil { |
| 5368 | return customView{}, fmt.Errorf("invalid: transform %s", strings.TrimPrefix(err.Error(), "invalid: ")) |
| 5369 | } |
| 5370 | if view.Trigger == "" { |
| 5371 | @@ -313,7 +428,7 @@ func dedupeInts(values []int) []int { |
| 5372 | return out |
| 5373 | } |
| 5374 | |
| 5375 | -func (t *Tenant) changeCustomView(ctx context.Context, actor, method string, params []json.RawMessage) (any, error) { |
| 5376 | +func (t *customViewService) changeCustomView(ctx context.Context, actor, method string, params []json.RawMessage) (any, error) { |
| 5377 | name := customViewNameParam(params) |
| 5378 | if name == "" { |
| 5379 | return nil, errors.New("invalid: view name required") |
| 5380 | @@ -345,7 +460,7 @@ func (t *Tenant) changeCustomView(ctx context.Context, actor, method string, par |
| 5381 | if err := t.community.Record(ctx, actor, method, name, strconv.Itoa(queued)); err != nil { |
| 5382 | return nil, err |
| 5383 | } |
| 5384 | - t.app.telemetry.Logger().Info("custom view run queued", "tenant", t.meta.Name, "queued", queued) |
| 5385 | + t.telemetry.Logger().Info("custom view run queued", "tenant", t.tenantName, "queued", queued) |
| 5386 | out := view.summary() |
| 5387 | out["queued"] = queued |
| 5388 | return out, nil |
| 5389 | @@ -367,7 +482,7 @@ func (t *Tenant) changeCustomView(ctx context.Context, actor, method string, par |
| 5390 | |
| 5391 | // removeCustomView deletes the definition and every artifact it produced, |
| 5392 | // including the stored public records. |
| 5393 | -func (t *Tenant) removeCustomView(ctx context.Context, view customView) error { |
| 5394 | +func (t *customViewService) removeCustomView(ctx context.Context, view customView) error { |
| 5395 | ids, err := t.customViewArtifactIDs(ctx, `SELECT event_id FROM custom_view_artifacts WHERE view=?`, view.Name) |
| 5396 | if err != nil { |
| 5397 | return err |
| 5398 | @@ -417,7 +532,7 @@ func customViewRefreshParam(params []json.RawMessage) bool { |
| 5399 | return options.Refresh |
| 5400 | } |
| 5401 | |
| 5402 | -func (t *Tenant) customViewByName(ctx context.Context, name string) (customView, error) { |
| 5403 | +func (t *customViewService) customViewByName(ctx context.Context, name string) (customView, error) { |
| 5404 | rows, err := t.customViewRows(ctx, name) |
| 5405 | if err != nil { |
| 5406 | return customView{}, err |
| 5407 | @@ -429,8 +544,8 @@ func (t *Tenant) customViewByName(ctx context.Context, name string) (customView, |
| 5408 | } |
| 5409 | |
| 5410 | // customViewRows lists definitions, narrowed to one name. |
| 5411 | -func (t *Tenant) customViewRows(ctx context.Context, names ...string) ([]customView, error) { |
| 5412 | - query := `SELECT name,kinds,transform,trigger,audience,languages,max_bytes,secret,enabled,created_at,last_run_at,last_status,failures FROM custom_views` |
| 5413 | +func (t *customViewService) customViewRows(ctx context.Context, names ...string) ([]customView, error) { |
| 5414 | + query := `SELECT name,kinds,transform,trigger,audience,languages,max_bytes,secret,generation,enabled,created_at,last_run_at,last_status,failures FROM custom_views` |
| 5415 | args := []any{} |
| 5416 | if len(names) > 0 { |
| 5417 | query += ` WHERE name=?` |
| 5418 | @@ -447,7 +562,7 @@ func (t *Tenant) customViewRows(ctx context.Context, names ...string) ([]customV |
| 5419 | var item customView |
| 5420 | var kinds, languages string |
| 5421 | var enabled int |
| 5422 | - if err := rows.Scan(&item.Name, &kinds, &item.Transform, &item.Trigger, &item.Audience, &languages, &item.MaxBytes, &item.secret, &enabled, &item.CreatedAt, &item.LastRunAt, &item.LastStatus, &item.Failures); err != nil { |
| 5423 | + if err := rows.Scan(&item.Name, &kinds, &item.Transform, &item.Trigger, &item.Audience, &languages, &item.MaxBytes, &item.secret, &item.generation, &enabled, &item.CreatedAt, &item.LastRunAt, &item.LastStatus, &item.Failures); err != nil { |
| 5424 | return nil, err |
| 5425 | } |
| 5426 | item.Enabled = enabled != 0 |
| 5427 | @@ -464,7 +579,7 @@ func (t *Tenant) customViewRows(ctx context.Context, names ...string) ([]customV |
| 5428 | |
| 5429 | // customViewIndex returns the enabled views indexed by kind, built on first |
| 5430 | // use and dropped whenever a view changes. |
| 5431 | -func (t *Tenant) customViewIndex(ctx context.Context) map[int][]customView { |
| 5432 | +func (t *customViewService) customViewIndex(ctx context.Context) map[int][]customView { |
| 5433 | t.viewMu.Lock() |
| 5434 | defer t.viewMu.Unlock() |
| 5435 | if t.viewIndex != nil { |
| 5436 | @@ -473,7 +588,7 @@ func (t *Tenant) customViewIndex(ctx context.Context) map[int][]customView { |
| 5437 | index := map[int][]customView{} |
| 5438 | rows, err := t.customViewRows(ctx) |
| 5439 | if err != nil { |
| 5440 | - t.app.telemetry.Logger().Error("load custom views", "tenant", t.meta.Name, "error", err) |
| 5441 | + t.telemetry.Logger().Error("load custom views", "tenant", t.tenantName, "error", err) |
| 5442 | return index |
| 5443 | } |
| 5444 | for _, row := range rows { |
| 5445 | @@ -488,7 +603,7 @@ func (t *Tenant) customViewIndex(ctx context.Context) map[int][]customView { |
| 5446 | return index |
| 5447 | } |
| 5448 | |
| 5449 | -func (t *Tenant) invalidateCustomViews() { |
| 5450 | +func (t *customViewService) invalidateCustomViews() { |
| 5451 | t.viewMu.Lock() |
| 5452 | t.viewIndex = nil |
| 5453 | t.viewMu.Unlock() |
| 5454 | @@ -496,7 +611,7 @@ func (t *Tenant) invalidateCustomViews() { |
| 5455 | |
| 5456 | // CustomViews is the summary the web UI renders with: the enabled views' |
| 5457 | // names and languages, nothing else. |
| 5458 | -func (t *Tenant) CustomViews() []views.View { |
| 5459 | +func (t *customViewService) CustomViews() []views.View { |
| 5460 | seen := map[string]bool{} |
| 5461 | var out []views.View |
| 5462 | for _, list := range t.customViewIndex(context.Background()) { |
| 5463 | @@ -514,45 +629,135 @@ func (t *Tenant) CustomViews() []views.View { |
| 5464 | // queueCustomViews runs after an event is stored or released. Each enabled |
| 5465 | // write-triggered view that watches the kind and finds a block it renders |
| 5466 | // gets one transform intent. It never fails the publish. |
| 5467 | -func (t *Tenant) queueCustomViews(ctx context.Context, e event.Event) { |
| 5468 | - candidates := t.customViewIndex(ctx)[e.Kind] |
| 5469 | - if len(candidates) == 0 { |
| 5470 | +func (t *customViewService) queueCustomViews(ctx context.Context, e event.Event) { |
| 5471 | + intents := t.PrepareReleased(ctx, e) |
| 5472 | + if len(intents) == 0 { |
| 5473 | return |
| 5474 | } |
| 5475 | - if e.Kind == event.KIND_REPO_STATE && (t.git == nil || t.statePending(ctx, e.ID)) { |
| 5476 | - // The README is read from the objects, which have not arrived yet; |
| 5477 | - // releaseGit queues the state once they have. |
| 5478 | + err := t.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 5479 | + return storage.AddIntents(ctx, tx, intents, time.Now().Unix()) |
| 5480 | + }) |
| 5481 | + if err != nil { |
| 5482 | + t.telemetry.Logger().Error("view transforms not queued", "tenant", t.tenantName, "error", err) |
| 5483 | return |
| 5484 | } |
| 5485 | - var intents []storage.Intent |
| 5486 | - for _, view := range candidates { |
| 5487 | - if view.Trigger != "write" { |
| 5488 | + t.telemetry.Logger().Debug("view transforms queued", "tenant", t.tenantName, "queued", len(intents)) |
| 5489 | +} |
| 5490 | + |
| 5491 | +// Prepare returns write-triggered transform intents for an event. The caller |
| 5492 | +// owns the transaction that persists the returned intents, allowing event and |
| 5493 | +// follow-up work to share one commit boundary. |
| 5494 | +func (t *customViewService) Prepare(ctx context.Context, e event.Event) []storage.Intent { |
| 5495 | + targets, err := t.CandidateTargets(ctx, e) |
| 5496 | + if err != nil { |
| 5497 | + t.telemetry.Logger().Error("plan custom view transforms", "tenant", t.tenantName, "error", err) |
| 5498 | + return nil |
| 5499 | + } |
| 5500 | + intents, err := t.PrepareTargets(ctx, e, targets, false) |
| 5501 | + if err != nil { |
| 5502 | + t.telemetry.Logger().Error("prepare custom view transforms", "tenant", t.tenantName, "error", err) |
| 5503 | + return nil |
| 5504 | + } |
| 5505 | + return intents |
| 5506 | +} |
| 5507 | + |
| 5508 | +// PrepareReleased returns intents after repository promotion has made Git |
| 5509 | +// objects available. Repository state events are intentionally accepted only |
| 5510 | +// through this path. |
| 5511 | +func (t *customViewService) PrepareReleased(ctx context.Context, e event.Event) []storage.Intent { |
| 5512 | + targets, err := t.CandidateTargets(ctx, e) |
| 5513 | + if err != nil { |
| 5514 | + t.telemetry.Logger().Error("plan released custom view transforms", "tenant", t.tenantName, "error", err) |
| 5515 | + return nil |
| 5516 | + } |
| 5517 | + intents, err := t.PrepareTargets(ctx, e, targets, true) |
| 5518 | + if err != nil { |
| 5519 | + t.telemetry.Logger().Error("prepare released custom view transforms", "tenant", t.tenantName, "error", err) |
| 5520 | + return nil |
| 5521 | + } |
| 5522 | + return intents |
| 5523 | +} |
| 5524 | + |
| 5525 | +// CandidateTargets captures names and registration identities for an event. |
| 5526 | +// The identity prevents a later registration with the same name from |
| 5527 | +// receiving work planned for an earlier registration. |
| 5528 | +func (t *customViewService) CandidateTargets(ctx context.Context, e event.Event) ([]customViewTarget, error) { |
| 5529 | + if eventExpired(e, time.Now().Unix()) { |
| 5530 | + return nil, nil |
| 5531 | + } |
| 5532 | + rows, err := t.customViewRows(ctx) |
| 5533 | + if err != nil { |
| 5534 | + return nil, err |
| 5535 | + } |
| 5536 | + targets := make([]customViewTarget, 0, len(rows)) |
| 5537 | + for _, view := range rows { |
| 5538 | + if !view.Enabled || view.Trigger != "write" || !view.watches(e.Kind) { |
| 5539 | continue |
| 5540 | } |
| 5541 | if e.Kind != event.KIND_REPO_STATE && len(views.Matching(views.Blocks(e.Content), view.Languages)) == 0 { |
| 5542 | continue |
| 5543 | } |
| 5544 | - intents = append(intents, t.viewIntent(view, e, "", false)) |
| 5545 | + targets = append(targets, customViewTarget{Name: view.Name, Fingerprint: customViewFingerprint(view)}) |
| 5546 | } |
| 5547 | - if len(intents) == 0 { |
| 5548 | - return |
| 5549 | + return targets, nil |
| 5550 | +} |
| 5551 | + |
| 5552 | +// PrepareTargets rechecks captured registration identities before creating |
| 5553 | +// intents. A removed and recreated view with the same name is rejected. |
| 5554 | +func (t *customViewService) PrepareTargets(ctx context.Context, e event.Event, targets []customViewTarget, released bool) ([]storage.Intent, error) { |
| 5555 | + if len(targets) == 0 || eventExpired(e, time.Now().Unix()) || (!released && e.Kind == event.KIND_REPO_STATE) { |
| 5556 | + return nil, nil |
| 5557 | } |
| 5558 | - err := t.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 5559 | - return storage.AddIntents(ctx, tx, intents, time.Now().Unix()) |
| 5560 | - }) |
| 5561 | + if e.Kind != event.KIND_REPO_STATE { |
| 5562 | + current, err := t.sourceCurrent(ctx, e) |
| 5563 | + if err != nil { |
| 5564 | + return nil, err |
| 5565 | + } |
| 5566 | + if !current { |
| 5567 | + return nil, nil |
| 5568 | + } |
| 5569 | + } |
| 5570 | + rows, err := t.customViewRows(ctx) |
| 5571 | if err != nil { |
| 5572 | - t.app.telemetry.Logger().Error("view transforms not queued", "tenant", t.meta.Name, "error", err) |
| 5573 | - return |
| 5574 | + return nil, err |
| 5575 | + } |
| 5576 | + candidates := make(map[string]customView, len(rows)) |
| 5577 | + for _, view := range rows { |
| 5578 | + candidates[view.Name] = view |
| 5579 | + } |
| 5580 | + if e.Kind == event.KIND_REPO_STATE { |
| 5581 | + if t.git == nil || t.statePending(ctx, e.ID) { |
| 5582 | + return nil, nil |
| 5583 | + } |
| 5584 | } |
| 5585 | - t.app.telemetry.Logger().Debug("view transforms queued", "tenant", t.meta.Name, "queued", len(intents)) |
| 5586 | + var intents []storage.Intent |
| 5587 | + for _, target := range targets { |
| 5588 | + view, ok := candidates[target.Name] |
| 5589 | + if !ok || !view.Enabled || view.Trigger != "write" || !view.watches(e.Kind) { |
| 5590 | + continue |
| 5591 | + } |
| 5592 | + if customViewFingerprint(view) != target.Fingerprint { |
| 5593 | + continue |
| 5594 | + } |
| 5595 | + if e.Kind != event.KIND_REPO_STATE && len(views.Matching(views.Blocks(e.Content), view.Languages)) == 0 { |
| 5596 | + continue |
| 5597 | + } |
| 5598 | + intents = append(intents, t.viewIntent(view, e, "", false)) |
| 5599 | + } |
| 5600 | + return intents, nil |
| 5601 | +} |
| 5602 | + |
| 5603 | +func eventExpired(e event.Event, now int64) bool { |
| 5604 | + expires := event.Expiration(e) |
| 5605 | + return expires > 0 && expires <= now |
| 5606 | } |
| 5607 | |
| 5608 | // viewIntent is the transform intent for one source. The event travels in |
| 5609 | // the payload; run makes a backfill or an hourly pass distinct from the |
| 5610 | // write-time intent for the same event. |
| 5611 | -func (t *Tenant) viewIntent(view customView, e event.Event, run string, refresh ...bool) storage.Intent { |
| 5612 | +func (t *customViewService) viewIntent(view customView, e event.Event, run string, refresh ...bool) storage.Intent { |
| 5613 | force := len(refresh) > 0 && refresh[0] |
| 5614 | - payload, _ := json.Marshal(viewPayload{Event: e, Attempt: 1, Run: run, Refresh: force}) |
| 5615 | + payload, _ := json.Marshal(viewPayload{Event: e, Attempt: 1, Run: run, Refresh: force, Generation: view.generation}) |
| 5616 | id := e.ID |
| 5617 | if run != "" { |
| 5618 | id += "@" + run |
| 5619 | @@ -562,7 +767,7 @@ func (t *Tenant) viewIntent(view customView, e event.Event, run string, refresh |
| 5620 | |
| 5621 | // statePending reports whether a repository state still waits for its |
| 5622 | // objects. |
| 5623 | -func (t *Tenant) statePending(ctx context.Context, id string) bool { |
| 5624 | +func (t *customViewService) statePending(ctx context.Context, id string) bool { |
| 5625 | var one int |
| 5626 | err := t.store.DB().QueryRowContext(ctx, `SELECT 1 FROM pending_events WHERE id=?`, id).Scan(&one) |
| 5627 | return err == nil || t.git.IsPending(id) |
| 5628 | @@ -571,7 +776,7 @@ func (t *Tenant) statePending(ctx context.Context, id string) bool { |
| 5629 | // queueCustomViewBackfill queues the newest sources of a view's kinds, at |
| 5630 | // most 500, written since the given time. Repository states are queued |
| 5631 | // whether or not their README has blocks; the handler reads it. |
| 5632 | -func (t *Tenant) queueCustomViewBackfill(ctx context.Context, view customView, since int64, run string, refresh ...bool) (int, error) { |
| 5633 | +func (t *customViewService) queueCustomViewBackfill(ctx context.Context, view customView, since int64, run string, refresh ...bool) (int, error) { |
| 5634 | force := len(refresh) > 0 && refresh[0] |
| 5635 | rows, err := t.store.Query(ctx, event.Filter{Kinds: view.Kinds, Tags: map[string][]string{}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: viewBackfillLimit}) |
| 5636 | if err != nil { |
| 5637 | @@ -602,7 +807,7 @@ func (t *Tenant) queueCustomViewBackfill(ctx context.Context, view customView, s |
| 5638 | |
| 5639 | // tickCustomViews queues the hourly views that are due. It is called from |
| 5640 | // the tenant scheduler on its own timer. |
| 5641 | -func (t *Tenant) tickCustomViews(ctx context.Context, now int64) error { |
| 5642 | +func (t *customViewService) tickCustomViews(ctx context.Context, now int64) error { |
| 5643 | rows, err := t.customViewRows(ctx) |
| 5644 | if err != nil { |
| 5645 | return err |
| 5646 | @@ -623,7 +828,7 @@ func (t *Tenant) tickCustomViews(ctx context.Context, now int64) error { |
| 5647 | return err |
| 5648 | } |
| 5649 | if queued > 0 { |
| 5650 | - t.app.telemetry.Logger().Info("custom view hourly run queued", "tenant", t.meta.Name, "queued", queued) |
| 5651 | + t.telemetry.Logger().Info("custom view hourly run queued", "tenant", t.tenantName, "queued", queued) |
| 5652 | } |
| 5653 | } |
| 5654 | return nil |
| 5655 | diff --git a/internal/daemon/custom_views_http.go b/internal/daemon/custom_views_http.go |
| 5656 | index a0ae190..03cb22f 100644 |
| 5657 | --- a/internal/daemon/custom_views_http.go |
| 5658 | +++ b/internal/daemon/custom_views_http.go |
| 5659 | @@ -18,8 +18,8 @@ func isViewArtifactPath(path string) bool { |
| 5660 | return strings.HasPrefix(path, "/views/") |
| 5661 | } |
| 5662 | |
| 5663 | -func (t *Tenant) viewArtifactHTTP(w http.ResponseWriter, r *http.Request) { |
| 5664 | - ctx, finish := t.app.telemetry.Start(r.Context(), "view") |
| 5665 | +func (t *customViewService) viewArtifactHTTP(w http.ResponseWriter, r *http.Request) { |
| 5666 | + ctx, finish := t.telemetry.Start(r.Context(), "view") |
| 5667 | r = r.WithContext(ctx) |
| 5668 | outcome := "error" |
| 5669 | defer func() { finish(outcome) }() |
| 5670 | @@ -41,7 +41,7 @@ func (t *Tenant) viewArtifactHTTP(w http.ResponseWriter, r *http.Request) { |
| 5671 | outcome = "invalid" |
| 5672 | return |
| 5673 | } |
| 5674 | - actor, err := t.resolveUIActor(r) |
| 5675 | + actor, err := t.resolveActor(r) |
| 5676 | if err != nil { |
| 5677 | http.Error(w, err.Error(), http.StatusUnauthorized) |
| 5678 | outcome = "unauthorized" |
| 5679 | diff --git a/internal/daemon/custom_views_targets_test.go b/internal/daemon/custom_views_targets_test.go |
| 5680 | new file mode 100644 |
| 5681 | index 0000000..5675aed |
| 5682 | --- /dev/null |
| 5683 | +++ b/internal/daemon/custom_views_targets_test.go |
| 5684 | @@ -0,0 +1,91 @@ |
| 5685 | +package daemon |
| 5686 | + |
| 5687 | +import ( |
| 5688 | + "context" |
| 5689 | + "testing" |
| 5690 | + "time" |
| 5691 | + |
| 5692 | + "github.com/FelineStateMachine/tinyrelay/internal/work" |
| 5693 | +) |
| 5694 | + |
| 5695 | +func TestCustomViewTargetsRejectRecreatedRegistration(t *testing.T) { |
| 5696 | + tenant, server, _ := viewTenant(t, []int{1}, nil) |
| 5697 | + ctx := context.Background() |
| 5698 | + note := signedEvent(t, testOwnerSecret, 1, time.Now().Unix(), nil, "```mermaid\ngraph TD; a-->b;\n```") |
| 5699 | + if err := publishAs(t, tenant, note); err != nil { |
| 5700 | + t.Fatal(err) |
| 5701 | + } |
| 5702 | + targets, err := tenant.customViews.CandidateTargets(ctx, note) |
| 5703 | + if err != nil || len(targets) != 1 { |
| 5704 | + t.Fatalf("candidate targets = %+v, %v", targets, err) |
| 5705 | + } |
| 5706 | + oldIntents, err := tenant.customViews.PrepareTargets(ctx, note, targets, false) |
| 5707 | + if err != nil || len(oldIntents) != 1 { |
| 5708 | + t.Fatalf("planned intents = %+v, %v", oldIntents, err) |
| 5709 | + } |
| 5710 | + if _, err := tenant.Execute(ctx, tenant.Policy().Owner, "removecustomview", viewParams(map[string]any{"name": "diagrams"})); err != nil { |
| 5711 | + t.Fatal(err) |
| 5712 | + } |
| 5713 | + addView(t, tenant, map[string]any{ |
| 5714 | + "name": "diagrams", |
| 5715 | + "kinds": []int{1}, |
| 5716 | + "transform": "https://render.example/tiny", |
| 5717 | + "languages": []string{"mermaid"}, |
| 5718 | + "secret": "same-secret-value-1234", |
| 5719 | + }) |
| 5720 | + targets, err = tenant.customViews.CandidateTargets(ctx, note) |
| 5721 | + if err != nil || len(targets) != 1 { |
| 5722 | + t.Fatalf("replacement candidate targets = %+v, %v", targets, err) |
| 5723 | + } |
| 5724 | + if _, err := tenant.Execute(ctx, tenant.Policy().Owner, "removecustomview", viewParams(map[string]any{"name": "diagrams"})); err != nil { |
| 5725 | + t.Fatal(err) |
| 5726 | + } |
| 5727 | + addView(t, tenant, map[string]any{ |
| 5728 | + "name": "diagrams", |
| 5729 | + "kinds": []int{1}, |
| 5730 | + "transform": "https://render.example/tiny", |
| 5731 | + "languages": []string{"mermaid"}, |
| 5732 | + "secret": "same-secret-value-1234", |
| 5733 | + }) |
| 5734 | + intents, err := tenant.customViews.PrepareTargets(ctx, note, targets, false) |
| 5735 | + if err != nil { |
| 5736 | + t.Fatal(err) |
| 5737 | + } |
| 5738 | + if len(intents) != 0 { |
| 5739 | + t.Fatalf("recreated view received old targets: %+v", intents) |
| 5740 | + } |
| 5741 | + if err := tenant.customViews.handleViewTransform(ctx, work.Intent{ID: "stale", Kind: oldIntents[0].Kind, EventID: oldIntents[0].EventID, Target: oldIntents[0].Target, Payload: oldIntents[0].Payload}); err != nil { |
| 5742 | + t.Fatal(err) |
| 5743 | + } |
| 5744 | + if server.count() != 0 { |
| 5745 | + t.Fatalf("stale generation contacted transform: %d requests", server.count()) |
| 5746 | + } |
| 5747 | +} |
| 5748 | + |
| 5749 | +func TestCustomViewGenerationBackfillIsStable(t *testing.T) { |
| 5750 | + tenant, _, _ := viewTenant(t, []int{1}, nil) |
| 5751 | + ctx := context.Background() |
| 5752 | + if _, err := tenant.store.DB().ExecContext(ctx, `ALTER TABLE custom_views DROP COLUMN generation`); err != nil { |
| 5753 | + t.Fatal(err) |
| 5754 | + } |
| 5755 | + if err := tenant.customViews.initCustomViews(ctx); err != nil { |
| 5756 | + t.Fatal(err) |
| 5757 | + } |
| 5758 | + var first string |
| 5759 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT generation FROM custom_views WHERE name='diagrams'`).Scan(&first); err != nil { |
| 5760 | + t.Fatal(err) |
| 5761 | + } |
| 5762 | + if first == "" { |
| 5763 | + t.Fatal("legacy view generation was not backfilled") |
| 5764 | + } |
| 5765 | + if err := tenant.customViews.initCustomViews(ctx); err != nil { |
| 5766 | + t.Fatal(err) |
| 5767 | + } |
| 5768 | + var second string |
| 5769 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT generation FROM custom_views WHERE name='diagrams'`).Scan(&second); err != nil { |
| 5770 | + t.Fatal(err) |
| 5771 | + } |
| 5772 | + if first != second { |
| 5773 | + t.Fatalf("generation changed on reopen: %q != %q", first, second) |
| 5774 | + } |
| 5775 | +} |
| 5776 | diff --git a/internal/daemon/custom_views_test.go b/internal/daemon/custom_views_test.go |
| 5777 | index 32fa8ed..4febce3 100644 |
| 5778 | --- a/internal/daemon/custom_views_test.go |
| 5779 | +++ b/internal/daemon/custom_views_test.go |
| 5780 | @@ -92,7 +92,7 @@ func viewTenant(t *testing.T, kinds []int, extra map[string]any) (*Tenant, *tran |
| 5781 | server.answer(http.StatusOK) |
| 5782 | service := httptest.NewTLSServer(server) |
| 5783 | t.Cleanup(service.Close) |
| 5784 | - tenant.viewClient = service.Client() |
| 5785 | + tenant.customViews.viewClient = service.Client() |
| 5786 | values := map[string]any{"name": "diagrams", "kinds": kinds, "transform": service.URL + "/render?token=t0k3n", "languages": []string{"mermaid", "dot"}} |
| 5787 | for key, value := range extra { |
| 5788 | values[key] = value |
| 5789 | @@ -126,7 +126,7 @@ func pendingViewIntents(t *testing.T, tenant *Tenant, name, eventID string) []wo |
| 5790 | // complete. |
| 5791 | func runViewIntent(t *testing.T, tenant *Tenant, intent work.Intent) { |
| 5792 | t.Helper() |
| 5793 | - if err := tenant.handleViewTransform(context.Background(), intent); err != nil { |
| 5794 | + if err := tenant.customViews.handleViewTransform(context.Background(), intent); err != nil { |
| 5795 | t.Fatalf("handleViewTransform: %v", err) |
| 5796 | } |
| 5797 | if _, err := tenant.store.DB().ExecContext(context.Background(), "UPDATE work_intents SET state='completed' WHERE id=?", intent.ID); err != nil { |
| 5798 | @@ -301,7 +301,7 @@ func TestCustomViewManagementIsOwnerOnlyAndValidated(t *testing.T) { |
| 5799 | if _, err := tenant.Execute(ctx, owner, "runcustomview", []json.RawMessage{json.RawMessage(`"diagrams"`)}); err == nil || !strings.Contains(err.Error(), "resume") { |
| 5800 | t.Fatalf("run while paused: %v", err) |
| 5801 | } |
| 5802 | - for _, view := range tenant.customViewIndex(ctx)[1] { |
| 5803 | + for _, view := range tenant.customViews.customViewIndex(ctx)[1] { |
| 5804 | if view.Name == "diagrams" { |
| 5805 | t.Fatal("paused view stayed in the kind index") |
| 5806 | } |
| 5807 | @@ -557,8 +557,8 @@ func TestCustomViewReplacementReusesAndDropsBlocks(t *testing.T) { |
| 5808 | t.Fatal("B survived the replacement") |
| 5809 | } |
| 5810 | // A stale intent for the replaced version renders nothing. |
| 5811 | - stale := tenant.viewIntent(customView{Name: "diagrams"}, first, "old") |
| 5812 | - if err := tenant.handleViewTransform(ctx, work.Intent{ID: "stale", Kind: stale.Kind, EventID: stale.EventID, Target: stale.Target, Payload: stale.Payload}); err != nil { |
| 5813 | + stale := tenant.customViews.viewIntent(customView{Name: "diagrams"}, first, "old") |
| 5814 | + if err := tenant.customViews.handleViewTransform(ctx, work.Intent{ID: "stale", Kind: stale.Kind, EventID: stale.EventID, Target: stale.Target, Payload: stale.Payload}); err != nil { |
| 5815 | t.Fatal(err) |
| 5816 | } |
| 5817 | if server.count() != 1 { |
| 5818 | @@ -832,7 +832,7 @@ func TestCustomViewHourlyTickQueuesRecentSources(t *testing.T) { |
| 5819 | if queued := pendingViewIntents(t, tenant, "diagrams", note.ID); len(queued) != 0 { |
| 5820 | t.Fatal("an hourly view queued at write time") |
| 5821 | } |
| 5822 | - if err := tenant.tickCustomViews(ctx, now); err != nil { |
| 5823 | + if err := tenant.customViews.tickCustomViews(ctx, now); err != nil { |
| 5824 | t.Fatal(err) |
| 5825 | } |
| 5826 | queued := pendingViewIntents(t, tenant, "diagrams", note.ID) |
| 5827 | @@ -844,7 +844,7 @@ func TestCustomViewHourlyTickQueuesRecentSources(t *testing.T) { |
| 5828 | t.Fatal("hourly artifact missing") |
| 5829 | } |
| 5830 | // Not due again for an hour; a backfill through runcustomview queues now. |
| 5831 | - if err := tenant.tickCustomViews(ctx, now+60); err != nil { |
| 5832 | + if err := tenant.customViews.tickCustomViews(ctx, now+60); err != nil { |
| 5833 | t.Fatal(err) |
| 5834 | } |
| 5835 | if queued := pendingViewIntents(t, tenant, "diagrams", note.ID); len(queued) != 0 { |
| 5836 | @@ -900,7 +900,7 @@ func TestCustomViewRendersTheReadmeAtTheStateHead(t *testing.T) { |
| 5837 | relay := httptest.NewServer(tenant) |
| 5838 | defer relay.Close() |
| 5839 | gitTest(t, "-C", work, "push", relay.URL+"/npub10xlxvlhemja6c4dqv22uapctqupfhlxm9h8z3k2e72q4k9hcz7vqpkge6d/docs.git", "refs/heads/main:refs/heads/main") |
| 5840 | - blocks, err := tenant.sourceBlocks(ctx, state) |
| 5841 | + blocks, err := tenant.customViews.sourceBlocks(ctx, state) |
| 5842 | if err != nil { |
| 5843 | t.Fatal(err) |
| 5844 | } |
| 5845 | diff --git a/internal/daemon/custom_views_transform.go b/internal/daemon/custom_views_transform.go |
| 5846 | index 2047f05..3d7151e 100644 |
| 5847 | --- a/internal/daemon/custom_views_transform.go |
| 5848 | +++ b/internal/daemon/custom_views_transform.go |
| 5849 | @@ -37,10 +37,11 @@ import ( |
| 5850 | // not depend on the event still being stored, the attempt number and the |
| 5851 | // run that queued it. |
| 5852 | type viewPayload struct { |
| 5853 | - Event event.Event `json:"event"` |
| 5854 | - Attempt int `json:"attempt"` |
| 5855 | - Run string `json:"run,omitempty"` |
| 5856 | - Refresh bool `json:"refresh,omitempty"` |
| 5857 | + Event event.Event `json:"event"` |
| 5858 | + Attempt int `json:"attempt"` |
| 5859 | + Run string `json:"run,omitempty"` |
| 5860 | + Refresh bool `json:"refresh,omitempty"` |
| 5861 | + Generation string `json:"generation,omitempty"` |
| 5862 | } |
| 5863 | |
| 5864 | // viewRequest is the body posted to a transform: only the blocks, with the |
| 5865 | @@ -95,18 +96,18 @@ func viewSourceTag(source string) []string { |
| 5866 | |
| 5867 | // viewHTTPClient refuses private addresses and redirects unless the relay |
| 5868 | // itself runs on a loopback address. |
| 5869 | -func (t *Tenant) viewHTTPClient() *http.Client { |
| 5870 | +func (t *customViewService) viewHTTPClient() *http.Client { |
| 5871 | if t.viewClient != nil { |
| 5872 | return t.viewClient |
| 5873 | } |
| 5874 | - if t.loopbackRelay() { |
| 5875 | + if t.loopback != nil && t.loopback() { |
| 5876 | return &http.Client{Timeout: viewTimeout, Transport: &http.Transport{TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}}, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| 5877 | } |
| 5878 | return replication.NewPinnedClient(viewTimeout) |
| 5879 | } |
| 5880 | |
| 5881 | // viewLock bounds work to one transform run per view at a time. |
| 5882 | -func (t *Tenant) viewLock(name string) *sync.Mutex { |
| 5883 | +func (t *customViewService) viewLock(name string) *sync.Mutex { |
| 5884 | t.viewMu.Lock() |
| 5885 | defer t.viewMu.Unlock() |
| 5886 | if t.viewLocks == nil { |
| 5887 | @@ -122,7 +123,7 @@ func (t *Tenant) viewLock(name string) *sync.Mutex { |
| 5888 | |
| 5889 | // sourceBlocks finds the blocks of a source: the fenced blocks of its |
| 5890 | // content, or of the README at the head of a repository state. |
| 5891 | -func (t *Tenant) sourceBlocks(ctx context.Context, e event.Event) ([]views.Block, error) { |
| 5892 | +func (t *customViewService) sourceBlocks(ctx context.Context, e event.Event) ([]views.Block, error) { |
| 5893 | if e.Kind != event.KIND_REPO_STATE { |
| 5894 | return views.Blocks(e.Content), nil |
| 5895 | } |
| 5896 | @@ -135,7 +136,7 @@ func (t *Tenant) sourceBlocks(ctx context.Context, e event.Event) ([]views.Block |
| 5897 | |
| 5898 | // stateReadme reads README.md at the head named by a repository state. A |
| 5899 | // missing README yields no content. |
| 5900 | -func (t *Tenant) stateReadme(ctx context.Context, e event.Event) (string, error) { |
| 5901 | +func (t *customViewService) stateReadme(ctx context.Context, e event.Event) (string, error) { |
| 5902 | if t.git == nil { |
| 5903 | return "", nil |
| 5904 | } |
| 5905 | @@ -168,7 +169,7 @@ func (t *Tenant) stateReadme(ctx context.Context, e event.Event) (string, error) |
| 5906 | // sourceCurrent reports whether the source is still the stored event: an |
| 5907 | // event that was deleted, or a state that a newer state replaced, renders |
| 5908 | // nothing. |
| 5909 | -func (t *Tenant) sourceCurrent(ctx context.Context, e event.Event) (bool, error) { |
| 5910 | +func (t *customViewService) sourceCurrent(ctx context.Context, e event.Event) (bool, error) { |
| 5911 | var id string |
| 5912 | var err error |
| 5913 | if e.Kind == event.KIND_REPO_STATE { |
| 5914 | @@ -186,8 +187,8 @@ func (t *Tenant) sourceCurrent(ctx context.Context, e event.Event) (bool, error) |
| 5915 | // view and the source right before the POST, attaches artifacts that |
| 5916 | // already exist for a block's hash and posts only the blocks that need |
| 5917 | // rendering. |
| 5918 | -func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (err error) { |
| 5919 | - ctx, finish := t.app.telemetry.Start(ctx, "view") |
| 5920 | +func (t *customViewService) handleViewTransform(ctx context.Context, intent work.Intent) (err error) { |
| 5921 | + ctx, finish := t.telemetry.Start(ctx, "view") |
| 5922 | outcome := "error" |
| 5923 | defer func() { finish(outcome) }() |
| 5924 | var payload viewPayload |
| 5925 | @@ -213,6 +214,10 @@ func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (e |
| 5926 | outcome = "paused" |
| 5927 | return nil |
| 5928 | } |
| 5929 | + if payload.Generation != "" && payload.Generation != view.generation { |
| 5930 | + outcome = "invalid" |
| 5931 | + return nil |
| 5932 | + } |
| 5933 | current, err := t.sourceCurrent(ctx, payload.Event) |
| 5934 | if err != nil { |
| 5935 | return err |
| 5936 | @@ -286,7 +291,7 @@ func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (e |
| 5937 | return err |
| 5938 | } |
| 5939 | // Counts only: no source, block or body leaves the relay in a log. |
| 5940 | - t.app.telemetry.Logger().Info("view transform delivered", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status, "blocks", len(pending), "artifacts", stored, "refused", refused, "errors", len(response.Errors)) |
| 5941 | + t.telemetry.Logger().Info("view transform delivered", "tenant", t.tenantName, "attempt", payload.Attempt, "status", status, "blocks", len(pending), "artifacts", stored, "refused", refused, "errors", len(response.Errors)) |
| 5942 | return t.pruneViewArtifacts(ctx, view.Name, intent.ID) |
| 5943 | } |
| 5944 | reason := postErr.Error() |
| 5945 | @@ -300,7 +305,7 @@ func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (e |
| 5946 | } else if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET last_run_at=?, failures=?, last_status=? WHERE name=?`, now, failures, reason, view.Name); err != nil { |
| 5947 | return err |
| 5948 | } |
| 5949 | - t.app.telemetry.Logger().Info("view transform failed", "tenant", t.meta.Name, "attempt", payload.Attempt, "status", status, "failures", failures, "paused", paused) |
| 5950 | + t.telemetry.Logger().Info("view transform failed", "tenant", t.tenantName, "attempt", payload.Attempt, "status", status, "failures", failures, "paused", paused) |
| 5951 | if paused || payload.Attempt >= viewMaxAttempts { |
| 5952 | return nil |
| 5953 | } |
| 5954 | @@ -309,7 +314,7 @@ func (t *Tenant) handleViewTransform(ctx context.Context, intent work.Intent) (e |
| 5955 | |
| 5956 | // pauseCustomView stops the view and records the count and reason, then |
| 5957 | // drops the index so no new intents are queued for it. |
| 5958 | -func (t *Tenant) pauseCustomView(ctx context.Context, name string, failures int, status string) error { |
| 5959 | +func (t *customViewService) pauseCustomView(ctx context.Context, name string, failures int, status string) error { |
| 5960 | if _, err := t.store.DB().ExecContext(ctx, `UPDATE custom_views SET enabled=0, failures=?, last_status=? WHERE name=?`, failures, status, name); err != nil { |
| 5961 | return err |
| 5962 | } |
| 5963 | @@ -319,7 +324,7 @@ func (t *Tenant) pauseCustomView(ctx context.Context, name string, failures int, |
| 5964 | |
| 5965 | // retryViewTransform queues the next attempt after its backoff as its own |
| 5966 | // intent so the durable queue's ordering and fencing still apply. |
| 5967 | -func (t *Tenant) retryViewTransform(ctx context.Context, intent work.Intent, payload viewPayload) error { |
| 5968 | +func (t *customViewService) retryViewTransform(ctx context.Context, intent work.Intent, payload viewPayload) error { |
| 5969 | delay := viewBackoff[len(viewBackoff)-1] |
| 5970 | if payload.Attempt-1 < len(viewBackoff) { |
| 5971 | delay = viewBackoff[payload.Attempt-1] |
| 5972 | @@ -342,7 +347,7 @@ func (t *Tenant) retryViewTransform(ctx context.Context, intent work.Intent, pay |
| 5973 | // postViewTransform sends the blocks and reads the answer. It returns the |
| 5974 | // HTTP status with a bounded reason on failure; a response that is not |
| 5975 | // JSON counts as a failure. |
| 5976 | -func (t *Tenant) postViewTransform(ctx context.Context, view customView, e event.Event, blocks []views.Block) (int, viewResponse, error) { |
| 5977 | +func (t *customViewService) postViewTransform(ctx context.Context, view customView, e event.Event, blocks []views.Block) (int, viewResponse, error) { |
| 5978 | body, err := json.Marshal(viewRequest{Relay: t.publicURL, View: view.Name, Source: viewSource{ID: viewSourceKey(e), Kind: e.Kind}, Blocks: blocks}) |
| 5979 | if err != nil { |
| 5980 | return 0, viewResponse{}, errors.New("encode request") |
| 5981 | @@ -455,7 +460,7 @@ func checkSVG(body string) error { |
| 5982 | |
| 5983 | // storeArtifact checks one artifact and keeps it as a signed record |
| 5984 | // attached to its source. |
| 5985 | -func (t *Tenant) storeArtifact(ctx context.Context, view customView, block views.Block, artifact viewArtifact, source string, expires int64, now int64) error { |
| 5986 | +func (t *customViewService) storeArtifact(ctx context.Context, view customView, block views.Block, artifact viewArtifact, source string, expires int64, now int64) error { |
| 5987 | content, err := checkArtifact(artifact, view.MaxBytes) |
| 5988 | if err != nil { |
| 5989 | return err |
| 5990 | @@ -480,7 +485,7 @@ func (t *Tenant) storeArtifact(ctx context.Context, view customView, block views |
| 5991 | |
| 5992 | // attachArtifact reports whether an artifact exists for the hash and, when |
| 5993 | // it does, attaches the source to it through a fresh signed record. |
| 5994 | -func (t *Tenant) attachArtifact(ctx context.Context, view customView, hash, source string, block int, expires int64) (bool, error) { |
| 5995 | +func (t *customViewService) attachArtifact(ctx context.Context, view customView, hash, source string, block int, expires int64) (bool, error) { |
| 5996 | var exists int |
| 5997 | err := t.store.DB().QueryRowContext(ctx, `SELECT 1 FROM custom_view_artifacts WHERE view=? AND hash=?`, view.Name, hash).Scan(&exists) |
| 5998 | if errors.Is(err, sql.ErrNoRows) { |
| 5999 | @@ -507,7 +512,7 @@ func (t *Tenant) attachArtifact(ctx context.Context, view customView, hash, sour |
| 6000 | |
| 6001 | // signArtifact signs the artifact with its current sources as a kind 30078 |
| 6002 | // record and stores the record when the view is public. |
| 6003 | -func (t *Tenant) signArtifact(ctx context.Context, view customView, hash string, now int64) error { |
| 6004 | +func (t *customViewService) signArtifact(ctx context.Context, view customView, hash string, now int64) error { |
| 6005 | var typ, engine, content string |
| 6006 | if err := t.store.DB().QueryRowContext(ctx, `SELECT type,engine,content FROM custom_view_artifacts WHERE view=? AND hash=?`, view.Name, hash).Scan(&typ, &engine, &content); err != nil { |
| 6007 | return err |
| 6008 | @@ -566,7 +571,7 @@ func (t *Tenant) signArtifact(ctx context.Context, view customView, hash string, |
| 6009 | |
| 6010 | // detachStaleArtifacts drops the source from artifacts whose block it no |
| 6011 | // longer carries, then removes artifacts left without a source. |
| 6012 | -func (t *Tenant) detachStaleArtifacts(ctx context.Context, name, source string, hashes []string) error { |
| 6013 | +func (t *customViewService) detachStaleArtifacts(ctx context.Context, name, source string, hashes []string) error { |
| 6014 | rows, err := t.store.DB().QueryContext(ctx, `SELECT hash FROM custom_view_sources WHERE view=? AND source=?`, name, source) |
| 6015 | if err != nil { |
| 6016 | return err |
| 6017 | @@ -600,7 +605,7 @@ func (t *Tenant) detachStaleArtifacts(ctx context.Context, name, source string, |
| 6018 | // pruneViewArtifacts detaches sources that no longer exist and removes the |
| 6019 | // artifacts that lose their last source. A view with transform work still |
| 6020 | // queued is left alone so a replaced source can reattach its blocks first. |
| 6021 | -func (t *Tenant) pruneViewArtifacts(ctx context.Context, name, except string) error { |
| 6022 | +func (t *customViewService) pruneViewArtifacts(ctx context.Context, name, except string) error { |
| 6023 | var pending int |
| 6024 | if err := t.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM work_intents WHERE kind=? AND target=? AND state IN ('pending','running') AND id<>?`, viewTransform, name, except).Scan(&pending); err != nil { |
| 6025 | return err |
| 6026 | @@ -646,7 +651,7 @@ func (t *Tenant) pruneViewArtifacts(ctx context.Context, name, except string) er |
| 6027 | |
| 6028 | // pruneAllViewArtifacts runs the prune for every view; the maintenance |
| 6029 | // sweep calls it after expired events are removed. |
| 6030 | -func (t *Tenant) pruneAllViewArtifacts(ctx context.Context) error { |
| 6031 | +func (t *customViewService) pruneAllViewArtifacts(ctx context.Context) error { |
| 6032 | rows, err := t.customViewRows(ctx) |
| 6033 | if err != nil { |
| 6034 | return err |
| 6035 | @@ -659,7 +664,7 @@ func (t *Tenant) pruneAllViewArtifacts(ctx context.Context) error { |
| 6036 | return nil |
| 6037 | } |
| 6038 | |
| 6039 | -func (t *Tenant) sourcePresent(ctx context.Context, source string) (bool, error) { |
| 6040 | +func (t *customViewService) sourcePresent(ctx context.Context, source string) (bool, error) { |
| 6041 | var one int |
| 6042 | var err error |
| 6043 | if tag := viewSourceTag(source); tag[0] == "a" { |
| 6044 | @@ -679,7 +684,7 @@ func (t *Tenant) sourcePresent(ctx context.Context, source string) (bool, error) |
| 6045 | |
| 6046 | // dropOrphanArtifacts removes artifacts without a source, and their stored |
| 6047 | // records, then re-signs the artifacts whose source list shrank. |
| 6048 | -func (t *Tenant) dropOrphanArtifacts(ctx context.Context, name string) error { |
| 6049 | +func (t *customViewService) dropOrphanArtifacts(ctx context.Context, name string) error { |
| 6050 | ids, err := t.customViewArtifactIDs(ctx, `SELECT event_id FROM custom_view_artifacts WHERE view=? AND NOT EXISTS(SELECT 1 FROM custom_view_sources WHERE custom_view_sources.view=custom_view_artifacts.view AND custom_view_sources.hash=custom_view_artifacts.hash)`, name) |
| 6051 | if err != nil { |
| 6052 | return err |
| 6053 | @@ -691,7 +696,7 @@ func (t *Tenant) dropOrphanArtifacts(ctx context.Context, name string) error { |
| 6054 | return err |
| 6055 | } |
| 6056 | if len(ids) > 0 { |
| 6057 | - t.app.telemetry.Logger().Info("view artifacts removed", "tenant", t.meta.Name, "removed", len(ids)) |
| 6058 | + t.telemetry.Logger().Info("view artifacts removed", "tenant", t.tenantName, "removed", len(ids)) |
| 6059 | } |
| 6060 | // Artifacts that kept a source but lost one carry a tag for it; a fresh |
| 6061 | // record drops it. |
| 6062 | @@ -744,7 +749,7 @@ func (t *Tenant) dropOrphanArtifacts(ctx context.Context, name string) error { |
| 6063 | return nil |
| 6064 | } |
| 6065 | |
| 6066 | -func (t *Tenant) artifactSources(ctx context.Context, name, hash string) ([]string, error) { |
| 6067 | +func (t *customViewService) artifactSources(ctx context.Context, name, hash string) ([]string, error) { |
| 6068 | rows, err := t.store.DB().QueryContext(ctx, `SELECT source FROM custom_view_sources WHERE view=? AND hash=?`, name, hash) |
| 6069 | if err != nil { |
| 6070 | return nil, err |
| 6071 | @@ -761,7 +766,7 @@ func (t *Tenant) artifactSources(ctx context.Context, name, hash string) ([]stri |
| 6072 | return out, rows.Err() |
| 6073 | } |
| 6074 | |
| 6075 | -func (t *Tenant) customViewArtifactIDs(ctx context.Context, query string, args ...any) ([]string, error) { |
| 6076 | +func (t *customViewService) customViewArtifactIDs(ctx context.Context, query string, args ...any) ([]string, error) { |
| 6077 | rows, err := t.store.DB().QueryContext(ctx, query, args...) |
| 6078 | if err != nil { |
| 6079 | return nil, err |
| 6080 | @@ -781,7 +786,7 @@ func (t *Tenant) customViewArtifactIDs(ctx context.Context, query string, args . |
| 6081 | } |
| 6082 | |
| 6083 | // deleteStoredArtifacts removes the public records of artifacts by id. |
| 6084 | -func (t *Tenant) deleteStoredArtifacts(ctx context.Context, ids []string) error { |
| 6085 | +func (t *customViewService) deleteStoredArtifacts(ctx context.Context, ids []string) error { |
| 6086 | for _, id := range ids { |
| 6087 | if _, err := t.store.DeleteEvent(ctx, id); err != nil { |
| 6088 | return err |
| 6089 | @@ -792,7 +797,7 @@ func (t *Tenant) deleteStoredArtifacts(ctx context.Context, ids []string) error |
| 6090 | |
| 6091 | // cascadeViewDeletion runs after a kind 5 deletion is stored: artifacts |
| 6092 | // attached to the deleted events or addresses lose that source at once. |
| 6093 | -func (t *Tenant) cascadeViewDeletion(ctx context.Context, e event.Event) { |
| 6094 | +func (t *customViewService) cascadeViewDeletion(ctx context.Context, e event.Event) { |
| 6095 | if e.Kind != event.KIND_DELETION { |
| 6096 | return |
| 6097 | } |
| 6098 | @@ -802,7 +807,7 @@ func (t *Tenant) cascadeViewDeletion(ctx context.Context, e event.Event) { |
| 6099 | } |
| 6100 | for _, view := range rows { |
| 6101 | if err := t.pruneViewArtifacts(ctx, view.Name, ""); err != nil { |
| 6102 | - t.app.telemetry.Logger().Error("view artifacts not pruned", "tenant", t.meta.Name, "error", err) |
| 6103 | + t.telemetry.Logger().Error("view artifacts not pruned", "tenant", t.tenantName, "error", err) |
| 6104 | } |
| 6105 | } |
| 6106 | } |
| 6107 | @@ -815,7 +820,7 @@ type artifactRecord struct { |
| 6108 | EventID string |
| 6109 | } |
| 6110 | |
| 6111 | -func (t *Tenant) customViewArtifact(ctx context.Context, name, hash string) (artifactRecord, error) { |
| 6112 | +func (t *customViewService) customViewArtifact(ctx context.Context, name, hash string) (artifactRecord, error) { |
| 6113 | var record artifactRecord |
| 6114 | err := t.store.DB().QueryRowContext(ctx, `SELECT type,audience,content,event_id FROM custom_view_artifacts WHERE view=? AND hash=?`, name, hash).Scan(&record.Type, &record.Audience, &record.Content, &record.EventID) |
| 6115 | if errors.Is(err, sql.ErrNoRows) { |
| 6116 | diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go |
| 6117 | index 84c713e..260fcac 100644 |
| 6118 | --- a/internal/daemon/daemon.go |
| 6119 | +++ b/internal/daemon/daemon.go |
| 6120 | @@ -186,20 +186,21 @@ func initializeTenant(ctx context.Context, s *storage.Store, p policy.Policy, na |
| 6121 | if err := storage.PutSetting(ctx, tx, "policy", p); err != nil { |
| 6122 | return err |
| 6123 | } |
| 6124 | + communityConfig := community.ConfigSnapshot{} |
| 6125 | for _, rule := range []struct { |
| 6126 | kinds []int |
| 6127 | name string |
| 6128 | }{{p.AllowedKinds, "allow"}, {p.BlockedKinds, "block"}} { |
| 6129 | for _, kind := range rule.kinds { |
| 6130 | - if _, err := tx.ExecContext(ctx, "INSERT INTO community_kind_rules(kind,rule) VALUES(?,?)", kind, rule.name); err != nil { |
| 6131 | - return err |
| 6132 | - } |
| 6133 | + communityConfig.KindRules = append(communityConfig.KindRules, community.ConfigKindRule{Kind: kind, Rule: rule.name}) |
| 6134 | } |
| 6135 | } |
| 6136 | for _, r := range p.Retention { |
| 6137 | - if _, err := tx.ExecContext(ctx, "INSERT INTO community_retention(kind,days) VALUES(?,?)", r.Kind, r.Days); err != nil { |
| 6138 | - return err |
| 6139 | - } |
| 6140 | + kind := r.Kind |
| 6141 | + communityConfig.Retention = append(communityConfig.Retention, community.ConfigRetention{Kind: &kind, Days: r.Days}) |
| 6142 | + } |
| 6143 | + if err := community.ApplyConfigTx(ctx, tx, communityConfig, false, false, false, false, true, true); err != nil { |
| 6144 | + return err |
| 6145 | } |
| 6146 | if _, err := tx.ExecContext(ctx, "CREATE TABLE IF NOT EXISTS config_connections(position INTEGER PRIMARY KEY,value TEXT NOT NULL)"); err != nil { |
| 6147 | return err |
| 6148 | diff --git a/internal/daemon/event_commit.go b/internal/daemon/event_commit.go |
| 6149 | new file mode 100644 |
| 6150 | index 0000000..b7fd3d6 |
| 6151 | --- /dev/null |
| 6152 | +++ b/internal/daemon/event_commit.go |
| 6153 | @@ -0,0 +1,122 @@ |
| 6154 | +package daemon |
| 6155 | + |
| 6156 | +import ( |
| 6157 | + "context" |
| 6158 | + "database/sql" |
| 6159 | + "encoding/json" |
| 6160 | + |
| 6161 | + "github.com/FelineStateMachine/tinyrelay/internal/community" |
| 6162 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 6163 | + "github.com/FelineStateMachine/tinyrelay/internal/gitrelay" |
| 6164 | + "github.com/FelineStateMachine/tinyrelay/internal/policy" |
| 6165 | + "github.com/FelineStateMachine/tinyrelay/internal/replication" |
| 6166 | + "github.com/FelineStateMachine/tinyrelay/internal/sites" |
| 6167 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 6168 | +) |
| 6169 | + |
| 6170 | +// eventCommitter prepares the projections common to client publication and |
| 6171 | +// replication. Admission and client-only protocol actions remain with their |
| 6172 | +// entry points; every projection here participates in the event transaction. |
| 6173 | +type eventCommitter struct { |
| 6174 | + policy policy.Policy |
| 6175 | + community *community.Service |
| 6176 | + sites *sites.Service |
| 6177 | + git *gitrelay.GitRelay |
| 6178 | +} |
| 6179 | + |
| 6180 | +type eventCommitPlan struct { |
| 6181 | + options storage.SaveOptions |
| 6182 | + repository gitrelay.Repository |
| 6183 | + gitMetadata bool |
| 6184 | + followups *storage.Intent |
| 6185 | +} |
| 6186 | + |
| 6187 | +func (c eventCommitter) prepare(ctx context.Context, e event.Event, origin replication.Origin, now int64) (eventCommitPlan, error) { |
| 6188 | + plan := eventCommitPlan{options: storage.SaveOptions{Now: now}} |
| 6189 | + if c.sites != nil { |
| 6190 | + plan.options = c.sites.SaveOptions(e, now) |
| 6191 | + } |
| 6192 | + plan.options.SearchMode = c.policy.Features.Search |
| 6193 | + if e.Kind == 30023 { |
| 6194 | + plan.options.Intents = append(plan.options.Intents, storage.Intent{Kind: "view-publish", EventID: e.ID, Target: "articles", Payload: "{}"}) |
| 6195 | + } |
| 6196 | + if e.Kind == event.KIND_AGENT_GRANT || e.Kind == event.KIND_DELETION { |
| 6197 | + plan.options.AddBeforeCommit(func(ctx context.Context, tx *sql.Tx) error { |
| 6198 | + return c.community.ApplyAgentEventTx(ctx, tx, e, now) |
| 6199 | + }) |
| 6200 | + } |
| 6201 | + if c.policy.Features.Grasp && (e.Kind == event.KIND_REPO || e.Kind == event.KIND_REPO_STATE) { |
| 6202 | + if err := c.prepareRepository(ctx, e, origin, &plan); err != nil { |
| 6203 | + return eventCommitPlan{}, err |
| 6204 | + } |
| 6205 | + } |
| 6206 | + return plan, nil |
| 6207 | +} |
| 6208 | + |
| 6209 | +func (c eventCommitter) prepareRepository(ctx context.Context, e event.Event, origin replication.Origin, plan *eventCommitPlan) error { |
| 6210 | + validate := c.git.ValidateImported |
| 6211 | + if origin == replication.OriginClient { |
| 6212 | + validate = c.git.Validate |
| 6213 | + } |
| 6214 | + repo, err := validate(ctx, e) |
| 6215 | + if err != nil { |
| 6216 | + return err |
| 6217 | + } |
| 6218 | + raw, err := json.Marshal(repo) |
| 6219 | + if err != nil { |
| 6220 | + return err |
| 6221 | + } |
| 6222 | + plan.repository, plan.gitMetadata = repo, true |
| 6223 | + plan.options.Intents = append(plan.options.Intents, storage.Intent{Kind: "git-metadata", EventID: e.ID, Target: repo.Owner + ":" + repo.Identifier, Payload: string(raw)}) |
| 6224 | + if e.Kind == event.KIND_REPO_STATE { |
| 6225 | + plan.options.AddBeforeCommit(func(ctx context.Context, tx *sql.Tx) error { |
| 6226 | + _, err := tx.ExecContext(ctx, "INSERT OR REPLACE INTO pending_events(id,reason) VALUES(?,'git objects')", e.ID) |
| 6227 | + return err |
| 6228 | + }) |
| 6229 | + } |
| 6230 | + return nil |
| 6231 | +} |
| 6232 | + |
| 6233 | +func (t *Tenant) validateStoredEvent(ctx context.Context, e event.Event) error { |
| 6234 | + if err := t.validatePrivateRepositoryPlacement(ctx, e); err != nil { |
| 6235 | + return err |
| 6236 | + } |
| 6237 | + if err := validateReviewAnchor(e); err != nil { |
| 6238 | + return err |
| 6239 | + } |
| 6240 | + if t.sites != nil { |
| 6241 | + return sites.ValidateManifest(e) |
| 6242 | + } |
| 6243 | + return nil |
| 6244 | +} |
| 6245 | + |
| 6246 | +func (t *Tenant) prepareStoredEvent(ctx context.Context, e event.Event, origin replication.Origin, now int64) (eventCommitPlan, error) { |
| 6247 | + committer := eventCommitter{policy: t.Policy(), community: t.community, sites: t.sites, git: t.git} |
| 6248 | + plan, err := committer.prepare(ctx, e, origin, now) |
| 6249 | + if err != nil { |
| 6250 | + return plan, err |
| 6251 | + } |
| 6252 | + var push *replicationPushFollowupPayload |
| 6253 | + if origin == replication.OriginClient && t.Policy().Features.Push { |
| 6254 | + payload, _ := t.prepareReplicationPushFollowup(ctx, e, now) |
| 6255 | + if len(payload.RegistrationIDs) != 0 || payload.Discover { |
| 6256 | + push = &payload |
| 6257 | + } |
| 6258 | + } |
| 6259 | + plan.followups = t.followups.prepareWithPush(ctx, e, now, false, push) |
| 6260 | + if plan.followups != nil { |
| 6261 | + plan.options.Intents = append(plan.options.Intents, *plan.followups) |
| 6262 | + } |
| 6263 | + return plan, nil |
| 6264 | +} |
| 6265 | + |
| 6266 | +func (t *Tenant) afterStoredEvent(ctx context.Context, e event.Event, followups *storage.Intent) { |
| 6267 | + t.notifyDevices(ctx, e) |
| 6268 | + t.followups.try(ctx, followups) |
| 6269 | + if event.IsEphemeral(e.Kind) { |
| 6270 | + t.notifyCallbacks(ctx, e) |
| 6271 | + } |
| 6272 | + t.customViews.cascadeViewDeletion(ctx, e) |
| 6273 | + t.notifyWikiMerge(ctx, e) |
| 6274 | + t.notifyWikiProposal(ctx, e) |
| 6275 | +} |
| 6276 | diff --git a/internal/daemon/event_followups.go b/internal/daemon/event_followups.go |
| 6277 | new file mode 100644 |
| 6278 | index 0000000..65804e4 |
| 6279 | --- /dev/null |
| 6280 | +++ b/internal/daemon/event_followups.go |
| 6281 | @@ -0,0 +1,212 @@ |
| 6282 | +package daemon |
| 6283 | + |
| 6284 | +import ( |
| 6285 | + "context" |
| 6286 | + "database/sql" |
| 6287 | + "encoding/json" |
| 6288 | + "errors" |
| 6289 | + "fmt" |
| 6290 | + "time" |
| 6291 | + |
| 6292 | + "github.com/FelineStateMachine/tinyrelay/internal/event" |
| 6293 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 6294 | + "github.com/FelineStateMachine/tinyrelay/internal/telemetry" |
| 6295 | + "github.com/FelineStateMachine/tinyrelay/internal/work" |
| 6296 | +) |
| 6297 | + |
| 6298 | +const ( |
| 6299 | + eventFollowupKind = "event-followups" |
| 6300 | + followupMaxAge = 15 * time.Minute |
| 6301 | + followupMaxAttempts = 3 |
| 6302 | +) |
| 6303 | + |
| 6304 | +// eventFollowups plans optional work without depending on a remote service. |
| 6305 | +// Its outbox record commits with the source event; delivery and transforms |
| 6306 | +// retain their own bounded retry policies and deterministic intent IDs. |
| 6307 | +type eventFollowups struct { |
| 6308 | + store *storage.Store |
| 6309 | + callbacks *callbackService |
| 6310 | + views *customViewService |
| 6311 | + telemetry *telemetry.Telemetry |
| 6312 | + push func(context.Context, replicationPushFollowupPayload) ([]storage.Intent, error) |
| 6313 | +} |
| 6314 | + |
| 6315 | +type eventFollowupPayload struct { |
| 6316 | + Event event.Event `json:"event"` |
| 6317 | + AcceptedAt int64 `json:"accepted_at"` |
| 6318 | + Released bool `json:"released,omitempty"` |
| 6319 | + Callbacks []string `json:"callbacks,omitempty"` |
| 6320 | + Views []customViewTarget `json:"views,omitempty"` |
| 6321 | + DiscoverCallbacks bool `json:"discover_callbacks,omitempty"` |
| 6322 | + DiscoverViews bool `json:"discover_views,omitempty"` |
| 6323 | + Push *replicationPushFollowupPayload `json:"push,omitempty"` |
| 6324 | + RecoverPush bool `json:"recover_push,omitempty"` |
| 6325 | +} |
| 6326 | + |
| 6327 | +func (f *eventFollowups) prepare(ctx context.Context, e event.Event, now int64, released bool) *storage.Intent { |
| 6328 | + return f.prepareWithPush(ctx, e, now, released, nil) |
| 6329 | +} |
| 6330 | + |
| 6331 | +func (f *eventFollowups) prepareWithPush(ctx context.Context, e event.Event, now int64, released bool, push *replicationPushFollowupPayload) *storage.Intent { |
| 6332 | + if event.IsEphemeral(e.Kind) { |
| 6333 | + return nil |
| 6334 | + } |
| 6335 | + payload := eventFollowupPayload{Event: e, AcceptedAt: now, Released: released, Push: push} |
| 6336 | + if released && push == nil { |
| 6337 | + var err error |
| 6338 | + payload.Push, err = f.releasedPush(ctx, e.ID) |
| 6339 | + payload.RecoverPush = err != nil |
| 6340 | + } |
| 6341 | + var callbackErr, viewErr error |
| 6342 | + if e.Kind != event.KIND_REPO_STATE || released { |
| 6343 | + payload.Callbacks, callbackErr = f.callbacks.CandidateIDs(ctx, e) |
| 6344 | + payload.Views, viewErr = f.views.CandidateTargets(ctx, e) |
| 6345 | + } |
| 6346 | + payload.DiscoverCallbacks = callbackErr != nil |
| 6347 | + payload.DiscoverViews = viewErr != nil |
| 6348 | + if len(payload.Callbacks) == 0 && len(payload.Views) == 0 && callbackErr == nil && viewErr == nil && payload.Push == nil && !payload.RecoverPush { |
| 6349 | + return nil |
| 6350 | + } |
| 6351 | + // Event and target fields contain no custom JSON marshalers. |
| 6352 | + raw, err := json.Marshal(payload) |
| 6353 | + if err != nil { |
| 6354 | + f.telemetry.Logger().Error("encode optional event follow-ups", "error", err) |
| 6355 | + return nil |
| 6356 | + } |
| 6357 | + target := "stored" |
| 6358 | + if released { |
| 6359 | + target = "released" |
| 6360 | + } |
| 6361 | + return &storage.Intent{Kind: eventFollowupKind, EventID: e.ID, Target: target, Payload: string(raw)} |
| 6362 | +} |
| 6363 | + |
| 6364 | +// try runs only local planning after commit. A failure leaves the outbox |
| 6365 | +// pending for the worker; success marks planning complete with its derived |
| 6366 | +// intents, so a later retry never replays already planned remote work. |
| 6367 | +func (f *eventFollowups) try(ctx context.Context, intent *storage.Intent) { |
| 6368 | + if intent == nil { |
| 6369 | + return |
| 6370 | + } |
| 6371 | + job := work.Intent{ID: storage.IntentID(intent.Kind, intent.EventID, intent.Target), Kind: intent.Kind, EventID: intent.EventID, Target: intent.Target, Payload: intent.Payload} |
| 6372 | + if err := f.plan(ctx, job, true); err != nil { |
| 6373 | + f.telemetry.Logger().Debug("optional event follow-ups deferred", "error", err) |
| 6374 | + } |
| 6375 | +} |
| 6376 | + |
| 6377 | +func (f *eventFollowups) handle(ctx context.Context, job work.Intent) error { |
| 6378 | + err := f.plan(ctx, job, false) |
| 6379 | + if err != nil && job.Attempts >= followupMaxAttempts { |
| 6380 | + return work.Stop(fmt.Errorf("optional follow-up planning exhausted: %w", err)) |
| 6381 | + } |
| 6382 | + return err |
| 6383 | +} |
| 6384 | + |
| 6385 | +func (f *eventFollowups) plan(ctx context.Context, job work.Intent, pending bool) error { |
| 6386 | + var payload eventFollowupPayload |
| 6387 | + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil || payload.Event.ID != job.EventID { |
| 6388 | + return work.Stop(errors.New("invalid optional event follow-up")) |
| 6389 | + } |
| 6390 | + if time.Now().Unix()-payload.AcceptedAt > int64(followupMaxAge/time.Second) { |
| 6391 | + return work.Stop(errors.New("optional event follow-up expired")) |
| 6392 | + } |
| 6393 | + active, err := f.sourceActive(ctx, payload.Event) |
| 6394 | + if err != nil { |
| 6395 | + return err |
| 6396 | + } |
| 6397 | + var intents []storage.Intent |
| 6398 | + var planningErr error |
| 6399 | + if active { |
| 6400 | + intents, planningErr = f.intents(ctx, payload) |
| 6401 | + } |
| 6402 | + err = f.store.WithTx(ctx, func(tx *sql.Tx) error { |
| 6403 | + if err := storage.AddIntents(ctx, tx, intents, time.Now().Unix()); err != nil { |
| 6404 | + return err |
| 6405 | + } |
| 6406 | + if pending && planningErr == nil { |
| 6407 | + _, err := work.CompletePendingTx(ctx, tx, job.ID, time.Now()) |
| 6408 | + return err |
| 6409 | + } |
| 6410 | + return nil |
| 6411 | + }) |
| 6412 | + return errors.Join(planningErr, err) |
| 6413 | +} |
| 6414 | + |
| 6415 | +func (f *eventFollowups) sourceActive(ctx context.Context, e event.Event) (bool, error) { |
| 6416 | + result, err := f.store.Query(ctx, event.Filter{IDs: []string{e.ID}}, storage.QueryOptions{Now: time.Now().Unix(), Access: storage.Access{All: true}, Limit: 1}) |
| 6417 | + return len(result.Events) != 0, err |
| 6418 | +} |
| 6419 | + |
| 6420 | +func (f *eventFollowups) intents(ctx context.Context, payload eventFollowupPayload) ([]storage.Intent, error) { |
| 6421 | + callbacks, callbackErr := f.callbackIntents(ctx, payload) |
| 6422 | + views, viewErr := f.viewIntents(ctx, payload) |
| 6423 | + intents := append(callbacks, views...) |
| 6424 | + var pushErr error |
| 6425 | + if payload.RecoverPush { |
| 6426 | + payload.Push, pushErr = f.releasedPush(ctx, payload.Event.ID) |
| 6427 | + } |
| 6428 | + if payload.Push != nil { |
| 6429 | + push, err := f.push(ctx, *payload.Push) |
| 6430 | + pushErr = err |
| 6431 | + intents = append(intents, push...) |
| 6432 | + } |
| 6433 | + return intents, errors.Join(callbackErr, viewErr, pushErr) |
| 6434 | +} |
| 6435 | + |
| 6436 | +// releasedPush preserves the registrations captured at client acceptance. |
| 6437 | +// Git promotion can happen later, after its initial planner has found the |
| 6438 | +// source still pending. Imports have no client push snapshot to recover. |
| 6439 | +func (f *eventFollowups) releasedPush(ctx context.Context, eventID string) (*replicationPushFollowupPayload, error) { |
| 6440 | + var raw string |
| 6441 | + err := f.store.DB().QueryRowContext(ctx, `SELECT payload FROM work_intents WHERE id=?`, storage.IntentID(eventFollowupKind, eventID, "stored")).Scan(&raw) |
| 6442 | + if errors.Is(err, sql.ErrNoRows) { |
| 6443 | + return nil, nil |
| 6444 | + } |
| 6445 | + if err != nil { |
| 6446 | + return nil, err |
| 6447 | + } |
| 6448 | + var payload eventFollowupPayload |
| 6449 | + if err := json.Unmarshal([]byte(raw), &payload); err != nil { |
| 6450 | + return nil, err |
| 6451 | + } |
| 6452 | + return payload.Push, nil |
| 6453 | +} |
| 6454 | + |
| 6455 | +// Discovery is retried only when the original local lookup failed. The |
| 6456 | +// acceptance cutoff prevents a new registration from receiving old events. |
| 6457 | +func (f *eventFollowups) callbackIntents(ctx context.Context, payload eventFollowupPayload) ([]storage.Intent, error) { |
| 6458 | + if payload.DiscoverCallbacks { |
| 6459 | + ids, err := f.callbacks.CandidateIDs(ctx, payload.Event) |
| 6460 | + if err != nil { |
| 6461 | + return nil, err |
| 6462 | + } |
| 6463 | + for _, id := range ids { |
| 6464 | + candidate, err := f.callbacks.callback(ctx, id) |
| 6465 | + if err != nil { |
| 6466 | + return nil, err |
| 6467 | + } |
| 6468 | + if candidate.CreatedAt < payload.AcceptedAt { |
| 6469 | + payload.Callbacks = append(payload.Callbacks, id) |
| 6470 | + } |
| 6471 | + } |
| 6472 | + } |
| 6473 | + return f.callbacks.PrepareFor(ctx, payload.Event, payload.Callbacks) |
| 6474 | +} |
| 6475 | + |
| 6476 | +func (f *eventFollowups) viewIntents(ctx context.Context, payload eventFollowupPayload) ([]storage.Intent, error) { |
| 6477 | + if payload.DiscoverViews { |
| 6478 | + targets, err := f.views.CandidateTargets(ctx, payload.Event) |
| 6479 | + if err != nil { |
| 6480 | + return nil, err |
| 6481 | + } |
| 6482 | + for _, target := range targets { |
| 6483 | + candidate, err := f.views.customViewByName(ctx, target.Name) |
| 6484 | + if err != nil { |
| 6485 | + return nil, err |
| 6486 | + } |
| 6487 | + if candidate.CreatedAt < payload.AcceptedAt { |
| 6488 | + payload.Views = append(payload.Views, target) |
| 6489 | + } |
| 6490 | + } |
| 6491 | + } |
| 6492 | + return f.views.PrepareTargets(ctx, payload.Event, payload.Views, payload.Released) |
| 6493 | +} |
| 6494 | diff --git a/internal/daemon/event_pipeline_test.go b/internal/daemon/event_pipeline_test.go |
| 6495 | new file mode 100644 |
| 6496 | index 0000000..80f7534 |
| 6497 | --- /dev/null |
| 6498 | +++ b/internal/daemon/event_pipeline_test.go |
| 6499 | @@ -0,0 +1,123 @@ |
| 6500 | +package daemon |
| 6501 | + |
| 6502 | +import ( |
| 6503 | + "context" |
| 6504 | + "encoding/json" |
| 6505 | + "testing" |
| 6506 | + "time" |
| 6507 | + |
| 6508 | + "github.com/FelineStateMachine/tinyrelay/internal/relay" |
| 6509 | + "github.com/FelineStateMachine/tinyrelay/internal/replication" |
| 6510 | + "github.com/FelineStateMachine/tinyrelay/internal/storage" |
| 6511 | + "github.com/FelineStateMachine/tinyrelay/internal/work" |
| 6512 | +) |
| 6513 | + |
| 6514 | +func TestOptionalFollowupFailurePreservesEventAndRecovery(t *testing.T) { |
| 6515 | + for _, source := range []string{"client", "import"} { |
| 6516 | + t.Run(source, func(t *testing.T) { |
| 6517 | + tenant, remote, _ := viewTenant(t, []int{30023}, nil) |
| 6518 | + ctx := context.Background() |
| 6519 | + e := signedEvent(t, testOwnerSecret, 30023, time.Now().Unix(), [][]string{{"d", "recover-view"}}, "```mermaid\ngraph LR; A-->B\n```") |
| 6520 | + if _, err := tenant.store.DB().ExecContext(ctx, `CREATE TRIGGER reject_view_work BEFORE INSERT ON work_intents WHEN NEW.kind='view-transform' BEGIN SELECT RAISE(ABORT,'view work unavailable'); END`); err != nil { |
| 6521 | + t.Fatal(err) |
| 6522 | + } |
| 6523 | + if source == "import" { |
| 6524 | + if err := tenant.commitImported(ctx, e, replication.OriginImport); err != nil { |
| 6525 | + t.Fatal(err) |
| 6526 | + } |
| 6527 | + } else if _, err := tenant.Publish(ctx, e, relay.Session{PubKeys: []string{e.PubKey}}); err != nil { |
| 6528 | + t.Fatal(err) |
| 6529 | + } |
| 6530 | + var stored int |
| 6531 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT count(*) FROM events WHERE id=?`, e.ID).Scan(&stored); err != nil { |
| 6532 | + t.Fatal(err) |
| 6533 | + } |
| 6534 | + if stored != 1 { |
| 6535 | + t.Fatal("optional planning failure rejected valid event") |
| 6536 | + } |
| 6537 | + job := followupJob(t, tenant, e.ID) |
| 6538 | + if remote.count() != 0 { |
| 6539 | + t.Fatal("planning contacted the remote transform") |
| 6540 | + } |
| 6541 | + if _, err := tenant.store.DB().ExecContext(ctx, `DROP TRIGGER reject_view_work`); err != nil { |
| 6542 | + t.Fatal(err) |
| 6543 | + } |
| 6544 | + intent := storage.Intent{Kind: job.Kind, EventID: job.EventID, Target: job.Target, Payload: job.Payload} |
| 6545 | + tenant.followups.try(ctx, &intent) |
| 6546 | + if intents := pendingViewIntents(t, tenant, "diagrams", e.ID); len(intents) != 1 { |
| 6547 | + t.Fatalf("recovery queued %d view intents, want one", len(intents)) |
| 6548 | + } |
| 6549 | + var state string |
| 6550 | + if err := tenant.store.DB().QueryRowContext(ctx, `SELECT state FROM work_intents WHERE id=?`, job.ID).Scan(&state); err != nil { |
| 6551 | + t.Fatal(err) |
| 6552 | + } |
| 6553 | + if state != "completed" { |
| 6554 | + t.Fatalf("recovered planning state=%s", state) |
| 6555 | + } |
| 6556 | + if _, err := tenant.store.DB().ExecContext(ctx, `UPDATE work_intents SET state='completed' WHERE kind=? AND event_id=?`, viewTransform, e.ID); err != nil { |
| 6557 | + t.Fatal(err) |
| 6558 | + } |
| 6559 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 6560 | + t.Fatal(err) |
| 6561 | + } |
| 6562 | + if intents := pendingViewIntents(t, tenant, "diagrams", e.ID); len(intents) != 0 { |
| 6563 | + t.Fatal("recovery replayed already completed transform work") |
| 6564 | + } |
| 6565 | + }) |
| 6566 | + } |
| 6567 | +} |
| 6568 | + |
| 6569 | +func TestFollowupPlanningSkipsRemovedSourcesAndNewTargets(t *testing.T) { |
| 6570 | + tenant, _, _ := viewTenant(t, []int{30023}, nil) |
| 6571 | + ctx := context.Background() |
| 6572 | + e := signedEvent(t, testOwnerSecret, 30023, time.Now().Unix(), [][]string{{"d", "captured-targets"}}, "```mermaid\ngraph LR; A-->B\n```") |
| 6573 | + intent := tenant.followups.prepare(ctx, e, time.Now().Unix(), false) |
| 6574 | + if intent == nil { |
| 6575 | + t.Fatal("matching view did not create planning intent") |
| 6576 | + } |
| 6577 | + if _, err := tenant.store.Save(ctx, e, storage.SaveOptions{Now: time.Now().Unix(), Intents: []storage.Intent{*intent}}); err != nil { |
| 6578 | + t.Fatal(err) |
| 6579 | + } |
| 6580 | + addView(t, tenant, map[string]any{"name": "later", "kinds": []int{30023}, "transform": "https://later.example/render", "languages": []string{"mermaid"}}) |
| 6581 | + job := followupJob(t, tenant, e.ID) |
| 6582 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 6583 | + t.Fatal(err) |
| 6584 | + } |
| 6585 | + if intents := pendingViewIntents(t, tenant, "later", e.ID); len(intents) != 0 { |
| 6586 | + t.Fatal("new view received an older captured event") |
| 6587 | + } |
| 6588 | + if _, err := tenant.store.DB().ExecContext(ctx, `DELETE FROM work_intents WHERE kind=? AND event_id=?`, viewTransform, e.ID); err != nil { |
| 6589 | + t.Fatal(err) |
| 6590 | + } |
| 6591 | + if _, err := tenant.store.DB().ExecContext(ctx, `DELETE FROM events WHERE id=?`, e.ID); err != nil { |
| 6592 | + t.Fatal(err) |
| 6593 | + } |
| 6594 | + if err := tenant.followups.handle(ctx, job); err != nil { |
| 6595 | + t.Fatal(err) |
| 6596 | + } |
| 6597 | + if intents := pendingViewIntents(t, tenant, "diagrams", e.ID); len(intents) != 0 { |
| 6598 | + t.Fatal("removed source was replanned") |
| 6599 | + } |
| 6600 | + var payload eventFollowupPayload |
| 6601 | + if err := json.Unmarshal([]byte(job.Payload), &payload); err != nil { |
| 6602 | + t.Fatal(err) |
| 6603 | + } |
| 6604 | + payload.AcceptedAt = time.Now().Add(-followupMaxAge - time.Minute).Unix() |
| 6605 | + raw, err := json.Marshal(payload) |
| 6606 | + if err != nil { |
| 6607 | + t.Fatal(err) |
| 6608 | + } |
| 6609 | + job.Payload = string(raw) |
| 6610 | + if err := tenant.followups.handle(ctx, job); err == nil { |
| 6611 | + t.Fatal("expired optional planning was not stopped") |
| 6612 | + } |
| 6613 | +} |
| 6614 | + |
| 6615 | +func followupJob(t *testing.T, tenant *Tenant, eventID string) work.Intent { |
| 6616 | + t.Helper() |
| 6617 | + job := work.Intent{Kind: eventFollowupKind, EventID: eventID, Attempts: 1} |
| 6618 | + if err := tenant.store.DB().QueryRow(`SELECT id,target,payload FROM work_intents WHERE kind=? AND event_id=?`, eventFollowupKind, eventID).Scan(&job.ID, &job.Target, &job.Payload); err != nil { |
| 6619 | + t.Fatal(err) |
| 6620 | + } |
| 6621 | + return job |
| 6622 | +} |
| 6623 | diff --git a/internal/daemon/git_services.go b/internal/daemon/git_services.go |
| 6624 | index 42a21bc..c99d8d0 100644 |
| 6625 | --- a/internal/daemon/git_services.go |
| 6626 | +++ b/internal/daemon/git_services.go |
| 6627 | @@ -265,7 +265,7 @@ func gitQueryContext(ctx context.Context, maximum time.Duration) (context.Contex |
| 6628 | return context.WithTimeout(ctx, maximum) |
| 6629 | } |
| 6630 | |
| 6631 | -func queryGitHead(ctx context.Context, transport *replication.NostrTransport, target string, filter event.Filter) ([]event.Event, error) { |
| 6632 | +func queryGitHead(ctx context.Context, transport replication.PullTransport, target string, filter event.Filter) ([]event.Event, error) { |
| 6633 | limit := gitHeadPageSize |
| 6634 | filter.Limit = &limit |
| 6635 | return transport.Query(ctx, target, filter) |
| 6636 | diff --git a/internal/daemon/http.go b/internal/daemon/http.go |
| 6637 | index 31035f1..02bde41 100644 |
| 6638 | --- a/internal/daemon/http.go |
| 6639 | +++ b/internal/daemon/http.go |
| 6640 | @@ -87,7 +87,7 @@ func (t *Tenant) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| 6641 | return |
| 6642 | } |
| 6643 | if isViewArtifactPath(r.URL.Path) { |
| 6644 | - t.viewArtifactHTTP(w, r) |
| 6645 | + t.customV |