| 1 | diff --git a/docs/app.md b/docs/app.md |
| 2 | index a0e00af..886ad76 100644 |
| 3 | --- a/docs/app.md |
| 4 | +++ b/docs/app.md |
| 5 | @@ -6,6 +6,10 @@ The relay installs as an app from the browser menu on Android, iOS and desktop. |
| 6 | |
| 7 | Share a photo, file or link from another app and choose the relay. Shared files wait in the Upload panel on the Files page until you press **Upload**, so nothing is stored without your signature. A shared link opens the **Import from URL** control with the address filled in. |
| 8 | |
| 9 | +## Large uploads |
| 10 | + |
| 11 | +On browsers that offer it, a file of 8 MiB or more uploads through the browser's background transfer, so the upload continues after the app closes. Every request is signed before the transfer starts, and the browser shows its own progress notice. Open the Files page later to see the result; for an encrypted file the share link appears there. On other browsers the upload runs while the page stays open. |
| 12 | + |
| 13 | ## Nostr links |
| 14 | |
| 15 | The app registers for `web+nostr:` links. A link to an `npub`, `nprofile`, `note`, `nevent` or `naddr` opens the matching page: the author's events, the event page or the article. Plain `nostr:` links open the same way through `/open?target=`. |
| 16 | diff --git a/internal/webui/blossom-upload.js b/internal/webui/blossom-upload.js |
| 17 | index 9c91678..ba1c180 100644 |
| 18 | --- a/internal/webui/blossom-upload.js |
| 19 | +++ b/internal/webui/blossom-upload.js |
| 20 | @@ -31,7 +31,10 @@ |
| 21 | return descriptor; |
| 22 | }; |
| 23 | |
| 24 | - const uploadTask = async (source, options = {}, controller) => { |
| 25 | + // resolveTarget checks the source and options and returns the blob URL |
| 26 | + // the upload addresses. Both the live uploader and the background plan |
| 27 | + // use it so they agree on every detail. |
| 28 | + const resolveTarget = async (source, options) => { |
| 29 | const bytes = await asBytes(source); |
| 30 | const hash = options.hash || hex(await sha256(bytes)); |
| 31 | if (!/^[0-9a-f]{64}$/.test(hash)) throw new Error("expected a lowercase SHA-256 hash"); |
| 32 | @@ -55,7 +58,38 @@ |
| 33 | const target = new RegExp("/" + hash + "(?:\\.[a-z0-9]{1,8})?/?$").test(path) |
| 34 | ? base |
| 35 | : base.replace(/\/$/, "") + "/" + hash + (extension ? "." + extension : ""); |
| 36 | - const url = target; |
| 37 | + return {bytes, hash, type, url: target}; |
| 38 | + }; |
| 39 | + |
| 40 | + // plan lists the signed chunk requests for one upload without sending |
| 41 | + // them, so a browser can hand the transfer to Background Fetch and finish |
| 42 | + // it after the page closes. Each chunk carries its own authorization. |
| 43 | + const plan = async (source, options = {}) => { |
| 44 | + const {bytes, hash, type, url} = await resolveTarget(source, options); |
| 45 | + const chunkSize = |
| 46 | + Number.isSafeInteger(options.chunkSize) && options.chunkSize > 0 ? options.chunkSize : 5 * 1024 * 1024; |
| 47 | + const requests = []; |
| 48 | + for (let offset = 0; offset < bytes.byteLength || (bytes.byteLength === 0 && offset === 0); offset += chunkSize) { |
| 49 | + const chunk = bytes.slice(offset, Math.min(offset + chunkSize, bytes.byteLength)); |
| 50 | + const headers = { |
| 51 | + "upload-type": type, |
| 52 | + "upload-length": String(bytes.byteLength), |
| 53 | + "upload-offset": String(offset), |
| 54 | + "content-type": "application/octet-stream" |
| 55 | + }; |
| 56 | + if (options.authorize) { |
| 57 | + const value = await options.authorize(url, "PATCH", chunk, {hash, type}); |
| 58 | + if (typeof value === "string") headers.authorization = value; |
| 59 | + else if (value && typeof value === "object") Object.assign(headers, value); |
| 60 | + } |
| 61 | + requests.push({url, method: "PATCH", headers, body: chunk}); |
| 62 | + if (bytes.byteLength === 0) break; |
| 63 | + } |
| 64 | + return {hash, url, size: bytes.byteLength, requests}; |
| 65 | + }; |
| 66 | + |
| 67 | + const uploadTask = async (source, options = {}, controller) => { |
| 68 | + const {bytes, hash, type, url} = await resolveTarget(source, options); |
| 69 | const fetcher = options.fetch || globalThis.fetch; |
| 70 | if (typeof fetcher !== "function") throw new Error("fetch is unavailable"); |
| 71 | const chunkSize = |
| 72 | @@ -248,7 +282,7 @@ |
| 73 | return task; |
| 74 | }; |
| 75 | |
| 76 | - const api = Object.freeze({upload}); |
| 77 | + const api = Object.freeze({upload, plan}); |
| 78 | globalThis.tiny = globalThis.tiny || {}; |
| 79 | globalThis.tiny.blossom = {...globalThis.tiny.blossom, upload: api}; |
| 80 | })(); |
| 81 | diff --git a/internal/webui/file-workspace.js b/internal/webui/file-workspace.js |
| 82 | index ffed562..06886fe 100644 |
| 83 | --- a/internal/webui/file-workspace.js |
| 84 | +++ b/internal/webui/file-workspace.js |
| 85 | @@ -145,6 +145,11 @@ |
| 86 | // file gets a fresh random key, a folder or a large file becomes encrypted |
| 87 | // manifests, and the share link carries the key only in its fragment. |
| 88 | const LARGE_FILE_BYTES = 64 * 1024 * 1024; |
| 89 | + // Transfers at least this large go through Background Fetch where the |
| 90 | + // browser offers it, so closing the app does not stop them. |
| 91 | + const BACKGROUND_BYTES = 8 * 1024 * 1024; |
| 92 | + const backgroundAvailable = () => |
| 93 | + "BackgroundFetchManager" in globalThis && Boolean(navigator.serviceWorker?.controller); |
| 94 | const {b64url} = globalThis.tiny.util; |
| 95 | class FileUpload extends HTMLElement { |
| 96 | connectedCallback() { |
| 97 | @@ -176,6 +181,116 @@ |
| 98 | this.querySelector("[data-cancel]").addEventListener("click", () => this.controller?.abort()); |
| 99 | this.querySelector("[data-retry]").addEventListener("click", () => this.run().catch(() => {})); |
| 100 | this.intake().catch(error => this.say(error.message, true)); |
| 101 | + this.resumeBackground().catch(() => {}); |
| 102 | + } |
| 103 | + // storeBackground hands one blob to the browser's Background Fetch with |
| 104 | + // every request signed up front, then waits for the parked descriptor. |
| 105 | + async storeBackground(bytes, {hash, type, name, fragment}) { |
| 106 | + const registration = await navigator.serviceWorker.ready; |
| 107 | + const base = new URL(tiny.localPath("/"), location.href).href; |
| 108 | + const authorize = (target, method, body) => tiny.authorization(target, method, body); |
| 109 | + let requests; |
| 110 | + const probe = await fetch(base, {method: "OPTIONS"}).catch(() => null); |
| 111 | + if (probe && /(?:^|,|\s)PATCH(?:,|\s|$)/i.test(probe.headers.get("allow") || "")) { |
| 112 | + const planned = await upload().plan(bytes, {url: base, hash, type, authorize}); |
| 113 | + requests = planned.requests.map( |
| 114 | + item => new Request(item.url, {method: item.method, headers: item.headers, body: item.body}) |
| 115 | + ); |
| 116 | + } else { |
| 117 | + const target = base.replace(/\/$/, "") + "/upload"; |
| 118 | + requests = [ |
| 119 | + new Request(target, { |
| 120 | + method: "PUT", |
| 121 | + headers: {"content-type": type, authorization: await authorize(target, "PUT", bytes)}, |
| 122 | + body: bytes |
| 123 | + }) |
| 124 | + ]; |
| 125 | + } |
| 126 | + const id = "tiny-upload-" + crypto.randomUUID(); |
| 127 | + localStorage.setItem("tiny.bg." + id, JSON.stringify({hash, fragment: fragment || "", name, at: Date.now()})); |
| 128 | + const task = await registration.backgroundFetch.fetch(id, requests, { |
| 129 | + title: "Uploading " + name, |
| 130 | + icons: [ |
| 131 | + {src: new URL(tiny.localPath("/icon-192.png"), location.href).href, sizes: "192x192", type: "image/png"} |
| 132 | + ], |
| 133 | + uploadTotal: bytes.byteLength |
| 134 | + }); |
| 135 | + this.say("Uploading in the background. You can close this page."); |
| 136 | + return this.awaitBackground(task, hash, id); |
| 137 | + } |
| 138 | + awaitBackground(task, hash, id) { |
| 139 | + return new Promise((resolve, reject) => { |
| 140 | + const settle = async () => { |
| 141 | + if (task.result === "") { |
| 142 | + if (task.uploadTotal) |
| 143 | + this.say("Uploading in the background " + Math.round((task.uploaded / task.uploadTotal) * 100) + "%…"); |
| 144 | + return false; |
| 145 | + } |
| 146 | + task.removeEventListener?.("progress", settle); |
| 147 | + try { |
| 148 | + resolve( |
| 149 | + await this.backgroundDescriptor(id, hash, task.result === "success" ? "" : task.failureReason || "failed") |
| 150 | + ); |
| 151 | + } catch (error) { |
| 152 | + reject(error); |
| 153 | + } |
| 154 | + return true; |
| 155 | + }; |
| 156 | + task.addEventListener("progress", settle); |
| 157 | + settle(); |
| 158 | + }); |
| 159 | + } |
| 160 | + // backgroundDescriptor reads the parked result, verifies the hash and |
| 161 | + // forgets the transfer. |
| 162 | + async backgroundDescriptor(id, hash, failure) { |
| 163 | + const cache = await caches.open("tiny-uploads"); |
| 164 | + const key = new URL(tiny.localPath("/uploads/" + id), location.href).href; |
| 165 | + const response = await cache.match(key); |
| 166 | + await cache.delete(key); |
| 167 | + localStorage.removeItem("tiny.bg." + id); |
| 168 | + if (failure) throw Error("Background upload " + failure + "."); |
| 169 | + const descriptor = response ? await response.json().catch(() => null) : null; |
| 170 | + if (!descriptor || descriptor.sha256 !== hash) throw Error("Relay returned an unexpected hash."); |
| 171 | + return descriptor; |
| 172 | + } |
| 173 | + // resumeBackground picks up transfers started on an earlier visit. |
| 174 | + async resumeBackground() { |
| 175 | + if (!backgroundAvailable()) return; |
| 176 | + const pending = Object.keys(localStorage).filter(key => key.startsWith("tiny.bg.")); |
| 177 | + if (!pending.length) return; |
| 178 | + const registration = await navigator.serviceWorker.ready; |
| 179 | + for (const key of pending) { |
| 180 | + const id = key.slice("tiny.bg.".length); |
| 181 | + let info; |
| 182 | + try { |
| 183 | + info = JSON.parse(localStorage.getItem(key)); |
| 184 | + } catch { |
| 185 | + localStorage.removeItem(key); |
| 186 | + continue; |
| 187 | + } |
| 188 | + const task = await registration.backgroundFetch.get(id).catch(() => null); |
| 189 | + const finish = descriptor => { |
| 190 | + if (!info.fragment) { |
| 191 | + this.say("Background upload of " + info.name + " finished."); |
| 192 | + return tiny.navigate?.(location.href); |
| 193 | + } |
| 194 | + const link = new URL(tiny.localPath("/file"), location.href); |
| 195 | + link.search = "?hash=" + descriptor.sha256; |
| 196 | + link.hash = info.fragment; |
| 197 | + return this.showLink(link.href); |
| 198 | + }; |
| 199 | + try { |
| 200 | + if (task && task.result === "") await this.awaitBackground(task, info.hash, id).then(finish); |
| 201 | + else |
| 202 | + await this.backgroundDescriptor( |
| 203 | + id, |
| 204 | + info.hash, |
| 205 | + task && task.result !== "success" ? task.failureReason || "failed" : "" |
| 206 | + ).then(finish); |
| 207 | + } catch (error) { |
| 208 | + this.say(error.message, true); |
| 209 | + } |
| 210 | + } |
| 211 | } |
| 212 | // intake collects files parked by the service worker for a share from |
| 213 | // another app. They wait here until the person presses Upload. |
| 214 | @@ -190,9 +305,14 @@ |
| 215 | const response = await cache.match("/share/" + id + "/" + index); |
| 216 | if (!response) continue; |
| 217 | const blob = await response.blob(); |
| 218 | - files.push(new File([blob], decodeURIComponent(response.headers.get("x-name") || "shared"), {type: response.headers.get("content-type") || blob.type})); |
| 219 | + files.push( |
| 220 | + new File([blob], decodeURIComponent(response.headers.get("x-name") || "shared"), { |
| 221 | + type: response.headers.get("content-type") || blob.type |
| 222 | + }) |
| 223 | + ); |
| 224 | } |
| 225 | - for (const key of await cache.keys()) if (new URL(key.url).pathname.startsWith("/share/" + id + "/")) await cache.delete(key); |
| 226 | + for (const key of await cache.keys()) |
| 227 | + if (new URL(key.url).pathname.startsWith("/share/" + id + "/")) await cache.delete(key); |
| 228 | history.replaceState?.(null, "", location.pathname + location.search); |
| 229 | if (files.length) { |
| 230 | this.choose(files); |
| 231 | @@ -311,10 +431,13 @@ |
| 232 | for (const [index, file] of files.entries()) { |
| 233 | checkAbort(this.controller.signal); |
| 234 | this.say("Uploading " + file.name + " (" + (index + 1) + " of " + files.length + ")…"); |
| 235 | - await tiny.signedFetch("/upload", "PUT", await file.arrayBuffer(), { |
| 236 | - contentType: file.type || "application/octet-stream", |
| 237 | - signal: this.controller.signal |
| 238 | - }); |
| 239 | + const bytes = new Uint8Array(await file.arrayBuffer()); |
| 240 | + const type = file.type || "application/octet-stream"; |
| 241 | + if (files.length === 1 && bytes.byteLength >= BACKGROUND_BYTES && backgroundAvailable()) { |
| 242 | + await this.storeBackground(bytes, {hash: await tiny.sha256hex(bytes), type, name: file.name}); |
| 243 | + continue; |
| 244 | + } |
| 245 | + await tiny.signedFetch("/upload", "PUT", bytes, {contentType: type, signal: this.controller.signal}); |
| 246 | } |
| 247 | } |
| 248 | // storeSealed encrypts one file with a fresh AES-GCM key. The ciphertext |
| 249 | @@ -345,7 +468,14 @@ |
| 250 | const {ciphertext, fragment, state} = this.pending; |
| 251 | const hash = await tiny.sha256hex(ciphertext); |
| 252 | let descriptor; |
| 253 | - if (upload()?.upload) { |
| 254 | + if (ciphertext.byteLength >= BACKGROUND_BYTES && backgroundAvailable() && upload()?.plan) { |
| 255 | + descriptor = await this.storeBackground(ciphertext, { |
| 256 | + hash, |
| 257 | + type: "application/octet-stream", |
| 258 | + name: file.name, |
| 259 | + fragment |
| 260 | + }); |
| 261 | + } else if (upload()?.upload) { |
| 262 | const task = upload().upload(ciphertext, { |
| 263 | url: new URL(tiny.localPath("/"), location.href).href, |
| 264 | hash, |
| 265 | diff --git a/internal/webui/screenshot-narrow.png b/internal/webui/screenshot-narrow.png |
| 266 | new file mode 100644 |
| 267 | index 0000000..78821be |
| 268 | Binary files /dev/null and b/internal/webui/screenshot-narrow.png differ |
| 269 | diff --git a/internal/webui/screenshot-wide.png b/internal/webui/screenshot-wide.png |
| 270 | new file mode 100644 |
| 271 | index 0000000..b113ece |
| 272 | Binary files /dev/null and b/internal/webui/screenshot-wide.png differ |
| 273 | diff --git a/internal/webui/sw.js b/internal/webui/sw.js |
| 274 | index 538dc82..14e8f2b 100644 |
| 275 | --- a/internal/webui/sw.js |
| 276 | +++ b/internal/webui/sw.js |
| 277 | @@ -41,6 +41,31 @@ const receiveShare = async (request, url) => { |
| 278 | return Response.redirect(url.pathname.replace(/\/share$/, "/files") + "#share=" + id, 303); |
| 279 | }; |
| 280 | |
| 281 | +// Background uploads finish after the page closes. The last response is the |
| 282 | +// blob descriptor; it is parked for the Files page and the person is told. |
| 283 | +const uploadResult = async (registration, ok, detail) => { |
| 284 | + const cache = await caches.open("tiny-uploads"); |
| 285 | + let response = Response.json({error: detail || "upload failed"}, {status: 500}); |
| 286 | + if (ok) { |
| 287 | + const records = await registration.matchAll(); |
| 288 | + const last = records[records.length - 1]; |
| 289 | + if (last) response = (await last.responseReady).clone(); |
| 290 | + } |
| 291 | + await cache.put(new Request(new URL("/uploads/" + registration.id, self.registration.scope).href), response); |
| 292 | +}; |
| 293 | +self.addEventListener("backgroundfetchsuccess", event => { |
| 294 | + event.waitUntil(uploadResult(event.registration, true).then(() => event.updateUI({title: "Upload complete"}))); |
| 295 | +}); |
| 296 | +self.addEventListener("backgroundfetchfail", event => { |
| 297 | + event.waitUntil(uploadResult(event.registration, false, event.registration.failureReason).then(() => event.updateUI({title: "Upload failed"}))); |
| 298 | +}); |
| 299 | +self.addEventListener("backgroundfetchabort", event => { |
| 300 | + event.waitUntil(uploadResult(event.registration, false, "aborted")); |
| 301 | +}); |
| 302 | +self.addEventListener("backgroundfetchclick", event => { |
| 303 | + event.waitUntil(self.clients.openWindow(new URL("files", self.registration.scope).href)); |
| 304 | +}); |
| 305 | + |
| 306 | // Notifications are opt-in per device; a push only arrives after the person |
| 307 | // enabled them on this browser. The payload is the relay's short summary. |
| 308 | self.addEventListener("push", event => { |
| 309 | diff --git a/scripts/blossom-upload.test.mjs b/scripts/blossom-upload.test.mjs |
| 310 | index 60054de..e678e93 100644 |
| 311 | --- a/scripts/blossom-upload.test.mjs |
| 312 | +++ b/scripts/blossom-upload.test.mjs |
| 313 | @@ -125,3 +125,17 @@ test("keeps the response body readable after headers arrive", async () => { |
| 314 | assert.equal(result.descriptor.sha256, hash); |
| 315 | } finally { await new Promise(resolve => server.close(resolve)); } |
| 316 | }); |
| 317 | + |
| 318 | +test("plan lists signed chunk requests without sending them", async () => { |
| 319 | + const bytes = new Uint8Array(12); |
| 320 | + const calls = []; |
| 321 | + const planned = await sandbox.tiny.blossom.upload.plan(bytes, {url: "https://blossom.example/", chunkSize: 5, authorize: async (url, method, body) => { calls.push([method, body.byteLength]); return "Nostr " + url; }}); |
| 322 | + assert.equal(planned.size, 12); |
| 323 | + assert.equal(planned.requests.length, 3); |
| 324 | + assert.equal(planned.requests.map(request => request.headers["upload-offset"]).join(","), "0,5,10"); |
| 325 | + assert.equal(planned.requests.map(request => request.body.byteLength).join(","), "5,5,2"); |
| 326 | + assert.equal(planned.requests[0].headers["upload-length"], "12"); |
| 327 | + assert.equal(planned.requests[0].url, "https://blossom.example/" + planned.hash); |
| 328 | + assert.equal(JSON.stringify(calls), JSON.stringify([["PATCH", 5], ["PATCH", 5], ["PATCH", 2]])); |
| 329 | + assert.equal(planned.requests[2].headers.authorization, "Nostr https://blossom.example/" + planned.hash); |
| 330 | +}); |
| 331 | diff --git a/scripts/file-workspace.test.mjs b/scripts/file-workspace.test.mjs |
| 332 | index 8cda48b..9714975 100644 |
| 333 | --- a/scripts/file-workspace.test.mjs |
| 334 | +++ b/scripts/file-workspace.test.mjs |
| 335 | @@ -285,3 +285,48 @@ test("retry keeps the original ciphertext and key after interrupted encrypted up |
| 336 | assert.equal(new URL(link).hash.slice(1), originalFragment); |
| 337 | assert.equal(upload.pending, null); |
| 338 | }); |
| 339 | + |
| 340 | +test("large uploads hand signed requests to Background Fetch and read the parked descriptor", async () => { |
| 341 | + const {window, FileUpload} = load(true); |
| 342 | + window.tiny.localPath = path => path; |
| 343 | + window.tiny.sha256hex = async bytes => Buffer.from(await webcrypto.subtle.digest("SHA-256", bytes)).toString("hex"); |
| 344 | + window.tiny.authorization = async (url, method, body) => "Nostr " + method + ":" + new URL(url).pathname + ":" + body.byteLength; |
| 345 | + window.tiny.navigate = async () => {}; |
| 346 | + const bytes = new Uint8Array(9 * 1024 * 1024); |
| 347 | + bytes[0] = 7; |
| 348 | + const hash = await window.tiny.sha256hex(bytes); |
| 349 | + const descriptor = {sha256: hash, size: bytes.byteLength}; |
| 350 | + let started; |
| 351 | + const task = {result: "", uploaded: 0, uploadTotal: bytes.byteLength, listeners: [], addEventListener(name, fn) { this.listeners.push(fn); }, removeEventListener() {}}; |
| 352 | + const parked = new Map(); |
| 353 | + globalThis.caches = {open: async () => ({match: async key => parked.get(key), delete: async key => parked.delete(key), put: async (key, value) => parked.set(key, value)})}; |
| 354 | + globalThis.BackgroundFetchManager = class {}; |
| 355 | + const store = new Map(); |
| 356 | + globalThis.localStorage = {getItem: key => store.get(key) ?? null, setItem: (key, value) => store.set(key, value), removeItem: key => store.delete(key), get length() { return store.size; }}; |
| 357 | + Object.defineProperty(globalThis, "localStorage", {configurable: true, value: globalThis.localStorage}); |
| 358 | + Object.keys = ((original => target => target === globalThis.localStorage ? [...store.keys()] : original(target)))(Object.keys); |
| 359 | + Object.defineProperty(globalThis, "navigator", {configurable: true, value: {clipboard: {writeText: async () => { throw Error("unavailable"); }}, serviceWorker: {controller: {}, ready: Promise.resolve({backgroundFetch: {fetch: async (id, requests, options) => { started = {id, requests, options}; return task; }, get: async () => null}})}}}); |
| 360 | + const originalFetch = globalThis.fetch; |
| 361 | + globalThis.fetch = async (_url, init = {}) => new Response(null, {status: 204, headers: {Allow: "PATCH, PUT"}}); |
| 362 | + try { |
| 363 | + const upload = new FileUpload(); window.document.append(upload); upload.connectedCallback(); |
| 364 | + const form = upload.querySelector("form"); |
| 365 | + form.elements.namedItem("file").files = [new File([bytes], "big.bin", {type: "application/octet-stream"})]; |
| 366 | + const run = upload.run(form); |
| 367 | + await new Promise(resolve => setTimeout(resolve, 50)); |
| 368 | + assert.ok(started, "background fetch was not started"); |
| 369 | + assert.equal(started.requests.length, 2); |
| 370 | + assert.equal(started.requests[0].method, "PATCH"); |
| 371 | + assert.equal(started.requests[1].headers.get("upload-offset"), String(5 * 1024 * 1024)); |
| 372 | + assert.match(started.requests[0].headers.get("authorization"), /^Nostr PATCH:/); |
| 373 | + assert.equal(started.options.uploadTotal, bytes.byteLength); |
| 374 | + assert.equal(store.get("tiny.bg." + started.id) !== undefined, true); |
| 375 | + parked.set("https://relay.test/uploads/" + started.id, Response.json(descriptor)); |
| 376 | + task.result = "success"; task.uploaded = bytes.byteLength; |
| 377 | + for (const fn of task.listeners) await fn(); |
| 378 | + await run; |
| 379 | + assert.equal(store.has("tiny.bg." + started.id), false); |
| 380 | + assert.equal(parked.size, 0); |
| 381 | + assert.equal(upload.out.textContent, "Stored 1 file."); |
| 382 | + } finally { globalThis.fetch = originalFetch; } |
| 383 | +}); |