tinyrelayfile
tinyrelay/internal/daemon/replication_services_test.go/
internal/daemon/replication_services_test.go raw
1package daemon
2
3import (
4 "bytes"
5 "compress/zlib"
6 "context"
7 "crypto/sha256"
8 "encoding/base64"
9 "encoding/json"
10 "fmt"
11 "io"
12 "os"
13 "os/exec"
14 "path/filepath"
15 "strings"
16 "testing"
17 "time"
18
19 "github.com/FelineStateMachine/tinyrelay/internal/catalog"
20 "github.com/FelineStateMachine/tinyrelay/internal/event"
21 "github.com/FelineStateMachine/tinyrelay/internal/replication"
22 "github.com/FelineStateMachine/tinyrelay/internal/storage"
23)
24
25func TestReplicationProviderSnapshotsSQLiteTablesWithoutWALCopy(t *testing.T) {
26 ctx := context.Background()
27 store, err := storage.Open(ctx, filepath.Join(t.TempDir(), "tenant.db"))
28 if err != nil {
29 t.Fatal(err)
30 }
31 defer store.Close()
32 if _, err := store.DB().Exec(`CREATE TABLE provider_fixture (id TEXT PRIMARY KEY, body TEXT NOT NULL); INSERT INTO provider_fixture VALUES ('one','value')`); err != nil {
33 t.Fatal(err)
34 }
35 tables, err := snapshotTables(ctx, store)
36 if err != nil {
37 t.Fatal(err)
38 }
39 var found bool
40 for _, table := range tables {
41 if table.Name == "provider_fixture" && len(table.Rows) == 1 {
42 found = true
43 }
44 }
45 if !found {
46 t.Fatalf("fixture missing from snapshot: %#v", tables)
47 }
48 root := t.TempDir()
49 if err := os.MkdirAll(filepath.Join(root, "blobs"), 0700); err != nil {
50 t.Fatal(err)
51 }
52 if err := os.WriteFile(filepath.Join(root, "blobs", "blob-a"), []byte("blob"), 0600); err != nil {
53 t.Fatal(err)
54 }
55 tenant := &Tenant{store: store, publicURL: "https://relay.example", meta: catalog.Tenant{Paths: catalog.Paths{Root: root, Git: filepath.Join(root, "git")}}}
56 provider := tenantReplicationProvider{tenant: tenant}
57 state, err := provider.Snapshot(ctx)
58 if err != nil {
59 t.Fatal(err)
60 }
61 if len(state.Blobs) != 1 {
62 t.Fatalf("blobs = %#v", state.Blobs)
63 }
64}
65
66func TestLegacyBindWSBackupRestoresGitCloneAndState(t *testing.T) {
67 ctx := context.Background()
68 _, tenant := testTenant(t)
69 owner := tenant.Policy().Owner
70 secret := strings.Repeat("0", 63) + "1"
71 e := event.Event{CreatedAt: 20, Kind: 1, Tags: [][]string{}, Content: "legacy event"}
72 if err := event.Sign(&e, secret); err != nil {
73 t.Fatal(err)
74 }
75 gitBackup := makeLegacyGitBackup(t, owner)
76 pendingUntil := time.Now().Unix() + 3600
77 blob := []byte("legacy blob")
78 blobSum := sha256Hex(blob)
79 fixture := map[string]any{
80 "format": "bind.ws/relay-backup/1",
81 "manifest": map[string]any{"createdAt": 20, "owner": owner, "relayIdentity": owner, "slug": "legacy", "archiveSha256": ""},
82 "config": map[string]any{
83 "format": "bind.ws/relay-config/2", "policy": map[string]any{"features": map[string]any{"search": "prose"}},
84 "members": []any{}, "bans": []any{}, "addresses": []any{}, "banned_events": []any{},
85 "kinds": map[string]any{"allow": []int{}, "block": []int{}}, "retention": []any{}, "connections": []any{}, "jobs": []any{},
86 },
87 "events": []string{mustJSON(t, e)},
88 "blobs": []any{map[string]any{"sha256": blobSum, "type": "application/octet-stream", "uploader": owner, "uploaded": int64(21), "data": base64.StdEncoding.EncodeToString(blob)}},
89 "git": []any{}, "sqlGit": gitBackup,
90 "state": map[string]any{"listHistory": []string{}, "hidden": []string{"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}, "hosted": []string{"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, "pending": []any{map[string]any{"id": e.ID, "until": pendingUntil}}, "prRefs": []any{}},
91 }
92 raw, err := json.Marshal(fixture)
93 if err != nil {
94 t.Fatal(err)
95 }
96 raw = []byte(legacyFixtureWithChecksum(string(raw)))
97 archive, err := replication.ReadCompatibleBackup(bytes.NewReader(raw))
98 if err != nil {
99 t.Fatal(err)
100 }
101 if _, err := replication.RestoreBackupWithProvider(ctx, tenant.store, tenant.ReplicationBackupProvider(), archive, 30); err != nil {
102 t.Fatal(err)
103 }
104 var events int
105 if err := tenant.store.DB().QueryRow(`SELECT count(*) FROM events WHERE id=?`, e.ID).Scan(&events); err != nil || events != 1 {
106 t.Fatalf("restored event = %d, %v", events, err)
107 }
108 var hidden int
109 if err := tenant.store.DB().QueryRow(`SELECT count(*) FROM hidden_events WHERE id='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'`).Scan(&hidden); err != nil || hidden != 1 {
110 t.Fatalf("hidden state = %d, %v", hidden, err)
111 }
112 var pending int
113 if err := tenant.store.DB().QueryRow(`SELECT count(*) FROM pending_events WHERE id=?`, e.ID).Scan(&pending); err != nil || pending != 1 {
114 t.Fatalf("pending state = %d, %v", pending, err)
115 }
116 var uploader string
117 var uploaded int64
118 if err := tenant.store.DB().QueryRow(`SELECT uploader,uploaded FROM blobs WHERE sha256=?`, blobSum).Scan(&uploader, &uploaded); err != nil || uploader != owner || uploaded != 21 {
119 t.Fatalf("blob metadata = %q/%d, %v", uploader, uploaded, err)
120 }
121 gitDir := filepath.Join(tenant.meta.Paths.Git, owner, "legacy.git")
122 if out, err := exec.Command("git", "--git-dir", gitDir, "rev-parse", "refs/heads/main").CombinedOutput(); err != nil || len(strings.TrimSpace(string(out))) != 40 {
123 t.Fatalf("restored git ref = %q, %v", out, err)
124 }
125 clone := filepath.Join(t.TempDir(), "clone")
126 if out, err := exec.Command("git", "clone", gitDir, clone).CombinedOutput(); err != nil {
127 t.Fatalf("clone failed: %v: %s", err, out)
128 }
129 content, err := os.ReadFile(filepath.Join(clone, "README.md"))
130 if err != nil || string(content) != "legacy git\n" {
131 t.Fatalf("cloned content = %q, %v", content, err)
132 }
133}
134
135func TestReplicationPendingAndPRDeadlinesExpireAtSchedulerSweep(t *testing.T) {
136 ctx := context.Background()
137 _, tenant := testTenant(t)
138 e := event.Event{CreatedAt: 20, Kind: 30618, Tags: [][]string{}, Content: "pending"}
139 if err := event.Sign(&e, strings.Repeat("0", 63)+"1"); err != nil {
140 t.Fatal(err)
141 }
142 if _, err := tenant.store.Save(ctx, e, storage.SaveOptions{Now: 20}); err != nil {
143 t.Fatal(err)
144 }
145 state, _ := json.Marshal(map[string]any{"pending": []any{map[string]any{"id": e.ID, "until": int64(10)}}, "prRefs": []any{map[string]any{"repo": "pr:" + tenant.Policy().Owner + ":demo", "ref": "refs/nostr/" + e.ID, "until": int64(10)}}})
146 if err := restoreRetainedState(ctx, tenant.store, state); err != nil {
147 t.Fatal(err)
148 }
149 if err := tenant.sweepReplicationState(ctx, 11); err != nil {
150 t.Fatal(err)
151 }
152 var events, refs int
153 if err := tenant.store.DB().QueryRow(`SELECT count(*) FROM events WHERE id=?`, e.ID).Scan(&events); err != nil {
154 t.Fatal(err)
155 }
156 if err := tenant.store.DB().QueryRow(`SELECT count(*) FROM replication_pr_refs`).Scan(&refs); err != nil {
157 t.Fatal(err)
158 }
159 if events != 0 || refs != 0 {
160 t.Fatalf("expired state survived: events=%d refs=%d", events, refs)
161 }
162}
163
164func makeLegacyGitBackup(t *testing.T, owner string) map[string]any {
165 t.Helper()
166 work := t.TempDir()
167 for _, args := range [][]string{{"init", work}, {"-C", work, "config", "user.email", "test@example.com"}, {"-C", work, "config", "user.name", "Test"}} {
168 if out, err := exec.Command("git", args...).CombinedOutput(); err != nil {
169 t.Fatalf("git %v: %v: %s", args, err, out)
170 }
171 }
172 if err := os.WriteFile(filepath.Join(work, "README.md"), []byte("legacy git\n"), 0600); err != nil {
173 t.Fatal(err)
174 }
175 for _, args := range [][]string{{"-C", work, "add", "README.md"}, {"-C", work, "commit", "-m", "legacy"}} {
176 if out, err := exec.Command("git", args...).CombinedOutput(); err != nil {
177 t.Fatalf("git %v: %v: %s", args, err, out)
178 }
179 }
180 refOut, err := exec.Command("git", "-C", work, "rev-parse", "HEAD").Output()
181 if err != nil {
182 t.Fatal(err)
183 }
184 commit := strings.TrimSpace(string(refOut))
185 objectsOut, err := exec.Command("git", "-C", work, "rev-list", "--objects", "HEAD").Output()
186 if err != nil {
187 t.Fatal(err)
188 }
189 objects := []any{}
190 for _, line := range strings.Split(strings.TrimSpace(string(objectsOut)), "\n") {
191 oid := strings.Fields(line)[0]
192 body := gitObjectBytes(t, work, oid)
193 typOut, err := exec.Command("git", "-C", work, "cat-file", "-t", oid).Output()
194 if err != nil {
195 t.Fatal(err)
196 }
197 typ := strings.TrimSpace(string(typOut))
198 var compressed bytes.Buffer
199 zw := zlib.NewWriter(&compressed)
200 _, _ = zw.Write(body)
201 _ = zw.Close()
202 objects = append(objects, map[string]any{"oid": oid, "type": typ, "size": len(body), "chunks": []string{base64.StdEncoding.EncodeToString(compressed.Bytes())}})
203 }
204 return map[string]any{"format": "bind.ws/git-sqlite/1", "repositories": []any{map[string]any{"key": strings.Repeat("a", 64), "owner": owner, "identifier": "legacy", "alternative": false, "objects": objects, "refs": map[string]string{"refs/heads/main": commit}}}}
205}
206
207func sha256Hex(body []byte) string {
208 sum := sha256.Sum256(body)
209 return fmt.Sprintf("%x", sum[:])
210}
211
212func legacyFixtureWithChecksum(raw string) string {
213 key := `"archiveSha256":"`
214 at := strings.Index(raw, key)
215 start := at + len(key)
216 end := strings.IndexByte(raw[start:], '"') + start
217 unsigned := raw[:start] + raw[end:]
218 return raw[:start] + sha256Hex([]byte(unsigned)) + raw[end:]
219}
220
221func mustJSON(t *testing.T, value any) string {
222 t.Helper()
223 raw, err := json.Marshal(value)
224 if err != nil {
225 t.Fatal(err)
226 }
227 return string(raw)
228}
229
230func gitObjectBytes(t *testing.T, work, oid string) []byte {
231 t.Helper()
232 cmd := exec.Command("git", "-C", work, "cat-file", "--batch")
233 stdin, err := cmd.StdinPipe()
234 if err != nil {
235 t.Fatal(err)
236 }
237 stdout, err := cmd.StdoutPipe()
238 if err != nil {
239 t.Fatal(err)
240 }
241 if err := cmd.Start(); err != nil {
242 t.Fatal(err)
243 }
244 _, _ = fmt.Fprintln(stdin, oid)
245 _ = stdin.Close()
246 header := make([]byte, 0, 128)
247 buf := make([]byte, 1)
248 for {
249 if _, err := stdout.Read(buf); err != nil {
250 t.Fatal(err)
251 }
252 header = append(header, buf[0])
253 if buf[0] == '\n' {
254 break
255 }
256 }
257 parts := strings.Fields(string(header))
258 if len(parts) < 3 {
259 t.Fatalf("bad git batch header %q", header)
260 }
261 n := 0
262 if _, err := fmt.Sscanf(parts[2], "%d", &n); err != nil {
263 t.Fatal(err)
264 }
265 body := make([]byte, n)
266 if _, err := io.ReadFull(stdout, body); err != nil {
267 t.Fatal(err)
268 }
269 if err := cmd.Wait(); err != nil {
270 t.Fatal(err)
271 }
272 return body
273}
274
275func TestReplicationProviderRejectsEscapingBackupFiles(t *testing.T) {
276 root := t.TempDir()
277 if err := restoreFiles(root, []replication.BackupObject{{Name: "../outside", SHA256: "", Content: []byte("bad")}}); err == nil {
278 t.Fatal("backup path escaped tenant root")
279 }
280 if _, err := os.Stat(filepath.Join(filepath.Dir(root), "outside")); err == nil {
281 t.Fatal("outside file created")
282 }
283}
284
285func TestTenantBackupRestoreRoundTripWithFilesAndFTS(t *testing.T) {
286 ctx := context.Background()
287 root := t.TempDir()
288 store, err := storage.Open(ctx, filepath.Join(root, "tenant.db"))
289 if err != nil {
290 t.Fatal(err)
291 }
292 defer store.Close()
293 e := event.Event{CreatedAt: 10, Kind: 1, Tags: [][]string{}, Content: "restorable phrase"}
294 if err := event.Sign(&e, "1111111111111111111111111111111111111111111111111111111111111111"); err != nil {
295 t.Fatal(err)
296 }
297 if _, err := store.Save(ctx, e, storage.SaveOptions{Now: 11, SearchMode: "full"}); err != nil {
298 t.Fatal(err)
299 }
300 if err := os.MkdirAll(filepath.Join(root, "blobs", "nested"), 0700); err != nil {
301 t.Fatal(err)
302 }
303 if err := os.MkdirAll(filepath.Join(root, "git"), 0700); err != nil {
304 t.Fatal(err)
305 }
306 if err := os.WriteFile(filepath.Join(root, "blobs", "nested", "blob"), []byte("blob-original"), 0600); err != nil {
307 t.Fatal(err)
308 }
309 if err := os.WriteFile(filepath.Join(root, "git", "object"), []byte("git-original"), 0600); err != nil {
310 t.Fatal(err)
311 }
312 tenant := &Tenant{store: store, publicURL: "https://relay.example", meta: catalog.Tenant{Paths: catalog.Paths{Root: root, Git: filepath.Join(root, "git")}}}
313 provider := tenantReplicationProvider{tenant: tenant}
314 state, err := provider.Snapshot(ctx)
315 if err != nil {
316 t.Fatal(err)
317 }
318 if _, err := store.DB().Exec(`DELETE FROM events`); err != nil {
319 t.Fatal(err)
320 }
321 if err := os.WriteFile(filepath.Join(root, "blobs", "nested", "blob"), []byte("mutated"), 0600); err != nil {
322 t.Fatal(err)
323 }
324 if err := os.WriteFile(filepath.Join(root, "blobs", "extra"), []byte("remove-me"), 0600); err != nil {
325 t.Fatal(err)
326 }
327 if err := os.WriteFile(filepath.Join(root, "git", "object"), []byte("git-mutated"), 0600); err != nil {
328 t.Fatal(err)
329 }
330 if err := provider.Restore(ctx, state); err != nil {
331 t.Fatal(err)
332 }
333 result, err := store.Query(ctx, event.Filter{Search: "restorable", Tags: map[string][]string{}}, storage.QueryOptions{Now: 11, Access: storage.Access{All: true}, Limit: 10})
334 if err != nil {
335 t.Fatal(err)
336 }
337 if len(result.Events) != 1 {
338 t.Fatalf("restored search results = %d", len(result.Events))
339 }
340 blob, _ := os.ReadFile(filepath.Join(root, "blobs", "nested", "blob"))
341 if string(blob) != "blob-original" {
342 t.Fatalf("blob = %q", blob)
343 }
344 git, _ := os.ReadFile(filepath.Join(root, "git", "object"))
345 if string(git) != "git-original" {
346 t.Fatalf("git = %q", git)
347 }
348 if _, err := os.Stat(filepath.Join(root, "blobs", "extra")); !os.IsNotExist(err) {
349 t.Fatal("extra blob survived restore")
350 }
351 bad := replication.BackupArchive{Format: replication.BackupFormat, Events: []event.Event{e}, State: state}
352 bad.State.Blobs = append(bad.State.Blobs, replication.BackupObject{Name: "broken", SHA256: "bad", Content: []byte("bad")})
353 raw, _ := json.Marshal(bad)
354 if _, err := replication.ReadBackup(bytes.NewReader(raw)); err == nil {
355 t.Fatal("bad archive passed integrity check")
356 }
357 if _, err := store.DB().Exec(`DELETE FROM events`); err != nil {
358 t.Fatal(err)
359 }
360 if err := os.WriteFile(filepath.Join(root, "blobs", "nested", "blob"), []byte("rollback-marker"), 0600); err != nil {
361 t.Fatal(err)
362 }
363 if err := provider.Restore(ctx, bad.State); err == nil {
364 t.Fatal("corrupt file restore unexpectedly succeeded")
365 }
366 rolledBack, err := store.Query(ctx, event.Filter{IDs: []string{e.ID}, Tags: map[string][]string{}}, storage.QueryOptions{Now: 11, Access: storage.Access{All: true}, Limit: 1})
367 if err != nil {
368 t.Fatal(err)
369 }
370 if len(rolledBack.Events) != 0 {
371 t.Fatal("database rollback restored the rejected archive")
372 }
373 marker, _ := os.ReadFile(filepath.Join(root, "blobs", "nested", "blob"))
374 if string(marker) != "rollback-marker" {
375 t.Fatalf("filesystem rollback = %q", marker)
376 }
377}
378
379func TestRestoreRetainedStateIsTransactionalAndPreservesAllFamilies(t *testing.T) {
380 ctx := context.Background()
381 store, err := storage.Open(ctx, filepath.Join(t.TempDir(), "tenant.db"))
382 if err != nil {
383 t.Fatal(err)
384 }
385 defer store.Close()
386 e := event.Event{CreatedAt: 10, Kind: 30023, Tags: [][]string{{"d", "post"}}, Content: "history"}
387 if err := event.Sign(&e, "1111111111111111111111111111111111111111111111111111111111111111"); err != nil {
388 t.Fatal(err)
389 }
390 rawEvent, _ := json.Marshal(e)
391 state, _ := json.Marshal(map[string]any{
392 "listHistory": []string{string(rawEvent)},
393 "hidden": []string{"hidden-id"},
394 "hosted": []string{"hosted-id"},
395 "pending": []map[string]any{{"id": "pending-id", "until": int64(42)}},
396 "prRefs": []map[string]any{{"repo": "repo", "ref": "refs/heads/main", "until": int64(43)}},
397 })
398 if err := restoreRetainedState(ctx, store, state); err != nil {
399 t.Fatal(err)
400 }
401 for _, query := range []string{
402 `SELECT count(*) FROM list_history`,
403 `SELECT count(*) FROM hidden_events WHERE id='hidden-id'`,
404 `SELECT count(*) FROM replication_hosted_events WHERE id='hosted-id'`,
405 `SELECT count(*) FROM replication_pending_until WHERE id='pending-id' AND until=42`,
406 `SELECT count(*) FROM replication_pr_refs WHERE repo='repo' AND ref='refs/heads/main' AND until=43`,
407 } {
408 var n int
409 if err := store.DB().QueryRowContext(ctx, query).Scan(&n); err != nil {
410 t.Fatal(err)
411 }
412 if n != 1 {
413 t.Fatalf("%s count=%d", query, n)
414 }
415 }
416 bad, _ := json.Marshal(map[string]any{"listHistory": []string{"not-json"}, "hidden": []string{"should-not-commit"}})
417 if err := restoreRetainedState(ctx, store, bad); err == nil {
418 t.Fatal("malformed retained state restored")
419 }
420 var n int
421 if err := store.DB().QueryRowContext(ctx, `SELECT count(*) FROM hidden_events WHERE id='should-not-commit'`).Scan(&n); err != nil {
422 t.Fatal(err)
423 }
424 if n != 0 {
425 t.Fatal("retained state partially committed")
426 }
427}

About

ownerc01503e8035f
visibilitypublic
headd730679639bf

Maintainers

  • c01503e8035f owner
  • ba1b5beed5b5 agent hermes

Clone

  • https://012.run/npub1cq2s86qrtadf36eqq3epm38wmhdynrrcwxf84eldk3f94dkv7ttsrulrz5/tinyrelay.git

Refs