anbot

d608636 / sync.mjs

View file at this commit

diff --cc sync.mjs
index 0000000,a8c6d5c..07f5150
mode 000000,100644..100644
--- a/sync.mjs
+++ b/sync.mjs
@@@ -1,0 -1,211 +1,214 @@@
+ // Isomorphic pull client: sync signed records and AndFS blocks from a peer
+ // over HTTP, verify everything before storing, and advance the selection
+ // only after the bundle verifies.
+ //
+ // pull(store, baseUrl, { verifyBundle }) fetches the peer's poll pages,
+ // verifies each record envelope and content before storing it, follows the
+ // repository's update chain back until it reaches updates already held,
+ // fetches the missing manifest and chunks for the new segment, and calls
+ // verifyBundle before writing the selection. The default verifyBundle
+ // reassembles the bundle from the store and verifies its size; Node passes
+ // a hook that runs the Git materialize path. Partial failure never advances
+ // the selection. Re-pulling is a no-op: records and blocks already held
+ // are skipped.
+ //
+ // No cursor is persisted: every pull starts from the beginning and relies
+ // on deduplication, which keeps re-pulls cheap at this scale. The peer URL
+ // is the node origin; a clone URL works too, since only its origin is used.
+ 
+ import {
+   chunkSize, concat, envelopeTimestamp, maxManifest, parseJson, recordAuthor,
+   requireThat, utf8Encode, validateManifest, verifyBlock, verifyRecord
+ } from './core.mjs'
++import { verifyPost } from './social.mjs'
+ 
+ const maxPages = 1000
+ const maxPollBytes = 8 * 1024 * 1024
+ const requestTimeout = 30000
+ 
+ async function fetchChecked(url) {
+   const response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(requestTimeout) })
+   requireThat(response.ok, `Sync request failed with status ${response.status}`)
+   return response
+ }
+ 
+ async function fetchPoll(base, cursor) {
+   const url = cursor ? `${base}/gossip/poll?since=${encodeURIComponent(cursor)}` : `${base}/gossip/poll`
+   const text = await (await fetchChecked(url)).text()
+   requireThat(text.length <= maxPollBytes, 'Poll response too large')
+   const body = parseJson(text)
+   requireThat(body && typeof body === 'object' && Array.isArray(body.records) &&
+     (body.nextSince === null || typeof body.nextSince === 'string'), 'Invalid poll response')
+   return body
+ }
+ 
+ async function fetchBlock(base, hash) {
+   return new Uint8Array(await (await fetchChecked(`${base}/blocks/${hash}`)).arrayBuffer())
+ }
+ 
+ // Reassemble a bundle from blocks held in the store, verifying the manifest
+ // structure, each chunk hash and length, and the reconstructed size.
+ export async function reassembleBundle(store, bundleId) {
+   const manifestBytes = await store.getBlock(bundleId, maxManifest)
+   requireThat(manifestBytes, 'Missing AndFS block')
+   requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+   const manifest = validateManifest(manifestBytes)
+   const parts = []
+   let remaining = manifest.size
+   for (const hash of manifest.chunks) {
+     const expected = Math.min(chunkSize, remaining)
+     const chunk = await store.getBlock(hash, expected)
+     requireThat(chunk, 'Missing AndFS block')
+     requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+     requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+     parts.push(chunk)
+     remaining -= chunk.length
+   }
+   requireThat(remaining === 0, 'Unexpected AndFS file size')
+   return concat(...parts)
+ }
+ 
+ function isGitRepo(body) {
+   return body?.anbot === 1 && body.type === 'git-repo' && typeof body.name === 'string'
+ }
+ 
+ function belongsToRepo(body, repoId) {
+   return body?.repo?.id === repoId
+ }
+ 
+ function isGitUpdate(body, repoId) {
+   return body?.anbot === 1 && body.type === 'git-update' &&
+     body.repo?.format === 'anproto' && body.repo.id === repoId &&
+     typeof body.ref === 'string' && typeof body.commit === 'string' &&
+     body.bundle?.format === 'andfs-v1' && typeof body.bundle.id === 'string' &&
+     (body.previous === undefined || body.previous === null || typeof body.previous === 'string')
+ }
+ 
+ export async function pull(store, baseUrl, options = {}) {
+   const base = baseUrl.replace(/\/+$/, '')
+ 
+   async function fetchVerified() {
+     const found = []
+     let cursor = null
+     for (let page = 0; page < maxPages; page++) {
+       const body = await fetchPoll(base, cursor)
+       for (const record of body.records) {
+         const content = await verifyRecord(record, recordAuthor(record))
+         found.push({ record, body: content })
+       }
+       if (body.records.length === 0 || body.nextSince === null) return found
+       cursor = body.nextSince
+     }
+     throw new Error('Poll page limit exceeded')
+   }
+ 
+   async function ensureBundle(update) {
+     const bundleId = update.body.bundle.id
+     let fetched = 0
+     let manifestBytes = await store.getBlock(bundleId, maxManifest)
+     if (manifestBytes === undefined) {
+       manifestBytes = await fetchBlock(base, bundleId)
+       requireThat(manifestBytes.length <= maxManifest, 'Block exceeds size limit')
+       requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+       await store.putBlock(bundleId, manifestBytes)
+       fetched++
+     } else {
+       requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+     }
+     const manifest = validateManifest(manifestBytes)
+     let remaining = manifest.size
+     for (const hash of manifest.chunks) {
+       const expected = Math.min(chunkSize, remaining)
+       let chunk = await store.getBlock(hash, expected)
+       if (chunk === undefined) {
+         chunk = await fetchBlock(base, hash)
+         requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+         requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+         await store.putBlock(hash, chunk)
+         fetched++
+       } else {
+         requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+         requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+       }
+       remaining -= chunk.length
+     }
+     requireThat(remaining === 0, 'Unexpected AndFS file size')
+     return fetched
+   }
+ 
+   const verified = await fetchVerified()
+   const repos = verified.filter(({ body }) => isGitRepo(body))
+   const previous = await store.getSelection()
+   let repoId
+   if (previous) {
+     repoId = previous.repoId
+     requireThat(repos.some(({ record }) => record.id === repoId),
+       'Remote serves a different repository')
+   } else {
+     requireThat(repos.length === 1, repos.length === 0
+       ? 'Remote serves no repository' : 'Remote serves multiple repositories')
+     repoId = repos[0].record.id
+   }
+ 
+   const held = verified.filter(({ record, body }) =>
 -    (isGitRepo(body) && record.id === repoId) || belongsToRepo(body, repoId))
++    (isGitRepo(body) && record.id === repoId) || belongsToRepo(body, repoId) ||
++    (body?.anbot === 1 && body.type === 'post'))
+   // Snapshot what was already held before storing: the chain walk stops at
+   // records held before this pull, not at records this pull just stored.
+   const heldBefore = new Set()
+   for (const { record } of held) {
+     if ((await store.getRecord(record.id)) !== undefined) heldBefore.add(record.id)
+   }
+   let stored = 0
 -  for (const { record } of held) {
++  for (const { record, body } of held) {
++    if (body?.type === 'post') await verifyPost(record)
+     if (await store.putRecord(record.id, utf8Encode(JSON.stringify(record)))) stored++
+   }
+ 
+   const updates = []
+   for (const entry of verified) {
+     if (!isGitUpdate(entry.body, repoId)) continue
+     updates.push({ ...entry, timestamp: envelopeTimestamp(entry.record.signature) })
+   }
+   requireThat(updates.length > 0, 'Remote has no updates')
+   const byId = new Map(updates.map(update => [update.record.id, update]))
+   const referenced = new Set()
+   for (const update of updates) {
 -    const prev = update.body.previous ?? null
++    const prev = update.body.previous ?? update.body.previousUpdate ?? null
+     if (prev !== null) referenced.add(prev)
+   }
+   const tips = updates.filter(update => !referenced.has(update.record.id))
+   requireThat(tips.length > 0, 'Update chain has no tip')
+   tips.sort((a, b) => (a.timestamp - b.timestamp) ||
+     (a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0))
+   const head = tips.at(-1)
+ 
+   const segment = []
+   const seen = new Set()
+   let current = head
+   while (current) {
+     requireThat(!seen.has(current.record.id), 'Update chain loops')
+     seen.add(current.record.id)
+     if (heldBefore.has(current.record.id)) break
+     segment.push(current)
 -    const prev = current.body.previous ?? null
++    const prev = current.body.previous ?? current.body.previousUpdate ?? null
+     if (prev === null) break
+     current = byId.get(prev)
+     if (current === undefined) {
+       requireThat((await store.getRecord(prev)) !== undefined, 'Update chain is incomplete')
+       break
+     }
+   }
+ 
+   if (previous?.updateId === head.record.id && segment.length === 0) {
+     return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched: 0 }
+   }
+   let fetched = 0
+   const targets = segment.includes(head) ? segment : [head, ...segment]
+   for (const update of targets) fetched += await ensureBundle(update)
+   const verify = options.verifyBundle ??
+     (async ({ body }) => { await reassembleBundle(store, body.bundle.id) })
+   await verify({ id: head.record.id, body: head.body, repoId })
+   await store.putSelection({ repoId, ref: head.body.ref, updateId: head.record.id })
+   return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched }
+ }

Git ยท Social feed