d608636 / sync.mjs
diff --cc sync.mjs
index 0000000,a8c6d5c..07f5150
mode 000000,100644..100644
--- a/sync.mjs
+++ b/sync.mjs
@@@ -1,0 -1,211 +1,214 @@@
+ // Isomorphic pull client: sync signed records and AndFS blocks from a peer
+ // over HTTP, verify everything before storing, and advance the selection
+ // only after the bundle verifies.
+ //
+ // pull(store, baseUrl, { verifyBundle }) fetches the peer's poll pages,
+ // verifies each record envelope and content before storing it, follows the
+ // repository's update chain back until it reaches updates already held,
+ // fetches the missing manifest and chunks for the new segment, and calls
+ // verifyBundle before writing the selection. The default verifyBundle
+ // reassembles the bundle from the store and verifies its size; Node passes
+ // a hook that runs the Git materialize path. Partial failure never advances
+ // the selection. Re-pulling is a no-op: records and blocks already held
+ // are skipped.
+ //
+ // No cursor is persisted: every pull starts from the beginning and relies
+ // on deduplication, which keeps re-pulls cheap at this scale. The peer URL
+ // is the node origin; a clone URL works too, since only its origin is used.
+
+ import {
+ chunkSize, concat, envelopeTimestamp, maxManifest, parseJson, recordAuthor,
+ requireThat, utf8Encode, validateManifest, verifyBlock, verifyRecord
+ } from './core.mjs'
++import { verifyPost } from './social.mjs'
+
+ const maxPages = 1000
+ const maxPollBytes = 8 * 1024 * 1024
+ const requestTimeout = 30000
+
+ async function fetchChecked(url) {
+ const response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(requestTimeout) })
+ requireThat(response.ok, `Sync request failed with status ${response.status}`)
+ return response
+ }
+
+ async function fetchPoll(base, cursor) {
+ const url = cursor ? `${base}/gossip/poll?since=${encodeURIComponent(cursor)}` : `${base}/gossip/poll`
+ const text = await (await fetchChecked(url)).text()
+ requireThat(text.length <= maxPollBytes, 'Poll response too large')
+ const body = parseJson(text)
+ requireThat(body && typeof body === 'object' && Array.isArray(body.records) &&
+ (body.nextSince === null || typeof body.nextSince === 'string'), 'Invalid poll response')
+ return body
+ }
+
+ async function fetchBlock(base, hash) {
+ return new Uint8Array(await (await fetchChecked(`${base}/blocks/${hash}`)).arrayBuffer())
+ }
+
+ // Reassemble a bundle from blocks held in the store, verifying the manifest
+ // structure, each chunk hash and length, and the reconstructed size.
+ export async function reassembleBundle(store, bundleId) {
+ const manifestBytes = await store.getBlock(bundleId, maxManifest)
+ requireThat(manifestBytes, 'Missing AndFS block')
+ requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+ const manifest = validateManifest(manifestBytes)
+ const parts = []
+ let remaining = manifest.size
+ for (const hash of manifest.chunks) {
+ const expected = Math.min(chunkSize, remaining)
+ const chunk = await store.getBlock(hash, expected)
+ requireThat(chunk, 'Missing AndFS block')
+ requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+ requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+ parts.push(chunk)
+ remaining -= chunk.length
+ }
+ requireThat(remaining === 0, 'Unexpected AndFS file size')
+ return concat(...parts)
+ }
+
+ function isGitRepo(body) {
+ return body?.anbot === 1 && body.type === 'git-repo' && typeof body.name === 'string'
+ }
+
+ function belongsToRepo(body, repoId) {
+ return body?.repo?.id === repoId
+ }
+
+ function isGitUpdate(body, repoId) {
+ return body?.anbot === 1 && body.type === 'git-update' &&
+ body.repo?.format === 'anproto' && body.repo.id === repoId &&
+ typeof body.ref === 'string' && typeof body.commit === 'string' &&
+ body.bundle?.format === 'andfs-v1' && typeof body.bundle.id === 'string' &&
+ (body.previous === undefined || body.previous === null || typeof body.previous === 'string')
+ }
+
+ export async function pull(store, baseUrl, options = {}) {
+ const base = baseUrl.replace(/\/+$/, '')
+
+ async function fetchVerified() {
+ const found = []
+ let cursor = null
+ for (let page = 0; page < maxPages; page++) {
+ const body = await fetchPoll(base, cursor)
+ for (const record of body.records) {
+ const content = await verifyRecord(record, recordAuthor(record))
+ found.push({ record, body: content })
+ }
+ if (body.records.length === 0 || body.nextSince === null) return found
+ cursor = body.nextSince
+ }
+ throw new Error('Poll page limit exceeded')
+ }
+
+ async function ensureBundle(update) {
+ const bundleId = update.body.bundle.id
+ let fetched = 0
+ let manifestBytes = await store.getBlock(bundleId, maxManifest)
+ if (manifestBytes === undefined) {
+ manifestBytes = await fetchBlock(base, bundleId)
+ requireThat(manifestBytes.length <= maxManifest, 'Block exceeds size limit')
+ requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+ await store.putBlock(bundleId, manifestBytes)
+ fetched++
+ } else {
+ requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
+ }
+ const manifest = validateManifest(manifestBytes)
+ let remaining = manifest.size
+ for (const hash of manifest.chunks) {
+ const expected = Math.min(chunkSize, remaining)
+ let chunk = await store.getBlock(hash, expected)
+ if (chunk === undefined) {
+ chunk = await fetchBlock(base, hash)
+ requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+ requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+ await store.putBlock(hash, chunk)
+ fetched++
+ } else {
+ requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
+ requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
+ }
+ remaining -= chunk.length
+ }
+ requireThat(remaining === 0, 'Unexpected AndFS file size')
+ return fetched
+ }
+
+ const verified = await fetchVerified()
+ const repos = verified.filter(({ body }) => isGitRepo(body))
+ const previous = await store.getSelection()
+ let repoId
+ if (previous) {
+ repoId = previous.repoId
+ requireThat(repos.some(({ record }) => record.id === repoId),
+ 'Remote serves a different repository')
+ } else {
+ requireThat(repos.length === 1, repos.length === 0
+ ? 'Remote serves no repository' : 'Remote serves multiple repositories')
+ repoId = repos[0].record.id
+ }
+
+ const held = verified.filter(({ record, body }) =>
- (isGitRepo(body) && record.id === repoId) || belongsToRepo(body, repoId))
++ (isGitRepo(body) && record.id === repoId) || belongsToRepo(body, repoId) ||
++ (body?.anbot === 1 && body.type === 'post'))
+ // Snapshot what was already held before storing: the chain walk stops at
+ // records held before this pull, not at records this pull just stored.
+ const heldBefore = new Set()
+ for (const { record } of held) {
+ if ((await store.getRecord(record.id)) !== undefined) heldBefore.add(record.id)
+ }
+ let stored = 0
- for (const { record } of held) {
++ for (const { record, body } of held) {
++ if (body?.type === 'post') await verifyPost(record)
+ if (await store.putRecord(record.id, utf8Encode(JSON.stringify(record)))) stored++
+ }
+
+ const updates = []
+ for (const entry of verified) {
+ if (!isGitUpdate(entry.body, repoId)) continue
+ updates.push({ ...entry, timestamp: envelopeTimestamp(entry.record.signature) })
+ }
+ requireThat(updates.length > 0, 'Remote has no updates')
+ const byId = new Map(updates.map(update => [update.record.id, update]))
+ const referenced = new Set()
+ for (const update of updates) {
- const prev = update.body.previous ?? null
++ const prev = update.body.previous ?? update.body.previousUpdate ?? null
+ if (prev !== null) referenced.add(prev)
+ }
+ const tips = updates.filter(update => !referenced.has(update.record.id))
+ requireThat(tips.length > 0, 'Update chain has no tip')
+ tips.sort((a, b) => (a.timestamp - b.timestamp) ||
+ (a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0))
+ const head = tips.at(-1)
+
+ const segment = []
+ const seen = new Set()
+ let current = head
+ while (current) {
+ requireThat(!seen.has(current.record.id), 'Update chain loops')
+ seen.add(current.record.id)
+ if (heldBefore.has(current.record.id)) break
+ segment.push(current)
- const prev = current.body.previous ?? null
++ const prev = current.body.previous ?? current.body.previousUpdate ?? null
+ if (prev === null) break
+ current = byId.get(prev)
+ if (current === undefined) {
+ requireThat((await store.getRecord(prev)) !== undefined, 'Update chain is incomplete')
+ break
+ }
+ }
+
+ if (previous?.updateId === head.record.id && segment.length === 0) {
+ return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched: 0 }
+ }
+ let fetched = 0
+ const targets = segment.includes(head) ? segment : [head, ...segment]
+ for (const update of targets) fetched += await ensureBundle(update)
+ const verify = options.verifyBundle ??
+ (async ({ body }) => { await reassembleBundle(store, body.bundle.id) })
+ await verify({ id: head.record.id, body: head.body, repoId })
+ await store.putSelection({ repoId, ref: head.body.ref, updateId: head.record.id })
+ return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched }
+ }
Git ยท Social feed