174e82a / sync.mjs
9959 bytes
// Isomorphic pull client: sync signed records and AndFS blocks from a peer
// over HTTP, verify everything before storing, and advance the selection
// only after the bundle verifies.
//
// pull(store, baseUrl, { verifyBundle }) fetches the peer's poll pages,
// verifies each record envelope and content before storing it, follows the
// repository's update chain back until it reaches updates already held,
// fetches the missing manifest and chunks for the new segment, and calls
// verifyBundle before writing the selection. The default verifyBundle
// reassembles the bundle from the store and verifies its size; Node passes
// a hook that runs the Git materialize path. Partial failure never advances
// the selection. Re-pulling is a no-op: records and blocks already held
// are skipped.
//
// Poll records that are social posts are verified with verifyPost and
// stored through the store's post operations. Posts have no chain, no
// blocks, and no effect on bundle verification or selection advancement:
// a pull that brings only new posts still stores them, and a re-pull
// stores nothing twice.
//
// No cursor is persisted: every pull starts from the beginning and relies
// on deduplication, which keeps re-pulls cheap at this scale. The peer URL
// is the node origin; a clone URL works too, since only its origin is used.
import {
chunkSize, concat, envelopeTimestamp, maxManifest, parseJson, recordAuthor,
requireThat, utf8Encode, validateManifest, verifyBlock, verifyRecord
} from './core.mjs'
import { verifyPost } from './social.mjs'
const maxPages = 1000
const maxPollBytes = 8 * 1024 * 1024
const requestTimeout = 30000
async function fetchChecked(url) {
const response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(requestTimeout) })
requireThat(response.ok, `Sync request failed with status ${response.status}`)
return response
}
async function fetchPoll(base, cursor) {
const url = cursor ? `${base}/gossip/poll?since=${encodeURIComponent(cursor)}` : `${base}/gossip/poll`
const text = await (await fetchChecked(url)).text()
requireThat(text.length <= maxPollBytes, 'Poll response too large')
const body = parseJson(text)
requireThat(body && typeof body === 'object' && Array.isArray(body.records) &&
(body.nextSince === null || typeof body.nextSince === 'string'), 'Invalid poll response')
return body
}
async function fetchBlock(base, hash) {
return new Uint8Array(await (await fetchChecked(`${base}/blocks/${hash}`)).arrayBuffer())
}
// Reassemble a bundle from blocks held in the store, verifying the manifest
// structure, each chunk hash and length, and the reconstructed size.
export async function reassembleBundle(store, bundleId) {
const manifestBytes = await store.getBlock(bundleId, maxManifest)
requireThat(manifestBytes, 'Missing AndFS block')
requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
const manifest = validateManifest(manifestBytes)
const parts = []
let remaining = manifest.size
for (const hash of manifest.chunks) {
const expected = Math.min(chunkSize, remaining)
const chunk = await store.getBlock(hash, expected)
requireThat(chunk, 'Missing AndFS block')
requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
parts.push(chunk)
remaining -= chunk.length
}
requireThat(remaining === 0, 'Unexpected AndFS file size')
return concat(...parts)
}
function isGitRepo(body) {
return body?.anbot === 1 && body.type === 'git-repo' && typeof body.name === 'string'
}
function belongsToRepo(body, repoId) {
return body?.repo?.id === repoId
}
function isGitUpdate(body, repoId) {
return body?.anbot === 1 && body.type === 'git-update' &&
body.repo?.format === 'anproto' && body.repo.id === repoId &&
typeof body.ref === 'string' && typeof body.commit === 'string' &&
body.bundle?.format === 'andfs-v1' && typeof body.bundle.id === 'string' &&
(body.previous === undefined || body.previous === null || typeof body.previous === 'string')
}
function isPost(body) {
return body?.anbot === 1 && body.type === 'post'
}
export async function pull(store, baseUrl, options = {}) {
const base = baseUrl.replace(/\/+$/, '')
async function fetchVerified() {
const found = []
let cursor = null
for (let page = 0; page < maxPages; page++) {
const body = await fetchPoll(base, cursor)
for (const record of body.records) {
const content = await verifyRecord(record, recordAuthor(record))
found.push({ record, body: content })
}
if (body.records.length === 0 || body.nextSince === null) return found
cursor = body.nextSince
}
throw new Error('Poll page limit exceeded')
}
async function ensureBundle(update) {
const bundleId = update.body.bundle.id
let fetched = 0
let manifestBytes = await store.getBlock(bundleId, maxManifest)
if (manifestBytes === undefined) {
manifestBytes = await fetchBlock(base, bundleId)
requireThat(manifestBytes.length <= maxManifest, 'Block exceeds size limit')
requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
await store.putBlock(bundleId, manifestBytes)
fetched++
} else {
requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
}
const manifest = validateManifest(manifestBytes)
let remaining = manifest.size
for (const hash of manifest.chunks) {
const expected = Math.min(chunkSize, remaining)
let chunk = await store.getBlock(hash, expected)
if (chunk === undefined) {
chunk = await fetchBlock(base, hash)
requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
await store.putBlock(hash, chunk)
fetched++
} else {
requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
}
remaining -= chunk.length
}
requireThat(remaining === 0, 'Unexpected AndFS file size')
return fetched
}
const verified = await fetchVerified()
const repos = verified.filter(({ body }) => isGitRepo(body))
const previous = await store.getSelection()
let repoId
if (previous) {
repoId = previous.repoId
requireThat(repos.some(({ record }) => record.id === repoId),
'Remote serves a different repository')
} else {
requireThat(repos.length === 1, repos.length === 0
? 'Remote serves no repository' : 'Remote serves multiple repositories')
repoId = repos[0].record.id
}
const publisher = recordAuthor(repos.find(({ record }) => record.id === repoId).record)
const held = verified.filter(({ record, body }) =>
(isGitRepo(body) && record.id === repoId) ||
(belongsToRepo(body, repoId) && recordAuthor(record) === publisher))
// Snapshot what was already held before storing: the chain walk stops at
// records held before this pull, not at records this pull just stored.
const heldBefore = new Set()
for (const { record } of held) {
if ((await store.getRecord(record.id)) !== undefined) heldBefore.add(record.id)
}
let stored = 0
for (const { record } of held) {
if (await store.putRecord(record.id, utf8Encode(JSON.stringify(record)))) stored++
}
// Posts replicate independently of the git chain: verify each post's
// shape, store it through the post operations, and never let it touch
// the chain walk or the selection below.
let posts = 0
for (const { record, body } of verified) {
if (!isPost(body)) continue
await verifyPost(record)
if (await store.putPost(record.id, utf8Encode(JSON.stringify(record)))) posts++
}
const updates = []
for (const entry of verified) {
if (!isGitUpdate(entry.body, repoId) || recordAuthor(entry.record) !== publisher) continue
updates.push({ ...entry, timestamp: envelopeTimestamp(entry.record.signature) })
}
requireThat(updates.length > 0, 'Remote has no updates')
const byId = new Map(updates.map(update => [update.record.id, update]))
const referenced = new Set()
for (const update of updates) {
const prev = update.body.previous ?? update.body.previousUpdate ?? null
if (prev !== null) referenced.add(prev)
}
const tips = updates.filter(update => !referenced.has(update.record.id))
requireThat(tips.length > 0, 'Update chain has no tip')
tips.sort((a, b) => (a.timestamp - b.timestamp) ||
(a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0))
const head = tips.at(-1)
const segment = []
const seen = new Set()
let current = head
while (current) {
requireThat(!seen.has(current.record.id), 'Update chain loops')
seen.add(current.record.id)
if (heldBefore.has(current.record.id)) break
segment.push(current)
const prev = current.body.previous ?? current.body.previousUpdate ?? null
if (prev === null) break
current = byId.get(prev)
if (current === undefined) {
requireThat((await store.getRecord(prev)) !== undefined, 'Update chain is incomplete')
break
}
}
if (previous?.updateId === head.record.id && segment.length === 0) {
return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched: 0, posts }
}
let fetched = 0
const targets = segment.includes(head) ? segment : [head, ...segment]
for (const update of targets) fetched += await ensureBundle(update)
const verify = options.verifyBundle ??
(async ({ body }) => { await reassembleBundle(store, body.bundle.id) })
await verify({ id: head.record.id, body: head.body, repoId })
await store.putSelection({ repoId, ref: head.body.ref, updateId: head.record.id })
return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched, posts }
}
Git ยท Social feed