anbot

9d3e7fc / sync.mjs

9266 bytes

// Isomorphic pull client: sync signed records and AndFS blocks from a peer
// over HTTP, verify everything before storing, and advance the selection
// only after the bundle verifies.
//
// pull(store, baseUrl, { verifyBundle }) fetches the peer's poll pages,
// verifies each record envelope and content before storing it, follows the
// repository's update chain back until it reaches updates already held,
// fetches the missing manifest and chunks for the new segment, and calls
// verifyBundle before writing the selection. The default verifyBundle
// reassembles the bundle from the store and verifies its size; Node passes
// a hook that runs the Git materialize path. Partial failure never advances
// the selection. Re-pulling is a no-op: records and blocks already held
// are skipped.
//
// No cursor is persisted: every pull starts from the beginning and relies
// on deduplication, which keeps re-pulls cheap at this scale. The peer URL
// is the node origin; a clone URL works too, since only its origin is used.

import {
  chunkSize, concat, envelopeTimestamp, maxManifest, parseJson, recordAuthor,
  requireThat, utf8Encode, validateManifest, verifyBlock, verifyRecord
} from './core.mjs'
import { verifyPost } from './social.mjs'

const maxPages = 1000
const maxPollBytes = 8 * 1024 * 1024
const requestTimeout = 30000

async function fetchChecked(url) {
  const response = await fetch(url, { redirect: 'error', signal: AbortSignal.timeout(requestTimeout) })
  requireThat(response.ok, `Sync request failed with status ${response.status}`)
  return response
}

async function fetchPoll(base, cursor) {
  const url = cursor ? `${base}/gossip/poll?since=${encodeURIComponent(cursor)}` : `${base}/gossip/poll`
  const text = await (await fetchChecked(url)).text()
  requireThat(text.length <= maxPollBytes, 'Poll response too large')
  const body = parseJson(text)
  requireThat(body && typeof body === 'object' && Array.isArray(body.records) &&
    (body.nextSince === null || typeof body.nextSince === 'string'), 'Invalid poll response')
  return body
}

async function fetchBlock(base, hash) {
  return new Uint8Array(await (await fetchChecked(`${base}/blocks/${hash}`)).arrayBuffer())
}

// Reassemble a bundle from blocks held in the store, verifying the manifest
// structure, each chunk hash and length, and the reconstructed size.
export async function reassembleBundle(store, bundleId) {
  const manifestBytes = await store.getBlock(bundleId, maxManifest)
  requireThat(manifestBytes, 'Missing AndFS block')
  requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
  const manifest = validateManifest(manifestBytes)
  const parts = []
  let remaining = manifest.size
  for (const hash of manifest.chunks) {
    const expected = Math.min(chunkSize, remaining)
    const chunk = await store.getBlock(hash, expected)
    requireThat(chunk, 'Missing AndFS block')
    requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
    requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
    parts.push(chunk)
    remaining -= chunk.length
  }
  requireThat(remaining === 0, 'Unexpected AndFS file size')
  return concat(...parts)
}

function isGitRepo(body) {
  return body?.anbot === 1 && body.type === 'git-repo' && typeof body.name === 'string'
}

function belongsToRepo(body, repoId) {
  return body?.repo?.id === repoId
}

function isGitUpdate(body, repoId) {
  return body?.anbot === 1 && body.type === 'git-update' &&
    body.repo?.format === 'anproto' && body.repo.id === repoId &&
    typeof body.ref === 'string' && typeof body.commit === 'string' &&
    body.bundle?.format === 'andfs-v1' && typeof body.bundle.id === 'string' &&
    (body.previous === undefined || body.previous === null || typeof body.previous === 'string')
}

export async function pull(store, baseUrl, options = {}) {
  const base = baseUrl.replace(/\/+$/, '')

  async function fetchVerified() {
    const found = []
    let cursor = null
    for (let page = 0; page < maxPages; page++) {
      const body = await fetchPoll(base, cursor)
      for (const record of body.records) {
        const content = await verifyRecord(record, recordAuthor(record))
        found.push({ record, body: content })
      }
      if (body.records.length === 0 || body.nextSince === null) return found
      cursor = body.nextSince
    }
    throw new Error('Poll page limit exceeded')
  }

  async function ensureBundle(update) {
    const bundleId = update.body.bundle.id
    let fetched = 0
    let manifestBytes = await store.getBlock(bundleId, maxManifest)
    if (manifestBytes === undefined) {
      manifestBytes = await fetchBlock(base, bundleId)
      requireThat(manifestBytes.length <= maxManifest, 'Block exceeds size limit')
      requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
      await store.putBlock(bundleId, manifestBytes)
      fetched++
    } else {
      requireThat(await verifyBlock(manifestBytes, bundleId), 'Corrupt AndFS block')
    }
    const manifest = validateManifest(manifestBytes)
    let remaining = manifest.size
    for (const hash of manifest.chunks) {
      const expected = Math.min(chunkSize, remaining)
      let chunk = await store.getBlock(hash, expected)
      if (chunk === undefined) {
        chunk = await fetchBlock(base, hash)
        requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
        requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
        await store.putBlock(hash, chunk)
        fetched++
      } else {
        requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
        requireThat(await verifyBlock(chunk, hash), 'Corrupt AndFS block')
      }
      remaining -= chunk.length
    }
    requireThat(remaining === 0, 'Unexpected AndFS file size')
    return fetched
  }

  const verified = await fetchVerified()
  const repos = verified.filter(({ body }) => isGitRepo(body))
  const previous = await store.getSelection()
  let repoId
  if (previous) {
    repoId = previous.repoId
    requireThat(repos.some(({ record }) => record.id === repoId),
      'Remote serves a different repository')
  } else {
    requireThat(repos.length === 1, repos.length === 0
      ? 'Remote serves no repository' : 'Remote serves multiple repositories')
    repoId = repos[0].record.id
  }

  const publisher = recordAuthor(repos.find(({ record }) => record.id === repoId).record)
  const held = verified.filter(({ record, body }) =>
    (isGitRepo(body) && record.id === repoId) ||
    (belongsToRepo(body, repoId) && recordAuthor(record) === publisher) ||
    (body?.anbot === 1 && body.type === 'post'))
  // Snapshot what was already held before storing: the chain walk stops at
  // records held before this pull, not at records this pull just stored.
  const heldBefore = new Set()
  for (const { record } of held) {
    if ((await store.getRecord(record.id)) !== undefined) heldBefore.add(record.id)
  }
  let stored = 0
  for (const { record, body } of held) {
    if (body?.type === 'post') await verifyPost(record)
    if (await store.putRecord(record.id, utf8Encode(JSON.stringify(record)))) stored++
  }

  const updates = []
  for (const entry of verified) {
    if (!isGitUpdate(entry.body, repoId) || recordAuthor(entry.record) !== publisher) continue
    updates.push({ ...entry, timestamp: envelopeTimestamp(entry.record.signature) })
  }
  requireThat(updates.length > 0, 'Remote has no updates')
  const byId = new Map(updates.map(update => [update.record.id, update]))
  const referenced = new Set()
  for (const update of updates) {
    const prev = update.body.previous ?? update.body.previousUpdate ?? null
    if (prev !== null) referenced.add(prev)
  }
  const tips = updates.filter(update => !referenced.has(update.record.id))
  requireThat(tips.length > 0, 'Update chain has no tip')
  tips.sort((a, b) => (a.timestamp - b.timestamp) ||
    (a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0))
  const head = tips.at(-1)

  const segment = []
  const seen = new Set()
  let current = head
  while (current) {
    requireThat(!seen.has(current.record.id), 'Update chain loops')
    seen.add(current.record.id)
    if (heldBefore.has(current.record.id)) break
    segment.push(current)
    const prev = current.body.previous ?? current.body.previousUpdate ?? null
    if (prev === null) break
    current = byId.get(prev)
    if (current === undefined) {
      requireThat((await store.getRecord(prev)) !== undefined, 'Update chain is incomplete')
      break
    }
  }

  if (previous?.updateId === head.record.id && segment.length === 0) {
    return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched: 0 }
  }
  let fetched = 0
  const targets = segment.includes(head) ? segment : [head, ...segment]
  for (const update of targets) fetched += await ensureBundle(update)
  const verify = options.verifyBundle ??
    (async ({ body }) => { await reassembleBundle(store, body.bundle.id) })
  await verify({ id: head.record.id, body: head.body, repoId })
  await store.putSelection({ repoId, ref: head.body.ref, updateId: head.record.id })
  return { repoId, ref: head.body.ref, updateId: head.record.id, stored, fetched }
}

Git ยท Social feed