anbot

root / storage.mjs

9375 bytes

import {
  closeSync, fsyncSync, linkSync, mkdirSync, mkdtempSync, openSync, readFileSync,
  readSync, readdirSync, renameSync, rmSync, writeFileSync, writeSync
} from 'node:fs'
import { join } from 'node:path'
import {
  authorFromPrivateKey, base64Encode, blockId, chunkSize, createManifest, createRecord,
  decodeHash, envelopeTimestamp, exportPrivateKeyPkcs8, generatePrivateKey,
  importPrivateKey, maxChunks, maxManifest, parseJson, pemDecodePkcs8, pemEncodePkcs8,
  recordAuthor, recordPath, requireThat, utf8Encode, validateManifest, verifyBlock,
  verifyRecord
} from './core.mjs'
import { verifyPost } from './social.mjs'

export { parseJson, recordPath, requireThat } from './core.mjs'

export function syncDirectory(directory) {
  const fd = openSync(directory, 'r')
  try {
    fsyncSync(fd)
  } finally {
    closeSync(fd)
  }
}

export function persist(directory, name, bytes, replace = false) {
  const temporary = mkdtempSync(join(directory, '.write-'))
  const staged = join(temporary, 'data')
  const target = join(directory, name)
  let installed = true
  try {
    writeFileSync(staged, bytes, { mode: 0o600, flush: true })
    if (replace) {
      renameSync(staged, target)
    } else {
      try {
        linkSync(staged, target)
      } catch (error) {
        if (error.code !== 'EEXIST') throw error
        installed = false
      }
    }
    syncDirectory(directory)
    return installed
  } finally {
    rmSync(temporary, { recursive: true, force: true })
  }
}

export function initializeStore(root, parent) {
  mkdirSync(root, { mode: 0o700, recursive: true })
  syncDirectory(parent)
  for (const name of ['blocks', 'records', 'posts', 'cache']) {
    mkdirSync(join(root, name), { mode: 0o700, recursive: true })
  }
  syncDirectory(root)
}

export function readOptional(path) {
  try {
    return readFileSync(path)
  } catch (error) {
    if (error.code !== 'ENOENT') throw error
    return undefined
  }
}

export async function loadIdentity(root, create = false) {
  const path = join(root, 'identity.pem')
  let pem = readOptional(path)
  if (!pem && create) {
    const key = await generatePrivateKey()
    persist(root, 'identity.pem', pemEncodePkcs8(await exportPrivateKeyPkcs8(key)))
    pem = readFileSync(path)
  }
  requireThat(pem, 'Publish a repository before starting this node')
  const privateKey = await importPrivateKey(pemDecodePkcs8(pem))
  return { author: await authorFromPrivateKey(privateKey), privateKey }
}

// The filesystem implementation of the store interface in store.mjs.
// Records live at records/<recordPath>, blocks at blocks/<hash>, posts at
// posts/<recordPath>, and the selection at selected.json. Puts keep the
// existing durability semantics: staged file, fsync, atomic link or
// rename install, directory sync.
export function createFsStore(root) {
  const records = join(root, 'records')
  const blocks = join(root, 'blocks')
  const posts = join(root, 'posts')
  return {
    async getRecord(id) {
      const bytes = readOptional(join(records, recordPath(id)))
      return bytes && new Uint8Array(bytes)
    },
    async putRecord(id, bytes) {
      return persist(records, recordPath(id), bytes)
    },
    async getBlock(hash, maxSize) {
      decodeHash(hash, 32, true)
      requireThat(Number.isSafeInteger(maxSize) && maxSize >= 0, 'Invalid block size limit')
      let fd
      try {
        fd = openSync(join(blocks, hash), 'r')
      } catch (error) {
        if (error.code === 'ENOENT') return undefined
        throw error
      }
      try {
        const bytes = Buffer.allocUnsafe(maxSize + 1)
        const length = readInto(fd, bytes)
        requireThat(length <= maxSize, 'Block exceeds size limit')
        return new Uint8Array(bytes.subarray(0, length))
      } finally {
        closeSync(fd)
      }
    },
    async putBlock(hash, bytes) {
      decodeHash(hash, 32, true)
      return persist(blocks, hash, bytes)
    },
    async getPost(id) {
      const bytes = readOptional(join(posts, recordPath(id)))
      return bytes && new Uint8Array(bytes)
    },
    async putPost(id, bytes) {
      return persist(posts, recordPath(id), bytes)
    },
    async getSelection() {
      const bytes = readOptional(join(root, 'selected.json'))
      return bytes && parseJson(bytes)
    },
    async putSelection(selection) {
      persist(root, 'selected.json', JSON.stringify(selection), true)
    }
  }
}

export async function writeRecord(root, identity, body) {
  const record = await createRecord(identity, body)
  await createFsStore(root).putRecord(record.id, utf8Encode(JSON.stringify(record)))
  return record.id
}

export async function readRecord(root, author, id) {
  const bytes = await createFsStore(root).getRecord(id)
  requireThat(bytes, 'Invalid record')
  const record = parseJson(bytes)
  requireThat(record?.id === id, 'Invalid record')
  return verifyRecord(record, author)
}

// The author claimed by a stored record, without verifying it. Serving
// uses this to find the repository's publisher key, then verifies the
// record against that key; signing still uses the local identity.
export async function readRecordAuthor(root, id) {
  const bytes = await createFsStore(root).getRecord(id)
  requireThat(bytes, 'Invalid record')
  return recordAuthor(parseJson(bytes))
}

// Every stored record that verifies, with its signer-asserted payload
// timestamp. Unverifiable entries are skipped and logged so sync stays
// resilient; a bad record is never served. The caller orders the result.
export async function listVerifiedRecords(root) {
  const store = createFsStore(root)
  let entries
  try {
    entries = readdirSync(join(root, 'records'), { withFileTypes: true })
  } catch (error) {
    if (error.code === 'ENOENT') return []
    throw error
  }
  const verified = []
  for (const entry of entries) {
    if (!entry.isFile()) continue
    let id
    try {
      id = base64Encode(decodeHash(entry.name, 32, true))
    } catch {
      continue
    }
    try {
      const bytes = await store.getRecord(id)
      requireThat(bytes, 'Invalid record')
      const record = parseJson(bytes)
      requireThat(record?.id === id, 'Invalid record')
      await verifyRecord(record, recordAuthor(record))
      verified.push({ record, timestamp: envelopeTimestamp(record.signature) })
    } catch (error) {
      console.error(`Skipping unverified record ${entry.name}: ${error.message}`)
    }
  }
  return verified
}

// Every stored post that verifies, with its signer-asserted payload
// timestamp. Unverifiable entries are skipped and logged, like records,
// so a bad post is never served. The caller orders the result.
export async function listVerifiedPosts(root) {
  const store = createFsStore(root)
  let entries
  try {
    entries = readdirSync(join(root, 'posts'), { withFileTypes: true })
  } catch (error) {
    if (error.code === 'ENOENT') return []
    throw error
  }
  const verified = []
  for (const entry of entries) {
    if (!entry.isFile()) continue
    let id
    try {
      id = base64Encode(decodeHash(entry.name, 32, true))
    } catch {
      continue
    }
    try {
      const bytes = await store.getPost(id)
      requireThat(bytes, 'Invalid post')
      const record = parseJson(bytes)
      requireThat(record?.id === id, 'Invalid post')
      const post = await verifyPost(record)
      verified.push({ record, timestamp: post.timestamp })
    } catch (error) {
      console.error(`Skipping unverified post ${entry.name}: ${error.message}`)
    }
  }
  return verified
}

async function writeBlock(root, bytes) {
  const id = await blockId(bytes)
  await createFsStore(root).putBlock(id, bytes)
  return id
}

async function readBlock(root, id, maxSize) {
  const block = await createFsStore(root).getBlock(id, maxSize)
  requireThat(block, 'Missing AndFS block')
  requireThat(await verifyBlock(block, id), 'Corrupt AndFS block')
  return block
}

function readInto(fd, buffer) {
  let offset = 0
  while (offset < buffer.length) {
    const length = readSync(fd, buffer, offset, buffer.length - offset, null)
    if (length === 0) break
    offset += length
  }
  return offset
}

export async function storeBundle(root, path) {
  const fd = openSync(path, 'r')
  try {
    const chunks = []
    const buffer = Buffer.allocUnsafe(chunkSize)
    let size = 0
    for (let length; (length = readInto(fd, buffer)) > 0;) {
      requireThat(chunks.length < maxChunks, 'Bundle exceeds AndFS size limit')
      chunks.push(await writeBlock(root, buffer.subarray(0, length)))
      size += length
    }
    return await writeBlock(root, createManifest(size, chunks))
  } finally {
    closeSync(fd)
  }
}

export async function recoverBundle(root, id, path) {
  const manifest = validateManifest(await readBlock(root, id, maxManifest))
  const fd = openSync(path, 'wx', 0o600)
  try {
    let remaining = manifest.size
    for (const hash of manifest.chunks) {
      const expected = Math.min(chunkSize, remaining)
      const chunk = await readBlock(root, hash, expected)
      requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
      let offset = 0
      while (offset < chunk.length) offset += writeSync(fd, chunk, offset, chunk.length - offset)
      remaining -= chunk.length
    }
    requireThat(remaining === 0, 'Unexpected AndFS file size')
  } finally {
    closeSync(fd)
  }
}

Git ยท Social feed