root / storage.mjs
9375 bytes
import {
closeSync, fsyncSync, linkSync, mkdirSync, mkdtempSync, openSync, readFileSync,
readSync, readdirSync, renameSync, rmSync, writeFileSync, writeSync
} from 'node:fs'
import { join } from 'node:path'
import {
authorFromPrivateKey, base64Encode, blockId, chunkSize, createManifest, createRecord,
decodeHash, envelopeTimestamp, exportPrivateKeyPkcs8, generatePrivateKey,
importPrivateKey, maxChunks, maxManifest, parseJson, pemDecodePkcs8, pemEncodePkcs8,
recordAuthor, recordPath, requireThat, utf8Encode, validateManifest, verifyBlock,
verifyRecord
} from './core.mjs'
import { verifyPost } from './social.mjs'
export { parseJson, recordPath, requireThat } from './core.mjs'
export function syncDirectory(directory) {
const fd = openSync(directory, 'r')
try {
fsyncSync(fd)
} finally {
closeSync(fd)
}
}
export function persist(directory, name, bytes, replace = false) {
const temporary = mkdtempSync(join(directory, '.write-'))
const staged = join(temporary, 'data')
const target = join(directory, name)
let installed = true
try {
writeFileSync(staged, bytes, { mode: 0o600, flush: true })
if (replace) {
renameSync(staged, target)
} else {
try {
linkSync(staged, target)
} catch (error) {
if (error.code !== 'EEXIST') throw error
installed = false
}
}
syncDirectory(directory)
return installed
} finally {
rmSync(temporary, { recursive: true, force: true })
}
}
export function initializeStore(root, parent) {
mkdirSync(root, { mode: 0o700, recursive: true })
syncDirectory(parent)
for (const name of ['blocks', 'records', 'posts', 'cache']) {
mkdirSync(join(root, name), { mode: 0o700, recursive: true })
}
syncDirectory(root)
}
export function readOptional(path) {
try {
return readFileSync(path)
} catch (error) {
if (error.code !== 'ENOENT') throw error
return undefined
}
}
export async function loadIdentity(root, create = false) {
const path = join(root, 'identity.pem')
let pem = readOptional(path)
if (!pem && create) {
const key = await generatePrivateKey()
persist(root, 'identity.pem', pemEncodePkcs8(await exportPrivateKeyPkcs8(key)))
pem = readFileSync(path)
}
requireThat(pem, 'Publish a repository before starting this node')
const privateKey = await importPrivateKey(pemDecodePkcs8(pem))
return { author: await authorFromPrivateKey(privateKey), privateKey }
}
// The filesystem implementation of the store interface in store.mjs.
// Records live at records/<recordPath>, blocks at blocks/<hash>, posts at
// posts/<recordPath>, and the selection at selected.json. Puts keep the
// existing durability semantics: staged file, fsync, atomic link or
// rename install, directory sync.
export function createFsStore(root) {
const records = join(root, 'records')
const blocks = join(root, 'blocks')
const posts = join(root, 'posts')
return {
async getRecord(id) {
const bytes = readOptional(join(records, recordPath(id)))
return bytes && new Uint8Array(bytes)
},
async putRecord(id, bytes) {
return persist(records, recordPath(id), bytes)
},
async getBlock(hash, maxSize) {
decodeHash(hash, 32, true)
requireThat(Number.isSafeInteger(maxSize) && maxSize >= 0, 'Invalid block size limit')
let fd
try {
fd = openSync(join(blocks, hash), 'r')
} catch (error) {
if (error.code === 'ENOENT') return undefined
throw error
}
try {
const bytes = Buffer.allocUnsafe(maxSize + 1)
const length = readInto(fd, bytes)
requireThat(length <= maxSize, 'Block exceeds size limit')
return new Uint8Array(bytes.subarray(0, length))
} finally {
closeSync(fd)
}
},
async putBlock(hash, bytes) {
decodeHash(hash, 32, true)
return persist(blocks, hash, bytes)
},
async getPost(id) {
const bytes = readOptional(join(posts, recordPath(id)))
return bytes && new Uint8Array(bytes)
},
async putPost(id, bytes) {
return persist(posts, recordPath(id), bytes)
},
async getSelection() {
const bytes = readOptional(join(root, 'selected.json'))
return bytes && parseJson(bytes)
},
async putSelection(selection) {
persist(root, 'selected.json', JSON.stringify(selection), true)
}
}
}
export async function writeRecord(root, identity, body) {
const record = await createRecord(identity, body)
await createFsStore(root).putRecord(record.id, utf8Encode(JSON.stringify(record)))
return record.id
}
export async function readRecord(root, author, id) {
const bytes = await createFsStore(root).getRecord(id)
requireThat(bytes, 'Invalid record')
const record = parseJson(bytes)
requireThat(record?.id === id, 'Invalid record')
return verifyRecord(record, author)
}
// The author claimed by a stored record, without verifying it. Serving
// uses this to find the repository's publisher key, then verifies the
// record against that key; signing still uses the local identity.
export async function readRecordAuthor(root, id) {
const bytes = await createFsStore(root).getRecord(id)
requireThat(bytes, 'Invalid record')
return recordAuthor(parseJson(bytes))
}
// Every stored record that verifies, with its signer-asserted payload
// timestamp. Unverifiable entries are skipped and logged so sync stays
// resilient; a bad record is never served. The caller orders the result.
export async function listVerifiedRecords(root) {
const store = createFsStore(root)
let entries
try {
entries = readdirSync(join(root, 'records'), { withFileTypes: true })
} catch (error) {
if (error.code === 'ENOENT') return []
throw error
}
const verified = []
for (const entry of entries) {
if (!entry.isFile()) continue
let id
try {
id = base64Encode(decodeHash(entry.name, 32, true))
} catch {
continue
}
try {
const bytes = await store.getRecord(id)
requireThat(bytes, 'Invalid record')
const record = parseJson(bytes)
requireThat(record?.id === id, 'Invalid record')
await verifyRecord(record, recordAuthor(record))
verified.push({ record, timestamp: envelopeTimestamp(record.signature) })
} catch (error) {
console.error(`Skipping unverified record ${entry.name}: ${error.message}`)
}
}
return verified
}
// Every stored post that verifies, with its signer-asserted payload
// timestamp. Unverifiable entries are skipped and logged, like records,
// so a bad post is never served. The caller orders the result.
export async function listVerifiedPosts(root) {
const store = createFsStore(root)
let entries
try {
entries = readdirSync(join(root, 'posts'), { withFileTypes: true })
} catch (error) {
if (error.code === 'ENOENT') return []
throw error
}
const verified = []
for (const entry of entries) {
if (!entry.isFile()) continue
let id
try {
id = base64Encode(decodeHash(entry.name, 32, true))
} catch {
continue
}
try {
const bytes = await store.getPost(id)
requireThat(bytes, 'Invalid post')
const record = parseJson(bytes)
requireThat(record?.id === id, 'Invalid post')
const post = await verifyPost(record)
verified.push({ record, timestamp: post.timestamp })
} catch (error) {
console.error(`Skipping unverified post ${entry.name}: ${error.message}`)
}
}
return verified
}
async function writeBlock(root, bytes) {
const id = await blockId(bytes)
await createFsStore(root).putBlock(id, bytes)
return id
}
async function readBlock(root, id, maxSize) {
const block = await createFsStore(root).getBlock(id, maxSize)
requireThat(block, 'Missing AndFS block')
requireThat(await verifyBlock(block, id), 'Corrupt AndFS block')
return block
}
function readInto(fd, buffer) {
let offset = 0
while (offset < buffer.length) {
const length = readSync(fd, buffer, offset, buffer.length - offset, null)
if (length === 0) break
offset += length
}
return offset
}
export async function storeBundle(root, path) {
const fd = openSync(path, 'r')
try {
const chunks = []
const buffer = Buffer.allocUnsafe(chunkSize)
let size = 0
for (let length; (length = readInto(fd, buffer)) > 0;) {
requireThat(chunks.length < maxChunks, 'Bundle exceeds AndFS size limit')
chunks.push(await writeBlock(root, buffer.subarray(0, length)))
size += length
}
return await writeBlock(root, createManifest(size, chunks))
} finally {
closeSync(fd)
}
}
export async function recoverBundle(root, id, path) {
const manifest = validateManifest(await readBlock(root, id, maxManifest))
const fd = openSync(path, 'wx', 0o600)
try {
let remaining = manifest.size
for (const hash of manifest.chunks) {
const expected = Math.min(chunkSize, remaining)
const chunk = await readBlock(root, hash, expected)
requireThat(chunk.length === expected, 'Unexpected AndFS chunk length')
let offset = 0
while (offset < chunk.length) offset += writeSync(fd, chunk, offset, chunk.length - offset)
remaining -= chunk.length
}
requireThat(remaining === 0, 'Unexpected AndFS file size')
} finally {
closeSync(fd)
}
}
Git ยท Social feed