0431c76 / anbot.mjs
23386 bytes
#!/usr/bin/env node
import { spawn } from 'node:child_process'
import { existsSync, readFileSync, realpathSync } from 'node:fs'
import { createServer } from 'node:http'
import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'
import { parseArgs } from 'node:util'
import {
checkoutRoot, clonePath, blobExists, commitDiff, commitFiles, gitEnv, gitOptions,
listCommits, listTree, publish, readBlob, readCommit, readPushAnnouncements,
repositoryName, selectedRepository, verifySelection
} from './repository.mjs'
import {
createFsStore, initializeStore, listVerifiedPosts, listVerifiedRecords, loadIdentity,
readRecordAuthor, recordPath, requireThat
} from './storage.mjs'
import {
base64Decode, base64Encode, maxManifest, parseJson, utf8Encode, verifyBlock
} from './core.mjs'
import { pushAuth, pushWithIdentity } from './push-auth.mjs'
import { receivePush } from './receive.mjs'
import { receivePost } from './post-http.mjs'
import { publishPost, readFeed, submitPost } from './social-store.mjs'
import { verifyPost } from './social.mjs'
import { pull } from './sync.mjs'
import { blobPage, commitPage, diffPage, feedPage, indexPage, notFoundPage, treePage } from './webui.mjs'
const feedScript = readFileSync(new URL('./feed-live.js', import.meta.url))
const help = `Usage:
node anbot.mjs publish --data DIR --source CHECKOUT --branch BRANCH [--name NAME] [--host HOST] [--port PORT]
node anbot.mjs serve --data DIR [--host HOST] [--port PORT] [--push-allowlist FILE] [--push-origin ORIGIN]
node anbot.mjs push --data DIR --url CLONE_URL [--source CHECKOUT] [--branch BRANCH]
node anbot.mjs sync --data DIR --url NODE_URL
node anbot.mjs identity --data DIR
node anbot.mjs post --data DIR --text TEXT [--reply-to POST_ID] [--url NODE_URL]
node anbot.mjs post --data DIR --id POST_ID --url NODE_URL
The default listen address is 127.0.0.1:8787. The data directory's parent must exist.`
function serveGit(request, response, bare, endpoint, query, write = false) {
const child = spawn('git', [...gitOptions, '-c', `http.receivepack=${write}`, 'http-backend'], {
env: {
...gitEnv,
GIT_PROJECT_ROOT: dirname(bare),
GIT_HTTP_EXPORT_ALL: '1',
PATH_INFO: `/${basename(bare)}${endpoint}`,
REQUEST_METHOD: request.method,
QUERY_STRING: query,
CONTENT_TYPE: request.headers['content-type'] ?? '',
CONTENT_LENGTH: request.headers['content-length'] ?? '',
HTTP_GIT_PROTOCOL: request.headers['git-protocol'] ?? '',
REMOTE_ADDR: request.socket.remoteAddress ?? ''
},
stdio: ['pipe', 'pipe', 'inherit']
})
function fail() {
if (response.headersSent) response.destroy()
else response.writeHead(502).end('Git backend failed\n')
child.kill()
}
let headers = Buffer.alloc(0)
child.stdout.on('data', function readHeaders(chunk) {
headers = Buffer.concat([headers, chunk])
const end = headers.indexOf('\r\n\r\n')
if (end === -1) {
if (headers.length > 16384) fail()
return
}
try {
let status = 200
for (const line of headers.subarray(0, end).toString('latin1').split('\r\n')) {
const colon = line.indexOf(':')
requireThat(colon > 0, 'Invalid Git backend header')
const name = line.slice(0, colon)
const value = line.slice(colon + 1).trim()
if (name.toLowerCase() === 'status') status = Number(value.split(' ')[0])
else response.setHeader(name, value)
}
response.writeHead(status)
child.stdout.off('data', readHeaders)
const body = headers.subarray(end + 4)
if (!response.write(body)) {
child.stdout.pause()
response.once('drain', () => child.stdout.resume())
}
child.stdout.pipe(response)
} catch {
fail()
}
})
child.on('error', fail)
child.on('close', () => {
if (!response.headersSent) fail()
})
child.stdin.on('error', error => {
if (error.code !== 'EPIPE') fail()
})
response.on('close', () => {
if (!response.writableFinished) child.kill()
})
request.on('error', () => child.kill())
request.pipe(child.stdin)
}
function serveHtml(response, status, html) {
response.writeHead(status, {
'Content-Type': 'text/html; charset=utf-8', 'Cache-Control': 'no-store'
}).end(html)
}
// The poll cursor is opaque to clients: the unpadded Base64url form of the
// JSON array [payloadTimestamp, recordId] of the last record served. The
// (timestamp, id) pair orders records totally and survives restarts with
// no server-side index, because both values come from verified records.
const pollPageSize = 100
function encodeCursor(timestamp, id) {
return base64Encode(utf8Encode(JSON.stringify([timestamp, id])), true)
}
function decodeCursor(since) {
const decoded = parseJson(base64Decode(since, true))
requireThat(Array.isArray(decoded) && decoded.length === 2 &&
Number.isSafeInteger(decoded[0]) && typeof decoded[1] === 'string', 'Invalid poll cursor')
return decoded
}
function compareRecords(a, b) {
return (a.timestamp - b.timestamp) ||
(a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0)
}
async function servePoll(root, response, query) {
let cursor = null
try {
const since = new URLSearchParams(query).get('since')
if (since) cursor = decodeCursor(since)
} catch {
response.writeHead(400).end('Invalid poll cursor\n')
return
}
try {
const [records, posts, selection] = await Promise.all([
listVerifiedRecords(root), listVerifiedPosts(root), createFsStore(root).getSelection()
])
const byId = new Map(records.map(entry => [entry.record.id, entry]))
const visible = new Set([selection.repoId])
for (let id = selection.updateId; id != null;) {
requireThat(!visible.has(id) && byId.has(id), 'Incomplete selected update history')
visible.add(id)
const { previous, previousUpdate } = parseJson(byId.get(id).record.content)
id = previous ?? previousUpdate
}
const shared = records.filter(({ record }) =>
visible.has(record.id))
const verified = [...new Map([...shared, ...posts].map(entry => [entry.record.id, entry])).values()]
verified.sort(compareRecords)
const after = cursor
? verified.filter(({ record, timestamp }) =>
timestamp > cursor[0] || (timestamp === cursor[0] && record.id > cursor[1]))
: verified
const page = after.slice(0, pollPageSize)
const last = page.at(-1)
response.writeHead(200, {
'Content-Type': 'application/json', 'Cache-Control': 'no-store',
'Access-Control-Allow-Origin': '*'
}).end(JSON.stringify({
records: page.map(({ record }) => record),
nextSince: last ? encodeCursor(last.timestamp, last.record.id) : null
}))
} catch (error) {
console.error(error.message)
response.writeHead(503).end('Publication unavailable\n')
}
}
async function serveBlock(root, response, hash) {
try {
const bytes = await createFsStore(root).getBlock(hash, maxManifest)
if (bytes === undefined) {
response.writeHead(404).end('Unknown block\n')
return
}
requireThat(await verifyBlock(bytes, hash), 'Corrupt AndFS block')
response.writeHead(200, {
'Content-Type': 'application/octet-stream',
'Cache-Control': 'public, max-age=31536000, immutable',
'Access-Control-Allow-Origin': '*'
}).end(bytes)
} catch (error) {
if (error.message === 'Invalid encoding') {
response.writeHead(404).end('Unknown block\n')
return
}
console.error(error.message)
response.writeHead(503).end('Block unavailable\n')
}
}
function parseWebPath(rest) {
const trimmed = rest.replace(/\/+$/, '')
if (!trimmed) return ''
const segments = []
for (const raw of trimmed.split('/')) {
let segment
try {
segment = decodeURIComponent(raw)
} catch {
return null
}
if (!segment || segment === '.' || segment === '..' ||
segment.includes('/') || segment.includes('\0')) return null
segments.push(segment)
}
return segments.join('/')
}
function matchWebRoute(pathname) {
if (pathname === '/git' || pathname === '/git/') return { kind: 'index' }
if (pathname === '/tree' || pathname.startsWith('/tree/')) {
return { kind: 'tree', path: parseWebPath(pathname.slice('/tree'.length + 1)) }
}
if (pathname.startsWith('/blob/')) {
const path = parseWebPath(pathname.slice('/blob/'.length))
return { kind: 'blob', path: path === '' ? null : path }
}
const commitDiff = /^\/commit\/([0-9a-f]{40})\/diff\/(.+)$/.exec(pathname)
if (commitDiff) {
const path = parseWebPath(commitDiff[2])
return { kind: 'diff', hash: commitDiff[1], path: path === '' ? null : path }
}
const commitBlob = /^\/commit\/([0-9a-f]{40})\/blob\/(.+)$/.exec(pathname)
if (commitBlob) {
const path = parseWebPath(commitBlob[2])
return { kind: 'commitBlob', hash: commitBlob[1], path: path === '' ? null : path }
}
const commit = /^\/commit\/([0-9a-f]{40})$/.exec(pathname)
if (commit) return { kind: 'commit', hash: commit[1] }
return null
}
async function serveFeed(root, query, response) {
const params = new URLSearchParams(query)
const author = params.get('author')
const postId = params.get('post')
const updateId = params.get('update')
try {
requireThat([author, postId, updateId].filter(id => id !== null).length <= 1, 'Choose one feed view')
const id = author ?? postId ?? updateId
if (id !== null) recordPath(id)
} catch {
serveHtml(response, 404, feedPage({ missing: true }))
return
}
const [social, pushes] = await Promise.all([readFeed(root), readPushAnnouncements(root)])
const posts = [...social, ...pushes].sort((a, b) => b.timestamp - a.timestamp || a.id.localeCompare(b.id))
const post = posts.find(item => item.id === postId || item.updateId === updateId)
const missing = (postId !== null || updateId !== null) && !post
const matching = posts.filter(item =>
post ? item.replyTo === post.id : author === null || item.author === author)
serveHtml(response, missing ? 404 : 200, feedPage({ posts: matching, author, post, missing }))
}
async function serveRawPost(root, query, response) {
const id = new URLSearchParams(query).get('post')
try {
recordPath(id)
} catch {
response.writeHead(404).end('Post not found\n')
return
}
try {
const stored = await createFsStore(root).getPost(id)
let record
if (stored) {
record = parseJson(stored)
const post = await verifyPost(record)
requireThat(post.id === id, 'Stored post does not match its record ID')
} else {
record = (await readPushAnnouncements(root)).find(post => post.id === id)?.record
}
if (!record) {
response.writeHead(404).end('Post not found\n')
return
}
response.writeHead(200, {
'Content-Type': 'application/json; charset=utf-8', 'Cache-Control': 'no-store',
'X-Content-Type-Options': 'nosniff'
}).end(JSON.stringify({ id: record.id, signature: record.signature, content: record.content }, null, 2) + '\n')
} catch (error) {
console.error(error.message)
response.writeHead(503).end('Post unavailable\n')
}
}
async function runServer(root, identity, host, port, baseUrl, allowlist, pushOrigin) {
const verified = new Map()
const { repoId } = await selectedRepository(root, verified)
const repoHash = recordPath(repoId)
const publisher = await readRecordAuthor(root, repoId)
const legacyPath = `/git/${repoHash}/${recordPath(publisher)}.git`
const displayOrigin = new URL(pushOrigin ?? baseUrl).origin
const auth = allowlist && pushAuth(resolve(allowlist), displayOrigin, repoId)
let receiving = false
const server = createServer(async (request, response) => {
const [pathname, query = ''] = request.url.split('?', 2)
if (request.method === 'POST' && pathname === '/posts' && !query) {
await receivePost(root, request, response, auth?.isAllowed)
return
}
if (request.method === 'GET') {
if (pathname === '/raw') {
await serveRawPost(root, query, response)
return
}
if (pathname === '/feed-live.js') {
response.writeHead(200, {
'Content-Type': 'text/javascript; charset=utf-8', 'Cache-Control': 'no-cache'
}).end(feedScript)
return
}
if (pathname === '/' || pathname === '/feed') {
try {
await serveFeed(root, query, response)
} catch (error) {
console.error(error.message)
response.writeHead(503).end('Feed unavailable\n')
}
return
}
if (pathname === '/gossip/poll') {
await servePoll(root, response, query)
return
}
const block = /^\/blocks\/([A-Za-z0-9_-]+)$/.exec(pathname)
if (block) {
await serveBlock(root, response, block[1])
return
}
const route = matchWebRoute(pathname)
if (route) {
try {
const selected = await selectedRepository(root, verified)
requireThat(selected.repoId === repoId, 'The served repository ID changed')
const name = await repositoryName(root, repoId)
const branch = selected.ref.slice('refs/heads/'.length)
if (route.path === null) {
serveHtml(response, 404, notFoundPage(name, 'That page'))
return
}
if (route.kind === 'index') {
const commits = listCommits(selected.bare, selected.ref, 50)
const entries = listTree(selected.bare, selected.ref, '')
requireThat(entries, 'Cannot list the repository root')
serveHtml(response, 200, indexPage({
name, branch, repoId, updateId: selected.updateId,
cloneUrl: displayOrigin + clonePath(repoId), commits, entries
}))
return
}
if (route.kind === 'tree') {
const entries = listTree(selected.bare, selected.ref, route.path)
if (!entries) {
serveHtml(response, 404, notFoundPage(name, `Directory ${route.path}`))
return
}
serveHtml(response, 200, treePage({ name, path: route.path, entries }))
return
}
if (route.kind === 'blob') {
const blob = readBlob(selected.bare, selected.ref, route.path)
if (!blob) {
serveHtml(response, 404, notFoundPage(name, `File ${route.path}`))
return
}
serveHtml(response, 200, blobPage({ name, path: route.path, ...blob }))
return
}
if (route.kind === 'diff' || route.kind === 'commitBlob') {
const commit = readCommit(selected.bare, route.hash)
if (!commit) {
serveHtml(response, 404, notFoundPage(name, `Commit ${route.hash}`))
return
}
if (route.kind === 'diff') {
const diff = commitDiff(selected.bare, route.hash, route.path)
if (!diff) {
serveHtml(response, 404, notFoundPage(name, `File ${route.path} in commit ${route.hash}`))
return
}
const hasBlob = blobExists(selected.bare, route.hash, route.path)
serveHtml(response, 200, diffPage({ name, commit, path: route.path, ...diff, hasBlob }))
return
}
const blob = readBlob(selected.bare, route.hash, route.path)
if (!blob) {
serveHtml(response, 404, notFoundPage(name, `File ${route.path} in commit ${route.hash}`))
return
}
serveHtml(response, 200, blobPage({ name, path: route.path, commit, ...blob }))
return
}
const commit = readCommit(selected.bare, route.hash)
if (!commit) {
serveHtml(response, 404, notFoundPage(name, `Commit ${route.hash}`))
return
}
serveHtml(response, 200, commitPage({
name, commit, files: commitFiles(selected.bare, route.hash)
}))
} catch (error) {
console.error(error.message)
if (!response.headersSent) response.writeHead(503).end('Publication unavailable\n')
}
return
}
}
const [, repositoryPath, endpoint] =
/^(.+\.git)(\/(?:info\/refs|git-upload-pack|git-receive-pack|anbot-auth))$/.exec(pathname) ?? []
const prefix = /^\/git\/([A-Za-z0-9_-]+)\.git$/.exec(repositoryPath)?.[1]
// This node exposes one repository, so every matching prefix is unique.
const matches = repositoryPath === legacyPath || (prefix && repoHash.startsWith(prefix))
const advertise = request.method === 'GET' && endpoint === '/info/refs'
const readAdvertisement = advertise && query === 'service=git-upload-pack'
const writeAdvertisement = advertise && query === 'service=git-receive-pack'
const upload = request.method === 'POST' && endpoint === '/git-upload-pack' && !query
const receive = request.method === 'POST' && endpoint === '/git-receive-pack' && !query
const authenticate = ['GET', 'POST'].includes(request.method) && endpoint === '/anbot-auth' && !query
const write = writeAdvertisement || receive
if (!matches || (!readAdvertisement && !upload && !(auth && (write || authenticate)))) {
response.writeHead(404).end('Unknown Git endpoint\n')
return
}
try {
if (authenticate) {
await auth.challenge(request, response)
return
}
if (write && !auth.authorized(request)) {
response.writeHead(403).end('Push access denied\n')
return
}
if (receive) {
if (receiving) {
response.writeHead(503).end('Another push is in progress. Retry shortly.\n')
return
}
receiving = true
try {
await receivePush(root, identity, request, response, () => auth.authorized(request))
} finally {
receiving = false
}
return
}
const selected = await selectedRepository(root, verified)
requireThat(selected.repoId === repoId, 'The served repository ID changed')
serveGit(request, response, selected.bare, endpoint, query, write)
} catch (error) {
console.error(error.message)
response.writeHead(503).end('Publication unavailable\n')
}
})
server.on('error', error => {
console.error(error.message)
process.exitCode = 1
})
server.listen(port, host, () => console.log(baseUrl + clonePath(repoId)))
}
async function main() {
const [command, ...args] = process.argv.slice(2)
if (!command || command === '--help' || args.includes('--help')) {
console.log(help)
return
}
requireThat(['publish', 'serve', 'push', 'sync', 'identity', 'post'].includes(command), help)
const options = {
data: { type: 'string' }, host: { type: 'string', default: '127.0.0.1' },
port: { type: 'string', default: '8787' }
}
if (command === 'publish') {
Object.assign(options, {
source: { type: 'string' }, branch: { type: 'string' }, name: { type: 'string' }
})
}
if (command === 'serve') {
options['push-allowlist'] = { type: 'string' }
options['push-origin'] = { type: 'string' }
}
if (command === 'push') {
Object.assign(options, {
source: { type: 'string', default: '.' },
branch: { type: 'string', default: 'main' },
url: { type: 'string' }
})
}
if (command === 'post') {
options.text = { type: 'string' }
options['reply-to'] = { type: 'string' }
options.id = { type: 'string' }
options.url = { type: 'string' }
}
if (command === 'sync') {
options.url = { type: 'string' }
}
const { values } = parseArgs({ args, options })
requireThat(values.data, '--data is required')
const data = resolve(values.data)
const parent = realpathSync(dirname(data))
const root = existsSync(data) ? realpathSync(data) : resolve(parent, basename(data))
if (command === 'identity') {
initializeStore(root, dirname(root))
console.log((await loadIdentity(root, true)).author)
return
}
if (command === 'push') {
requireThat(values.url, '--url is required')
await pushWithIdentity(await loadIdentity(root), values.url, values.source, values.branch)
return
}
if (command === 'post') {
const retry = values.id !== undefined
requireThat(retry ? values.url && values.text === undefined && values['reply-to'] === undefined
: values.text !== undefined, 'Supply --text, or --id with --url to retry')
const remote = values.url === undefined ? null : new URL(values.url)
requireThat(!remote || ((remote.protocol === 'https:' ||
(remote.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(remote.hostname))) &&
!remote.username && !remote.password && !remote.search && !remote.hash),
'Use an HTTPS node URL or local loopback URL')
const identity = await loadIdentity(root)
const id = retry ? values.id : await publishPost(root, identity, values.text, values['reply-to'])
if (remote) {
try {
await submitPost(root, identity, id, remote)
} catch (error) {
throw new Error(`${error.message}. Retry an existing local post with --id ${id} --url ${remote.origin}`)
}
}
console.log(JSON.stringify({ id }))
return
}
if (command === 'sync') {
requireThat(values.url, '--url is required')
const remote = new URL(values.url)
requireThat((remote.protocol === 'https:' ||
(remote.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(remote.hostname))) &&
!remote.username && !remote.password && !remote.search && !remote.hash,
'Use an HTTPS node URL or local loopback URL')
initializeStore(root, dirname(root))
await loadIdentity(root, true)
const result = await pull(createFsStore(root), remote.origin, {
async verifyBundle(update) {
await verifySelection(root, { repoId: update.repoId, ref: update.body.ref, updateId: update.id })
}
})
console.log(JSON.stringify(result))
return
}
const port = Number(values.port)
requireThat(/^\d+$/.test(values.port) && port >= 1 && port <= 65535, 'Invalid port')
requireThat(/^[a-zA-Z0-9.-]+$/.test(values.host), 'Use an IPv4 address or hostname')
const baseUrl = `http://${values.host}:${port}`
if (command === 'serve') {
await runServer(root, await loadIdentity(root), values.host, port, baseUrl,
values['push-allowlist'], values['push-origin'])
return
}
requireThat(values.source && values.branch, '--source and --branch are required')
const source = realpathSync(checkoutRoot(realpathSync(values.source)))
const fromSource = relative(source, root)
requireThat(fromSource === '..' || fromSource.startsWith('..' + sep) ||
isAbsolute(fromSource), 'Keep the data directory outside the source checkout')
initializeStore(root, dirname(root))
const identity = await loadIdentity(root, true)
const result = await publish(root, identity, source, values.branch, values.name ?? basename(source))
console.log(JSON.stringify({ ...result, cloneUrl: baseUrl + clonePath(result.repoId) }))
}
try {
await main()
} catch (error) {
console.error(error.message)
process.exitCode = 1
}
Git ยท Social feed