anbot

78ef64a / anbot.mjs

22187 bytes

#!/usr/bin/env node
import { spawn } from 'node:child_process'
import { existsSync, readFileSync, realpathSync } from 'node:fs'
import { createServer } from 'node:http'
import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'
import { parseArgs } from 'node:util'
import {
  checkoutRoot, clonePath, blobExists, commitDiff, commitFiles, gitEnv, gitOptions,
  listCommits, listTree, publish, readBlob, readCommit, readPushAnnouncements,
  repositoryName, selectedRepository, verifySelection
} from './repository.mjs'
import {
  createFsStore, initializeStore, listVerifiedPosts, listVerifiedRecords, loadIdentity,
  readRecordAuthor, recordPath, requireThat
} from './storage.mjs'
import {
  base64Decode, base64Encode, maxManifest, parseJson, utf8Encode, verifyBlock
} from './core.mjs'
import { pushAuth, pushWithIdentity } from './push-auth.mjs'
import { receivePush } from './receive.mjs'
import { receivePost } from './post-http.mjs'
import { publishPost, readFeed, submitPost } from './social-store.mjs'
import { pull } from './sync.mjs'
import { blobPage, commitPage, diffPage, feedPage, indexPage, notFoundPage, treePage } from './webui.mjs'

const feedScript = readFileSync(new URL('./feed-live.js', import.meta.url))

const help = `Usage:
  node anbot.mjs publish --data DIR --source CHECKOUT --branch BRANCH [--name NAME] [--host HOST] [--port PORT]
  node anbot.mjs serve   --data DIR [--host HOST] [--port PORT] [--push-allowlist FILE] [--push-origin ORIGIN]
  node anbot.mjs push    --data DIR --url CLONE_URL [--source CHECKOUT] [--branch BRANCH]
  node anbot.mjs sync    --data DIR --url NODE_URL
  node anbot.mjs identity --data DIR
  node anbot.mjs post    --data DIR --text TEXT [--reply-to POST_ID] [--url NODE_URL]
  node anbot.mjs post    --data DIR --id POST_ID --url NODE_URL

The default listen address is 127.0.0.1:8787. The data directory's parent must exist.`

function serveGit(request, response, bare, endpoint, query, write = false) {
  const child = spawn('git', [...gitOptions, '-c', `http.receivepack=${write}`, 'http-backend'], {
    env: {
      ...gitEnv,
      GIT_PROJECT_ROOT: dirname(bare),
      GIT_HTTP_EXPORT_ALL: '1',
      PATH_INFO: `/${basename(bare)}${endpoint}`,
      REQUEST_METHOD: request.method,
      QUERY_STRING: query,
      CONTENT_TYPE: request.headers['content-type'] ?? '',
      CONTENT_LENGTH: request.headers['content-length'] ?? '',
      HTTP_GIT_PROTOCOL: request.headers['git-protocol'] ?? '',
      REMOTE_ADDR: request.socket.remoteAddress ?? ''
    },
    stdio: ['pipe', 'pipe', 'inherit']
  })

  function fail() {
    if (response.headersSent) response.destroy()
    else response.writeHead(502).end('Git backend failed\n')
    child.kill()
  }

  let headers = Buffer.alloc(0)
  child.stdout.on('data', function readHeaders(chunk) {
    headers = Buffer.concat([headers, chunk])
    const end = headers.indexOf('\r\n\r\n')
    if (end === -1) {
      if (headers.length > 16384) fail()
      return
    }
    try {
      let status = 200
      for (const line of headers.subarray(0, end).toString('latin1').split('\r\n')) {
        const colon = line.indexOf(':')
        requireThat(colon > 0, 'Invalid Git backend header')
        const name = line.slice(0, colon)
        const value = line.slice(colon + 1).trim()
        if (name.toLowerCase() === 'status') status = Number(value.split(' ')[0])
        else response.setHeader(name, value)
      }
      response.writeHead(status)
      child.stdout.off('data', readHeaders)
      const body = headers.subarray(end + 4)
      if (!response.write(body)) {
        child.stdout.pause()
        response.once('drain', () => child.stdout.resume())
      }
      child.stdout.pipe(response)
    } catch {
      fail()
    }
  })
  child.on('error', fail)
  child.on('close', () => {
    if (!response.headersSent) fail()
  })
  child.stdin.on('error', error => {
    if (error.code !== 'EPIPE') fail()
  })
  response.on('close', () => {
    if (!response.writableFinished) child.kill()
  })
  request.on('error', () => child.kill())
  request.pipe(child.stdin)
}

function serveHtml(response, status, html) {
  response.writeHead(status, {
    'Content-Type': 'text/html; charset=utf-8', 'Cache-Control': 'no-store'
  }).end(html)
}

// The poll cursor is opaque to clients: the unpadded Base64url form of the
// JSON array [payloadTimestamp, recordId] of the last record served. The
// (timestamp, id) pair orders records totally and survives restarts with
// no server-side index, because both values come from verified records.
const pollPageSize = 100

function encodeCursor(timestamp, id) {
  return base64Encode(utf8Encode(JSON.stringify([timestamp, id])), true)
}

function decodeCursor(since) {
  const decoded = parseJson(base64Decode(since, true))
  requireThat(Array.isArray(decoded) && decoded.length === 2 &&
    Number.isSafeInteger(decoded[0]) && typeof decoded[1] === 'string', 'Invalid poll cursor')
  return decoded
}

function compareRecords(a, b) {
  return (a.timestamp - b.timestamp) ||
    (a.record.id < b.record.id ? -1 : a.record.id > b.record.id ? 1 : 0)
}

async function servePoll(root, response, query) {
  let cursor = null
  try {
    const since = new URLSearchParams(query).get('since')
    if (since) cursor = decodeCursor(since)
  } catch {
    response.writeHead(400).end('Invalid poll cursor\n')
    return
  }
  try {
    const [records, posts, selection] = await Promise.all([
      listVerifiedRecords(root), listVerifiedPosts(root), createFsStore(root).getSelection()
    ])
    const byId = new Map(records.map(entry => [entry.record.id, entry]))
    const visible = new Set([selection.repoId])
    for (let id = selection.updateId; id != null;) {
      requireThat(!visible.has(id) && byId.has(id), 'Incomplete selected update history')
      visible.add(id)
      const { previous, previousUpdate } = parseJson(byId.get(id).record.content)
      id = previous ?? previousUpdate
    }
    const shared = records.filter(({ record }) =>
      visible.has(record.id))
    const verified = [...new Map([...shared, ...posts].map(entry => [entry.record.id, entry])).values()]
    verified.sort(compareRecords)
    const after = cursor
      ? verified.filter(({ record, timestamp }) =>
        timestamp > cursor[0] || (timestamp === cursor[0] && record.id > cursor[1]))
      : verified
    const page = after.slice(0, pollPageSize)
    const last = page.at(-1)
    response.writeHead(200, {
      'Content-Type': 'application/json', 'Cache-Control': 'no-store',
      'Access-Control-Allow-Origin': '*'
    }).end(JSON.stringify({
      records: page.map(({ record }) => record),
      nextSince: last ? encodeCursor(last.timestamp, last.record.id) : null
    }))
  } catch (error) {
    console.error(error.message)
    response.writeHead(503).end('Publication unavailable\n')
  }
}

async function serveBlock(root, response, hash) {
  try {
    const bytes = await createFsStore(root).getBlock(hash, maxManifest)
    if (bytes === undefined) {
      response.writeHead(404).end('Unknown block\n')
      return
    }
    requireThat(await verifyBlock(bytes, hash), 'Corrupt AndFS block')
    response.writeHead(200, {
      'Content-Type': 'application/octet-stream',
      'Cache-Control': 'public, max-age=31536000, immutable',
      'Access-Control-Allow-Origin': '*'
    }).end(bytes)
  } catch (error) {
    if (error.message === 'Invalid encoding') {
      response.writeHead(404).end('Unknown block\n')
      return
    }
    console.error(error.message)
    response.writeHead(503).end('Block unavailable\n')
  }
}

function parseWebPath(rest) {
  const trimmed = rest.replace(/\/+$/, '')
  if (!trimmed) return ''
  const segments = []
  for (const raw of trimmed.split('/')) {
    let segment
    try {
      segment = decodeURIComponent(raw)
    } catch {
      return null
    }
    if (!segment || segment === '.' || segment === '..' ||
      segment.includes('/') || segment.includes('\0')) return null
    segments.push(segment)
  }
  return segments.join('/')
}

function matchWebRoute(pathname) {
  if (pathname === '/git' || pathname === '/git/') return { kind: 'index' }
  if (pathname === '/tree' || pathname.startsWith('/tree/')) {
    return { kind: 'tree', path: parseWebPath(pathname.slice('/tree'.length + 1)) }
  }
  if (pathname.startsWith('/blob/')) {
    const path = parseWebPath(pathname.slice('/blob/'.length))
    return { kind: 'blob', path: path === '' ? null : path }
  }
  const commitDiff = /^\/commit\/([0-9a-f]{40})\/diff\/(.+)$/.exec(pathname)
  if (commitDiff) {
    const path = parseWebPath(commitDiff[2])
    return { kind: 'diff', hash: commitDiff[1], path: path === '' ? null : path }
  }
  const commitBlob = /^\/commit\/([0-9a-f]{40})\/blob\/(.+)$/.exec(pathname)
  if (commitBlob) {
    const path = parseWebPath(commitBlob[2])
    return { kind: 'commitBlob', hash: commitBlob[1], path: path === '' ? null : path }
  }
  const commit = /^\/commit\/([0-9a-f]{40})$/.exec(pathname)
  if (commit) return { kind: 'commit', hash: commit[1] }
  return null
}

async function serveFeed(root, query, response) {
  const params = new URLSearchParams(query)
  const author = params.get('author')
  const postId = params.get('post')
  const updateId = params.get('update')
  try {
    requireThat([author, postId, updateId].filter(id => id !== null).length <= 1, 'Choose one feed view')
    const id = author ?? postId ?? updateId
    if (id !== null) recordPath(id)
  } catch {
    serveHtml(response, 404, feedPage({ missing: true }))
    return
  }
  const [social, pushes] = await Promise.all([readFeed(root), readPushAnnouncements(root)])
  const posts = [...social, ...pushes].sort((a, b) => b.timestamp - a.timestamp || a.id.localeCompare(b.id))
  const post = posts.find(item => item.id === postId || item.updateId === updateId)
  const missing = (postId !== null || updateId !== null) && !post
  const matching = posts.filter(item =>
    post ? item.replyTo === post.id : author === null || item.author === author)
  serveHtml(response, missing ? 404 : 200, feedPage({ posts: matching, author, post, missing }))
}

async function runServer(root, identity, host, port, baseUrl, allowlist, pushOrigin) {
  const verified = new Map()
  const { repoId } = await selectedRepository(root, verified)
  const repoHash = recordPath(repoId)
  const publisher = await readRecordAuthor(root, repoId)
  const legacyPath = `/git/${repoHash}/${recordPath(publisher)}.git`
  const displayOrigin = new URL(pushOrigin ?? baseUrl).origin
  const auth = allowlist && pushAuth(resolve(allowlist), displayOrigin, repoId)
  let receiving = false

  const server = createServer(async (request, response) => {
    const [pathname, query = ''] = request.url.split('?', 2)
    if (request.method === 'POST' && pathname === '/posts' && !query) {
      await receivePost(root, request, response, auth?.isAllowed)
      return
    }
    if (request.method === 'GET') {
      if (pathname === '/feed-live.js') {
        response.writeHead(200, {
          'Content-Type': 'text/javascript; charset=utf-8', 'Cache-Control': 'no-cache'
        }).end(feedScript)
        return
      }
      if (pathname === '/' || pathname === '/feed') {
        try {
          await serveFeed(root, query, response)
        } catch (error) {
          console.error(error.message)
          response.writeHead(503).end('Feed unavailable\n')
        }
        return
      }
      if (pathname === '/gossip/poll') {
        await servePoll(root, response, query)
        return
      }
      const block = /^\/blocks\/([A-Za-z0-9_-]+)$/.exec(pathname)
      if (block) {
        await serveBlock(root, response, block[1])
        return
      }
      const route = matchWebRoute(pathname)
      if (route) {
        try {
          const selected = await selectedRepository(root, verified)
          requireThat(selected.repoId === repoId, 'The served repository ID changed')
          const name = await repositoryName(root, repoId)
          const branch = selected.ref.slice('refs/heads/'.length)
          if (route.path === null) {
            serveHtml(response, 404, notFoundPage(name, 'That page'))
            return
          }
          if (route.kind === 'index') {
            const commits = listCommits(selected.bare, selected.ref, 50)
            const entries = listTree(selected.bare, selected.ref, '')
            requireThat(entries, 'Cannot list the repository root')
            serveHtml(response, 200, indexPage({
              name, branch, repoId, updateId: selected.updateId,
              cloneUrl: displayOrigin + clonePath(repoId), commits, entries
            }))
            return
          }
          if (route.kind === 'tree') {
            const entries = listTree(selected.bare, selected.ref, route.path)
            if (!entries) {
              serveHtml(response, 404, notFoundPage(name, `Directory ${route.path}`))
              return
            }
            serveHtml(response, 200, treePage({ name, path: route.path, entries }))
            return
          }
          if (route.kind === 'blob') {
            const blob = readBlob(selected.bare, selected.ref, route.path)
            if (!blob) {
              serveHtml(response, 404, notFoundPage(name, `File ${route.path}`))
              return
            }
            serveHtml(response, 200, blobPage({ name, path: route.path, ...blob }))
            return
          }
          if (route.kind === 'diff' || route.kind === 'commitBlob') {
            const commit = readCommit(selected.bare, route.hash)
            if (!commit) {
              serveHtml(response, 404, notFoundPage(name, `Commit ${route.hash}`))
              return
            }
            if (route.kind === 'diff') {
              const diff = commitDiff(selected.bare, route.hash, route.path)
              if (!diff) {
                serveHtml(response, 404, notFoundPage(name, `File ${route.path} in commit ${route.hash}`))
                return
              }
              const hasBlob = blobExists(selected.bare, route.hash, route.path)
              serveHtml(response, 200, diffPage({ name, commit, path: route.path, ...diff, hasBlob }))
              return
            }
            const blob = readBlob(selected.bare, route.hash, route.path)
            if (!blob) {
              serveHtml(response, 404, notFoundPage(name, `File ${route.path} in commit ${route.hash}`))
              return
            }
            serveHtml(response, 200, blobPage({ name, path: route.path, commit, ...blob }))
            return
          }
          const commit = readCommit(selected.bare, route.hash)
          if (!commit) {
            serveHtml(response, 404, notFoundPage(name, `Commit ${route.hash}`))
            return
          }
          serveHtml(response, 200, commitPage({
            name, commit, files: commitFiles(selected.bare, route.hash)
          }))
        } catch (error) {
          console.error(error.message)
          if (!response.headersSent) response.writeHead(503).end('Publication unavailable\n')
        }
        return
      }
    }
    const [, repositoryPath, endpoint] =
      /^(.+\.git)(\/(?:info\/refs|git-upload-pack|git-receive-pack|anbot-auth))$/.exec(pathname) ?? []
    const prefix = /^\/git\/([A-Za-z0-9_-]+)\.git$/.exec(repositoryPath)?.[1]
    // This node exposes one repository, so every matching prefix is unique.
    const matches = repositoryPath === legacyPath || (prefix && repoHash.startsWith(prefix))
    const advertise = request.method === 'GET' && endpoint === '/info/refs'
    const readAdvertisement = advertise && query === 'service=git-upload-pack'
    const writeAdvertisement = advertise && query === 'service=git-receive-pack'
    const upload = request.method === 'POST' && endpoint === '/git-upload-pack' && !query
    const receive = request.method === 'POST' && endpoint === '/git-receive-pack' && !query
    const authenticate = ['GET', 'POST'].includes(request.method) && endpoint === '/anbot-auth' && !query
    const write = writeAdvertisement || receive
    if (!matches || (!readAdvertisement && !upload && !(auth && (write || authenticate)))) {
      response.writeHead(404).end('Unknown Git endpoint\n')
      return
    }

    try {
      if (authenticate) {
        await auth.challenge(request, response)
        return
      }
      if (write && !auth.authorized(request)) {
        response.writeHead(403).end('Push access denied\n')
        return
      }
      if (receive) {
        if (receiving) {
          response.writeHead(503).end('Another push is in progress. Retry shortly.\n')
          return
        }
        receiving = true
        try {
          await receivePush(root, identity, request, response, () => auth.authorized(request))
        } finally {
          receiving = false
        }
        return
      }
      const selected = await selectedRepository(root, verified)
      requireThat(selected.repoId === repoId, 'The served repository ID changed')
      serveGit(request, response, selected.bare, endpoint, query, write)
    } catch (error) {
      console.error(error.message)
      response.writeHead(503).end('Publication unavailable\n')
    }
  })
  server.on('error', error => {
    console.error(error.message)
    process.exitCode = 1
  })
  server.listen(port, host, () => console.log(baseUrl + clonePath(repoId)))
}

async function main() {
  const [command, ...args] = process.argv.slice(2)
  if (!command || command === '--help' || args.includes('--help')) {
    console.log(help)
    return
  }
  requireThat(['publish', 'serve', 'push', 'sync', 'identity', 'post'].includes(command), help)
  const options = {
    data: { type: 'string' }, host: { type: 'string', default: '127.0.0.1' },
    port: { type: 'string', default: '8787' }
  }
  if (command === 'publish') {
    Object.assign(options, {
      source: { type: 'string' }, branch: { type: 'string' }, name: { type: 'string' }
    })
  }
  if (command === 'serve') {
    options['push-allowlist'] = { type: 'string' }
    options['push-origin'] = { type: 'string' }
  }
  if (command === 'push') {
    Object.assign(options, {
      source: { type: 'string', default: '.' },
      branch: { type: 'string', default: 'main' },
      url: { type: 'string' }
    })
  }
  if (command === 'post') {
    options.text = { type: 'string' }
    options['reply-to'] = { type: 'string' }
    options.id = { type: 'string' }
    options.url = { type: 'string' }
  }
  if (command === 'sync') {
    options.url = { type: 'string' }
  }
  const { values } = parseArgs({ args, options })
  requireThat(values.data, '--data is required')
  const data = resolve(values.data)
  const parent = realpathSync(dirname(data))
  const root = existsSync(data) ? realpathSync(data) : resolve(parent, basename(data))
  if (command === 'identity') {
    initializeStore(root, dirname(root))
    console.log((await loadIdentity(root, true)).author)
    return
  }
  if (command === 'push') {
    requireThat(values.url, '--url is required')
    await pushWithIdentity(await loadIdentity(root), values.url, values.source, values.branch)
    return
  }
  if (command === 'post') {
    const retry = values.id !== undefined
    requireThat(retry ? values.url && values.text === undefined && values['reply-to'] === undefined
      : values.text !== undefined, 'Supply --text, or --id with --url to retry')
    const remote = values.url === undefined ? null : new URL(values.url)
    requireThat(!remote || ((remote.protocol === 'https:' ||
      (remote.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(remote.hostname))) &&
      !remote.username && !remote.password && !remote.search && !remote.hash),
    'Use an HTTPS node URL or local loopback URL')
    const identity = await loadIdentity(root)
    const id = retry ? values.id : await publishPost(root, identity, values.text, values['reply-to'])
    if (remote) {
      try {
        await submitPost(root, identity, id, remote)
      } catch (error) {
        throw new Error(`${error.message}. Retry an existing local post with --id ${id} --url ${remote.origin}`)
      }
    }
    console.log(JSON.stringify({ id }))
    return
  }
  if (command === 'sync') {
    requireThat(values.url, '--url is required')
    const remote = new URL(values.url)
    requireThat((remote.protocol === 'https:' ||
      (remote.protocol === 'http:' && ['127.0.0.1', 'localhost', '[::1]'].includes(remote.hostname))) &&
      !remote.username && !remote.password && !remote.search && !remote.hash,
    'Use an HTTPS node URL or local loopback URL')
    initializeStore(root, dirname(root))
    await loadIdentity(root, true)
    const result = await pull(createFsStore(root), remote.origin, {
      async verifyBundle(update) {
        await verifySelection(root, { repoId: update.repoId, ref: update.body.ref, updateId: update.id })
      }
    })
    console.log(JSON.stringify(result))
    return
  }
  const port = Number(values.port)
  requireThat(/^\d+$/.test(values.port) && port >= 1 && port <= 65535, 'Invalid port')
  requireThat(/^[a-zA-Z0-9.-]+$/.test(values.host), 'Use an IPv4 address or hostname')
  const baseUrl = `http://${values.host}:${port}`

  if (command === 'serve') {
    await runServer(root, await loadIdentity(root), values.host, port, baseUrl,
      values['push-allowlist'], values['push-origin'])
    return
  }

  requireThat(values.source && values.branch, '--source and --branch are required')
  const source = realpathSync(checkoutRoot(realpathSync(values.source)))
  const fromSource = relative(source, root)
  requireThat(fromSource === '..' || fromSource.startsWith('..' + sep) ||
    isAbsolute(fromSource), 'Keep the data directory outside the source checkout')
  initializeStore(root, dirname(root))
  const identity = await loadIdentity(root, true)
  const result = await publish(root, identity, source, values.branch, values.name ?? basename(source))
  console.log(JSON.stringify({ ...result, cloneUrl: baseUrl + clonePath(result.repoId) }))
}

try {
  await main()
} catch (error) {
  console.error(error.message)
  process.exitCode = 1
}

Git ยท Social feed