#!/usr/bin/env node import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { lstat, mkdir, readdir, readFile, realpath, rename, rm, stat, writeFile, } from "node:fs/promises"; import path from "node:path"; import process from "node:process"; import { parseDocument, stringify } from "yaml"; type CatalogFile = { path?: unknown; }; type ProgramIndex = { type?: unknown; versions?: Array<{ files?: CatalogFile[] }>; additional_files?: CatalogFile[]; }; type FileDetails = { size: number; sha256: string; }; type FileManifest = { format: 1; files: Record; }; type Options = { catalogDirectory: string; check: boolean; }; type ResolvedCatalogFile = { key: string; path: string; size: number; }; const ignoredDirectories = new Set([".git", "node_modules", "files", "img"]); function usage(): void { console.log(`Usage: pnpm fill-metadata [--check] [catalog-directory] The default catalog directory is ./catalog. The generated file is files.yaml in the catalog root. --check verify files.yaml without modifying it --help show this help message`); } function parseArguments(args: string[]): Options { const targets: string[] = []; let check = false; for (const argument of args) { if (argument === "--check") { check = true; } else if (argument === "--help" || argument === "-h") { usage(); process.exit(0); } else if (argument.startsWith("-")) { throw new Error(`Unknown option: ${argument}`); } else { targets.push(argument); } } if (targets.length > 1) { throw new Error("Expected at most one catalog directory"); } return { catalogDirectory: path.resolve(targets[0] ?? "catalog"), check, }; } async function findIndexes(directory: string, result: string[]): Promise { const entries = await readdir(directory, { withFileTypes: true }); for (const entry of entries) { const entryPath = path.join(directory, entry.name); if (entry.isFile() && entry.name === "index.yaml") { result.push(entryPath); } else if (entry.isDirectory() && !ignoredDirectories.has(entry.name)) { await findIndexes(entryPath, result); } } } function collectFileReferences(data: ProgramIndex): CatalogFile[] { const references: CatalogFile[] = []; for (const version of data.versions ?? []) { references.push(...(version.files ?? [])); } references.push(...(data.additional_files ?? [])); return references; } async function resolveCatalogFile( catalogDirectory: string, indexPath: string, value: unknown, ): Promise { if (typeof value !== "string" || value.length === 0) { throw new Error("the path field is missing or is not a string"); } if (path.isAbsolute(value) || value.includes("\\")) { throw new Error(`invalid path: ${value}`); } const parts = value.split("/"); if (parts[0] !== "files" || parts.some((part) => part === ".." || part === "")) { throw new Error(`path must be inside files/: ${value}`); } const programDirectory = path.dirname(indexPath); const filesDirectory = path.resolve(programDirectory, "files"); const filePath = path.resolve(programDirectory, ...parts); const relativePath = path.relative(filesDirectory, filePath); if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) { throw new Error(`path escapes files/: ${value}`); } const [realFilesDirectory, realFilePath] = await Promise.all([ realpath(filesDirectory), realpath(filePath), ]); const realRelativePath = path.relative(realFilesDirectory, realFilePath); if (realRelativePath.startsWith("..") || path.isAbsolute(realRelativePath)) { throw new Error(`symbolic link escapes files/: ${value}`); } const fileStat = await stat(realFilePath); if (!fileStat.isFile()) { throw new Error(`path does not point to a regular file: ${value}`); } const key = path.relative(catalogDirectory, filePath).split(path.sep).join("/"); if (key.startsWith("../") || path.isAbsolute(key)) { throw new Error(`path escapes the catalog: ${value}`); } return { key, path: realFilePath, size: fileStat.size }; } async function calculateSha256(filePath: string): Promise { const hash = createHash("sha256"); for await (const chunk of createReadStream(filePath)) { hash.update(chunk); } return hash.digest("hex"); } async function writeAtomically(filePath: string, contents: string): Promise { const directory = path.dirname(filePath); const temporaryPath = path.join( directory, `.${path.basename(filePath)}.${process.pid}.tmp`, ); await mkdir(directory, { recursive: true }); try { await writeFile(temporaryPath, contents, "utf8"); await rename(temporaryPath, filePath); } finally { await rm(temporaryPath, { force: true }); } } async function readProgram(indexPath: string): Promise { const source = await readFile(indexPath, "utf8"); const document = parseDocument(source, { prettyErrors: true }); if (document.errors.length > 0) { throw new Error(document.errors.map((error) => error.message).join("; ")); } return document.toJS() as ProgramIndex; } async function generateManifest(catalogDirectory: string): Promise<{ indexes: number; manifest: FileManifest; }> { const catalogStat = await lstat(catalogDirectory); if (!catalogStat.isDirectory()) { throw new Error(`Not a directory: ${catalogDirectory}`); } const indexes: string[] = []; await findIndexes(catalogDirectory, indexes); indexes.sort(); const resolvedFiles = new Map(); const errors: string[] = []; for (const indexPath of indexes) { try { const data = await readProgram(indexPath); if (data.type !== "program") { continue; } for (const reference of collectFileReferences(data)) { try { const file = await resolveCatalogFile( catalogDirectory, indexPath, reference.path, ); resolvedFiles.set(file.key, file); } catch (error) { const message = error instanceof Error ? error.message : String(error); errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`); } } } catch (error) { const message = error instanceof Error ? error.message : String(error); errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`); } } if (errors.length > 0) { throw new Error(errors.join("\n")); } const entries = await Promise.all( [...resolvedFiles.values()].map(async (file) => [ file.key, { size: file.size, sha256: await calculateSha256(file.path), }, ] as const), ); entries.sort(([left], [right]) => left.localeCompare(right, "en")); return { indexes: indexes.length, manifest: { format: 1, files: Object.fromEntries(entries), }, }; } async function main(): Promise { const options = parseArguments(process.argv.slice(2)); const manifestPath = path.join(options.catalogDirectory, "files.yaml"); const { indexes, manifest } = await generateManifest(options.catalogDirectory); const contents = stringify(manifest, { lineWidth: 0 }); let currentContents: string | undefined; try { currentContents = await readFile(manifestPath, "utf8"); } catch (error) { const code = error instanceof Error && "code" in error ? error.code : undefined; if (code !== "ENOENT") { throw error; } } const displayPath = path.relative(process.cwd(), manifestPath) || manifestPath; if (currentContents === contents) { console.log( `Done. ${displayPath} is up to date. Files: ${Object.keys(manifest.files).length}; index.yaml files: ${indexes}`, ); return; } if (options.check) { console.error(`${displayPath}: update required`); process.exitCode = 1; return; } await writeAtomically(manifestPath, contents); console.log( `Updated ${displayPath}. Files: ${Object.keys(manifest.files).length}; index.yaml files: ${indexes}`, ); } try { await main(); } catch (error) { const message = error instanceof Error ? error.message : String(error); console.error(`Error: ${message}`); process.exitCode = 1; }