soft/scripts/fillMetadata.ts

#!/usr/bin/env node

import { createHash } from "node:crypto";
import { createReadStream } from "node:fs";
import {
  lstat,
  mkdir,
  readdir,
  readFile,
  realpath,
  rename,
  rm,
  stat,
  writeFile,
} from "node:fs/promises";
import path from "node:path";
import process from "node:process";
import { stringify } from "yaml";
import { readCatalogIndex } from "./catalogIndex.js";

type CatalogFile = {
  path?: unknown;
};

type ProgramIndex = {
  type?: unknown;
  versions?: Array<{ files?: CatalogFile[] }>;
  additional_files?: CatalogFile[];
};

type FileDetails = {
  size: number;
  sha256: string;
};

type FileManifest = {
  format: 1;
  files: Record<string, FileDetails>;
};

type Options = {
  catalogDirectory: string;
  check: boolean;
};

type ResolvedCatalogFile = {
  key: string;
  path: string;
  size: number;
};

const ignoredDirectories = new Set([".git", "node_modules", "files", "img"]);

function usage(): void {
  console.log(`Usage: pnpm fill-metadata [--check] [catalog-directory]

The default catalog directory is ./catalog. The generated file is files.yaml
in the catalog root.

  --check  verify files.yaml without modifying it
  --help   show this help message`);
}

function parseArguments(args: string[]): Options {
  const targets: string[] = [];
  let check = false;

  for (const argument of args) {
    if (argument === "--check") {
      check = true;
    } else if (argument === "--help" || argument === "-h") {
      usage();
      process.exit(0);
    } else if (argument.startsWith("-")) {
      throw new Error(`Unknown option: ${argument}`);
    } else {
      targets.push(argument);
    }
  }

  if (targets.length > 1) {
    throw new Error("Expected at most one catalog directory");
  }

  return {
    catalogDirectory: path.resolve(targets[0] ?? "catalog"),
    check,
  };
}

async function findIndexes(directory: string, result: string[]): Promise<void> {
  const entries = await readdir(directory, { withFileTypes: true });
  for (const entry of entries) {
    const entryPath = path.join(directory, entry.name);
    if (entry.isFile() && entry.name === "index.md") {
      result.push(entryPath);
    } else if (entry.isDirectory() && !ignoredDirectories.has(entry.name)) {
      await findIndexes(entryPath, result);
    }
  }
}

function collectFileReferences(data: ProgramIndex): CatalogFile[] {
  const references: CatalogFile[] = [];

  for (const version of data.versions ?? []) {
    references.push(...(version.files ?? []));
  }
  references.push(...(data.additional_files ?? []));

  return references;
}

async function resolveCatalogFile(
  catalogDirectory: string,
  indexPath: string,
  value: unknown,
): Promise<ResolvedCatalogFile> {
  if (typeof value !== "string" || value.length === 0) {
    throw new Error("the path field is missing or is not a string");
  }
  if (path.isAbsolute(value) || value.includes("\\")) {
    throw new Error(`invalid path: ${value}`);
  }

  const parts = value.split("/");
  if (parts[0] !== "files" || parts.some((part) => part === ".." || part === "")) {
    throw new Error(`path must be inside files/: ${value}`);
  }

  const programDirectory = path.dirname(indexPath);
  const filesDirectory = path.resolve(programDirectory, "files");
  const filePath = path.resolve(programDirectory, ...parts);
  const relativePath = path.relative(filesDirectory, filePath);
  if (relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
    throw new Error(`path escapes files/: ${value}`);
  }

  const [realFilesDirectory, realFilePath] = await Promise.all([
    realpath(filesDirectory),
    realpath(filePath),
  ]);
  const realRelativePath = path.relative(realFilesDirectory, realFilePath);
  if (realRelativePath.startsWith("..") || path.isAbsolute(realRelativePath)) {
    throw new Error(`symbolic link escapes files/: ${value}`);
  }

  const fileStat = await stat(realFilePath);
  if (!fileStat.isFile()) {
    throw new Error(`path does not point to a regular file: ${value}`);
  }

  const key = path.relative(catalogDirectory, filePath).split(path.sep).join("/");
  if (key.startsWith("../") || path.isAbsolute(key)) {
    throw new Error(`path escapes the catalog: ${value}`);
  }

  return { key, path: realFilePath, size: fileStat.size };
}

async function calculateSha256(filePath: string): Promise<string> {
  const hash = createHash("sha256");
  for await (const chunk of createReadStream(filePath)) {
    hash.update(chunk);
  }
  return hash.digest("hex");
}

async function writeAtomically(filePath: string, contents: string): Promise<void> {
  const directory = path.dirname(filePath);
  const temporaryPath = path.join(
    directory,
    `.${path.basename(filePath)}.${process.pid}.tmp`,
  );

  await mkdir(directory, { recursive: true });
  try {
    await writeFile(temporaryPath, contents, "utf8");
    await rename(temporaryPath, filePath);
  } finally {
    await rm(temporaryPath, { force: true });
  }
}

async function readProgram(indexPath: string): Promise<ProgramIndex> {
  return (await readCatalogIndex<ProgramIndex>(indexPath)).metadata;
}

async function generateManifest(catalogDirectory: string): Promise<{
  indexes: number;
  manifest: FileManifest;
}> {
  const catalogStat = await lstat(catalogDirectory);
  if (!catalogStat.isDirectory()) {
    throw new Error(`Not a directory: ${catalogDirectory}`);
  }

  const indexes: string[] = [];
  await findIndexes(catalogDirectory, indexes);
  indexes.sort();

  const resolvedFiles = new Map<string, ResolvedCatalogFile>();
  const errors: string[] = [];

  for (const indexPath of indexes) {
    try {
      const data = await readProgram(indexPath);
      if (data.type !== "program") {
        continue;
      }

      for (const reference of collectFileReferences(data)) {
        try {
          const file = await resolveCatalogFile(
            catalogDirectory,
            indexPath,
            reference.path,
          );
          resolvedFiles.set(file.key, file);
        } catch (error) {
          const message = error instanceof Error ? error.message : String(error);
          errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`);
        }
      }
    } catch (error) {
      const message = error instanceof Error ? error.message : String(error);
      errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`);
    }
  }

  if (errors.length > 0) {
    throw new Error(errors.join("\n"));
  }

  const entries = await Promise.all(
    [...resolvedFiles.values()].map(async (file) => [
      file.key,
      {
        size: file.size,
        sha256: await calculateSha256(file.path),
      },
    ] as const),
  );
  entries.sort(([left], [right]) => left.localeCompare(right, "en"));

  return {
    indexes: indexes.length,
    manifest: {
      format: 1,
      files: Object.fromEntries(entries),
    },
  };
}

async function main(): Promise<void> {
  const options = parseArguments(process.argv.slice(2));
  const manifestPath = path.join(options.catalogDirectory, "files.yaml");
  const { indexes, manifest } = await generateManifest(options.catalogDirectory);
  const contents = stringify(manifest, { lineWidth: 0 });

  let currentContents: string | undefined;
  try {
    currentContents = await readFile(manifestPath, "utf8");
  } catch (error) {
    const code = error instanceof Error && "code" in error ? error.code : undefined;
    if (code !== "ENOENT") {
      throw error;
    }
  }

  const displayPath = path.relative(process.cwd(), manifestPath) || manifestPath;
  if (currentContents === contents) {
    console.log(
      `Done. ${displayPath} is up to date. Files: ${Object.keys(manifest.files).length}; index.md files: ${indexes}`,
    );
    return;
  }

  if (options.check) {
    console.error(`${displayPath}: update required`);
    process.exitCode = 1;
    return;
  }

  await writeAtomically(manifestPath, contents);
  console.log(
    `Updated ${displayPath}. Files: ${Object.keys(manifest.files).length}; index.md files: ${indexes}`,
  );
}

try {
  await main();
} catch (error) {
  const message = error instanceof Error ? error.message : String(error);
  console.error(`Error: ${message}`);
  process.exitCode = 1;
}