Repack software and centralize file metadata
This commit is contained in:
+132
-149
@@ -15,12 +15,10 @@ import {
|
||||
} from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { isMap, parseDocument } from "yaml";
|
||||
import { parseDocument, stringify } from "yaml";
|
||||
|
||||
type CatalogFile = {
|
||||
path?: unknown;
|
||||
size?: unknown;
|
||||
sha256?: unknown;
|
||||
};
|
||||
|
||||
type ProgramIndex = {
|
||||
@@ -29,17 +27,23 @@ type ProgramIndex = {
|
||||
additional_files?: CatalogFile[];
|
||||
};
|
||||
|
||||
type Options = {
|
||||
check: boolean;
|
||||
targets: string[];
|
||||
type FileDetails = {
|
||||
size: number;
|
||||
sha256: string;
|
||||
};
|
||||
|
||||
type FileReference = {
|
||||
data: CatalogFile;
|
||||
yamlPath: Array<string | number>;
|
||||
type FileManifest = {
|
||||
format: 1;
|
||||
files: Record<string, FileDetails>;
|
||||
};
|
||||
|
||||
type Options = {
|
||||
catalogDirectory: string;
|
||||
check: boolean;
|
||||
};
|
||||
|
||||
type ResolvedCatalogFile = {
|
||||
key: string;
|
||||
path: string;
|
||||
size: number;
|
||||
};
|
||||
@@ -47,12 +51,12 @@ type ResolvedCatalogFile = {
|
||||
const ignoredDirectories = new Set([".git", "node_modules", "files", "img"]);
|
||||
|
||||
function usage(): void {
|
||||
console.log(`Usage: pnpm fill-metadata [--check] [path ...]
|
||||
console.log(`Usage: pnpm fill-metadata [--check] [catalog-directory]
|
||||
|
||||
With no paths, the script scans the current directory. A path may point to an
|
||||
index.yaml file or a directory containing index.yaml files.
|
||||
The default catalog directory is ./catalog. The generated file is files.yaml
|
||||
in the catalog root.
|
||||
|
||||
--check verify checksums and file sizes without modifying YAML
|
||||
--check verify files.yaml without modifying it
|
||||
--help show this help message`);
|
||||
}
|
||||
|
||||
@@ -73,59 +77,41 @@ function parseArguments(args: string[]): Options {
|
||||
}
|
||||
}
|
||||
|
||||
return { check, targets: targets.length > 0 ? targets : ["."] };
|
||||
if (targets.length > 1) {
|
||||
throw new Error("Expected at most one catalog directory");
|
||||
}
|
||||
|
||||
return {
|
||||
catalogDirectory: path.resolve(targets[0] ?? "catalog"),
|
||||
check,
|
||||
};
|
||||
}
|
||||
|
||||
async function findIndexes(target: string, result: Set<string>): Promise<void> {
|
||||
const absoluteTarget = path.resolve(target);
|
||||
const targetStat = await lstat(absoluteTarget);
|
||||
|
||||
if (targetStat.isFile()) {
|
||||
if (path.basename(absoluteTarget) !== "index.yaml") {
|
||||
throw new Error(`Expected an index.yaml file: ${target}`);
|
||||
}
|
||||
result.add(absoluteTarget);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!targetStat.isDirectory()) {
|
||||
throw new Error(`Path is neither a file nor a directory: ${target}`);
|
||||
}
|
||||
|
||||
const entries = await readdir(absoluteTarget, { withFileTypes: true });
|
||||
async function findIndexes(directory: string, result: string[]): Promise<void> {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(absoluteTarget, entry.name);
|
||||
const entryPath = path.join(directory, entry.name);
|
||||
if (entry.isFile() && entry.name === "index.yaml") {
|
||||
result.add(entryPath);
|
||||
result.push(entryPath);
|
||||
} else if (entry.isDirectory() && !ignoredDirectories.has(entry.name)) {
|
||||
await findIndexes(entryPath, result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function collectFileReferences(data: ProgramIndex): FileReference[] {
|
||||
const references: FileReference[] = [];
|
||||
function collectFileReferences(data: ProgramIndex): CatalogFile[] {
|
||||
const references: CatalogFile[] = [];
|
||||
|
||||
for (const [versionIndex, version] of (data.versions ?? []).entries()) {
|
||||
for (const [fileIndex, file] of (version.files ?? []).entries()) {
|
||||
references.push({
|
||||
data: file,
|
||||
yamlPath: ["versions", versionIndex, "files", fileIndex],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const [fileIndex, file] of (data.additional_files ?? []).entries()) {
|
||||
references.push({
|
||||
data: file,
|
||||
yamlPath: ["additional_files", fileIndex],
|
||||
});
|
||||
for (const version of data.versions ?? []) {
|
||||
references.push(...(version.files ?? []));
|
||||
}
|
||||
references.push(...(data.additional_files ?? []));
|
||||
|
||||
return references;
|
||||
}
|
||||
|
||||
async function resolveCatalogFile(
|
||||
catalogDirectory: string,
|
||||
indexPath: string,
|
||||
value: unknown,
|
||||
): Promise<ResolvedCatalogFile> {
|
||||
@@ -157,12 +143,18 @@ async function resolveCatalogFile(
|
||||
if (realRelativePath.startsWith("..") || path.isAbsolute(realRelativePath)) {
|
||||
throw new Error(`symbolic link escapes files/: ${value}`);
|
||||
}
|
||||
|
||||
const fileStat = await stat(realFilePath);
|
||||
if (!fileStat.isFile()) {
|
||||
throw new Error(`path does not point to a regular file: ${value}`);
|
||||
}
|
||||
|
||||
return { path: realFilePath, size: fileStat.size };
|
||||
const key = path.relative(catalogDirectory, filePath).split(path.sep).join("/");
|
||||
if (key.startsWith("../") || path.isAbsolute(key)) {
|
||||
throw new Error(`path escapes the catalog: ${value}`);
|
||||
}
|
||||
|
||||
return { key, path: realFilePath, size: fileStat.size };
|
||||
}
|
||||
|
||||
async function calculateSha256(filePath: string): Promise<string> {
|
||||
@@ -189,130 +181,121 @@ async function writeAtomically(filePath: string, contents: string): Promise<void
|
||||
}
|
||||
}
|
||||
|
||||
function setFileSize(
|
||||
document: ReturnType<typeof parseDocument>,
|
||||
yamlPath: Array<string | number>,
|
||||
size: number,
|
||||
): void {
|
||||
document.setIn([...yamlPath, "size"], size);
|
||||
|
||||
const fileMap = document.getIn(yamlPath, true);
|
||||
if (!isMap(fileMap)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const keyName = (pair: (typeof fileMap.items)[number]): string => String(pair.key);
|
||||
const sizeIndex = fileMap.items.findIndex((pair) => keyName(pair) === "size");
|
||||
const sha256Index = fileMap.items.findIndex((pair) => keyName(pair) === "sha256");
|
||||
if (sizeIndex < 0 || sha256Index < 0 || sizeIndex < sha256Index) {
|
||||
return;
|
||||
}
|
||||
|
||||
const [sizePair] = fileMap.items.splice(sizeIndex, 1);
|
||||
fileMap.items.splice(sha256Index, 0, sizePair);
|
||||
}
|
||||
|
||||
async function processIndex(indexPath: string, check: boolean): Promise<number> {
|
||||
async function readProgram(indexPath: string): Promise<ProgramIndex> {
|
||||
const source = await readFile(indexPath, "utf8");
|
||||
const document = parseDocument(source, { prettyErrors: true });
|
||||
if (document.errors.length > 0) {
|
||||
throw new Error(document.errors.map((error) => error.message).join("; "));
|
||||
}
|
||||
return document.toJS() as ProgramIndex;
|
||||
}
|
||||
|
||||
const data = document.toJS() as ProgramIndex;
|
||||
if (data.type !== "program") {
|
||||
return 0;
|
||||
async function generateManifest(catalogDirectory: string): Promise<{
|
||||
indexes: number;
|
||||
manifest: FileManifest;
|
||||
}> {
|
||||
const catalogStat = await lstat(catalogDirectory);
|
||||
if (!catalogStat.isDirectory()) {
|
||||
throw new Error(`Not a directory: ${catalogDirectory}`);
|
||||
}
|
||||
|
||||
let changed = 0;
|
||||
const errors: string[] = [];
|
||||
const references = collectFileReferences(data);
|
||||
const indexes: string[] = [];
|
||||
await findIndexes(catalogDirectory, indexes);
|
||||
indexes.sort();
|
||||
|
||||
const results = await Promise.all(
|
||||
references.map(async (reference) => {
|
||||
try {
|
||||
const file = await resolveCatalogFile(indexPath, reference.data.path);
|
||||
return {
|
||||
reference,
|
||||
digest: await calculateSha256(file.path),
|
||||
size: file.size,
|
||||
};
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push(`${String(reference.data.path)}: ${message}`);
|
||||
return undefined;
|
||||
const resolvedFiles = new Map<string, ResolvedCatalogFile>();
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const indexPath of indexes) {
|
||||
try {
|
||||
const data = await readProgram(indexPath);
|
||||
if (data.type !== "program") {
|
||||
continue;
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
for (const reference of collectFileReferences(data)) {
|
||||
try {
|
||||
const file = await resolveCatalogFile(
|
||||
catalogDirectory,
|
||||
indexPath,
|
||||
reference.path,
|
||||
);
|
||||
resolvedFiles.set(file.key, file);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
errors.push(`${path.relative(process.cwd(), indexPath)}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
throw new Error(errors.join("; "));
|
||||
throw new Error(errors.join("\n"));
|
||||
}
|
||||
|
||||
for (const result of results) {
|
||||
if (result === undefined) {
|
||||
continue;
|
||||
}
|
||||
const entries = await Promise.all(
|
||||
[...resolvedFiles.values()].map(async (file) => [
|
||||
file.key,
|
||||
{
|
||||
size: file.size,
|
||||
sha256: await calculateSha256(file.path),
|
||||
},
|
||||
] as const),
|
||||
);
|
||||
entries.sort(([left], [right]) => left.localeCompare(right, "en"));
|
||||
|
||||
if (result.reference.data.size !== result.size) {
|
||||
changed += 1;
|
||||
if (!check) {
|
||||
setFileSize(document, result.reference.yamlPath, result.size);
|
||||
}
|
||||
}
|
||||
if (result.reference.data.sha256 !== result.digest) {
|
||||
changed += 1;
|
||||
if (!check) {
|
||||
document.setIn([...result.reference.yamlPath, "sha256"], result.digest);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const displayPath = path.relative(process.cwd(), indexPath) || indexPath;
|
||||
if (changed > 0 && check) {
|
||||
console.error(`${displayPath}: metadata update required (${changed} fields)`);
|
||||
} else if (changed > 0) {
|
||||
await writeAtomically(indexPath, document.toString({ lineWidth: 0 }));
|
||||
console.log(`${displayPath}: updated file metadata (${changed} fields)`);
|
||||
}
|
||||
|
||||
return changed;
|
||||
return {
|
||||
indexes: indexes.length,
|
||||
manifest: {
|
||||
format: 1,
|
||||
files: Object.fromEntries(entries),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseArguments(process.argv.slice(2));
|
||||
const indexes = new Set<string>();
|
||||
const manifestPath = path.join(options.catalogDirectory, "files.yaml");
|
||||
const { indexes, manifest } = await generateManifest(options.catalogDirectory);
|
||||
const contents = stringify(manifest, { lineWidth: 0 });
|
||||
|
||||
for (const target of options.targets) {
|
||||
await findIndexes(target, indexes);
|
||||
}
|
||||
|
||||
let changed = 0;
|
||||
let failed = 0;
|
||||
for (const indexPath of [...indexes].sort()) {
|
||||
try {
|
||||
changed += await processIndex(indexPath, options.check);
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
const displayPath = path.relative(process.cwd(), indexPath) || indexPath;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`${displayPath}: ${message}`);
|
||||
let currentContents: string | undefined;
|
||||
try {
|
||||
currentContents = await readFile(manifestPath, "utf8");
|
||||
} catch (error) {
|
||||
const code = error instanceof Error && "code" in error ? error.code : undefined;
|
||||
if (code !== "ENOENT") {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (failed > 0) {
|
||||
console.error(`Errors: ${failed}`);
|
||||
process.exitCode = 1;
|
||||
} else if (options.check && changed > 0) {
|
||||
console.error(`File metadata mismatches: ${changed}`);
|
||||
process.exitCode = 1;
|
||||
} else {
|
||||
const displayPath = path.relative(process.cwd(), manifestPath) || manifestPath;
|
||||
if (currentContents === contents) {
|
||||
console.log(
|
||||
changed > 0
|
||||
? `Done. Updated fields: ${changed}`
|
||||
: `Done. All SHA-256 checksums and file sizes are up to date. Checked index.yaml files: ${indexes.size}`,
|
||||
`Done. ${displayPath} is up to date. Files: ${Object.keys(manifest.files).length}; index.yaml files: ${indexes}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.check) {
|
||||
console.error(`${displayPath}: update required`);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
await writeAtomically(manifestPath, contents);
|
||||
console.log(
|
||||
`Updated ${displayPath}. Files: ${Object.keys(manifest.files).length}; index.yaml files: ${indexes}`,
|
||||
);
|
||||
}
|
||||
|
||||
await main();
|
||||
try {
|
||||
await main();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`Error: ${message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user