Reorganize software catalog
This commit is contained in:
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFile } from "node:child_process";
|
||||
import { lstat, mkdir, mkdtemp, readdir, rm, stat } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
type Options = {
|
||||
dryRun: boolean;
|
||||
outputDirectory: string;
|
||||
sourceDirectory: string;
|
||||
temporaryDirectory: string;
|
||||
};
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
function usage(): void {
|
||||
console.log(`Usage: pnpm extract-archives [options] <output-directory>
|
||||
|
||||
Recursively extracts every ZIP from catalog into the output directory. Category and
|
||||
program paths are preserved; each archive is extracted into a directory named after it.
|
||||
Existing destination directories and files are skipped.
|
||||
|
||||
--dry-run list operations without extracting files
|
||||
--source <path> read ZIP archives from a directory other than catalog
|
||||
--temp-dir <path> use a specific directory for temporary files
|
||||
--help show this help message`);
|
||||
}
|
||||
|
||||
function parseArguments(args: string[]): Options {
|
||||
const positional: string[] = [];
|
||||
let dryRun = false;
|
||||
let sourceDirectory = path.resolve("catalog");
|
||||
let temporaryDirectory = "/tmp/codex/extract-archives";
|
||||
|
||||
for (let index = 0; index < args.length; index += 1) {
|
||||
const argument = args[index];
|
||||
if (argument === "--help" || argument === "-h") {
|
||||
usage();
|
||||
process.exit(0);
|
||||
} else if (argument === "--dry-run") {
|
||||
dryRun = true;
|
||||
} else if (argument === "--source") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined) {
|
||||
throw new Error("--source requires a path");
|
||||
}
|
||||
sourceDirectory = path.resolve(value);
|
||||
index += 1;
|
||||
} else if (argument === "--temp-dir") {
|
||||
const value = args[index + 1];
|
||||
if (value === undefined) {
|
||||
throw new Error("--temp-dir requires a path");
|
||||
}
|
||||
temporaryDirectory = path.resolve(value);
|
||||
index += 1;
|
||||
} else if (argument.startsWith("-")) {
|
||||
throw new Error(`Unknown option: ${argument}`);
|
||||
} else {
|
||||
positional.push(argument);
|
||||
}
|
||||
}
|
||||
|
||||
if (positional.length !== 1) {
|
||||
throw new Error("Expected one output directory path");
|
||||
}
|
||||
|
||||
return {
|
||||
dryRun,
|
||||
outputDirectory: path.resolve(positional[0]),
|
||||
sourceDirectory,
|
||||
temporaryDirectory,
|
||||
};
|
||||
}
|
||||
|
||||
async function run(command: string, args: string[]): Promise<string> {
|
||||
const { stdout } = await execFileAsync(command, args, {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
});
|
||||
return stdout;
|
||||
}
|
||||
|
||||
async function collectArchives(directory: string): Promise<string[]> {
|
||||
const archives: string[] = [];
|
||||
|
||||
async function visit(currentDirectory: string): Promise<void> {
|
||||
const entries = await readdir(currentDirectory, { withFileTypes: true });
|
||||
entries.sort((left, right) => left.name.localeCompare(right.name, "en"));
|
||||
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(currentDirectory, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
await visit(entryPath);
|
||||
} else if (entry.isFile() && path.extname(entry.name).toLowerCase() === ".zip") {
|
||||
archives.push(entryPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await visit(directory);
|
||||
return archives;
|
||||
}
|
||||
|
||||
function destinationFor(archive: string, sourceDirectory: string, outputDirectory: string): string {
|
||||
const basename = path.basename(archive);
|
||||
const stem = basename.slice(0, -path.extname(basename).length);
|
||||
if (stem.length === 0) {
|
||||
throw new Error(`Archive has no usable name: ${archive}`);
|
||||
}
|
||||
|
||||
const relativeArchive = path.relative(sourceDirectory, archive);
|
||||
if (
|
||||
relativeArchive === ".." ||
|
||||
relativeArchive.startsWith(`..${path.sep}`) ||
|
||||
path.isAbsolute(relativeArchive)
|
||||
) {
|
||||
throw new Error(`Archive is outside the source directory: ${archive}`);
|
||||
}
|
||||
|
||||
let relativeParent = path.dirname(relativeArchive);
|
||||
if (path.basename(relativeParent) === "files") {
|
||||
relativeParent = path.dirname(relativeParent);
|
||||
}
|
||||
return path.join(outputDirectory, relativeParent, stem);
|
||||
}
|
||||
|
||||
async function pathExists(target: string): Promise<boolean> {
|
||||
try {
|
||||
await lstat(target);
|
||||
return true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return false;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function validateArchivePaths(archive: string, listing: string): void {
|
||||
for (const originalEntry of listing.split(/\r?\n/)) {
|
||||
if (originalEntry.length === 0) {
|
||||
continue;
|
||||
}
|
||||
const entry = originalEntry.replaceAll("\\", "/");
|
||||
const parts = entry.split("/");
|
||||
if (
|
||||
entry.startsWith("/") ||
|
||||
/^[a-z]:($|\/)/i.test(entry) ||
|
||||
parts.includes("..")
|
||||
) {
|
||||
throw new Error(`${archive}: unsafe path in ZIP: ${originalEntry}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function validateExtractedTree(directory: string): Promise<void> {
|
||||
for (const entry of await readdir(directory, { withFileTypes: true })) {
|
||||
const entryPath = path.join(directory, entry.name);
|
||||
const entryStat = await lstat(entryPath);
|
||||
if (entryStat.isSymbolicLink()) {
|
||||
throw new Error(`ZIP contains a symbolic link: ${entryPath}`);
|
||||
}
|
||||
if (entryStat.isDirectory()) {
|
||||
await validateExtractedTree(entryPath);
|
||||
} else if (!entryStat.isFile()) {
|
||||
throw new Error(`ZIP contains an unsupported special file: ${entryPath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function extractArchive(archive: string, destination: string, temporaryRoot: string): Promise<void> {
|
||||
await run("7z", ["t", archive, "-bso0", "-bsp0"]);
|
||||
validateArchivePaths(archive, await run("zipinfo", ["-1", archive]));
|
||||
|
||||
const temporaryDirectory = await mkdtemp(path.join(temporaryRoot, "archive-"));
|
||||
let destinationCreated = false;
|
||||
try {
|
||||
await run("7z", ["x", archive, `-o${temporaryDirectory}`, "-y", "-bso0", "-bsp0"]);
|
||||
await validateExtractedTree(temporaryDirectory);
|
||||
await mkdir(path.dirname(destination), { recursive: true });
|
||||
await mkdir(destination);
|
||||
destinationCreated = true;
|
||||
await run("cp", ["-a", "--", `${temporaryDirectory}${path.sep}.`, destination]);
|
||||
} catch (error) {
|
||||
if (destinationCreated) {
|
||||
await rm(destination, { recursive: true, force: true });
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<void> {
|
||||
const options = parseArguments(process.argv.slice(2));
|
||||
const rootStat = await stat(options.sourceDirectory);
|
||||
if (!rootStat.isDirectory()) {
|
||||
throw new Error(`Expected a source directory: ${options.sourceDirectory}`);
|
||||
}
|
||||
|
||||
const relativeOutput = path.relative(options.sourceDirectory, options.outputDirectory);
|
||||
if (
|
||||
relativeOutput === "" ||
|
||||
(!relativeOutput.startsWith(`..${path.sep}`) && relativeOutput !== ".." && !path.isAbsolute(relativeOutput))
|
||||
) {
|
||||
throw new Error("Output directory must not be inside the source directory");
|
||||
}
|
||||
|
||||
const archives = await collectArchives(options.sourceDirectory);
|
||||
if (archives.length === 0) {
|
||||
console.log(`No ZIP archives found in ${options.sourceDirectory}`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!options.dryRun) {
|
||||
await mkdir(options.outputDirectory, { recursive: true });
|
||||
await mkdir(options.temporaryDirectory, { recursive: true });
|
||||
}
|
||||
|
||||
let extracted = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
for (const archive of archives) {
|
||||
try {
|
||||
const destination = destinationFor(
|
||||
archive,
|
||||
options.sourceDirectory,
|
||||
options.outputDirectory,
|
||||
);
|
||||
if (await pathExists(destination)) {
|
||||
skipped += 1;
|
||||
console.log(`Skipped: ${archive} (destination exists: ${destination})`);
|
||||
} else if (options.dryRun) {
|
||||
console.log(`Would extract: ${archive} -> ${destination}`);
|
||||
} else {
|
||||
await extractArchive(archive, destination, options.temporaryDirectory);
|
||||
extracted += 1;
|
||||
console.log(`Extracted: ${archive} -> ${destination}`);
|
||||
}
|
||||
} catch (error) {
|
||||
failed += 1;
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`Failed: ${archive}: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Done. Archives: ${archives.length}; extracted: ${extracted}; skipped: ${skipped}; failed: ${failed}`,
|
||||
);
|
||||
if (failed > 0) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await main();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
console.error(`Error: ${message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
Reference in New Issue
Block a user