diff --git a/package-lock.json b/package-lock.json index 41dcda91..f1e1349d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -82,6 +82,7 @@ "husky": "^9.1.7", "jimp": "0.22.12", "jsdom": "^29.0.0", + "jszip": "^3.10.1", "otpauth": "^9.5.0", "prettier": "2.8.8", "sass-embedded": "^1.81.0", @@ -10087,6 +10088,12 @@ "integrity": "sha512-QpLcX9ZSsq3YYUUnD3nFDY8H7wctAhQj/TFKL8Ya8v5fMm3CFXxo8zStsLAl780ltoYoo1WvKUVGBQK+1ifr7g==", "license": "MIT" }, + "node_modules/immediate": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/immediate/-/immediate-3.0.6.tgz", + "integrity": "sha512-XXOFtyqDjNDAQxVfYxuF7g9Il/IbWmmlQg2MYKOH8ExIT1qg6xc4zyS3HaEEATgs1btfzxq15ciUiY7gjSXRGQ==", + "dev": true + }, "node_modules/immutable": { "version": "5.1.5", "resolved": "https://registry.npmjs.org/immutable/-/immutable-5.1.5.tgz", @@ -10994,6 +11001,18 @@ "node": ">=0.10.0" } }, + "node_modules/jszip": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/jszip/-/jszip-3.10.1.tgz", + "integrity": "sha512-xXDvecyTpGLrqFrvkrUSoxxfJI5AH7U8zxxtVclpsUtMCq4JQ290LY8AW5c7Ggnr/Y/oK+bQMbqK2qmtk3pN4g==", + "dev": true, + "dependencies": { + "lie": "~3.3.0", + "pako": "~1.0.2", + "readable-stream": "~2.3.6", + "setimmediate": "^1.0.5" + } + }, "node_modules/jwa": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", @@ -11062,6 +11081,15 @@ "node": ">= 0.8.0" } }, + "node_modules/lie": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/lie/-/lie-3.3.0.tgz", + "integrity": "sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==", + "dev": true, + "dependencies": { + "immediate": "~3.0.5" + } + }, "node_modules/lightningcss": { "version": "1.32.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", @@ -13627,6 +13655,12 @@ "node": ">= 0.4" } }, + "node_modules/setimmediate": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", + "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==", + "dev": true + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", diff --git a/package.json b/package.json index a004381d..d871e997 100644 --- a/package.json +++ b/package.json @@ -154,6 +154,7 @@ "husky": "^9.1.7", "jimp": "0.22.12", "jsdom": "^29.0.0", + "jszip": "^3.10.1", "otpauth": "^9.5.0", "prettier": "2.8.8", "sass-embedded": "^1.81.0", diff --git a/scripts/release/package-dist.mjs b/scripts/release/package-dist.mjs index 72be5453..50ade690 100644 --- a/scripts/release/package-dist.mjs +++ b/scripts/release/package-dist.mjs @@ -1,14 +1,14 @@ import crypto from "node:crypto"; -import { execFile } from "node:child_process"; import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; -import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; +import JSZip from "jszip"; -const execFileAsync = promisify(execFile); const SHA256_PATTERN = /^[0-9a-f]{64}$/i; const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}$/i; +const MAX_DIST_FILES = 10_000; +const MAX_DIST_BYTES = 256 * 1024 * 1024; const SERVER_EXECUTABLE_EXTENSION_PATTERN = /(?:^|\.)(?:php\d*|phtml|phar|cgi|fcgi|pl|pm|py|rb|sh|bash|zsh|fish|cmd|bat|ps1|exe|com|dll|so|dylib|jsp|jspx|asp|aspx)(?:\.|$)/i; @@ -100,6 +100,7 @@ export async function collectDistInventory(distDirectory) { } const files = []; + let totalBytes = 0; const walk = async (relativeDirectory = "") => { const absoluteDirectory = relativeDirectory ? path.join(root, relativeDirectory) : root; const names = (await fsp.readdir(absoluteDirectory)).sort(comparePaths); @@ -122,7 +123,17 @@ export async function collectDistInventory(distDirectory) { } assertSafeFileName(relativePath); + if (files.length >= MAX_DIST_FILES) { + throw new Error(`dist contains more than the ${MAX_DIST_FILES}-file packaging limit.`); + } + if (totalBytes + stat.size > MAX_DIST_BYTES) { + throw new Error(`dist exceeds the ${MAX_DIST_BYTES}-byte packaging limit.`); + } const contents = await fsp.readFile(absolutePath); + totalBytes += contents.length; + if (totalBytes > MAX_DIST_BYTES) { + throw new Error(`dist exceeds the ${MAX_DIST_BYTES}-byte packaging limit.`); + } files.push({ path: `dist/${relativePath}`, bytes: contents.length, @@ -238,23 +249,111 @@ function assertMatchingInventories(expected, actual, label) { } } +function zipEntryDate() { + return new Date("2020-01-01T00:00:00.000Z"); +} + +function archiveDirectoryPaths(inventory) { + const directories = new Set(["dist/"]); + for (const file of inventory.files) { + const segments = file.path.split("/"); + for (let depth = 1; depth < segments.length; depth += 1) { + directories.add(`${segments.slice(0, depth).join("/")}/`); + } + } + return Array.from(directories).sort(comparePaths); +} + +async function createZipArchive(distDirectory, inventory, archivePath) { + const archive = new JSZip(); + for (const directoryPath of archiveDirectoryPaths(inventory)) { + archive.file(directoryPath, null, { + createFolders: false, + date: zipEntryDate(), + dir: true, + unixPermissions: 0o40755, + }); + } + for (const file of inventory.files) { + const relativePath = file.path.slice("dist/".length); + assertSafeFileName(relativePath); + const contents = await fsp.readFile(path.join(distDirectory, ...relativePath.split("/"))); + archive.file(file.path, contents, { + binary: true, + createFolders: false, + date: zipEntryDate(), + unixPermissions: 0o100644, + }); + } + + const contents = await archive.generateAsync({ + type: "nodebuffer", + compression: "DEFLATE", + compressionOptions: { level: 6 }, + platform: "UNIX", + streamFiles: false, + }); + await fsp.writeFile(archivePath, contents, { flag: "wx" }); +} + +function archiveEntryName(entry) { + return entry.unsafeOriginalName || entry.name; +} + async function validateArchiveEntries(archivePath) { - const { stdout } = await execFileAsync("unzip", ["-Z1", archivePath], { maxBuffer: 10 * 1024 * 1024 }); - const entries = stdout.split(/\r?\n/).filter(Boolean); - if (entries.length === 0) { + let archive; + try { + archive = await JSZip.loadAsync(await fsp.readFile(archivePath), { checkCRC32: true }); + } catch (error) { + throw new Error(`release archive could not be read: ${error instanceof Error ? error.message : error}`, { + cause: error, + }); + } + + const entries = Object.values(archive.files); + if (entries.length === 0 || entries.every((entry) => entry.dir)) { throw new Error("release archive is empty."); } - for (const archiveEntry of entries) { + for (const entry of entries) { + const archiveEntry = archiveEntryName(entry); const normalizedEntry = archiveEntry.endsWith("/") ? archiveEntry.slice(0, -1) : archiveEntry; if (normalizedEntry === "dist") { + if (!entry.dir) { + throw new Error("release archive contains a file at the dist root path."); + } continue; } if (!normalizedEntry.startsWith("dist/")) { throw new Error(`release archive contains an entry outside dist/: ${archiveEntry}`); } assertSafeRelativePath(normalizedEntry, "archive entry"); + if (!entry.dir) { + assertSafeFileName(normalizedEntry.slice("dist/".length)); + } } + + return archive; +} + +async function collectArchiveInventory(archive) { + const files = []; + for (const entry of Object.values(archive.files)) { + if (entry.dir) { + continue; + } + const archiveEntry = archiveEntryName(entry); + const relativePath = archiveEntry.slice("dist/".length); + assertSafeFileName(relativePath); + const contents = await entry.async("nodebuffer"); + files.push({ + path: `dist/${relativePath}`, + bytes: contents.length, + sha256: sha256(contents), + }); + } + files.sort((left, right) => comparePaths(left.path, right.path)); + return { schema_version: 1, files }; } function appendGithubOutputs(values, githubOutput = process.env.GITHUB_OUTPUT) { @@ -330,25 +429,16 @@ export async function createReleaseArchive({ const sourceRoot = path.join(temporaryDirectory, "source"); const copiedDistDirectory = path.join(sourceRoot, "dist"); const temporaryArchivePath = path.join(temporaryDirectory, archiveName); - const extractionRoot = path.join(temporaryDirectory, "extracted"); await fsp.mkdir(sourceRoot, { recursive: true }); await fsp.cp(resolvedDistDirectory, copiedDistDirectory, { recursive: true, errorOnExist: true }); const copiedInventory = await collectDistInventory(copiedDistDirectory); assertMatchingInventories(inventory, copiedInventory, "archive source"); - await execFileAsync("zip", ["-X", "-q", "-r", temporaryArchivePath, "dist"], { - cwd: sourceRoot, - maxBuffer: 10 * 1024 * 1024, - }); - await validateArchiveEntries(temporaryArchivePath); - - await fsp.mkdir(extractionRoot); - await execFileAsync("unzip", ["-q", temporaryArchivePath, "-d", extractionRoot], { - maxBuffer: 10 * 1024 * 1024, - }); - const extractedInventory = await collectDistInventory(path.join(extractionRoot, "dist")); - assertMatchingInventories(inventory, extractedInventory, "round-trip extracted archive"); + await createZipArchive(copiedDistDirectory, copiedInventory, temporaryArchivePath); + const archive = await validateArchiveEntries(temporaryArchivePath); + const archivedInventory = await collectArchiveInventory(archive); + assertMatchingInventories(inventory, archivedInventory, "round-trip archive"); const archiveContents = await fsp.readFile(temporaryArchivePath); const archiveSha256 = sha256(archiveContents); diff --git a/tests/unit/release-package-dist.spec.js b/tests/unit/release-package-dist.spec.js index 480ee733..e28970a1 100644 --- a/tests/unit/release-package-dist.spec.js +++ b/tests/unit/release-package-dist.spec.js @@ -1,9 +1,9 @@ import crypto from "node:crypto"; -import { execFileSync } from "node:child_process"; import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import JSZip from "jszip"; import { collectDistInventory, @@ -90,25 +90,50 @@ describe("release dist packager", () => { }); const outputDirectory = path.join(temporaryRoot, "artifacts"); const githubOutput = path.join(temporaryRoot, "github-output.txt"); + const emptyPath = path.join(temporaryRoot, "empty-path"); await fs.writeFile(githubOutput, "existing=value\n"); + await fs.mkdir(emptyPath); - const result = await createReleaseArchive({ - distDirectory, - outputDirectory, - expectedCommitSha: COMMIT_SHA, - expectedBuildId: BUILD_ID, - runId: "123456", - runAttempt: "2", - githubOutput, - }); + const originalPath = process.env.PATH; + let result; + try { + process.env.PATH = emptyPath; + result = await createReleaseArchive({ + distDirectory, + outputDirectory, + expectedCommitSha: COMMIT_SHA, + expectedBuildId: BUILD_ID, + runId: "123456", + runAttempt: "2", + githubOutput, + }); + } finally { + if (originalPath === undefined) { + delete process.env.PATH; + } else { + process.env.PATH = originalPath; + } + } expect(result.archive_name).toBe(`pleno-vue-${COMMIT_SHA}-123456-2.zip`); expect(result.release_id).toBe(`${COMMIT_SHA}-123456-2`); expect(result.archive_sha256).toMatch(/^[0-9a-f]{64}$/); - expect(execFileSync("unzip", ["-Z1", result.archive_path], { encoding: "utf8" })).toContain("dist/.htaccess"); + const writtenInventory = JSON.parse(await fs.readFile(result.inventory_path, "utf8")); + const archive = await JSZip.loadAsync(await fs.readFile(result.archive_path), { checkCRC32: true }); + expect(archive.file("dist/.htaccess")).not.toBeNull(); + expect( + Object.values(archive.files) + .filter((entry) => !entry.dir) + .map((entry) => entry.name) + .sort() + ).toEqual(writtenInventory.files.map((file) => file.path)); + expect(archive.files["dist/"].dir).toBe(true); + expect(archive.files["dist/assets/"].dir).toBe(true); + for (const entry of Object.values(archive.files)) { + expect(Number(entry.unixPermissions) & 0o777).toBe(entry.dir ? 0o755 : 0o644); + } expect(await fs.readFile(result.checksum_path, "utf8")).toBe(`${result.archive_sha256} ${result.archive_name}\n`); - const writtenInventory = JSON.parse(await fs.readFile(result.inventory_path, "utf8")); expect(writtenInventory).toEqual(await collectDistInventory(distDirectory)); expect(writtenInventory.files.map((file) => file.path)).toEqual( [...writtenInventory.files.map((file) => file.path)].sort() @@ -124,6 +149,43 @@ describe("release dist packager", () => { expect(outputText).toContain(`archive_sha256=${result.archive_sha256}\n`); }); + it("creates byte-identical archives independently of source mtimes", async () => { + const distDirectory = await writeFixtureDist(temporaryRoot); + const options = { + distDirectory, + expectedCommitSha: COMMIT_SHA, + expectedBuildId: BUILD_ID, + runId: "123456", + runAttempt: "2", + }; + + const originalTimezone = process.env.TZ; + let first; + let second; + try { + process.env.TZ = "UTC"; + first = await createReleaseArchive({ + ...options, + outputDirectory: path.join(temporaryRoot, "first"), + }); + await fs.utimes(path.join(distDirectory, "index.html"), new Date(1_000_000), new Date(2_000_000)); + process.env.TZ = "America/New_York"; + second = await createReleaseArchive({ + ...options, + outputDirectory: path.join(temporaryRoot, "second"), + }); + } finally { + if (originalTimezone === undefined) { + delete process.env.TZ; + } else { + process.env.TZ = originalTimezone; + } + } + + expect(second.archive_sha256).toBe(first.archive_sha256); + expect(await fs.readFile(second.archive_path)).toEqual(await fs.readFile(first.archive_path)); + }); + it("rejects release metadata that does not identify the exact tested commit and build", async () => { const distDirectory = await writeFixtureDist(temporaryRoot, { commitSha: "fedcba9876543210fedcba9876543210fedcba98", @@ -162,6 +224,16 @@ describe("release dist packager", () => { ); }); + it("rejects an oversized sparse file before loading it into memory", async () => { + const distDirectory = await writeFixtureDist(temporaryRoot); + const oversizedPath = path.join(distDirectory, "oversized.bin"); + const handle = await fs.open(oversizedPath, "w"); + await handle.truncate(256 * 1024 * 1024 + 1); + await handle.close(); + + await expect(collectDistInventory(distDirectory)).rejects.toThrow(/268435456-byte packaging limit/); + }); + it("rejects symbolic links instead of following them into the archive", async () => { const distDirectory = await writeFixtureDist(temporaryRoot); await fs.symlink(path.join(distDirectory, "index.html"), path.join(distDirectory, "linked-index.html"));