mirror of
https://github.com/wekan/wekan.git
synced 2026-11-05 01:20:07 -05:00
Make sharp load on every WeKan platform: add each bundle's own sharp package.
sharp makes the attachment thumbnails. Every bundle except amd64 is made from the amd64 Linux bundle, and npm installed only @img/sharp-linux-x64 there. Reading the published v12.32 zips showed that every other bundle carried that one package and nothing it could load: arm64, ppc64le, s390x, riscv64, i386, armhf, armv7, armv6, mac-arm64 and mac-x64, as well as the Docker images and snaps built from them. Thumbnails fell back to the full-size image, and nothing failed. releases/windows-native-packages.mjs becomes platform-native-packages.mjs, <bundle> <win32|linux|darwin> <arch>. It removes the packages npm would not install on that target (os, cpu, and on Linux libc), downloads @img/sharp-<platform>-<arch> and its own libvips at the versions pinned, checks each against the registry's sha512, and checks every native addon's header for the target. sharp publishes nothing native for Linux i386 or loong64, so there it uses @img/sharp-wasm32. That package is now a root optional dependency in package.json, at sharp's own range. It was run under Node 26 with no native package: it made a 512 px WebP thumbnail and a GIF. check-addon-arch.mjs now reads ELF (with byte order) and Mach-O (including universal binaries) as well as PE, and follows npm's libc rule on Linux. The script now runs in build-amd64, build-arm64 (in its arm64 container), both macOS jobs, all three Windows jobs, and build-extra-arches through install-node-for-arch.sh. There it runs with the target node's own process.arch, so armv6/armv7/armhf are arm, ppc64le is ppc64 and i386 is ia32. build-release-bundle.sh runs it for the local machine. Against the published v12.32 zips, given their native addons, it added the right sharp and libvips for all eleven Linux and macOS bundles, kept wasm32 for i386, and passed every bcrypt, argon2 and sharp binary. Checked the other way, the s390x bundle was refused as ppc64. tests/platformNativePackages.test.cjs covers the script with a mocked registry (libvips, wasm fallback, wrong CPU, tampered tarball, path traversal). tests/sharpEveryPlatform.test.cjs requires every job that makes a bundle zip to run it for its own target, after the prune and before the zip, and the wasm32 dependency to stay at sharp's version. Thanks to xet7 !
This commit is contained in:
Lauri Ojansivu committed
1 parent
0e9e62cfa1
commit
943bfd9197
12 files changed
+835
-452
No files matched your search
@@ -669,6 +669,11 @@ jobs:
|
||||
# nothing requires, and refuses any policy entry the graph says IS reached.
|
||||
node "$GITHUB_WORKSPACE/releases/prune-unreachable-npm.mjs" \
|
||||
"$GITHUB_WORKSPACE/.build/bundle"
|
||||
# sharp (attachment thumbnails) for this platform, and every native
|
||||
# addon's header checked against it. Every other bundle starts from
|
||||
# this one and runs the same script for its own platform and CPU.
|
||||
node "$GITHUB_WORKSPACE/releases/platform-native-packages.mjs" \
|
||||
"$GITHUB_WORKSPACE/.build/bundle" linux x64
|
||||
|
||||
- name: Does the bundle actually start?
|
||||
# THE CHECK THAT WAS MISSING. v10.96 shipped a bundle whose boot.js died
|
||||
@@ -964,7 +969,7 @@ jobs:
|
||||
-v "$(pwd)/Dockerfile:/Dockerfile:ro" \
|
||||
-w /bundle/programs/server \
|
||||
node:${NODE_VERSION}-slim \
|
||||
sh -c "bash /releases/apt-install.sh libatomic1 build-essential python3 && bash /releases/use-release-npm.sh && bash /releases/npm-retry.sh npm install && node /releases/prune-build-only-modules.mjs /bundle && node /releases/bump-bundle-npm-deps.mjs /bundle && node /releases/bundle-trim.mjs /bundle --transport sockjs --drop-legacy-client --trim-prebuilds --platform linux --arch arm64 && node /releases/prune-unreachable-npm.mjs /bundle"
|
||||
sh -c "bash /releases/apt-install.sh libatomic1 build-essential python3 && bash /releases/use-release-npm.sh && bash /releases/npm-retry.sh npm install && node /releases/prune-build-only-modules.mjs /bundle && node /releases/bump-bundle-npm-deps.mjs /bundle && node /releases/bundle-trim.mjs /bundle --transport sockjs --drop-legacy-client --trim-prebuilds --platform linux --arch arm64 && node /releases/prune-unreachable-npm.mjs /bundle && node /releases/platform-native-packages.mjs /bundle linux arm64"
|
||||
|
||||
- name: Check the binaries the arm64 bundle needs exist
|
||||
if: steps.preflight.outputs.skip != 'true'
|
||||
@@ -1458,7 +1463,7 @@ jobs:
|
||||
run: |
|
||||
SRC="$PWD/src"
|
||||
set -euo pipefail
|
||||
node "$SRC/releases/windows-native-packages.mjs" bundle x64
|
||||
node "$SRC/releases/platform-native-packages.mjs" bundle win32 x64
|
||||
node "$SRC/releases/check-addon-arch.mjs" bundle x64 bcrypt argon2 @img/sharp-win32-x64
|
||||
|
||||
- name: Check the binaries the win64 bundle needs exist
|
||||
@@ -1885,7 +1890,7 @@ jobs:
|
||||
run: |
|
||||
SRC="$PWD/src"
|
||||
set -euo pipefail
|
||||
node "$SRC/releases/windows-native-packages.mjs" bundle arm64
|
||||
node "$SRC/releases/platform-native-packages.mjs" bundle win32 arm64
|
||||
node "$SRC/releases/check-addon-arch.mjs" bundle arm64 bcrypt argon2 @img/sharp-win32-arm64
|
||||
|
||||
- name: Check the binaries the win-arm64 bundle needs exist
|
||||
@@ -2326,7 +2331,7 @@ jobs:
|
||||
run: |
|
||||
SRC="$PWD/src"
|
||||
set -euo pipefail
|
||||
node "$SRC/releases/windows-native-packages.mjs" bundle ia32
|
||||
node "$SRC/releases/platform-native-packages.mjs" bundle win32 ia32
|
||||
node "$SRC/releases/check-addon-arch.mjs" bundle ia32 bcrypt argon2 @img/sharp-win32-ia32
|
||||
|
||||
- name: Check the binaries the win32 bundle needs exist
|
||||
@@ -2604,6 +2609,9 @@ jobs:
|
||||
# packages load it through Npm.require(). This removes only what it can prove
|
||||
# nothing requires, and refuses any policy entry the graph says IS reached.
|
||||
node "$GITHUB_WORKSPACE/releases/prune-unreachable-npm.mjs" ../..
|
||||
# sharp (attachment thumbnails): the amd64 bundle has only its Linux
|
||||
# package. Add the macOS arm64 one and check every native addon.
|
||||
node "$GITHUB_WORKSPACE/releases/platform-native-packages.mjs" ../.. darwin arm64
|
||||
cd ../../..
|
||||
|
||||
- name: Check the binaries the mac-arm64 bundle needs exist
|
||||
@@ -2889,6 +2897,9 @@ jobs:
|
||||
# packages load it through Npm.require(). This removes only what it can prove
|
||||
# nothing requires, and refuses any policy entry the graph says IS reached.
|
||||
node "$GITHUB_WORKSPACE/releases/prune-unreachable-npm.mjs" ../..
|
||||
# sharp (attachment thumbnails): the amd64 bundle has only its Linux
|
||||
# package. Add the macOS x64 one and check every native addon.
|
||||
node "$GITHUB_WORKSPACE/releases/platform-native-packages.mjs" ../.. darwin x64
|
||||
cd ../../..
|
||||
|
||||
- name: Check the binaries the mac-x64 bundle needs exist
|
||||
|
||||
Generated
+4
@@ -17,6 +17,7 @@
|
||||
"@babel/runtime": "^8.0.0",
|
||||
"@fastify/busboy": "^3.2.2",
|
||||
"@google-cloud/storage": "^8.0.1",
|
||||
"@img/sharp-wasm32": "^0.35.5",
|
||||
"@meteorjs/reify": "^0.25.4",
|
||||
"@node-saml/node-saml": "^5.1.0",
|
||||
"@rwap/jquery-ui-touch-punch": "^1.0.11",
|
||||
@@ -90,6 +91,9 @@
|
||||
"sinon": "^22.1.0",
|
||||
"sinon-chai": "^4.0.1",
|
||||
"style-loader": "^4.0.0"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@img/sharp-wasm32": "^0.35.5"
|
||||
}
|
||||
},
|
||||
"node_modules/@asamuzakjp/css-color": {
|
||||
|
||||
@@ -133,5 +133,8 @@
|
||||
"qs": "6.16.0"
|
||||
}
|
||||
}
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@img/sharp-wasm32": "^0.35.5"
|
||||
}
|
||||
}
|
||||
@@ -162,6 +162,11 @@ node "$ROOT/releases/bundle-trim.mjs" "$BUNDLE" --transport sockjs --drop-legacy
|
||||
|
||||
say "3/6 prune-unreachable-npm"
|
||||
node "$ROOT/releases/prune-unreachable-npm.mjs" "$BUNDLE" || fail "prune-unreachable-npm failed."
|
||||
# The release workflow's own step: this machine's sharp package (attachment
|
||||
# thumbnails), and every native addon's header checked against this machine.
|
||||
node "$ROOT/releases/platform-native-packages.mjs" "$BUNDLE" \
|
||||
"$(node -p process.platform)" "$(node -p process.arch)" \
|
||||
|| fail "platform-native-packages failed: a native module in the bundle is not for this machine."
|
||||
|
||||
# ── 2. Does it start? ────────────────────────────────────────────────────────
|
||||
#
|
||||
|
||||
@@ -1,44 +1,65 @@
|
||||
#!/usr/bin/env node
|
||||
// Check that a Windows bundle's native addons are built for its own CPU.
|
||||
// Check that a bundle's native addons are built for its own platform and CPU.
|
||||
//
|
||||
// node releases/check-addon-arch.mjs <bundle dir> <ia32|x64|arm64> [package ...]
|
||||
// node releases/check-addon-arch.mjs <bundle dir> <arch> [package ...] [--platform win32|linux|darwin]
|
||||
//
|
||||
// Why. The win32 bundle is assembled on a 64-bit runner from the amd64 bundle,
|
||||
// and its addons (bcrypt and argon2, which Meteor's accounts-password imports
|
||||
// with no fallback) are cross-compiled with npm_config_arch=ia32. If any step
|
||||
// of that quietly does nothing - a skipped rebuild, a nodedir node-gyp did not
|
||||
// use - the zip still builds, still passes `7z t`, and ships a 64-bit or Linux
|
||||
// .node that a 32-bit node.exe cannot load. Nobody would see it until the
|
||||
// first person tried to sign in. So this reads the binary itself.
|
||||
// <arch> is Node's name for the CPU (process.arch): x64, arm64, ia32, arm,
|
||||
// ppc64, s390x, riscv64, loong64. --platform defaults to win32.
|
||||
//
|
||||
// Why. Every bundle except amd64 is assembled from the amd64 Linux bundle and
|
||||
// then has its native modules replaced for its own platform. If any step of
|
||||
// that quietly does nothing - a skipped rebuild, a platform package that never
|
||||
// arrived - the zip still builds and still passes `7z t`, and ships a .node
|
||||
// its node cannot load. bcrypt and argon2 (accounts-password imports both with
|
||||
// no fallback) would fail the first sign-in; sharp would silently turn every
|
||||
// thumbnail into the full-size image. So this reads each binary's header.
|
||||
//
|
||||
// What it checks. Every *.node outside a prebuilds/ directory is what Node's
|
||||
// require() or node-gyp-build's build/Release lookup loads, so each must be a
|
||||
// PE file for the wanted machine. prebuilds/ holds one directory per platform
|
||||
// and node-gyp-build opens only the matching one, so other platforms' files
|
||||
// there are expected and ignored. Each package named on the command line must
|
||||
// have at least one loadable addon for the wanted CPU, in build output or in
|
||||
// its own prebuilds/win32-<arch>/.
|
||||
// binary for this platform's format (PE, ELF, Mach-O) and CPU. prebuilds/
|
||||
// holds one directory per platform and node-gyp-build opens only the matching
|
||||
// one, so other platforms' files there are ignored. Each package named on the
|
||||
// command line must have at least one loadable addon for this platform/CPU.
|
||||
//
|
||||
// A package whose package.json `os` or `cpu` excludes this target is skipped
|
||||
// as well, by the same rule npm itself applies: the amd64 bundle carries
|
||||
// platform packages such as @img/sharp-linux-x64 (os: linux), which their
|
||||
// parent package only ever loads on that platform.
|
||||
// A package whose package.json `os`, `cpu` or (on Linux) `libc` excludes this
|
||||
// target is skipped, by the same rule npm itself installs by: the amd64 bundle
|
||||
// carries platform packages such as @img/sharp-linux-x64, which their parent
|
||||
// only ever loads on that platform.
|
||||
|
||||
import { readdirSync, openSync, readSync, closeSync, statSync, readFileSync, existsSync } from 'node:fs';
|
||||
import { join, relative, sep, resolve, dirname } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
|
||||
export const PE_MACHINE = { ia32: 0x014c, x64: 0x8664, arm64: 0xaa64 };
|
||||
export const ELF_MACHINE = {
|
||||
ia32: 3, x64: 62, arm: 40, arm64: 183, ppc64: 21, s390x: 22, riscv64: 243, loong64: 258,
|
||||
};
|
||||
export const MACHO_CPU = { x64: 0x01000007, arm64: 0x0100000c };
|
||||
const FORMAT = { win32: 'PE', linux: 'ELF', darwin: 'Mach-O' };
|
||||
const MACHINES = { win32: PE_MACHINE, linux: ELF_MACHINE, darwin: MACHO_CPU };
|
||||
|
||||
// The binary format and, for PE, its machine field.
|
||||
// The binary format, and the CPU(s) it is for.
|
||||
export function readBinaryKind(file) {
|
||||
const fd = openSync(file, 'r');
|
||||
try {
|
||||
const head = Buffer.alloc(64);
|
||||
const n = readSync(fd, head, 0, 64, 0);
|
||||
if (n >= 4 && head.readUInt32BE(0) === 0x7f454c46) return { format: 'ELF' };
|
||||
if (n >= 4 && [0xcffaedfe, 0xcefaedfe, 0xcafebabe].includes(head.readUInt32BE(0))) {
|
||||
return { format: 'Mach-O' };
|
||||
if (n >= 20 && head.readUInt32BE(0) === 0x7f454c46) {
|
||||
// e_machine is in the file's own byte order (EI_DATA: 1 little, 2 big).
|
||||
const machine = head[5] === 2 ? head.readUInt16BE(18) : head.readUInt16LE(18);
|
||||
return { format: 'ELF', machine };
|
||||
}
|
||||
if (n >= 8 && [0xcffaedfe, 0xcefaedfe].includes(head.readUInt32BE(0))) {
|
||||
return { format: 'Mach-O', cpus: [head.readUInt32LE(4)] };
|
||||
}
|
||||
if (n >= 8 && head.readUInt32BE(0) === 0xcafebabe) {
|
||||
// A universal binary: a count, then 20-byte entries starting with cputype.
|
||||
const count = head.readUInt32BE(4);
|
||||
const table = Buffer.alloc(count * 20);
|
||||
readSync(fd, table, 0, table.length, 8);
|
||||
const cpus = [];
|
||||
for (let i = 0; i < count; i++) cpus.push(table.readUInt32BE(i * 20));
|
||||
return { format: 'Mach-O', cpus };
|
||||
}
|
||||
if (n < 64 || head.toString('latin1', 0, 2) !== 'MZ') return { format: 'unknown' };
|
||||
const peOffset = head.readUInt32LE(0x3c);
|
||||
@@ -52,8 +73,14 @@ export function readBinaryKind(file) {
|
||||
}
|
||||
}
|
||||
|
||||
// npm's os/cpu rule: a list excludes a value when it names it with "!", or
|
||||
// when it names only positive values and this is not one of them.
|
||||
export function isFor(kind, platform, arch) {
|
||||
const want = MACHINES[platform]?.[arch];
|
||||
if (want === undefined || kind.format !== FORMAT[platform]) return false;
|
||||
return kind.format === 'Mach-O' ? kind.cpus.includes(want) : kind.machine === want;
|
||||
}
|
||||
|
||||
// npm's os/cpu/libc rule: a list excludes a value when it names it with "!",
|
||||
// or when it names only positive values and this is not one of them.
|
||||
export function listAllows(list, value) {
|
||||
if (!Array.isArray(list) || list.length === 0) return true;
|
||||
if (list.includes(`!${value}`)) return false;
|
||||
@@ -61,16 +88,21 @@ export function listAllows(list, value) {
|
||||
return positive.length === 0 || positive.includes(value);
|
||||
}
|
||||
|
||||
export function packageAllows(pkg, platform, arch) {
|
||||
return listAllows(pkg.os, platform) && listAllows(pkg.cpu, arch) &&
|
||||
(platform !== 'linux' || listAllows(pkg.libc, 'glibc'));
|
||||
}
|
||||
|
||||
// The package a file belongs to, if its package.json declares a platform
|
||||
// this target is not: such a package is never loaded here.
|
||||
function foreignPlatformPackage(bundle, file, arch) {
|
||||
function foreignPlatformPackage(bundle, file, platform, arch) {
|
||||
let dir = dirname(file);
|
||||
while (dir.length >= bundle.length) {
|
||||
const manifest = join(dir, 'package.json');
|
||||
if (existsSync(manifest)) {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(manifest, 'utf8'));
|
||||
if (!listAllows(pkg.os, 'win32') || !listAllows(pkg.cpu, arch)) return relative(bundle, dir);
|
||||
if (!packageAllows(pkg, platform, arch)) return relative(bundle, dir);
|
||||
} catch {
|
||||
// An unreadable manifest declares nothing.
|
||||
}
|
||||
@@ -82,6 +114,12 @@ function foreignPlatformPackage(bundle, file, arch) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// node-gyp-build's prebuilds/<platform>-<arch>[+<arch>...] directory names.
|
||||
function prebuildMatches(dirName, platform, arch) {
|
||||
const [p, archs = ''] = dirName.split('-');
|
||||
return p === platform && archs.split('+').includes(arch);
|
||||
}
|
||||
|
||||
function walk(dir, out = []) {
|
||||
for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
const p = join(dir, e.name);
|
||||
@@ -91,44 +129,48 @@ function walk(dir, out = []) {
|
||||
return out;
|
||||
}
|
||||
|
||||
const describe = (k) =>
|
||||
k.format === 'PE'
|
||||
? `PE machine 0x${k.machine.toString(16).padStart(4, '0')}`
|
||||
: k.format;
|
||||
const describe = (k) => {
|
||||
if (k.format === 'PE') return `PE machine 0x${k.machine.toString(16).padStart(4, '0')}`;
|
||||
if (k.format === 'ELF') return `ELF machine ${k.machine}`;
|
||||
if (k.format === 'Mach-O') return `Mach-O cpu ${k.cpus.map((c) => `0x${c.toString(16)}`).join('+')}`;
|
||||
return k.format;
|
||||
};
|
||||
const PLATFORM_NAME = { win32: 'Windows', linux: 'Linux', darwin: 'macOS' };
|
||||
|
||||
export function checkAddonArch(bundle, arch, packages = []) {
|
||||
const want = PE_MACHINE[arch];
|
||||
if (!want) throw new Error(`unknown arch ${arch}; expected one of ${Object.keys(PE_MACHINE).join(', ')}`);
|
||||
export function checkAddonArch(bundle, arch, packages = [], { platform = 'win32' } = {}) {
|
||||
if (MACHINES[platform]?.[arch] === undefined) {
|
||||
throw new Error(`unknown target ${platform}-${arch}; ${platform} CPUs: ${Object.keys(MACHINES[platform] || {}).join(', ')}`);
|
||||
}
|
||||
const target = `${arch} ${PLATFORM_NAME[platform]}`;
|
||||
const errors = [];
|
||||
const good = [];
|
||||
const skipped = [];
|
||||
const files = walk(bundle);
|
||||
for (const f of files) {
|
||||
const foreign = foreignPlatformPackage(bundle, f, arch);
|
||||
if (foreign) {
|
||||
if (foreignPlatformPackage(bundle, f, platform, arch)) {
|
||||
skipped.push(relative(bundle, f));
|
||||
continue;
|
||||
}
|
||||
const parts = relative(bundle, f).split(sep);
|
||||
const pi = parts.lastIndexOf('prebuilds');
|
||||
const kind = readBinaryKind(f);
|
||||
const ok = kind.format === 'PE' && kind.machine === want;
|
||||
const ok = isFor(kind, platform, arch);
|
||||
if (pi === -1) {
|
||||
if (ok) good.push(f);
|
||||
else errors.push(`${relative(bundle, f)} is ${describe(kind)}, not a ${arch} Windows addon`);
|
||||
} else if (parts[pi + 1] === `win32-${arch}`) {
|
||||
else errors.push(`${relative(bundle, f)} is ${describe(kind)}, not a ${target} addon`);
|
||||
} else if (prebuildMatches(parts[pi + 1] || '', platform, arch)) {
|
||||
if (ok) good.push(f);
|
||||
else errors.push(`${relative(bundle, f)} is ${describe(kind)}, in the ${arch} prebuilds`);
|
||||
else errors.push(`${relative(bundle, f)} is ${describe(kind)}, in the ${target} prebuilds`);
|
||||
}
|
||||
}
|
||||
for (const name of packages) {
|
||||
// Every copy of the package in the bundle, by its node_modules/<name> path.
|
||||
const seg = name.split('/');
|
||||
const dirs = new Set(
|
||||
files
|
||||
.map((f) => relative(bundle, f).split(sep))
|
||||
.map((parts) => {
|
||||
// A scoped name (@img/sharp-win32-x64) is two path segments.
|
||||
const seg = name.split('/');
|
||||
for (let i = parts.length - 1 - seg.length; i >= 1; i--) {
|
||||
if (parts[i - 1] === 'node_modules' && seg.every((s, k) => parts[i + k] === s)) {
|
||||
return parts.slice(0, i + seg.length).join(sep);
|
||||
@@ -144,7 +186,7 @@ export function checkAddonArch(bundle, arch, packages = []) {
|
||||
}
|
||||
for (const d of dirs) {
|
||||
if (!good.some((g) => relative(bundle, g).startsWith(d + sep))) {
|
||||
errors.push(`${d} has no ${arch} Windows addon to load`);
|
||||
errors.push(`${d} has no ${target} addon to load`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -152,15 +194,18 @@ export function checkAddonArch(bundle, arch, packages = []) {
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const [bundle, arch, ...packages] = process.argv.slice(2);
|
||||
const args = process.argv.slice(2);
|
||||
const pi = args.indexOf('--platform');
|
||||
const platform = pi === -1 ? 'win32' : args.splice(pi, 2)[1];
|
||||
const [bundle, arch, ...packages] = args;
|
||||
if (!bundle || !arch || !statSync(bundle, { throwIfNoEntry: false })?.isDirectory()) {
|
||||
console.error('usage: check-addon-arch.mjs <bundle dir> <ia32|x64|arm64> [package ...]');
|
||||
console.error('usage: check-addon-arch.mjs <bundle dir> <arch> [package ...] [--platform win32|linux|darwin]');
|
||||
process.exit(2);
|
||||
}
|
||||
const { errors, good, skipped } = checkAddonArch(bundle, arch, packages);
|
||||
const { errors, good, skipped } = checkAddonArch(bundle, arch, packages, { platform });
|
||||
for (const g of good) console.log(`ok ${g}`);
|
||||
for (const s of skipped) console.log(`-- ${s} (its package is for another platform)`);
|
||||
for (const e of errors) console.log(`::error::${e}`);
|
||||
if (errors.length) process.exit(1);
|
||||
console.log(`check-addon-arch: ${good.length} ${arch} addon(s), all loadable by a ${arch} node.exe.`);
|
||||
console.log(`check-addon-arch: ${good.length} ${platform}-${arch} addon(s), all loadable there.`);
|
||||
}
|
||||
@@ -185,6 +185,12 @@ node "$(dirname "$0")/bump-bundle-npm-deps.mjs" /bundle
|
||||
node "$(dirname "$0")/bundle-trim.mjs" /bundle --transport sockjs --drop-legacy-client
|
||||
# And the npm tree rspack cannot tree-shake: only what it can prove is unreachable.
|
||||
node "$(dirname "$0")/prune-unreachable-npm.mjs" /bundle
|
||||
# sharp (attachment thumbnails): the amd64 bundle this one came from has only
|
||||
# @img/sharp-linux-x64, which cannot load here. Add this CPU's package, or keep
|
||||
# the WebAssembly fallback where sharp publishes none (i386, loong64), and check
|
||||
# every native addon's header. `process.arch` of THIS node is the name sharp
|
||||
# itself looks up: arm for armv6/armv7/armhf, ppc64 for ppc64le, ia32 for i386.
|
||||
node "$(dirname "$0")/platform-native-packages.mjs" /bundle linux "$(node -p process.arch)"
|
||||
|
||||
# Bundle the target-arch Node.js for the self-contained launcher. /bundle is the
|
||||
# host's bundle/ directory, mounted by the workflow.
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env node
|
||||
// Give a bundle the platform packages its own platform and CPU load.
|
||||
//
|
||||
// node releases/platform-native-packages.mjs <bundle dir> <win32|linux|darwin> <arch> [--dry-run]
|
||||
//
|
||||
// <arch> is Node's name for the CPU (process.arch): x64, arm64, ia32, arm,
|
||||
// ppc64, s390x, riscv64, loong64. ppc64le is `ppc64` to Node, and armv6,
|
||||
// armv7 and armhf are all `arm`.
|
||||
//
|
||||
// Why. Every bundle except amd64 is made from the amd64 Linux bundle, and npm
|
||||
// installed that one's optional platform packages for linux-x64 only. sharp,
|
||||
// which makes the attachment thumbnails (server/lib/imageThumbnail.js), loads
|
||||
// `@img/sharp-<platform>-<arch>/sharp.node`, so on every other platform it
|
||||
// found nothing and every thumbnail fell back to the full-size original - in
|
||||
// v12.32 that was arm64, ppc64le, s390x, riscv64, i386, armhf, armv7, armv6,
|
||||
// both macOS and all three Windows bundles, and every Docker image and snap
|
||||
// made from them.
|
||||
//
|
||||
// What it does.
|
||||
// 1. Removes every package under programs/server whose package.json `os`,
|
||||
// `cpu` or (on Linux) `libc` excludes this target - the rule npm installs
|
||||
// by, so these are exactly the packages an install here would not have
|
||||
// (@img/sharp-linux-x64, its libvips, @napi-rs/canvas-linux-x64-gnu...).
|
||||
// 2. When the bundled sharp lists @img/sharp-<platform>-<arch>, downloads it
|
||||
// and its own libvips package at exactly the versions pinned, checks each
|
||||
// against the npm registry's sha512, and unpacks them beside sharp.
|
||||
// sharp publishes these for linux x64/arm64/arm/ppc64/s390x/riscv64,
|
||||
// darwin x64/arm64 and win32 x64/arm64/ia32.
|
||||
// 3. For a CPU sharp publishes nothing for (Linux i386 and loong64), sharp
|
||||
// falls back to @img/sharp-wasm32, which WeKan's package.json installs
|
||||
// into every bundle. That is checked to be there and to match sharp.
|
||||
// 4. Reads every remaining native addon's header (check-addon-arch.mjs), and
|
||||
// fails unless the sharp package from step 2 is loadable here.
|
||||
//
|
||||
// All in Node, on purpose: no shell `tar`, which in Git Bash reads a Windows
|
||||
// drive letter as a remote host (see the build-win32 headers step).
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, normalize, relative, resolve, sep } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import { checkAddonArch, packageAllows, ELF_MACHINE, MACHO_CPU, PE_MACHINE } from './check-addon-arch.mjs';
|
||||
|
||||
export const TARGETS = { win32: Object.keys(PE_MACHINE), linux: Object.keys(ELF_MACHINE), darwin: Object.keys(MACHO_CPU) };
|
||||
const REGISTRY = 'https://registry.npmjs.org';
|
||||
const WASM = '@img/sharp-wasm32';
|
||||
|
||||
// The package sharp loads on <platform>-<arch>, at the version sharp pins -
|
||||
// or null when sharp publishes none and falls back to WebAssembly.
|
||||
export function sharpPlatformSpec(sharpPackageJson, platform, arch) {
|
||||
const name = `@img/sharp-${platform}-${arch}`;
|
||||
const version = sharpPackageJson.optionalDependencies?.[name];
|
||||
return version ? { name, version } : null;
|
||||
}
|
||||
|
||||
// A minimal ustar reader: npm tarballs are ustar, with long paths in the
|
||||
// prefix field. Entries are returned with the leading `package/` removed.
|
||||
// Anything that would land outside the package directory is refused.
|
||||
export function untar(buffer) {
|
||||
const files = [];
|
||||
let off = 0;
|
||||
let longName = null;
|
||||
while (off + 512 <= buffer.length) {
|
||||
const header = buffer.subarray(off, off + 512);
|
||||
if (header.every((b) => b === 0)) break;
|
||||
const str = (a, b) => header.toString('utf8', a, b).replace(/\0.*$/s, '');
|
||||
const size = parseInt(str(124, 136).trim() || '0', 8);
|
||||
const type = str(156, 157) || '0';
|
||||
const prefix = str(345, 500);
|
||||
const name = longName ?? (prefix ? `${prefix}/${str(0, 100)}` : str(0, 100));
|
||||
longName = null;
|
||||
const body = buffer.subarray(off + 512, off + 512 + size);
|
||||
off += 512 + Math.ceil(size / 512) * 512;
|
||||
if (type === 'L') {
|
||||
longName = body.toString('utf8').replace(/\0.*$/s, '');
|
||||
continue;
|
||||
}
|
||||
if (type === 'x' || type === 'g') {
|
||||
const path = /\d+ path=([^\n]*)\n/.exec(body.toString('utf8'));
|
||||
if (path) longName = path[1];
|
||||
continue;
|
||||
}
|
||||
if (type !== '0' && type !== '\0' && type !== '5') continue;
|
||||
const rel = name.replace(/^[^/]+\//, '');
|
||||
const clean = normalize(rel);
|
||||
if (!rel || clean.startsWith('..') || clean.startsWith(sep) || clean.startsWith('/') || /^[a-zA-Z]:/.test(clean)) {
|
||||
throw new Error(`refusing tar entry outside the package: ${name}`);
|
||||
}
|
||||
if (type === '5') continue;
|
||||
files.push({ path: rel, data: Buffer.from(body) });
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export function verifyIntegrity(buffer, integrity) {
|
||||
const m = /^sha512-(.+)$/.exec(integrity || '');
|
||||
if (!m) throw new Error(`no sha512 integrity to check against (got ${integrity})`);
|
||||
const got = createHash('sha512').update(buffer).digest('base64');
|
||||
if (got !== m[1]) throw new Error(`integrity mismatch: registry ${m[1]}, downloaded ${got}`);
|
||||
}
|
||||
|
||||
async function fetchRetry(url, as) {
|
||||
let last;
|
||||
for (let attempt = 1; attempt <= 5; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) return as === 'json' ? res.json() : Buffer.from(await res.arrayBuffer());
|
||||
last = new Error(`${url}: HTTP ${res.status}`);
|
||||
if (res.status < 500 && res.status !== 429) throw last;
|
||||
} catch (e) {
|
||||
last = e;
|
||||
if (/HTTP 4\d\d/.test(e.message) && !/HTTP 429/.test(e.message)) throw e;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, attempt * 5000));
|
||||
}
|
||||
throw last;
|
||||
}
|
||||
|
||||
// Package directories: a node_modules/<name> or node_modules/@scope/<name>.
|
||||
function packageDirs(root, out = []) {
|
||||
if (!existsSync(root)) return out;
|
||||
for (const e of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const p = join(root, e.name);
|
||||
if (e.name === 'node_modules') {
|
||||
for (const m of readdirSync(p, { withFileTypes: true })) {
|
||||
if (!m.isDirectory() || m.name.startsWith('.')) continue;
|
||||
const q = join(p, m.name);
|
||||
if (m.name.startsWith('@')) {
|
||||
for (const s of readdirSync(q, { withFileTypes: true })) {
|
||||
if (s.isDirectory()) out.push(join(q, s.name));
|
||||
}
|
||||
} else {
|
||||
out.push(q);
|
||||
}
|
||||
}
|
||||
}
|
||||
packageDirs(p, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Packages npm would not install on <platform>-<arch>.
|
||||
export function foreignPackages(serverDir, platform, arch) {
|
||||
const foreign = [];
|
||||
for (const dir of new Set(packageDirs(serverDir))) {
|
||||
const manifest = join(dir, 'package.json');
|
||||
if (!existsSync(manifest)) continue;
|
||||
let pkg;
|
||||
try {
|
||||
pkg = JSON.parse(readFileSync(manifest, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!packageAllows(pkg, platform, arch)) foreign.push(dir);
|
||||
}
|
||||
// A removed package takes its own nested node_modules with it.
|
||||
return foreign.filter((d) => !foreign.some((o) => o !== d && d.startsWith(o + sep)));
|
||||
}
|
||||
|
||||
async function download(name, version) {
|
||||
if (!/^\d+\.\d+\.\d+(?:-[\w.]+)?$/.test(version)) {
|
||||
throw new Error(`${name}: expected an exact version, got ${version}`);
|
||||
}
|
||||
const meta = await fetchRetry(`${REGISTRY}/${name.replace('/', '%2F')}/${version}`, 'json');
|
||||
const tarball = await fetchRetry(meta.dist.tarball, 'buffer');
|
||||
verifyIntegrity(tarball, meta.dist.integrity);
|
||||
return { meta, files: untar(gunzipSync(tarball)) };
|
||||
}
|
||||
|
||||
function writePackage(dest, files) {
|
||||
rmSync(dest, { recursive: true, force: true });
|
||||
for (const f of files) {
|
||||
const out = resolve(dest, f.path);
|
||||
if (!out.startsWith(resolve(dest) + sep)) throw new Error(`refusing ${f.path}`);
|
||||
mkdirSync(dirname(out), { recursive: true });
|
||||
writeFileSync(out, f.data);
|
||||
}
|
||||
}
|
||||
|
||||
export async function addPlatformNativePackages(bundle, platform, arch, { dryRun = false, log = console.log } = {}) {
|
||||
if (!TARGETS[platform]?.includes(arch)) {
|
||||
throw new Error(`unknown target ${platform}-${arch}; expected one of ${Object.entries(TARGETS).map(([p, a]) => `${p}-{${a.join(',')}}`).join(' ')}`);
|
||||
}
|
||||
const server = join(bundle, 'programs', 'server');
|
||||
const modules = join(server, 'npm', 'node_modules');
|
||||
const sharpManifest = join(modules, 'sharp', 'package.json');
|
||||
if (!existsSync(sharpManifest)) throw new Error(`no sharp in ${relative(bundle, modules)}`);
|
||||
const sharp = JSON.parse(readFileSync(sharpManifest, 'utf8'));
|
||||
|
||||
const removed = foreignPackages(server, platform, arch);
|
||||
for (const dir of removed) {
|
||||
log(`remove ${relative(bundle, dir)} (its package.json names another platform)`);
|
||||
if (!dryRun) rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const added = [];
|
||||
const spec = sharpPlatformSpec(sharp, platform, arch);
|
||||
if (spec) {
|
||||
const { meta, files } = await download(spec.name, spec.version);
|
||||
const install = [{ ...spec, files }];
|
||||
// Its libvips: an optionalDependency of the platform package, pinned exactly.
|
||||
for (const [dep, version] of Object.entries(meta.optionalDependencies || {})) {
|
||||
install.push({ name: dep, version, files: (await download(dep, version)).files });
|
||||
}
|
||||
for (const p of install) {
|
||||
const dest = join(modules, ...p.name.split('/'));
|
||||
if (!dryRun) writePackage(dest, p.files);
|
||||
log(`add ${relative(bundle, dest)} ${p.version} (${p.files.length} files, sha512 verified)`);
|
||||
added.push({ name: p.name, version: p.version });
|
||||
}
|
||||
} else {
|
||||
// sharp publishes nothing native for this CPU; it falls back to WebAssembly.
|
||||
const wasm = join(modules, ...WASM.split('/'), 'package.json');
|
||||
if (!existsSync(wasm)) {
|
||||
throw new Error(`sharp ${sharp.version} has no ${platform}-${arch} package, and ${WASM} is not in the bundle to fall back to - is it in package.json's optionalDependencies?`);
|
||||
}
|
||||
const wasmVersion = JSON.parse(readFileSync(wasm, 'utf8')).version;
|
||||
if (wasmVersion !== sharp.version) {
|
||||
throw new Error(`${WASM} ${wasmVersion} does not match sharp ${sharp.version}`);
|
||||
}
|
||||
if (!existsSync(join(modules, '@emnapi', 'runtime', 'package.json'))) {
|
||||
throw new Error(`${WASM} needs @emnapi/runtime, and it is not in the bundle`);
|
||||
}
|
||||
log(`keep ${WASM} ${wasmVersion}: sharp publishes no ${platform}-${arch} binary, so it runs as WebAssembly`);
|
||||
}
|
||||
|
||||
if (dryRun) return { removed: removed.map((d) => relative(bundle, d)), added, errors: [] };
|
||||
const { errors, good, skipped } = checkAddonArch(bundle, arch, spec ? [spec.name] : [], { platform });
|
||||
for (const g of good) log(`ok ${g}`);
|
||||
for (const s of skipped) log(`-- ${s} (another platform's package)`);
|
||||
return { removed: removed.map((d) => relative(bundle, d)), added, errors };
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const args = process.argv.slice(2).filter((a) => a !== '--dry-run');
|
||||
const [bundle, platform, arch] = args;
|
||||
if (!bundle || !platform || !arch) {
|
||||
console.error('usage: platform-native-packages.mjs <bundle dir> <win32|linux|darwin> <arch> [--dry-run]');
|
||||
process.exit(2);
|
||||
}
|
||||
addPlatformNativePackages(bundle, platform, arch, { dryRun: process.argv.includes('--dry-run') })
|
||||
.then(({ errors }) => {
|
||||
for (const e of errors) console.log(`::error::${e}`);
|
||||
if (errors.length) process.exit(1);
|
||||
console.log(`platform-native-packages: ${platform}-${arch} bundle has the native packages it loads.`);
|
||||
})
|
||||
.catch((e) => {
|
||||
console.error(`::error::platform-native-packages: ${e.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// Give a Windows bundle the platform packages its own CPU loads.
|
||||
//
|
||||
// node releases/windows-native-packages.mjs <bundle dir> <ia32|x64|arm64> [--dry-run]
|
||||
//
|
||||
// Why. Every Windows bundle is made from the amd64 Linux bundle, and npm
|
||||
// installed that one's optional platform packages for linux-x64 only. sharp,
|
||||
// which makes the attachment thumbnails (server/lib/imageThumbnail.js), loads
|
||||
// `@img/sharp-<platform>-<arch>/sharp.node` - so on Windows it found nothing,
|
||||
// and every thumbnail fell back to the full-size original. sharp publishes
|
||||
// @img/sharp-win32-ia32, -x64 and -arm64, libvips DLLs included, so nothing
|
||||
// has to be compiled: the bundle only needs the right package.
|
||||
//
|
||||
// What it does.
|
||||
// 1. Reads the bundled sharp's version and optionalDependencies, and takes
|
||||
// @img/sharp-win32-<arch> at exactly the version sharp lists. Downloads its
|
||||
// tarball from the npm registry, checks it against the registry's sha512
|
||||
// integrity, and unpacks it beside sharp in node_modules/@img/.
|
||||
// 2. Removes every package under programs/server whose package.json `os` or
|
||||
// `cpu` excludes win32/<arch> - the rule npm itself installs by, so these
|
||||
// are exactly the packages an install on this platform would not have.
|
||||
// @img/sharp-linux-x64, its libvips and @napi-rs/canvas-linux-x64-gnu are
|
||||
// what this removes today.
|
||||
//
|
||||
// All in Node, on purpose: no shell `tar`, which in Git Bash reads a Windows
|
||||
// drive letter as a remote host (see the build-win32 headers step).
|
||||
|
||||
import { createHash } from 'node:crypto';
|
||||
import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, join, normalize, relative, resolve, sep } from 'node:path';
|
||||
import { pathToFileURL } from 'node:url';
|
||||
import { gunzipSync } from 'node:zlib';
|
||||
import { listAllows } from './check-addon-arch.mjs';
|
||||
|
||||
export const ARCHES = ['ia32', 'x64', 'arm64'];
|
||||
const REGISTRY = 'https://registry.npmjs.org';
|
||||
|
||||
// The package sharp loads on win32/<arch>, at the version sharp itself pins.
|
||||
export function sharpPlatformSpec(sharpPackageJson, arch) {
|
||||
const name = `@img/sharp-win32-${arch}`;
|
||||
const version = sharpPackageJson.optionalDependencies?.[name];
|
||||
if (!version) {
|
||||
throw new Error(`sharp ${sharpPackageJson.version} lists no ${name}: it does not support win32-${arch}`);
|
||||
}
|
||||
return { name, version };
|
||||
}
|
||||
|
||||
// A minimal ustar reader: npm tarballs are ustar, with long paths in the
|
||||
// prefix field. Entries are returned with the leading `package/` removed.
|
||||
// Anything that would land outside the package directory is refused.
|
||||
export function untar(buffer) {
|
||||
const files = [];
|
||||
let off = 0;
|
||||
let longName = null;
|
||||
while (off + 512 <= buffer.length) {
|
||||
const header = buffer.subarray(off, off + 512);
|
||||
if (header.every((b) => b === 0)) break;
|
||||
const str = (a, b) => header.toString('utf8', a, b).replace(/\0.*$/s, '');
|
||||
const size = parseInt(str(124, 136).trim() || '0', 8);
|
||||
const type = str(156, 157) || '0';
|
||||
const prefix = str(345, 500);
|
||||
let name = longName ?? (prefix ? `${prefix}/${str(0, 100)}` : str(0, 100));
|
||||
longName = null;
|
||||
const body = buffer.subarray(off + 512, off + 512 + size);
|
||||
off += 512 + Math.ceil(size / 512) * 512;
|
||||
if (type === 'L') {
|
||||
longName = body.toString('utf8').replace(/\0.*$/s, '');
|
||||
continue;
|
||||
}
|
||||
if (type === 'x' || type === 'g') {
|
||||
const path = /\d+ path=([^\n]*)\n/.exec(body.toString('utf8'));
|
||||
if (path) longName = path[1];
|
||||
continue;
|
||||
}
|
||||
if (type !== '0' && type !== '\0' && type !== '5') continue;
|
||||
const rel = name.replace(/^[^/]+\//, '');
|
||||
const clean = normalize(rel);
|
||||
if (!rel || clean.startsWith('..') || clean.startsWith(sep) || /^[a-zA-Z]:/.test(clean)) {
|
||||
throw new Error(`refusing tar entry outside the package: ${name}`);
|
||||
}
|
||||
if (type === '5') continue;
|
||||
files.push({ path: rel, data: Buffer.from(body) });
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
export function verifyIntegrity(buffer, integrity) {
|
||||
const m = /^sha512-(.+)$/.exec(integrity || '');
|
||||
if (!m) throw new Error(`no sha512 integrity to check against (got ${integrity})`);
|
||||
const got = createHash('sha512').update(buffer).digest('base64');
|
||||
if (got !== m[1]) throw new Error(`integrity mismatch: registry ${m[1]}, downloaded ${got}`);
|
||||
}
|
||||
|
||||
async function fetchRetry(url, as) {
|
||||
let last;
|
||||
for (let attempt = 1; attempt <= 5; attempt++) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) return as === 'json' ? res.json() : Buffer.from(await res.arrayBuffer());
|
||||
last = new Error(`${url}: HTTP ${res.status}`);
|
||||
if (res.status < 500 && res.status !== 429) throw last;
|
||||
} catch (e) {
|
||||
last = e;
|
||||
if (/HTTP 4\d\d/.test(e.message) && !/HTTP 429/.test(e.message)) throw e;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, attempt * 5000));
|
||||
}
|
||||
throw last;
|
||||
}
|
||||
|
||||
// Package directories: a node_modules/<name> or node_modules/@scope/<name>.
|
||||
function packageDirs(root, out = []) {
|
||||
if (!existsSync(root)) return out;
|
||||
for (const e of readdirSync(root, { withFileTypes: true })) {
|
||||
if (!e.isDirectory()) continue;
|
||||
const p = join(root, e.name);
|
||||
if (e.name === 'node_modules') {
|
||||
for (const m of readdirSync(p, { withFileTypes: true })) {
|
||||
if (!m.isDirectory() || m.name.startsWith('.')) continue;
|
||||
const q = join(p, m.name);
|
||||
if (m.name.startsWith('@')) {
|
||||
for (const s of readdirSync(q, { withFileTypes: true })) {
|
||||
if (s.isDirectory()) out.push(join(q, s.name));
|
||||
}
|
||||
} else {
|
||||
out.push(q);
|
||||
}
|
||||
}
|
||||
}
|
||||
packageDirs(p, out);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Packages npm would not install on win32/<arch>.
|
||||
export function foreignPackages(serverDir, arch) {
|
||||
const seen = new Set();
|
||||
const foreign = [];
|
||||
for (const dir of packageDirs(serverDir)) {
|
||||
if (seen.has(dir)) continue;
|
||||
seen.add(dir);
|
||||
const manifest = join(dir, 'package.json');
|
||||
if (!existsSync(manifest)) continue;
|
||||
let pkg;
|
||||
try {
|
||||
pkg = JSON.parse(readFileSync(manifest, 'utf8'));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
if (!listAllows(pkg.os, 'win32') || !listAllows(pkg.cpu, arch)) foreign.push(dir);
|
||||
}
|
||||
// A removed package takes its own nested node_modules with it.
|
||||
return foreign.filter((d) => !foreign.some((o) => o !== d && d.startsWith(o + sep)));
|
||||
}
|
||||
|
||||
export async function addWindowsNativePackages(bundle, arch, { dryRun = false, log = console.log } = {}) {
|
||||
if (!ARCHES.includes(arch)) throw new Error(`unknown arch ${arch}; expected one of ${ARCHES.join(', ')}`);
|
||||
const server = join(bundle, 'programs', 'server');
|
||||
const modules = join(server, 'npm', 'node_modules');
|
||||
const sharpManifest = join(modules, 'sharp', 'package.json');
|
||||
if (!existsSync(sharpManifest)) throw new Error(`no sharp in ${relative(bundle, modules)}`);
|
||||
|
||||
const removed = foreignPackages(server, arch);
|
||||
for (const dir of removed) {
|
||||
log(`remove ${relative(bundle, dir)} (its package.json names another platform)`);
|
||||
if (!dryRun) rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
const { name, version } = sharpPlatformSpec(JSON.parse(readFileSync(sharpManifest, 'utf8')), arch);
|
||||
const meta = await fetchRetry(`${REGISTRY}/${name.replace('/', '%2F')}/${version}`, 'json');
|
||||
const tarball = await fetchRetry(meta.dist.tarball, 'buffer');
|
||||
verifyIntegrity(tarball, meta.dist.integrity);
|
||||
const files = untar(gunzipSync(tarball));
|
||||
const dest = join(modules, ...name.split('/'));
|
||||
if (!dryRun) {
|
||||
rmSync(dest, { recursive: true, force: true });
|
||||
for (const f of files) {
|
||||
const out = resolve(dest, f.path);
|
||||
if (!out.startsWith(resolve(dest) + sep)) throw new Error(`refusing ${f.path}`);
|
||||
mkdirSync(dirname(out), { recursive: true });
|
||||
writeFileSync(out, f.data);
|
||||
}
|
||||
}
|
||||
log(`add ${relative(bundle, dest)} ${version} (${files.length} files, sha512 verified)`);
|
||||
return { removed: removed.map((d) => relative(bundle, d)), added: { name, version, dest } };
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
|
||||
const args = process.argv.slice(2).filter((a) => a !== '--dry-run');
|
||||
const [bundle, arch] = args;
|
||||
if (!bundle || !arch) {
|
||||
console.error('usage: windows-native-packages.mjs <bundle dir> <ia32|x64|arm64> [--dry-run]');
|
||||
process.exit(2);
|
||||
}
|
||||
addWindowsNativePackages(bundle, arch, { dryRun: process.argv.includes('--dry-run') }).catch((e) => {
|
||||
console.error(`::error::windows-native-packages: ${e.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
'use strict';
|
||||
(async () => {
|
||||
|
||||
// Plain-Node unit test for releases/platform-native-packages.mjs.
|
||||
// Run: node tests/platformNativePackages.test.cjs
|
||||
//
|
||||
// Every bundle except amd64 is made from the amd64 Linux bundle, so sharp - the
|
||||
// attachment thumbnails - had only @img/sharp-linux-x64 in every one of them,
|
||||
// and could not load on any other platform or CPU (v12.32: arm64, ppc64le,
|
||||
// s390x, riscv64, i386, armhf, armv7, armv6, both macOS, all three Windows).
|
||||
// The script puts in @img/sharp-<platform>-<arch> and its libvips at the
|
||||
// versions pinned, checked against the registry's sha512, keeps the
|
||||
// WebAssembly fallback where sharp publishes nothing (Linux i386, loong64),
|
||||
// and removes the packages npm would not install on that platform. The
|
||||
// network is replaced here by a fetch that serves tarballs built in the test.
|
||||
|
||||
const assert = require('assert');
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const zlib = require('zlib');
|
||||
|
||||
const {
|
||||
sharpPlatformSpec,
|
||||
untar,
|
||||
verifyIntegrity,
|
||||
foreignPackages,
|
||||
addPlatformNativePackages,
|
||||
} = await import('../releases/platform-native-packages.mjs');
|
||||
|
||||
let passed = 0;
|
||||
async function test(name, fn) {
|
||||
await fn();
|
||||
passed += 1;
|
||||
console.log(' ok -', name);
|
||||
}
|
||||
|
||||
// A ustar archive, the way npm packs: every entry under package/.
|
||||
function tar(entries) {
|
||||
const blocks = [];
|
||||
for (const [name, content] of entries) {
|
||||
const data = Buffer.from(content);
|
||||
const h = Buffer.alloc(512);
|
||||
h.write(name, 0, 100, 'utf8');
|
||||
h.write('0000644\0', 100);
|
||||
h.write('0000000\0', 108);
|
||||
h.write('0000000\0', 116);
|
||||
h.write(data.length.toString(8).padStart(11, '0') + '\0', 124);
|
||||
h.write('00000000000\0', 136);
|
||||
h.write(' ', 148);
|
||||
h.write('0', 156);
|
||||
h.write('ustar\0' + '00', 257);
|
||||
let sum = 0;
|
||||
for (const b of h) sum += b;
|
||||
h.write(sum.toString(8).padStart(6, '0') + '\0 ', 148);
|
||||
blocks.push(h, data, Buffer.alloc((512 - (data.length % 512)) % 512));
|
||||
}
|
||||
blocks.push(Buffer.alloc(1024));
|
||||
return Buffer.concat(blocks);
|
||||
}
|
||||
const integrity = (buf) => 'sha512-' + crypto.createHash('sha512').update(buf).digest('base64');
|
||||
|
||||
// Real headers, so the addon check at the end of a run reads them.
|
||||
function pe(machine) {
|
||||
const b = Buffer.alloc(0x90);
|
||||
b.write('MZ', 0, 'latin1');
|
||||
b.writeUInt32LE(0x80, 0x3c);
|
||||
b.write('PE\0\0', 0x80, 'latin1');
|
||||
b.writeUInt16LE(machine, 0x84);
|
||||
return b;
|
||||
}
|
||||
function elf(machine, bigEndian = false) {
|
||||
const b = Buffer.alloc(64);
|
||||
b.write('\x7fELF', 0, 'latin1');
|
||||
b[4] = 2;
|
||||
b[5] = bigEndian ? 2 : 1;
|
||||
if (bigEndian) b.writeUInt16BE(machine, 18); else b.writeUInt16LE(machine, 18);
|
||||
return b;
|
||||
}
|
||||
|
||||
function tree(files) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'platform-native-'));
|
||||
for (const [rel, content] of Object.entries(files)) {
|
||||
const f = path.join(dir, rel);
|
||||
fs.mkdirSync(path.dirname(f), { recursive: true });
|
||||
fs.writeFileSync(f, content);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
const NM = 'programs/server/npm/node_modules';
|
||||
const pkg = (o) => JSON.stringify(o);
|
||||
const SHARP = {
|
||||
name: 'sharp', version: '0.35.5',
|
||||
optionalDependencies: {
|
||||
'@img/sharp-linux-x64': '0.35.5', '@img/sharp-linux-arm': '0.35.5', '@img/sharp-linux-s390x': '0.35.5',
|
||||
'@img/sharp-darwin-arm64': '0.35.5', '@img/sharp-win32-ia32': '0.35.5',
|
||||
'@img/sharp-libvips-linux-arm': '1.3.4',
|
||||
},
|
||||
};
|
||||
|
||||
// A registry: { 'name@version': { files: [[path, content]], optionalDependencies } }.
|
||||
function registry(packages) {
|
||||
const served = [];
|
||||
const tarballs = {};
|
||||
for (const [id, p] of Object.entries(packages)) {
|
||||
tarballs[id] = zlib.gzipSync(tar(p.files.map(([f, c]) => [`package/${f}`, c])));
|
||||
}
|
||||
const fetch = async (url) => {
|
||||
served.push(url);
|
||||
const m = /^https:\/\/registry\.npmjs\.org\/(.+)\/([^/]+)$/.exec(url);
|
||||
if (m) {
|
||||
const id = `${decodeURIComponent(m[1])}@${m[2]}`;
|
||||
if (!packages[id]) return new Response('nope', { status: 404 });
|
||||
return new Response(JSON.stringify({
|
||||
optionalDependencies: packages[id].optionalDependencies,
|
||||
dist: { tarball: `https://tar.example/${encodeURIComponent(id)}.tgz`, integrity: packages[id].integrity || integrity(tarballs[id]) },
|
||||
}));
|
||||
}
|
||||
const t = /^https:\/\/tar\.example\/(.+)\.tgz$/.exec(url);
|
||||
if (t) return new Response(tarballs[decodeURIComponent(t[1])]);
|
||||
return new Response('nope', { status: 404 });
|
||||
};
|
||||
return { fetch, served };
|
||||
}
|
||||
async function withFetch(fetch, fn) {
|
||||
const real = globalThis.fetch;
|
||||
globalThis.fetch = fetch;
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
globalThis.fetch = real;
|
||||
}
|
||||
}
|
||||
const quiet = { log: () => {} };
|
||||
|
||||
console.log('platformNativePackages:');
|
||||
|
||||
await test('the sharp package is the one sharp itself pins for that platform and CPU', () => {
|
||||
assert.deepStrictEqual(sharpPlatformSpec(SHARP, 'linux', 'arm'), { name: '@img/sharp-linux-arm', version: '0.35.5' });
|
||||
assert.deepStrictEqual(sharpPlatformSpec(SHARP, 'darwin', 'arm64'), { name: '@img/sharp-darwin-arm64', version: '0.35.5' });
|
||||
assert.deepStrictEqual(sharpPlatformSpec(SHARP, 'win32', 'ia32'), { name: '@img/sharp-win32-ia32', version: '0.35.5' });
|
||||
// sharp publishes nothing for Linux i386: no guess, the fallback is used.
|
||||
assert.strictEqual(sharpPlatformSpec(SHARP, 'linux', 'ia32'), null);
|
||||
});
|
||||
|
||||
await test('the tarball is unpacked without its package/ prefix', () => {
|
||||
const files = untar(tar([['package/package.json', '{"name":"x"}'], ['package/lib/sharp.node', 'MZ']]));
|
||||
assert.deepStrictEqual(files.map((f) => f.path), ['package.json', 'lib/sharp.node']);
|
||||
assert.strictEqual(files[1].data.toString(), 'MZ');
|
||||
});
|
||||
|
||||
await test('a tar entry outside the package is refused', () => {
|
||||
assert.throws(() => untar(tar([['package/../../evil.so', 'x']])), /outside the package/);
|
||||
assert.throws(() => untar(tar([['package//etc/passwd', 'x']])), /outside the package/);
|
||||
});
|
||||
|
||||
await test('the download must match the registry sha512', () => {
|
||||
const buf = Buffer.from('tarball');
|
||||
verifyIntegrity(buf, integrity(buf));
|
||||
assert.throws(() => verifyIntegrity(Buffer.from('tampered'), integrity(buf)), /integrity mismatch/);
|
||||
assert.throws(() => verifyIntegrity(buf, 'sha1-abc'), /no sha512/);
|
||||
});
|
||||
|
||||
await test('only packages npm would not install on the target are removed', () => {
|
||||
const dir = tree({
|
||||
[`${NM}/@img/sharp-linux-x64/package.json`]: pkg({ os: ['linux'], cpu: ['x64'], libc: ['glibc'] }),
|
||||
[`${NM}/@img/sharp-linuxmusl-arm/package.json`]: pkg({ os: ['linux'], cpu: ['arm'], libc: ['musl'] }),
|
||||
[`${NM}/@img/sharp-win32-x64/package.json`]: pkg({ os: ['win32'], cpu: ['x64'] }),
|
||||
[`${NM}/@img/sharp-wasm32/package.json`]: pkg({ name: '@img/sharp-wasm32' }),
|
||||
[`${NM}/sharp/package.json`]: pkg({ name: 'sharp' }),
|
||||
[`${NM}/not-on-aix/package.json`]: pkg({ os: ['!aix'] }),
|
||||
[`${NM}/linux-only/package.json`]: pkg({ os: ['linux'] }),
|
||||
[`${NM}/linux-only/node_modules/inner/package.json`]: pkg({ os: ['linux'] }),
|
||||
});
|
||||
try {
|
||||
const server = path.join(dir, 'programs/server');
|
||||
const rel = (p, a) => foreignPackages(server, p, a)
|
||||
.map((d) => path.relative(dir, d).split(path.sep).join('/')).sort();
|
||||
assert.deepStrictEqual(rel('linux', 'arm'), [
|
||||
`${NM}/@img/sharp-linux-x64`,
|
||||
`${NM}/@img/sharp-linuxmusl-arm`,
|
||||
`${NM}/@img/sharp-win32-x64`,
|
||||
], 'another CPU, another libc and another OS go; wasm32 and plain packages stay');
|
||||
assert.deepStrictEqual(rel('win32', 'x64'), [
|
||||
`${NM}/@img/sharp-linux-x64`,
|
||||
`${NM}/@img/sharp-linuxmusl-arm`,
|
||||
`${NM}/linux-only`,
|
||||
], 'a nested package goes with its parent, and is not listed twice');
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('Linux ARM: sharp and its libvips arrive, and the x64 ones go', async () => {
|
||||
const dir = tree({
|
||||
[`${NM}/sharp/package.json`]: pkg(SHARP),
|
||||
[`${NM}/@img/sharp-linux-x64/package.json`]: pkg({ os: ['linux'], cpu: ['x64'] }),
|
||||
[`${NM}/@img/sharp-linux-x64/lib/sharp-linux-x64.node`]: elf(62),
|
||||
});
|
||||
const reg = registry({
|
||||
'@img/sharp-linux-arm@0.35.5': {
|
||||
optionalDependencies: { '@img/sharp-libvips-linux-arm': '1.3.4' },
|
||||
files: [['package.json', pkg({ os: ['linux'], cpu: ['arm'] })], ['lib/sharp-linux-arm-0.35.5.node', elf(40)]],
|
||||
},
|
||||
'@img/sharp-libvips-linux-arm@1.3.4': {
|
||||
files: [['package.json', pkg({ os: ['linux'], cpu: ['arm'] })], ['lib/libvips-cpp.so.8.18.7', 'so']],
|
||||
},
|
||||
});
|
||||
try {
|
||||
const r = await withFetch(reg.fetch, () => addPlatformNativePackages(dir, 'linux', 'arm', quiet));
|
||||
assert.deepStrictEqual(r.errors, []);
|
||||
assert.deepStrictEqual(r.added.map((a) => `${a.name}@${a.version}`),
|
||||
['@img/sharp-linux-arm@0.35.5', '@img/sharp-libvips-linux-arm@1.3.4']);
|
||||
assert.ok(fs.existsSync(path.join(dir, NM, '@img/sharp-libvips-linux-arm/lib/libvips-cpp.so.8.18.7')));
|
||||
assert.ok(!fs.existsSync(path.join(dir, NM, '@img/sharp-linux-x64')));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('Windows 32-bit: the PE package arrives and passes the check', async () => {
|
||||
const dir = tree({ [`${NM}/sharp/package.json`]: pkg(SHARP) });
|
||||
const reg = registry({
|
||||
'@img/sharp-win32-ia32@0.35.5': {
|
||||
files: [['package.json', pkg({ os: ['win32'], cpu: ['ia32'] })],
|
||||
['lib/sharp-win32-ia32-0.35.5.node', pe(0x014c)], ['lib/libvips-42.dll', 'MZ']],
|
||||
},
|
||||
});
|
||||
try {
|
||||
const r = await withFetch(reg.fetch, () => addPlatformNativePackages(dir, 'win32', 'ia32', quiet));
|
||||
assert.deepStrictEqual(r.errors, []);
|
||||
assert.ok(fs.existsSync(path.join(dir, NM, '@img/sharp-win32-ia32/lib/libvips-42.dll')));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Negative: a package that arrives built for the wrong CPU fails the run.
|
||||
await test('a sharp package for the wrong CPU is reported', async () => {
|
||||
const dir = tree({ [`${NM}/sharp/package.json`]: pkg(SHARP) });
|
||||
const reg = registry({
|
||||
'@img/sharp-linux-s390x@0.35.5': {
|
||||
files: [['package.json', pkg({ os: ['linux'], cpu: ['s390x'] })], ['lib/sharp.node', elf(21, true)]],
|
||||
},
|
||||
});
|
||||
try {
|
||||
const r = await withFetch(reg.fetch, () => addPlatformNativePackages(dir, 'linux', 's390x', quiet));
|
||||
assert.ok(r.errors.some((e) => /ELF machine 21, not a s390x Linux addon/.test(e)), r.errors.join('\n'));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('a tampered tarball is not unpacked', async () => {
|
||||
const dir = tree({ [`${NM}/sharp/package.json`]: pkg(SHARP) });
|
||||
const reg = registry({
|
||||
'@img/sharp-darwin-arm64@0.35.5': {
|
||||
integrity: integrity(Buffer.from('something else')),
|
||||
files: [['package.json', pkg({ os: ['darwin'], cpu: ['arm64'] })]],
|
||||
},
|
||||
});
|
||||
try {
|
||||
await assert.rejects(withFetch(reg.fetch, () => addPlatformNativePackages(dir, 'darwin', 'arm64', quiet)),
|
||||
/integrity mismatch/);
|
||||
assert.ok(!fs.existsSync(path.join(dir, NM, '@img/sharp-darwin-arm64')));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('Linux i386 keeps the WebAssembly sharp, and downloads nothing', async () => {
|
||||
const dir = tree({
|
||||
[`${NM}/sharp/package.json`]: pkg(SHARP),
|
||||
[`${NM}/@img/sharp-wasm32/package.json`]: pkg({ name: '@img/sharp-wasm32', version: '0.35.5' }),
|
||||
[`${NM}/@emnapi/runtime/package.json`]: pkg({ name: '@emnapi/runtime', version: '1.11.3' }),
|
||||
});
|
||||
const reg = registry({});
|
||||
try {
|
||||
const r = await withFetch(reg.fetch, () => addPlatformNativePackages(dir, 'linux', 'ia32', quiet));
|
||||
assert.deepStrictEqual(r.errors, []);
|
||||
assert.deepStrictEqual(r.added, []);
|
||||
assert.deepStrictEqual(reg.served, []);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// Negative: no native package and no fallback is an error, not a silent pass.
|
||||
await test('no native sharp and no usable WebAssembly fallback is an error', async () => {
|
||||
const cases = [
|
||||
[{}, /is not in the bundle to fall back to/],
|
||||
[{ [`${NM}/@img/sharp-wasm32/package.json`]: pkg({ version: '0.34.0' }) }, /does not match sharp 0\.35\.5/],
|
||||
[{ [`${NM}/@img/sharp-wasm32/package.json`]: pkg({ version: '0.35.5' }) }, /needs @emnapi\/runtime/],
|
||||
];
|
||||
for (const [extra, error] of cases) {
|
||||
const dir = tree({ [`${NM}/sharp/package.json`]: pkg(SHARP), ...extra });
|
||||
try {
|
||||
await assert.rejects(addPlatformNativePackages(dir, 'linux', 'loong64', quiet), error);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
await test('a bundle with no sharp, or an unknown target, is an error', async () => {
|
||||
const dir = tree({ [`${NM}/x/package.json`]: pkg({}) });
|
||||
try {
|
||||
await assert.rejects(addPlatformNativePackages(dir, 'linux', 'arm', quiet), /no sharp/);
|
||||
await assert.rejects(addPlatformNativePackages(dir, 'linux', 'mips', quiet), /unknown target/);
|
||||
await assert.rejects(addPlatformNativePackages(dir, 'darwin', 'ia32', quiet), /unknown target/);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nplatformNativePackages: all ${passed} tests passed`);
|
||||
})().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
'use strict';
|
||||
|
||||
// Guard: every WeKan bundle carries a sharp it can load.
|
||||
// Run: node tests/sharpEveryPlatform.test.cjs
|
||||
//
|
||||
// sharp makes the attachment thumbnails (server/lib/imageThumbnail.js). Every
|
||||
// bundle except amd64 is made from the amd64 Linux bundle, so in v12.32 every
|
||||
// one of them - arm64, ppc64le, s390x, riscv64, i386, armhf, armv7, armv6,
|
||||
// mac-arm64, mac-x64, win64, win-arm64, and the Docker images and snaps built
|
||||
// from them - had only @img/sharp-linux-x64 and showed full-size images in
|
||||
// place of thumbnails. Nothing failed; it was found by reading the zips.
|
||||
//
|
||||
// So the guard is about the PIPELINE, not one job: every job that makes a
|
||||
// bundle zip must run releases/platform-native-packages.mjs for its own
|
||||
// platform and CPU, after the last step that deletes files and before the zip,
|
||||
// and the WebAssembly fallback (for Linux i386 and loong64, where sharp
|
||||
// publishes no binary) must be in package.json at sharp's own version.
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const ROOT = path.join(__dirname, '..');
|
||||
const read = (rel) => fs.readFileSync(path.join(ROOT, rel), 'utf8');
|
||||
const code = (text) => text.split('\n').filter((l) => !/^\s*#/.test(l)).join('\n');
|
||||
|
||||
let passed = 0;
|
||||
function test(name, fn) {
|
||||
fn();
|
||||
passed += 1;
|
||||
console.log(' ok -', name);
|
||||
}
|
||||
|
||||
const workflow = read('.github/workflows/release-all.yml');
|
||||
function jobs(text) {
|
||||
const out = {};
|
||||
const starts = [...text.matchAll(/\n ([a-z0-9-]+):\n/g)].map((m) => ({ name: m[1], at: m.index }));
|
||||
starts.forEach((s, i) => {
|
||||
out[s.name] = code(text.slice(s.at, i + 1 < starts.length ? starts[i + 1].at : undefined));
|
||||
});
|
||||
return out;
|
||||
}
|
||||
const all = jobs(workflow);
|
||||
const SCRIPT = 'platform-native-packages.mjs';
|
||||
|
||||
console.log('sharpEveryPlatform:');
|
||||
|
||||
// The jobs that make a bundle zip, and what each must name. build-extra-arches
|
||||
// runs it inside the target container (install-node-for-arch.sh), where the
|
||||
// target node's own process.arch is the name sharp looks up.
|
||||
const EXPECTED = {
|
||||
'build-amd64': /platform-native-packages\.mjs" \\\n\s*"\$GITHUB_WORKSPACE\/\.build\/bundle" linux x64/,
|
||||
'build-arm64': /platform-native-packages\.mjs \/bundle linux arm64/,
|
||||
'build-mac-arm64': /platform-native-packages\.mjs" \.\.\/\.\. darwin arm64/,
|
||||
'build-mac-x64': /platform-native-packages\.mjs" \.\.\/\.\. darwin x64/,
|
||||
'build-win64': /platform-native-packages\.mjs" bundle win32 x64/,
|
||||
'build-win-arm64': /platform-native-packages\.mjs" bundle win32 arm64/,
|
||||
'build-win32': /platform-native-packages\.mjs" bundle win32 ia32/,
|
||||
'build-extra-arches': /install-node-for-arch\.sh/,
|
||||
};
|
||||
|
||||
test('every job that makes a bundle zip is one of the expected ones', () => {
|
||||
const zipping = Object.entries(all)
|
||||
.filter(([, body]) => /- name: Create .* zip\b/.test(body))
|
||||
.map(([name]) => name)
|
||||
.sort();
|
||||
assert.deepStrictEqual(zipping, Object.keys(EXPECTED).sort(),
|
||||
'a new bundle job must be added here, with the platform/arch it gives platform-native-packages.mjs');
|
||||
});
|
||||
|
||||
test('each bundle job runs platform-native-packages.mjs for its own target', () => {
|
||||
for (const [name, re] of Object.entries(EXPECTED)) {
|
||||
assert.ok(re.test(all[name]), `${name} must run ${SCRIPT} for its own platform and CPU`);
|
||||
}
|
||||
});
|
||||
|
||||
test('it runs after everything that deletes files, and before the zip', () => {
|
||||
for (const name of Object.keys(EXPECTED)) {
|
||||
if (name === 'build-extra-arches') continue;
|
||||
const body = all[name];
|
||||
const at = body.indexOf(SCRIPT);
|
||||
const prune = body.lastIndexOf('prune-unreachable-npm.mjs', at);
|
||||
assert.ok(prune !== -1 && prune < at, `${name}: after prune-unreachable-npm`);
|
||||
assert.ok(body.lastIndexOf('bundle-trim.mjs', at) !== -1, `${name}: after bundle-trim`);
|
||||
const zip = body.search(/- name: Create .* zip\b/);
|
||||
assert.ok(at < zip, `${name}: before the zip is made`);
|
||||
}
|
||||
const install = code(read('releases/install-node-for-arch.sh'));
|
||||
const at = install.indexOf(`${SCRIPT}" /bundle linux "$(node -p process.arch)"`);
|
||||
assert.ok(at !== -1, 'install-node-for-arch.sh names the target node\'s own process.arch');
|
||||
assert.ok(install.indexOf('prune-unreachable-npm.mjs') < at, 'install-node-for-arch.sh: after the prune');
|
||||
assert.ok(/install-node-for-arch\.sh/.test(all['build-extra-arches']), 'build-extra-arches runs it');
|
||||
});
|
||||
|
||||
test('the local release build runs the same step for its own machine', () => {
|
||||
const local = code(read('releases/build-release-bundle.sh'));
|
||||
assert.ok(/platform-native-packages\.mjs" "\$BUNDLE" \\\n\s*"\$\(node -p process\.platform\)" "\$\(node -p process\.arch\)"/.test(local));
|
||||
});
|
||||
|
||||
test('the WebAssembly fallback is a dependency, at sharp\'s own version', () => {
|
||||
const pkg = JSON.parse(read('package.json'));
|
||||
assert.ok(pkg.dependencies.sharp, 'sharp is a dependency');
|
||||
assert.strictEqual(pkg.optionalDependencies?.['@img/sharp-wasm32'], pkg.dependencies.sharp,
|
||||
'@img/sharp-wasm32 uses the same range as sharp, so they move together');
|
||||
const lock = JSON.parse(read('package-lock.json'));
|
||||
assert.strictEqual(lock.packages['']?.optionalDependencies?.['@img/sharp-wasm32'], pkg.dependencies.sharp,
|
||||
'package-lock.json records it as a root optional dependency');
|
||||
const sharp = lock.packages['node_modules/sharp'];
|
||||
const wasm = lock.packages['node_modules/@img/sharp-wasm32'];
|
||||
assert.ok(sharp && wasm, 'both are locked');
|
||||
assert.strictEqual(wasm.version, sharp.version, 'and locked to the same version');
|
||||
assert.ok(lock.packages['node_modules/@emnapi/runtime'], 'with its @emnapi/runtime');
|
||||
});
|
||||
|
||||
// Negative: nothing else may install a sharp platform package by hand, or the
|
||||
// version could drift from the one sharp pins.
|
||||
test('no other release script or job installs a sharp platform package itself', () => {
|
||||
const files = ['.github/workflows/release-all.yml', 'Dockerfile',
|
||||
...fs.readdirSync(path.join(ROOT, 'releases')).filter((f) => /\.(sh|mjs|py)$/.test(f)).map((f) => `releases/${f}`)];
|
||||
for (const f of files) {
|
||||
if (f === `releases/${SCRIPT}`) continue;
|
||||
const body = code(read(f));
|
||||
assert.ok(!/npm (?:i|install|pack)\b[^\n]*@img\/sharp-/.test(body), `${f} installs an @img/sharp- package itself`);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nsharpEveryPlatform: all ${passed} tests passed`);
|
||||
@@ -105,7 +105,7 @@ test('each Windows bundle job adds its own sharp and checks its addons', () => {
|
||||
const job = windowsJobs.find((j) => j.name === name);
|
||||
assert.ok(job, `${name} exists and runs on Windows`);
|
||||
const at = (s) => job.body.indexOf(s);
|
||||
const add = at(`releases/windows-native-packages.mjs" bundle ${arch}`);
|
||||
const add = at(`releases/platform-native-packages.mjs" bundle win32 ${arch}`);
|
||||
const check = at(`releases/check-addon-arch.mjs" bundle ${arch} bcrypt argon2 @img/sharp-win32-${arch}`);
|
||||
assert.ok(add !== -1, `${name} adds @img/sharp-win32-${arch}`);
|
||||
assert.ok(check > add, `${name} checks bcrypt, argon2 and sharp for ${arch}, after adding sharp`);
|
||||
@@ -144,7 +144,14 @@ function pe(machine) {
|
||||
b.writeUInt16LE(machine, 0x84);
|
||||
return b;
|
||||
}
|
||||
const ELF = Buffer.concat([Buffer.from([0x7f, 0x45, 0x4c, 0x46]), Buffer.alloc(60)]);
|
||||
// A little-endian x86-64 ELF header: e_machine 62 at offset 18.
|
||||
const ELF = (() => {
|
||||
const b = Buffer.alloc(64);
|
||||
b.write('\x7fELF', 0, 'latin1');
|
||||
b[4] = 2; b[5] = 1;
|
||||
b.writeUInt16LE(62, 18);
|
||||
return b;
|
||||
})();
|
||||
|
||||
function bundle(files) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'addon-arch-'));
|
||||
@@ -169,7 +176,7 @@ test('the PE machine field is read, and ELF is told apart', () => {
|
||||
withBundle({ 'a.node': pe(PE_MACHINE.ia32), 'b.node': pe(PE_MACHINE.x64), 'c.node': ELF }, (d) => {
|
||||
assert.deepStrictEqual(readBinaryKind(path.join(d, 'a.node')), { format: 'PE', machine: 0x014c });
|
||||
assert.deepStrictEqual(readBinaryKind(path.join(d, 'b.node')), { format: 'PE', machine: 0x8664 });
|
||||
assert.deepStrictEqual(readBinaryKind(path.join(d, 'c.node')), { format: 'ELF' });
|
||||
assert.deepStrictEqual(readBinaryKind(path.join(d, 'c.node')), { format: 'ELF', machine: 62 });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -212,7 +219,7 @@ test('a build for the wrong CPU, or a Linux one, is refused', () => {
|
||||
assert.ok(checkAddonArch(d, 'ia32', ['argon2']).errors.some((e) => /is ELF/.test(e)));
|
||||
});
|
||||
withBundle({ [`${AP}/bcrypt/prebuilds/win32-ia32/bcrypt.node`]: pe(PE_MACHINE.x64) }, (d) => {
|
||||
assert.ok(checkAddonArch(d, 'ia32', []).errors.some((e) => /in the ia32 prebuilds/.test(e)));
|
||||
assert.ok(checkAddonArch(d, 'ia32', []).errors.some((e) => /in the ia32 Windows prebuilds/.test(e)));
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,200 +0,0 @@
|
||||
'use strict';
|
||||
(async () => {
|
||||
|
||||
// Plain-Node unit test for releases/windows-native-packages.mjs.
|
||||
// Run: node tests/windowsNativePackages.test.cjs
|
||||
//
|
||||
// Every Windows bundle is made from the amd64 Linux bundle, so sharp - the
|
||||
// attachment thumbnails - had only @img/sharp-linux-x64 there and could never
|
||||
// load on Windows. The script puts in @img/sharp-win32-<arch> at the version
|
||||
// sharp pins, checked against the registry's sha512, and removes the packages
|
||||
// npm would not install on that platform. The network is replaced here by a
|
||||
// fetch that serves a tarball built in the test.
|
||||
|
||||
const assert = require('assert');
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const zlib = require('zlib');
|
||||
|
||||
const {
|
||||
sharpPlatformSpec,
|
||||
untar,
|
||||
verifyIntegrity,
|
||||
foreignPackages,
|
||||
addWindowsNativePackages,
|
||||
} = await import('../releases/windows-native-packages.mjs');
|
||||
|
||||
let passed = 0;
|
||||
async function test(name, fn) {
|
||||
await fn();
|
||||
passed += 1;
|
||||
console.log(' ok -', name);
|
||||
}
|
||||
|
||||
// A ustar archive, the way npm packs: every entry under package/.
|
||||
function tar(entries) {
|
||||
const blocks = [];
|
||||
for (const [name, content] of entries) {
|
||||
const data = Buffer.from(content);
|
||||
const h = Buffer.alloc(512);
|
||||
h.write(name, 0, 100, 'utf8');
|
||||
h.write('0000644\0', 100);
|
||||
h.write('0000000\0', 108);
|
||||
h.write('0000000\0', 116);
|
||||
h.write(data.length.toString(8).padStart(11, '0') + '\0', 124);
|
||||
h.write('00000000000\0', 136);
|
||||
h.write(' ', 148);
|
||||
h.write('0', 156);
|
||||
h.write('ustar\0' + '00', 257);
|
||||
let sum = 0;
|
||||
for (const b of h) sum += b;
|
||||
h.write(sum.toString(8).padStart(6, '0') + '\0 ', 148);
|
||||
blocks.push(h, data, Buffer.alloc((512 - (data.length % 512)) % 512));
|
||||
}
|
||||
blocks.push(Buffer.alloc(1024));
|
||||
return Buffer.concat(blocks);
|
||||
}
|
||||
const integrity = (buf) => 'sha512-' + crypto.createHash('sha512').update(buf).digest('base64');
|
||||
|
||||
function tree(files) {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'win-native-'));
|
||||
for (const [rel, content] of Object.entries(files)) {
|
||||
const f = path.join(dir, rel);
|
||||
fs.mkdirSync(path.dirname(f), { recursive: true });
|
||||
fs.writeFileSync(f, content);
|
||||
}
|
||||
return dir;
|
||||
}
|
||||
const NM = 'programs/server/npm/node_modules';
|
||||
const pkg = (o) => JSON.stringify(o);
|
||||
|
||||
console.log('windowsNativePackages:');
|
||||
|
||||
await test('the sharp package is the one sharp itself pins for that CPU', () => {
|
||||
const sharp = {
|
||||
version: '0.35.5',
|
||||
optionalDependencies: {
|
||||
'@img/sharp-win32-ia32': '0.35.5',
|
||||
'@img/sharp-win32-x64': '0.35.5',
|
||||
'@img/sharp-linux-x64': '0.35.5',
|
||||
},
|
||||
};
|
||||
assert.deepStrictEqual(sharpPlatformSpec(sharp, 'ia32'), { name: '@img/sharp-win32-ia32', version: '0.35.5' });
|
||||
assert.deepStrictEqual(sharpPlatformSpec(sharp, 'x64'), { name: '@img/sharp-win32-x64', version: '0.35.5' });
|
||||
// Negative: a sharp that does not support the CPU is an error, not a guess.
|
||||
assert.throws(() => sharpPlatformSpec(sharp, 'arm64'), /lists no @img\/sharp-win32-arm64/);
|
||||
});
|
||||
|
||||
await test('the tarball is unpacked without its package/ prefix', () => {
|
||||
const files = untar(tar([
|
||||
['package/package.json', '{"name":"x"}'],
|
||||
['package/lib/sharp.node', 'MZ'],
|
||||
]));
|
||||
assert.deepStrictEqual(files.map((f) => f.path), ['package.json', 'lib/sharp.node']);
|
||||
assert.strictEqual(files[1].data.toString(), 'MZ');
|
||||
});
|
||||
|
||||
await test('a tar entry outside the package is refused', () => {
|
||||
assert.throws(() => untar(tar([['package/../../evil.dll', 'x']])), /outside the package/);
|
||||
assert.throws(() => untar(tar([['package//etc/passwd', 'x']])), /outside the package/);
|
||||
});
|
||||
|
||||
await test('the download must match the registry sha512', () => {
|
||||
const buf = Buffer.from('tarball');
|
||||
verifyIntegrity(buf, integrity(buf));
|
||||
assert.throws(() => verifyIntegrity(Buffer.from('tampered'), integrity(buf)), /integrity mismatch/);
|
||||
assert.throws(() => verifyIntegrity(buf, 'sha1-abc'), /no sha512/);
|
||||
});
|
||||
|
||||
await test('only packages npm would not install on win32/<arch> are removed', () => {
|
||||
const dir = tree({
|
||||
[`${NM}/@img/sharp-linux-x64/package.json`]: pkg({ os: ['linux'], cpu: ['x64'] }),
|
||||
[`${NM}/@img/sharp-libvips-linux-x64/package.json`]: pkg({ os: ['linux'], cpu: ['x64'] }),
|
||||
[`${NM}/@img/sharp-win32-x64/package.json`]: pkg({ os: ['win32'], cpu: ['x64'] }),
|
||||
[`${NM}/@napi-rs/canvas-linux-x64-gnu/package.json`]: pkg({ os: ['linux'], cpu: ['x64'] }),
|
||||
[`${NM}/sharp/package.json`]: pkg({ name: 'sharp' }),
|
||||
[`${NM}/not-on-aix/package.json`]: pkg({ os: ['!aix'] }),
|
||||
[`${NM}/linux-only/package.json`]: pkg({ os: ['linux'] }),
|
||||
[`${NM}/linux-only/node_modules/inner/package.json`]: pkg({ os: ['linux'] }),
|
||||
});
|
||||
try {
|
||||
const server = path.join(dir, 'programs/server');
|
||||
const rel = (arch) => foreignPackages(server, arch).map((d) => path.relative(dir, d).split(path.sep).join('/')).sort();
|
||||
assert.deepStrictEqual(rel('x64'), [
|
||||
`${NM}/@img/sharp-libvips-linux-x64`,
|
||||
`${NM}/@img/sharp-linux-x64`,
|
||||
`${NM}/@napi-rs/canvas-linux-x64-gnu`,
|
||||
`${NM}/linux-only`,
|
||||
], 'a nested package goes with its parent, and is not listed twice');
|
||||
// On ia32 the x64 Windows package is foreign as well.
|
||||
assert.ok(rel('ia32').includes(`${NM}/@img/sharp-win32-x64`));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('the whole run, against a registry that serves a built tarball', async () => {
|
||||
const dir = tree({
|
||||
[`${NM}/sharp/package.json`]: pkg({
|
||||
name: 'sharp', version: '0.35.5',
|
||||
optionalDependencies: { '@img/sharp-win32-ia32': '0.35.5', '@img/sharp-linux-x64': '0.35.5' },
|
||||
}),
|
||||
[`${NM}/@img/sharp-linux-x64/package.json`]: pkg({ os: ['linux'], cpu: ['x64'] }),
|
||||
[`${NM}/@img/sharp-linux-x64/lib/sharp-linux-x64.node`]: 'ELF',
|
||||
});
|
||||
const tgz = zlib.gzipSync(tar([
|
||||
['package/package.json', pkg({ name: '@img/sharp-win32-ia32', os: ['win32'], cpu: ['ia32'] })],
|
||||
['package/lib/sharp-win32-ia32-0.35.5.node', 'MZ'],
|
||||
['package/lib/libvips-42.dll', 'MZ'],
|
||||
]));
|
||||
const realFetch = globalThis.fetch;
|
||||
const asked = [];
|
||||
globalThis.fetch = async (url) => {
|
||||
asked.push(url);
|
||||
if (url === 'https://registry.npmjs.org/@img%2Fsharp-win32-ia32/0.35.5') {
|
||||
return new Response(JSON.stringify({ dist: { tarball: 'https://tar.example/x.tgz', integrity: integrity(tgz) } }));
|
||||
}
|
||||
if (url === 'https://tar.example/x.tgz') return new Response(tgz);
|
||||
return new Response('nope', { status: 404 });
|
||||
};
|
||||
try {
|
||||
const r = await addWindowsNativePackages(dir, 'ia32', { log: () => {} });
|
||||
assert.deepStrictEqual(r.removed.map((d) => d.split(path.sep).join('/')), [`${NM}/@img/sharp-linux-x64`]);
|
||||
assert.strictEqual(r.added.version, '0.35.5');
|
||||
const added = path.join(dir, NM, '@img/sharp-win32-ia32');
|
||||
assert.ok(fs.existsSync(path.join(added, 'lib/sharp-win32-ia32-0.35.5.node')));
|
||||
assert.ok(fs.existsSync(path.join(added, 'lib/libvips-42.dll')), 'the libvips DLLs come along');
|
||||
assert.ok(!fs.existsSync(path.join(dir, NM, '@img/sharp-linux-x64')));
|
||||
assert.strictEqual(asked.length, 2);
|
||||
|
||||
// Negative: a tarball that does not match the registry's sha512 is not
|
||||
// unpacked, and the run fails.
|
||||
globalThis.fetch = async (url) => url.startsWith('https://registry')
|
||||
? new Response(JSON.stringify({ dist: { tarball: 'https://tar.example/x.tgz', integrity: integrity(Buffer.from('other')) } }))
|
||||
: new Response(tgz);
|
||||
fs.rmSync(added, { recursive: true, force: true });
|
||||
await assert.rejects(addWindowsNativePackages(dir, 'ia32', { log: () => {} }), /integrity mismatch/);
|
||||
assert.ok(!fs.existsSync(added), 'nothing is written from a tarball that failed its check');
|
||||
} finally {
|
||||
globalThis.fetch = realFetch;
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
await test('a bundle with no sharp, or an unknown CPU, is an error', async () => {
|
||||
const dir = tree({ [`${NM}/x/package.json`]: pkg({}) });
|
||||
try {
|
||||
await assert.rejects(addWindowsNativePackages(dir, 'ia32', { log: () => {} }), /no sharp/);
|
||||
await assert.rejects(addWindowsNativePackages(dir, 'mips', { log: () => {} }), /unknown arch/);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nwindowsNativePackages: all ${passed} tests passed`);
|
||||
})().catch((err) => {
|
||||
console.error(err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in new issue
Block a user