Files
wekan/server/lib/zipSource.js
T
Lauri Ojansivu 32c24ed84e Import page uploads stream to the server's disk, checked against the free space and an Admin Panel limit.
A file chosen on the Import page for the generalized importer's sources
is POSTed to /api/import/upload/:source and written to
WRITABLE_PATH/import-uploads a part at a time, instead of being read in
the browser and sent whole, as text or base64, through importBoard. The
size the browser announces is checked before the first byte against the
largest import set in Admin Panel / Settings / Security (none: no limit)
and against the free space of that disk less a reserve, and the space is
checked again every 64 MB while it is written; a refused or interrupted
upload leaves no file behind. importBoard then reads the file by its
id: a .zip export (Slack, Google Takeout, Vikunja, Notion, Plane) is
opened from the disk, its entries inflating up to the limit or the free
space, and a mailbox is read a message at a time, also inside a Takeout
zip. The upload is removed once read, or after an hour.

The HTML-only mail text is read through the parsed document rather than
stripped with a pattern. TODO.md says what is left of large files, and
which locale files hold another language.

Thanks to xet7 !
2026-11-20 05:42:27 +02:00

45 lines
2.0 KiB
JavaScript

'use strict';
// Server-only: open an uploaded .zip from memory (a Buffer, as the Import page
// sent it in a method call) or from the local disk ({ file }: an upload
// streamed there by server/lib/importUpload.js), with the same checks - not
// empty, within its size, a zip by its first four bytes - and the same
// messages. A zip on disk is opened with unzipper.Open.file, which reads its
// central directory and each entry from the file as asked, so the archive is
// never in memory whole. A file's size limit is the import limit the caller
// passes (the administrator's, or none: the disk has room for it already).
const fs = require('fs');
const isZipMagic = head => {
const magic = head && head.length >= 4 ? head.readUInt32LE(0) : 0;
return magic === 0x04034b50 || magic === 0x06054b50;
};
async function openZipSource(source, { what, maxBytes }) {
const unzipper = require('unzipper');
if (source && typeof source.file === 'string') {
const { size } = await fs.promises.stat(source.file);
if (!size) throw new Error(`${what} is empty`);
const limit = Number(source.maxBytes);
if (limit > 0 && size > limit) throw new Error(`${what} is larger than ${limit} bytes`);
const fd = await fs.promises.open(source.file, 'r');
const head = Buffer.alloc(4);
try { await fd.read(head, 0, 4, 0); } finally { await fd.close(); }
if (!isZipMagic(head)) throw new Error(`${what} is not a .zip file`);
try {
return await unzipper.Open.file(source.file);
} catch (error) {
throw new Error(`${what} is not a readable .zip file`);
}
}
if (!Buffer.isBuffer(source) || !source.length) throw new Error(`${what} is empty`);
if (source.length > maxBytes) throw new Error(`${what} is larger than ${maxBytes} bytes`);
if (!isZipMagic(source)) throw new Error(`${what} is not a .zip file`);
try {
return await unzipper.Open.buffer(source);
} catch (error) {
throw new Error(`${what} is not a readable .zip file`);
}
}
module.exports = { openZipSource, isZipMagic };