Files
wekan/models/lib/fileStoreStrategy.js
T
Ahmed Lukman 628a69c086 Keep renamed upload paths consistent before validation
Update the original and version paths in memory and in the file record, and wait for metadata writes before upload hooks continue. Cover rename failures, persistence ordering, normalized uploads, original bytes and card rendering.
2026-11-15 14:11:10 +03:00

1381 lines
53 KiB
JavaScript

import { ReactiveCache } from '/imports/reactiveCache';
import { Random } from 'meteor/random';
import Attachments from '/models/attachments';
import fs from 'fs';
import path from 'path';
import os from 'os';
import { PassThrough } from 'stream';
import { Meteor } from 'meteor/meteor';
import { createObjectId } from './grid/createObjectId';
import { httpStreamOutput } from './httpStream.js';
import {
STORAGE_NAME_FILESYSTEM,
STORAGE_NAME_GRIDFS,
STORAGE_NAME_S3,
STORAGE_NAME_AZURE,
STORAGE_NAME_GCS,
CLOUD_STORAGE_NAMES,
} from './fileStoreConstants';
import { getCloudAdapter, isCloudConfigured } from './cloudStorage';
import { ObjectId } from 'bson';
const { isFilesRoot } = require('./filesRoot');
// Re-export constants from shared module (keeps existing import paths working)
export {
STORAGE_NAME_FILESYSTEM,
STORAGE_NAME_GRIDFS,
STORAGE_NAME_S3,
STORAGE_NAME_AZURE,
STORAGE_NAME_GCS,
CLOUD_STORAGE_NAMES,
} from './fileStoreConstants';
// Pure filename helpers (path traversal + length cap) live in a Meteor-free
// module so they can be unit tested directly with Node. See
// models/lib/filenameSanitizer.js (#6412).
const { sanitizeFilename } = require('./filenameSanitizer');
const { hasEnoughDiskSpace } = require('./diskSpace');
// GHSA-4mxf-m8pq-xc9p: this containment check used to be written out here and
// nowhere else, so the board exporter - which reads the very same stored paths -
// had none. It lives in models/lib/storagePathContainment.js now, where the
// exporter uses it too and where plain Node can test it.
const { isPathInsideBase: isPathInside } = require('./storagePathContainment');
// #6898: drop the integrity baselines of files this module deletes or moves.
function forgetBaselines(paths) {
if (!Meteor.isServer) return;
try {
require('/server/lib/fileIntegrityForget').forgetIntegrityBaselinesSoon(paths);
} catch (e) { /* bookkeeping must never break a delete */ }
}
function tryRealPath(inputPath) {
try {
return fs.realpathSync(inputPath);
} catch (err) {
return null;
}
}
/*
* `literalRootPath` is the storage root as WRITTEN; `storageRootPath` is that
* root with every symlink resolved. Both are needed, and comparing a path
* against the wrong one of them is what broke reading here.
*
* The candidates are built by joining names onto the LITERAL root, so the cheap
* lexical pre-filter below has to use that same root. It used to compare them
* against the RESOLVED one, which is identical on a system where nothing in the
* path is a symlink and different everywhere else: on macOS `os.tmpdir()` is
* `/var/folders/...`, a symlink to `/private/var/folders/...`, so every
* candidate was rejected before it was ever looked at and WeKan served none of
* its own attachments. A deployment whose data directory is a symlink - a very
* ordinary arrangement - has the same fault on Linux.
*
* The security property is unchanged, because it was never the lexical check
* that carried it: what a caller must not be able to do is reach a file OUTSIDE
* the storage root, and that is decided below by resolving the candidate and
* requiring the result to be inside the RESOLVED root. That check is what
* refuses `../../etc/passwd` and a symlink pointing out of the tree
* (GHSA-4mxf-m8pq-xc9p), and it still runs on every candidate.
*/
function isSafeReadableFile(candidatePath, storageRootPath, literalRootPath) {
if (!candidatePath || !storageRootPath) {
return false;
}
const lexicalRoot = literalRootPath || storageRootPath;
if (!isPathInside(lexicalRoot, candidatePath)) {
return false;
}
try {
const candidateRealPath = tryRealPath(candidatePath) || candidatePath;
if (!isPathInside(storageRootPath, candidateRealPath)) {
return false;
}
const stats = fs.statSync(candidateRealPath);
return stats.isFile();
} catch (err) {
return false;
}
}
/** Factory for FileStoreStrategy */
export default class FileStoreStrategyFactory {
/** constructor
* @param classFileStoreStrategyFilesystem use this strategy for filesystem storage
* @param storagePath file storage path
* @param classFileStoreStrategyGridFs use this strategy for GridFS storage
* @param gridFsBucket use this GridFS Bucket as GridFS Storage
* @param classFileStoreStrategyCloud strategy class for cloud backends (S3/Azure/GCS), optional
*/
constructor(classFileStoreStrategyFilesystem, storagePath, classFileStoreStrategyGridFs, gridFsBucket, classFileStoreStrategyCloud, collection) {
this.classFileStoreStrategyFilesystem = classFileStoreStrategyFilesystem;
this.storagePath = storagePath;
this.classFileStoreStrategyGridFs = classFileStoreStrategyGridFs;
this.gridFsBucket = gridFsBucket;
// Single strategy class handling every cloud provider (S3/Azure/GCS) via
// @tweedegolf/storage-abstraction. Optional: when omitted (or the npm
// packages are absent) cloud storage falls back to the filesystem.
this.classFileStoreStrategyCloud = classFileStoreStrategyCloud;
// The FilesCollection (Attachments or Avatars) whose documents this factory
// operates on. Strategies persist storage/path changes to this collection so
// the same code can move both attachments and avatars. Defaults to the
// global Attachments for backward compatibility.
this.collection = collection;
}
/** returns the right FileStoreStrategy
* @param fileObj the current file object
* @param versionName the current version
* @param use this storage, or if not set, get the storage from fileObj
*/
getFileStrategy(fileObj, versionName, storage) {
if (!storage) {
// Every one of these can be absent, and each absence used to throw here.
// A freshly uploaded file has no `meta` on its version (only a GridFS one
// does), so `versions[v].meta.gridFsFileId` threw "Cannot read properties
// of undefined (reading 'gridFsFileId')" on EVERY upload - which
// onAfterUpload caught and logged as "filename hardening failed", so the
// mime detection and filename correction were skipped for every new file
// while the upload itself appeared to succeed. `fileObj.meta` can be
// absent in the same way, and so can the version itself when a caller asks
// for one this file does not have.
const version = (fileObj && fileObj.versions && fileObj.versions[versionName]) || {};
const versionMeta = version.meta || {};
const fileMeta = (fileObj && fileObj.meta) || {};
storage = version.storage;
if (!storage) {
if (fileMeta.source == "import" || versionMeta.gridFsFileId) {
// uploaded by import, so it's in GridFS (MongoDB)
storage = STORAGE_NAME_GRIDFS;
} else {
// newly uploaded, so it's at the filesystem
storage = STORAGE_NAME_FILESYSTEM;
}
}
}
let ret;
if (storage == STORAGE_NAME_FILESYSTEM) {
ret = new this.classFileStoreStrategyFilesystem(fileObj, versionName, this.collection);
} else if (storage == STORAGE_NAME_GRIDFS) {
ret = new this.classFileStoreStrategyGridFs(this.gridFsBucket, fileObj, versionName, this.collection);
} else if (CLOUD_STORAGE_NAMES.includes(storage)) {
if (this.classFileStoreStrategyCloud && isCloudConfigured(storage)) {
ret = new this.classFileStoreStrategyCloud(storage, fileObj, versionName, this.collection);
} else {
// Cloud strategy unavailable or provider not configured — fall back to
// filesystem so reads of files that never actually reached the cloud
// still resolve instead of throwing.
console.warn(`Cloud storage "${storage}" is not configured; falling back to filesystem storage`);
ret = new this.classFileStoreStrategyFilesystem(fileObj, versionName, this.collection);
}
}
return ret;
}
}
/** Strategy to store files */
class FileStoreStrategy {
/** constructor
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(fileObj, versionName, collection) {
this.fileObj = fileObj;
this.versionName = versionName;
// FilesCollection to persist changes to (Attachments or Avatars). Falls
// back to the global Attachments when not supplied.
this.collection = collection || Attachments;
}
/** The record of this version, or {} when the file has none: a record
* written without `versions` (an import, a hand-made document) made every
* reader throw "reading 'original'", and the file route answered 500 for
* what is a missing file - a 404.
*/
version() {
const versions = (this.fileObj && this.fileObj.versions) || {};
return versions[this.versionName] || {};
}
/** after successfull upload */
onAfterUpload() {
}
/** download the file
* @param http the current http request
* @param cacheControl cacheControl of FilesCollection
*
* The BASE implementation refuses, and that is deliberate. An empty body
* returns undefined, which Meteor-Files reads as "not handled" and answers by
* serving the file itself - from the stored path, with the stored
* Content-Type, and none of the policy httpStreamOutput applies. Every
* concrete strategy below overrides this, so the default is only ever reached
* by one that forgot to; a strategy that forgets should serve nothing rather
* than serve it unfiltered.
*/
interceptDownload(http, cacheControl) {
http.response.statusCode = 404;
http.response.end('not found');
return true;
}
/** after file remove */
onAfterRemove() {
}
/** returns a read stream
* @return the read stream
*/
getReadStream() {
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
}
/** returns the new file path
* @param storagePath use this storage path
* @return the new file path
*/
getNewPath(storagePath, name) {
if (typeof name !== 'string') {
name = this.fileObj.name;
}
// Sanitize filename to prevent path traversal attacks
const safeName = sanitizeFilename(name);
const ret = path.join(storagePath, this.fileObj._id + "-" + this.versionName + "-" + safeName);
return ret;
}
/** remove the file */
unlink() {
}
/** rename the file (physical)
* @li at database the filename is updated after this method
* @param newFilePath the new file path
*/
rename(newFilePath) {
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
}
}
/** Strategy to store attachments at GridFS (MongoDB) */
export class FileStoreStrategyGridFs extends FileStoreStrategy {
/** constructor
* @param gridFsBucket use this GridFS Bucket
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(gridFsBucket, fileObj, versionName, collection) {
super(fileObj, versionName, collection);
this.gridFsBucket = gridFsBucket;
}
/** download the file
* @param http the current http request
* @param cacheControl cacheControl of FilesCollection
*/
/*
* FAIL CLOSED when the strategy will not serve the file.
*
* Returning false here means "not handled", and Meteor-Files then serves the
* file ITSELF - from the stored path, with the stored Content-Type, and none
* of the policy httpStreamOutput applies. A stored attachment of type
* text/html came back as text/html, status 200, inline: precisely the stored
* XSS that policy exists to stop.
*
* What makes that a bypass rather than a fallback is WHY the strategy
* declines. getReadStream() returns nothing when the file cannot be resolved
* INSIDE the storage root - the containment check of GHSA-4mxf-m8pq-xc9p - so
* "false" means "this is not a file WeKan may serve", and handing it to a
* server with no containment check answers the refusal with the file.
*
* A file that is genuinely absent gets the same 404 it would have got anyway.
*/
interceptDownload(http, cacheControl) {
const readStream = this.getReadStream();
const downloadFlag = http?.params?.query?.download;
if (!readStream) {
http.response.statusCode = 404;
http.response.end('not found');
return true;
}
httpStreamOutput(readStream, this.fileObj.name, http, downloadFlag, cacheControl, this.fileObj);
return true;
}
/** after file remove */
onAfterRemove() {
this.unlink();
super.onAfterRemove();
}
/** returns a read stream
* @return the read stream
*/
getReadStream(range) {
const gfsId = this.getGridFsObjectId();
let ret;
if (gfsId) {
// #6745: a byte range (inclusive end, models/lib/httpRange.js) streams
// only those bytes; GridFS takes an exclusive end.
ret = range
? this.gridFsBucket.openDownloadStream(gfsId, { start: range.start, end: range.end + 1 })
: this.gridFsBucket.openDownloadStream(gfsId);
}
return ret;
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
const fileObj = this.fileObj;
const versionName = this.versionName;
const metadata = { ...fileObj.meta, versionName, fileId: fileObj._id };
const ret = this.gridFsBucket.openUploadStream(this.fileObj.name, {
contentType: fileObj.type || 'binary/octet-stream',
metadata,
});
// Keep a reference so writeStreamFinished can read the uploaded file's id
// from the stream — the mongodb driver's 'finish' event no longer passes the
// stored file document.
this.gridFsWriteStream = ret;
return ret;
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
const gridFsFileIdName = this.getGridFsFileIdName();
// Older mongodb drivers passed the stored file document to the 'finish'
// event; current drivers emit it with no argument (so finishedData is
// undefined and `finishedData._id` threw, crashing the server). Resolve the
// GridFS file id from the finish data when present, otherwise from the write
// stream itself (`id` is assigned by openUploadStream; `gridFSFile._id` is
// set once the upload finishes).
const stream = this.gridFsWriteStream;
const gridFsId =
(finishedData && finishedData._id) ||
(stream && stream.gridFSFile && stream.gridFSFile._id) ||
(stream && stream.id);
if (!gridFsId) {
console.error(
'GridFS write finished but no file id was available for',
this.fileObj && this.fileObj._id,
);
return;
}
const hexId =
typeof gridFsId.toHexString === 'function' ? gridFsId.toHexString() : String(gridFsId);
this.collection.updateAsync(
{ _id: this.fileObj._id },
{ $set: { [gridFsFileIdName]: hexId } },
).catch(error => {
console.error('Failed to persist GridFS file id:', error);
});
}
/** remove the file */
unlink() {
const gfsId = this.getGridFsObjectId();
if (gfsId) {
this.gridFsBucket.delete(gfsId, err => {
if (err) {
console.error("error on gfs bucket.delete: ", err);
}
});
}
const gridFsFileIdName = this.getGridFsFileIdName();
this.collection.updateAsync(
{ _id: this.fileObj._id },
{ $unset: { [gridFsFileIdName]: 1 } },
).catch(error => {
console.error('Failed to clear GridFS file id:', error);
});
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
return STORAGE_NAME_GRIDFS;
}
/** returns the GridFS Object-Id
* @return the GridFS Object-Id
*/
getGridFsObjectId() {
let ret;
const gridFsFileId = this.getGridFsFileId();
if (gridFsFileId) {
ret = createObjectId({ gridFsFileId });
}
return ret;
}
/** returns the GridFS Object-Id
* @return the GridFS Object-Id
*/
getGridFsFileId() {
const ret = (this.version().meta || {})
.gridFsFileId;
return ret;
}
/** returns the property name of gridFsFileId
* @return the property name of gridFsFileId
*/
getGridFsFileIdName() {
const ret = `versions.${this.versionName}.meta.gridFsFileId`;
return ret;
}
}
/** Strategy to store attachments at filesystem */
export class FileStoreStrategyFilesystem extends FileStoreStrategy {
/** constructor
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(fileObj, versionName, collection) {
super(fileObj, versionName, collection);
}
/** Serve filesystem files through WeKan's response backstop instead of
* falling through to Meteor-Files, which trusts the stored MIME type and
* defaults to inline disposition. */
interceptDownload(http, cacheControl) {
const readStream = this.getReadStream();
// Fail closed, for the reason given above the filesystem strategy's
// interceptDownload: `false` hands the file to Meteor-Files unfiltered.
if (!readStream) {
http.response.statusCode = 404;
http.response.end('not found');
return true;
}
const downloadFlag = http?.params?.query?.download;
httpStreamOutput(
readStream,
this.fileObj.name,
http,
downloadFlag,
cacheControl,
this.fileObj,
);
return true;
}
/** returns a read stream
* @return the read stream
*/
/** every place the physical file could be, in priority order
*
* The recorded `versions[<v>].path` is a GUESS about the past: it was written
* by whichever WeKan stored the file, and the layout has changed - `<id>`,
* `<id>.<ext>`, `<id>-<version>-<name>` - while a migration, a storage move or
* an extension correction could rewrite one of the two and not the other.
*
* #6589 is what that costs when only the reader knows: a .drawio attachment
* was served fine (this search found it) and could not be RENAMED, because
* rename used the recorded path alone:
*
* Error: ENOENT: no such file or directory, rename
* '/data/files/attachments/6a7d66369c6aee799e857d36.drawio' -> ...
*
* So the search is a method now, and reading, renaming and deleting all use it.
* @return array of absolute candidate paths
*/
candidatePaths() {
const v = this.version();
const originalPath = v.path || '';
const normalized = (originalPath || '').replace(/\\/g, '/');
const isAvatar = normalized.includes('/avatars/') || (this.fileObj.collectionName === 'avatars');
const baseDir = isAvatar ? 'avatars' : 'attachments';
const writableBase = process.env.WRITABLE_PATH || process.cwd();
const endsWithFiles = isFilesRoot(writableBase);
const storageRoot = endsWithFiles
? path.join(writableBase, baseDir)
: path.join(writableBase, 'files', baseDir);
const resolvedStorageRoot = tryRealPath(storageRoot) || path.resolve(storageRoot);
// Build candidate list in priority order
const candidates = [];
// 0) Try to find project root and resolve from there
let projectRoot = null;
if (originalPath) {
// Find project root by looking for .meteor directory
let current = process.cwd();
let maxLevels = 10; // Safety limit
while (maxLevels-- > 0) {
const meteorPath = path.join(current, '.meteor');
const packagePath = path.join(current, 'package.json');
if (fs.existsSync(meteorPath) || fs.existsSync(packagePath)) {
projectRoot = current;
break;
}
const parent = path.dirname(current);
if (parent === current) break; // Reached filesystem root
current = parent;
}
if (projectRoot) {
// Try resolving originalPath from project root
const fromProjectRoot = path.resolve(projectRoot, originalPath);
candidates.push(fromProjectRoot);
// Also try direct path: projectRoot/attachments/filename
const baseName = path.basename(normalized || this.fileObj._id || '');
if (baseName) {
const directPath = path.join(projectRoot, baseDir, baseName);
candidates.push(directPath);
}
}
}
// 1) Original as-is (absolute or relative resolved to CWD)
if (originalPath) {
candidates.push(originalPath);
if (!path.isAbsolute(originalPath)) {
candidates.push(path.resolve(process.cwd(), originalPath));
candidates.push(path.resolve(storageRoot, originalPath));
}
}
// 2) Same basename in storageRoot
const baseName = path.basename(normalized || this.fileObj._id || '');
if (baseName) {
candidates.push(path.join(storageRoot, baseName));
}
// 3) Only ObjectID (no extension) in storageRoot
if (this.fileObj && this.fileObj._id) {
candidates.push(path.join(storageRoot, String(this.fileObj._id)));
}
// 3) Old naming: {id}-{version}-{originalName}
if (this.fileObj.name) {
const safeName = sanitizeFilename(this.fileObj.name);
candidates.push(path.join(storageRoot, `${this.fileObj._id}-${this.versionName}-${safeName}`));
}
// 4) Fallback by prefix: {id}-{version}-* (covers unknown/changed original names)
if (this.fileObj && this.fileObj._id) {
try {
const prefix = `${this.fileObj._id}-${this.versionName}-`;
const dirEntries = fs.readdirSync(storageRoot);
const prefixedMatch = dirEntries.find((entry) => entry.startsWith(prefix));
if (prefixedMatch) {
candidates.push(path.join(storageRoot, prefixedMatch));
}
} catch (err) {
// Ignore listing errors and continue with other candidates
}
}
return { candidates, resolvedStorageRoot, storageRoot };
}
/** the path the file is ACTUALLY at, or undefined when none of them exists
* @return absolute path or undefined
*/
resolveExistingPath() {
const { candidates, resolvedStorageRoot, storageRoot } = this.candidatePaths();
for (const c of candidates) {
if (isSafeReadableFile(c, resolvedStorageRoot, storageRoot)) return c;
}
return undefined;
}
getReadStream(range) {
const chosen = this.resolveExistingPath();
if (!chosen) {
return undefined;
}
// #6745: a byte range (inclusive end, like fs) streams only those bytes.
return range
? fs.createReadStream(chosen, { start: range.start, end: range.end })
: fs.createReadStream(chosen);
}
/** returns a write stream
* @param filePath if set, use this path
* @return the write stream
*/
getWriteStream(filePath) {
if (typeof filePath !== 'string') {
filePath = this.version().path;
}
const ret = fs.createWriteStream(filePath);
return ret;
}
/** writing finished
* @param finishedData the data of the write stream finish event
*/
writeStreamFinished(finishedData) {
}
/** remove the file */
unlink() {
// The file that is THERE, not the one the database remembers - see
// candidatePaths(). A delete that misses leaves the bytes on disk forever.
const filePath = this.resolveExistingPath()
|| this.version().path;
if (filePath) {
fs.unlink(filePath, () => {});
// #6898: a file WeKan deleted is accounted for; without this the daily
// integrity scan reported it as missing on every run from then on.
forgetBaselines([filePath, this.version().path]);
}
}
/** rename the file (physical)
* @li at database the filename is updated after this method
* @param newFilePath the new file path
*/
rename(newFilePath) {
const current = this.resolveExistingPath();
if (!current) {
// #6589: this used to be `fs.renameSync(<recorded path>, ...)` and the
// recorded path did not exist, so every rename of that attachment threw
// ENOENT and it stayed stuck under the wrong name for good. Say what is
// actually wrong - the file is gone, not the rename - and name the
// attachment, because the ENOENT named a path nobody recognised.
const recorded = this.version().path || '(none recorded)';
throw new Error(
`Attachment ${this.fileObj._id} (${this.fileObj.name || 'unnamed'}), version `
+ `${this.versionName}: no file found on disk. The database says ${recorded}, `
+ `and neither that nor any of the older layouts is there, so there is nothing to rename.`);
}
if (path.resolve(current) === path.resolve(newFilePath)) return;
fs.renameSync(current, newFilePath);
// #6898: the old path is gone on purpose; the scan records the new one as
// a first sighting instead of reporting the old one missing forever.
forgetBaselines([current, this.version().path]);
}
/** return the storage name
* @return the storage name
*/
getStorageName() {
return STORAGE_NAME_FILESYSTEM;
}
}
/**
* Strategy to store attachments on a cloud backend (S3-compatible, Azure Blob,
* Google Cloud Storage) through @tweedegolf/storage-abstraction.
*
* The abstraction is asynchronous and result-object based, while the rest of
* WeKan expects synchronous, pipe-able streams. We bridge the two with
* PassThrough streams:
* - getReadStream() returns a PassThrough immediately and fills it once the
* async getFileAsStream() resolves.
* - getWriteStream() returns a PassThrough that the async addFileFromStream()
* consumes; waitUntilStored() exposes the upload's completion promise so
* moveToStorage() can confirm the upload before deleting the source file.
*
* The object key is `${id}-${version}-${name}` and is stored in
* `versions.<version>.path` (mirrors the filesystem strategy) so reads can find
* the object again regardless of provider.
*/
// #6745: where a file bound for cloud storage is spooled before its upload
// (FileStoreStrategyCloud.getWriteStream). A file there lives only for the
// length of one upload.
function cloudSpoolDir() {
const dir = path.join(os.tmpdir(), 'wekan-cloud-upload');
fs.mkdirSync(dir, { recursive: true });
return dir;
}
// S3 needs the body length up front to stream a body; the other providers
// stream a file without it. S3 also gets the server-side encryption the
// administrator chose (models/lib/s3Encryption.js), on every write.
function cloudUploadOptions(provider, size, adapter) {
if (provider !== STORAGE_NAME_S3) return {};
const options = { ...((adapter && adapter.uploadOptions) || {}) };
if (size !== undefined) options.ContentLength = size;
return options;
}
export class FileStoreStrategyCloud extends FileStoreStrategy {
/** constructor
* @param provider cloud storage name ('s3' | 'azure' | 'gcs')
* @param fileObj the current file object
* @param versionName the current version
*/
constructor(provider, fileObj, versionName, collection) {
super(fileObj, versionName, collection);
this.provider = provider;
this._uploadPromise = null;
this._key = null;
}
/** download the file
* @param http the current http request
* @param cacheControl cacheControl of FilesCollection
*/
/*
* FAIL CLOSED when the strategy will not serve the file.
*
* Returning false here means "not handled", and Meteor-Files then serves the
* file ITSELF - from the stored path, with the stored Content-Type, and none
* of the policy httpStreamOutput applies. A stored attachment of type
* text/html came back as text/html, status 200, inline: precisely the stored
* XSS that policy exists to stop.
*
* What makes that a bypass rather than a fallback is WHY the strategy
* declines. getReadStream() returns nothing when the file cannot be resolved
* INSIDE the storage root - the containment check of GHSA-4mxf-m8pq-xc9p - so
* "false" means "this is not a file WeKan may serve", and handing it to a
* server with no containment check answers the refusal with the file.
*
* A file that is genuinely absent gets the same 404 it would have got anyway.
*/
interceptDownload(http, cacheControl) {
const readStream = this.getReadStream();
const downloadFlag = http?.params?.query?.download;
if (!readStream) {
http.response.statusCode = 404;
http.response.end('not found');
return true;
}
httpStreamOutput(readStream, this.fileObj.name, http, downloadFlag, cacheControl, this.fileObj);
return true;
}
/** after file remove */
onAfterRemove() {
this.unlink();
super.onAfterRemove();
}
/** returns a read stream (filled asynchronously from the cloud backend)
* @return the read stream
*/
getReadStream(range) {
const pass = new PassThrough();
const adapter = getCloudAdapter(this.provider);
if (!adapter) {
process.nextTick(() => pass.destroy(new Error(`Cloud storage "${this.provider}" is not configured`)));
return pass;
}
const key = this.getObjectKey();
// #6745: a byte range (inclusive end) is fetched from the backend as such.
const rangeArgs = range ? [{ start: range.start, end: range.end }] : [];
adapter.storage.getFileAsStream(adapter.bucketName, key, ...rangeArgs)
.then(result => {
if (!result || result.error || !result.value) {
pass.destroy(new Error(result && result.error ? result.error : 'No cloud read stream'));
return;
}
result.value.on('error', error => pass.destroy(error));
result.value.pipe(pass);
})
.catch(error => pass.destroy(error));
return pass;
}
/** returns a write stream that uploads to the cloud backend
* @param filePath the object key to write to (from getNewPath)
* @return the write stream
*/
getWriteStream(filePath) {
const pass = new PassThrough();
this._key = (typeof filePath === 'string' && filePath) ? filePath : this.getObjectKey();
this._uploadError = null;
const adapter = getCloudAdapter(this.provider);
if (!adapter) {
this._uploadError = new Error(`Cloud storage "${this.provider}" is not configured`);
this._uploadPromise = Promise.resolve();
process.nextTick(() => pass.destroy(this._uploadError));
return pass;
}
// #6745: spool the incoming bytes to a temporary FILE, then upload that
// file as a stream of known length. Streaming a live body of unknown length
// to S3 fails with
// Invalid value "undefined" for header "x-amz-decoded-content-length"
// and, if the socket drops mid-upload, the AWS SDK's body-stream promise
// rejects UNHANDLED and crashes the server (SyncedCron treats it as fatal).
// This used to be avoided by collecting the whole file in one Buffer, which
// held every byte of a file moved to the cloud in server memory (no cap:
// the upload limit defaults to unlimited) - a 2 GB video needed 2 GB of
// RAM. A file on disk has a known length (ContentLength for S3) and is
// re-readable, so neither problem returns and memory stays at a stream
// buffer. The upload promise NEVER rejects - any failure is captured in
// this._uploadError (read by waitUntilStored()).
const spoolPath = path.join(cloudSpoolDir(), `${Random.id()}-cloud-upload`);
const spool = fs.createWriteStream(spoolPath);
const removeSpool = () => fs.promises.unlink(spoolPath).catch(() => {});
this._uploadPromise = new Promise(resolve => {
const fail = error => {
if (!this._uploadError) {
this._uploadError = error instanceof Error ? error : new Error(String(error));
}
try { spool.destroy(); } catch (e) { /* already closed */ }
// removeSpool() catches its own error, so it never rejects.
void removeSpool().then(resolve);
};
pass.on('error', fail);
spool.on('error', fail);
spool.on('finish', () => {
if (this._uploadError) return;
// Every failure is caught below and the tail never rejects.
void fs.promises.stat(spoolPath)
.then(stat => adapter.storage.addFileFromPath({
origPath: spoolPath,
bucketName: adapter.bucketName,
targetPath: this._key,
options: cloudUploadOptions(this.provider, stat.size, adapter),
}))
.then(result => {
if (result && result.error) {
this._uploadError = new Error(result.error);
}
})
.catch(error => {
this._uploadError = error instanceof Error ? error : new Error(String(error));
})
.then(removeSpool)
.then(resolve);
});
pass.pipe(spool);
});
return pass;
}
/** Promise resolving once the upload has durably completed on the backend.
* moveToStorage() awaits this before deleting the source file.
*/
waitUntilStored() {
// Wait for the (non-rejecting) upload promise, then re-throw any captured
// error so moveToStorage's try/catch keeps the source file intact instead of
// deleting it after a failed upload.
return Promise.resolve(this._uploadPromise).then(() => {
if (this._uploadError) {
throw this._uploadError;
}
});
}
/** writing finished — persist the object key in the version meta */
writeStreamFinished(finishedData) {
const field = this.getCloudFileIdName();
this.collection.updateAsync(
{ _id: this.fileObj._id },
{ $set: { [field]: this._key } },
).catch(error => {
console.error('Failed to persist cloud file id:', error);
});
}
/** remove the file from the cloud backend */
unlink() {
const adapter = getCloudAdapter(this.provider);
if (!adapter) {
return;
}
const key = this.getObjectKey();
Promise.resolve(adapter.storage.removeFile(adapter.bucketName, key)).catch(error => {
console.error('Failed to remove cloud file:', error);
});
const field = this.getCloudFileIdName();
this.collection.updateAsync(
{ _id: this.fileObj._id },
{ $unset: { [field]: 1 } },
).catch(error => {
console.error('Failed to clear cloud file id:', error);
});
}
/** the object key used in the cloud bucket */
getObjectKey() {
const version = this.version();
if (version.path) {
return version.path;
}
const meta = version.meta || {};
if (meta[`${this.provider}FileId`]) {
return meta[`${this.provider}FileId`];
}
const safeName = sanitizeFilename(this.fileObj.name);
return `${this.fileObj._id}-${this.versionName}-${safeName}`;
}
/** the object key for a fresh write (no leading storage directory) */
getNewPath(storagePath, name) {
if (typeof name !== 'string') {
name = this.fileObj.name;
}
const safeName = sanitizeFilename(name);
return `${this.fileObj._id}-${this.versionName}-${safeName}`;
}
/** the database field storing this provider's object key */
getCloudFileIdName() {
return `versions.${this.versionName}.meta.${this.provider}FileId`;
}
/** rename the file: cloud objects are keyed by id+version+name, so a rename
* is a server-side copy to the new key followed by removal of the old one.
*/
rename(newFilePath) {
const adapter = getCloudAdapter(this.provider);
if (!adapter) {
return;
}
const oldKey = this.getObjectKey();
const newKey = newFilePath;
if (!oldKey || !newKey || oldKey === newKey) {
return;
}
adapter.storage.getFileAsStream(adapter.bucketName, oldKey)
.then(result => {
if (!result || result.error || !result.value) {
throw new Error(result && result.error ? result.error : 'No cloud read stream for rename');
}
return adapter.storage.addFileFromStream({
stream: result.value,
bucketName: adapter.bucketName,
targetPath: newKey,
// A renamed object is a new object: it is encrypted like the old one.
options: cloudUploadOptions(this.provider, undefined, adapter),
});
})
.then(() => adapter.storage.removeFile(adapter.bucketName, oldKey))
.catch(error => console.error('Failed to rename cloud file:', error));
}
/** return the storage name (the provider) */
getStorageName() {
return this.provider;
}
}
/** move the fileObj to another storage
* @param fileObj move this fileObj to another storage
* @param storageDestination the storage destination (fs or gridfs)
* @param fileStoreStrategyFactory get FileStoreStrategy from this factory
* @return a Promise that resolves once every version has finished moving.
* Existing callers may ignore the return value (fire-and-forget); the
* server-side bulk move job awaits it to move attachments sequentially.
*/
export const moveToStorage = async function(fileObj, storageDestination, fileStoreStrategyFactory) {
const collection = fileStoreStrategyFactory.collection || Attachments;
// When migrating/moving a file, sanitize known exploits from its CONTENT with the
// same general function used on upload (strip JavaScript + XML-loop DOCTYPE/ENTITY
// from SVG), so the sanitized bytes are what get written to the destination. Runs
// in place on the local filesystem source (the common migration direction); the
// serve-time sanitizer is the backstop for files still on a remote backend.
try {
const { sanitizeUploadedFileExploits } = require('./sanitizeUploadedFile');
if (await sanitizeUploadedFileExploits(fileObj)) {
await collection.updateAsync({ _id: fileObj._id }, { $set: { versions: fileObj.versions } })
.catch(error => console.error('Failed to persist sanitized versions before move:', error));
try {
await require('/server/lib/filenameSanitizeLog').logContentSanitized({
fileObj, source: 'storageMigration',
});
} catch (e) { /* best effort */ }
}
} catch (error) {
console.error('[moveToStorage] content exploit sanitize failed (continuing):', error);
}
// When migrating/moving a file to its destination storage, fix the filename with
// the SAME general function used on upload: detect the real file type and correct
// the extension, strip invisible/exploit characters, fold confusable homoglyphs,
// cap the length to a portable maximum, keep it path-safe, and disambiguate a
// different-content same-name collision with increasing numbering. The corrected
// name is saved as the filename at the destination.
try {
const { finalizeStoredFileName } = require('./fileTypeCorrection');
const originalName = fileObj.name;
const { name: finalName, changed, detectedMime } = await finalizeStoredFileName(collection, fileObj, fileStoreStrategyFactory);
if (changed && finalName) {
const lastDot = finalName.lastIndexOf('.');
const extension = lastDot === -1 ? '' : finalName.substring(lastDot + 1).toLowerCase();
await collection.updateAsync({ _id: fileObj._id }, { $set: {
name: finalName,
extension,
extensionWithDot: extension ? `.${extension}` : '',
} }).catch(error => console.error('Failed to persist migrated attachment name:', error));
// Log to Admin Panel / Problems: who uploaded, why it was sanitized, when.
try {
const { sanitizationReasons } = require('./uploadFileName');
await require('/server/lib/filenameSanitizeLog').logFilenameSanitized({
fileObj, source: 'storageMigration',
reasons: sanitizationReasons(originalName, detectedMime || fileObj.type, finalName),
from: originalName, to: finalName,
});
} catch (e) { /* best effort */ }
fileObj.name = finalName;
}
} catch (error) {
// Never block a move because name-finalization failed; fall back to the basic
// path-traversal sanitizer so a malicious stored name is still cleaned up.
console.error('[moveToStorage] filename finalization failed, using basic sanitizer:', error);
const safeName = sanitizeFilename(fileObj.name);
if (safeName !== fileObj.name) {
collection.updateAsync({ _id: fileObj._id }, { $set: { name: safeName } }).catch(e => {
console.error('Failed to persist sanitized attachment name:', e);
});
fileObj.name = safeName;
}
}
const versionPromises = Object.keys(fileObj.versions).map(versionName => new Promise(resolve => {
const strategyRead = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName);
const strategyWrite = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName, storageDestination);
// Compare by storage name rather than class name: every cloud provider
// shares the same strategy class, so constructor.name would wrongly treat
// an S3 -> Azure move as "already there".
if (strategyRead.getStorageName() == strategyWrite.getStorageName()) {
// Already on the destination storage, nothing to move for this version.
resolve();
return;
}
const readStream = strategyRead.getReadStream();
const filePath = strategyWrite.getNewPath(fileStoreStrategyFactory.storagePath);
// Before writing to local-disk (filesystem) storage, make sure there is enough
// free space for this version — so a large move cannot fill the disk. When the
// platform does not expose free-space info (e.g. a sandbox), hasEnoughDiskSpace
// returns true and we proceed with chunked streaming, relying on the write-error
// handler below to stop and remove any partial output. Never delete the source.
if (strategyWrite.getStorageName() === STORAGE_NAME_FILESYSTEM) {
const versionSize = ((fileObj.versions || {})[versionName] || {}).size || fileObj.size || 0;
if (!hasEnoughDiskSpace(fileStoreStrategyFactory.storagePath, versionSize)) {
console.error(
'[moveToStorage] not enough free disk space to move attachment',
fileObj._id, `version "${versionName}" (${versionSize} bytes) — skipping, source left intact.`,
);
resolve();
return;
}
}
const writeStream = strategyWrite.getWriteStream(filePath);
// The source binary may be missing at its recorded location (e.g. a file left
// half-moved by an earlier interrupted run: storage says "gridfs" but the
// GridFS id reference is gone, or a filesystem path that no longer exists). In
// that case getReadStream()/getWriteStream() return undefined. Skip this
// version cleanly instead of throwing "Cannot read properties of undefined
// (reading 'on')", and crucially do NOT delete the source — leave it intact so
// no data is lost and the admin can investigate.
if (!readStream || !writeStream) {
console.error(
'[moveToStorage] cannot move attachment',
fileObj._id,
`version "${versionName}" from ${strategyRead.getStorageName()} to ${strategyWrite.getStorageName()}:`,
!readStream
? 'source file not found at its recorded location'
: 'could not open the destination write stream',
'— skipping this file, source left intact.',
);
resolve();
return;
}
let ended = false;
let finished = false;
let settled = false;
// Persist the new storage location and delete the source — but only once
// the write is durably stored. For cloud backends waitUntilStored() awaits
// the actual upload, so a failed upload never deletes the source file.
const finalize = async () => {
if (settled) {
return;
}
settled = true;
try {
if (typeof strategyWrite.waitUntilStored === 'function') {
await strategyWrite.waitUntilStored();
}
await (fileStoreStrategyFactory.collection || Attachments).updateAsync({ _id: fileObj._id }, { $set: {
[`versions.${versionName}.storage`]: strategyWrite.getStorageName(),
[`versions.${versionName}.path`]: filePath,
} });
strategyRead.unlink();
} catch (error) {
console.error('[moveToStorage] write not confirmed, keeping source intact: ', error, fileObj._id);
}
resolve();
};
// https://forums.meteor.com/t/meteor-code-must-always-run-within-a-fiber-try-wrapping-callbacks-that-you-pass-to-non-meteor-libraries-with-meteor-bindenvironmen/40099/8
const settle = () => {
if (ended && finished) {
// finalize() handles its own errors, but guard the floating promise so a
// stray rejection can never become an unhandled rejection (fatal to
// SyncedCron).
finalize().catch(error => {
console.error('[moveToStorage] finalize failed: ', error, fileObj._id);
if (!settled) {
settled = true;
}
resolve();
});
}
};
const fail = (error, label) => {
console.error(`[${label}]: `, error, fileObj._id);
// Stop streaming immediately and remove the partial output we already wrote,
// so a failed move never leaves a half-written destination file behind. Only
// the destination is removed (filesystem); the SOURCE is always left intact.
try { readStream.destroy(); } catch (e) { /* ignore */ }
try { writeStream.destroy(); } catch (e) { /* ignore */ }
if (strategyWrite.getStorageName() === STORAGE_NAME_FILESYSTEM) {
fs.promises.unlink(filePath).catch(() => {}); // remove partial destination
}
if (!settled) {
settled = true;
resolve();
}
};
writeStream.on('error', error => fail(error, 'writeStream error'));
readStream.on('error', error => fail(error, 'readStream error'));
writeStream.on('finish', finishedData => {
try {
strategyWrite.writeStreamFinished(finishedData);
} catch (error) {
// Persisting the storage id must never crash the process via this
// (synchronous) event callback; the move is finalized below regardless.
console.error('[writeStreamFinished error]: ', error, fileObj._id);
}
finished = true;
settle();
});
readStream.on('end', () => {
ended = true;
settle();
});
readStream.pipe(writeStream);
}));
return Promise.all(versionPromises);
};
// Add an attachment FROM A STREAM: the bytes go to a temp file as they arrive
// and the collection takes that file, so nothing is ever held whole in memory.
//
// This is what `copyFile` below does per version, lifted out so the .zip import
// can do it too (#1173): a 500 MB attachment inside an archive is piped from the
// archive to disk and handed over, never buffered. `Attachments.addFile` is the
// Meteor-Files call that takes a path rather than a Buffer - `write()` takes a
// Buffer, which is the thing to avoid here.
export const addAttachmentFromStream = function(
readStream, { fileName, type, meta, userId, size }, factory,
) {
return new Promise((resolve, reject) => {
const collection = (factory && factory.collection) || Attachments;
const storagePath = (factory && factory.storagePath) || '/tmp';
const safeName = sanitizeFilename(fileName || 'attachment');
const tempPath = path.join(storagePath, `${Random.id()}-import-${safeName}`);
const writeStream = fs.createWriteStream(tempPath);
const fail = error => {
try { readStream.destroy(); } catch (e) { /* already gone */ }
try { writeStream.destroy(); } catch (e) { /* already gone */ }
fs.promises.unlink(tempPath).catch(() => {});
reject(error);
};
readStream.on('error', fail);
writeStream.on('error', fail);
// Meteor-Files 3's addFile is async - addFile(path, opts, proceedAfterUpload)
// returning the file - and takes no callback. A callback passed as its third
// argument was never called, so every streamed attachment (a .zip import,
// a clone, the REST raw upload, a storage move) waited forever and the
// import stopped at the first file. copyFile below awaits it the same way.
writeStream.on('finish', () => {
Promise.resolve(collection.addFile(
tempPath,
{
fileName: fileName || 'attachment',
type: type || 'application/octet-stream',
meta,
userId,
size,
fileId: new ObjectId().toString(),
},
true,
)).then(resolve, fail);
});
readStream.pipe(writeStream);
});
};
// `fileIdFor(versionName)`, when given, names the copy: a durable rule copy
// (server/lib/syncRuleCopyCardCommand.js) derives it so a replay can find the
// copy it already made. Without it every copy gets a fresh id, as before.
export const copyFile = async function(fileObj, newCardId, fileStoreStrategyFactory, { fileIdFor = null } = {}) {
const newCard = await ReactiveCache.getCard(newCardId);
// Sanitize known exploits from the source content, and fix + sanitize the copied
// filename with the same general upload function (fold homoglyphs, strip
// invisible/exploit chars, correct the extension for the type, cap the length).
try {
const { sanitizeUploadedFileExploits } = require('./sanitizeUploadedFile');
await sanitizeUploadedFileExploits(fileObj);
} catch (e) { /* best effort */ }
let copyName = fileObj.name;
try {
const { sanitizeUploadFileName } = require('./uploadFileName');
copyName = sanitizeUploadFileName(fileObj.name, fileObj.type) || fileObj.name;
} catch (e) { /* keep original name on failure */ }
return Promise.all(Object.keys(fileObj.versions).map(versionName => new Promise((resolve, reject) => {
const strategyRead = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName);
const readStream = strategyRead.getReadStream();
const strategyWrite = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName, STORAGE_NAME_FILESYSTEM);
const safeName = sanitizeFilename(fileObj.name);
const tempPath = path.join(fileStoreStrategyFactory.storagePath, Random.id() + "-" + versionName + "-" + safeName);
const writeStream = strategyWrite.getWriteStream(tempPath);
// Source binary missing at its recorded location — skip this version instead
// of throwing "Cannot read properties of undefined (reading 'on')".
if (!readStream || !writeStream) {
console.error(
'[copyFile] cannot copy attachment',
fileObj._id,
`version "${versionName}": source file not found at its recorded location — skipping.`,
);
if (readStream) readStream.destroy();
if (writeStream) writeStream.destroy();
fs.promises.unlink(tempPath).catch(() => {});
resolve(null);
return;
}
// On any error, stop streaming and remove the partial temp file so a failed
// copy never leaves a half-written file behind (the source is untouched).
const cleanupPartial = (error, label) => {
console.error(`[${label}]: `, error, fileObj._id);
try { readStream.destroy(); } catch (e) { /* ignore */ }
try { writeStream.destroy(); } catch (e) { /* ignore */ }
fs.promises.unlink(tempPath).catch(() => {});
reject(error);
};
writeStream.on('error', error => cleanupPartial(error, 'writeStream error'));
readStream.on('error', error => cleanupPartial(error, 'readStream error'));
// The destination must be fully flushed before addFile reads it.
writeStream.on('finish', async () => {
try {
const collection = fileStoreStrategyFactory.collection || Attachments;
const fileRef = await collection.addFile(
tempPath,
{
fileName: copyName,
type: fileObj.type,
meta: {
boardId: newCard.boardId,
cardId: newCardId,
listId: newCard.listId,
swimlaneId: newCard.swimlaneId,
source: 'copy',
copyFrom: fileObj._id,
copyStorage: strategyRead.getStorageName(),
},
userId: fileObj.userId,
size: fileObj.size,
fileId: (fileIdFor && fileIdFor(versionName)) || new ObjectId().toString(),
},
true,
);
await collection.updateAsync({ _id: fileRef._id }, { $set: { userId: fileObj.userId } });
resolve(fileRef._id);
} catch (error) { cleanupPartial(error, 'copyFile addFile error'); }
});
readStream.pipe(writeStream);
})));
};
export const rename = async function(fileObj, newName, fileStoreStrategyFactory) {
// Sanitize the new name to prevent path traversal
const safeName = sanitizeFilename(newName);
const lastDot = safeName.lastIndexOf('.');
const extension = lastDot === -1 ? '' : safeName.substring(lastDot + 1).toLowerCase();
const extensionWithDot = extension ? `.${extension}` : '';
for (const versionName of Object.keys(fileObj.versions)) {
const strategy = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName);
const newFilePath = strategy.getNewPath(fileStoreStrategyFactory.storagePath, safeName);
strategy.rename(newFilePath);
// Upload hooks keep using this object and persist its versions again before
// validation. Keep it in sync with the renamed bytes and database record.
fileObj.versions[versionName].path = newFilePath;
const set = {
"name": safeName,
"extension": extension,
"extensionWithDot": extensionWithDot,
[`versions.${versionName}.path`]: newFilePath,
};
// Meteor-Files also records the original at the top level; the upload
// validator reads that path rather than versions.original.path.
if (versionName === 'original') {
fileObj.path = newFilePath;
set.path = newFilePath;
}
await (fileStoreStrategyFactory.collection || Attachments).updateAsync(
{ _id: fileObj._id }, { $set: set },
);
}
};