Several issues in the bulk move: - The GridFS source matcher required `meta.gridFsFileId` to be *absent*, but Meteor-Files always sets it, so selecting "MongoDB Meteor-Files" matched zero files and "nothing happened". The matcher now recognizes real GridFS files (`versions.*.storage === 'gridfs'` or a `meta.gridFsFileId` reference), and a "nothing to move" message is shown when a source is empty instead of silently doing nothing. - Moving files crashed the whole server with `FilesCollection#findOne() not available in server` — `ReactiveCache.getAttachment` used a synchronous `findOne()`; it now uses `findOneAsync()`. The background job is also hardened so a single failing file is skipped instead of crashing the server via an unhandled rejection. - The attachment **copy** API now honours the admin API upload limits (it previously skipped them). - **Fixed the "MongoDB Meteor-Files" file-count statistic** in Admin Panel / Attachments, which counted *every* attachment metadata document (so files on the Filesystem were wrongly reported as being in GridFS). It now counts only attachments actually stored in GridFS, consistent with the move tool. Also renamed the mislabeled "Mongo-Files" column to "Meteor-Files". Thanks to Claude. - **Unified attachment/avatar storage migration: move any → any.** Admin Panel / Attachments / Move Attachment can now move **Attachments, Avatars, or both**, from **any source to any destination** across Filesystem, Meteor-Files GridFS, Cloud (S3/Azure/GCS) and legacy CollectionFS GridFS. The default source is "All Read-enabled storages" (every backend whose Read flag is enabled and whose settings work), so everything can be consolidated into one destination in a single run. All metadata is preserved (board / swimlane / list / card / user / uploaded date / name / type / size), attachment cover references (`cards.coverId`) are remapped when an id changes, and the legacy source is deleted only after the new copy is verified. Thanks to Claude. - **Legacy CollectionFS GridFS is now a first-class storage backend** (read, migrate-from, and export-to) for both attachments and avatars, via the new `models/lib/collectionFsStore.js` (binary keyed by `copies.<coll>.key` in the `cfs_gridfs.<coll>` bucket, metadata in `cfs.<coll>.filerecord`). Thanks to Claude. - **The storage strategy layer is now collection-aware.** `moveToStorage` and the GridFS/Cloud strategies previously hard-coded the `Attachments` collection, so moving avatars to GridFS/cloud would have updated the wrong collection. The factory now carries its collection (`Attachments` or `Avatars`), so **avatars can be stored in Meteor-Files GridFS and cloud**, not only on the filesystem. Attachment behavior is unchanged. Thanks to Claude. - **Read legacy CollectionFS attachments and avatars in place** (without migrating). The backward-compatibility layer (`models/lib/attachmentBackwardCompatibility.js`) was broken — it looked up the GridFS binary by the filerecord `_id` and by filename instead of by `ObjectId(copies.<coll>.key)`, so legacy files were never found. It is fixed and generalized for attachments and avatars. Legacy attachments now appear in the card attachment gallery (new `legacyBoardAttachments` publication) and stream from the `cfs_gridfs.attachments` bucket, and legacy avatars (`/cfs/files/avatars/<id>` URLs) are served from the `cfs_gridfs.avatars` bucket instead of redirecting to a 404. Thanks to Claude.
198 lines
7.0 KiB
JavaScript
198 lines
7.0 KiB
JavaScript
import { ReactiveCache } from '/imports/reactiveCache';
|
|
import { Meteor } from 'meteor/meteor';
|
|
import { isFileValid } from './fileValidation';
|
|
import { isSvgFile, sanitizeSvgFileSync } from './lib/sanitizeSvg';
|
|
import { createBucket } from './lib/grid/createBucket';
|
|
import fs from 'fs';
|
|
import path from 'path';
|
|
import FileStoreStrategyFactory, { FileStoreStrategyFilesystem, FileStoreStrategyGridFs, FileStoreStrategyCloud, STORAGE_NAME_FILESYSTEM } from '/models/lib/fileStoreStrategy';
|
|
import { generateUniversalAvatarUrl } from '/models/lib/universalUrlGenerator';
|
|
import Avatars, { normalizeRemovedFiles, setAvatarsUploadSize } from './avatars';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Server-only configuration
|
|
// ---------------------------------------------------------------------------
|
|
|
|
let avatarsUploadExternalProgram;
|
|
let avatarsUploadMimeTypes = [];
|
|
let avatarsUploadSize = 72000;
|
|
|
|
if (process.env.AVATARS_UPLOAD_MIME_TYPES) {
|
|
avatarsUploadMimeTypes = process.env.AVATARS_UPLOAD_MIME_TYPES.split(',');
|
|
avatarsUploadMimeTypes = avatarsUploadMimeTypes.map(value => value.trim());
|
|
}
|
|
|
|
if (process.env.AVATARS_UPLOAD_MAX_SIZE) {
|
|
const avatarsUploadSize_ = parseInt(process.env.AVATARS_UPLOAD_MAX_SIZE);
|
|
|
|
if (typeof avatarsUploadSize_ === 'number' && avatarsUploadSize_ > 0) {
|
|
avatarsUploadSize = avatarsUploadSize_;
|
|
// Sync the upload size to the shared module for onBeforeUpload validation
|
|
setAvatarsUploadSize(avatarsUploadSize);
|
|
}
|
|
}
|
|
|
|
if (process.env.AVATARS_UPLOAD_EXTERNAL_PROGRAM) {
|
|
avatarsUploadExternalProgram = process.env.AVATARS_UPLOAD_EXTERNAL_PROGRAM;
|
|
|
|
if (!avatarsUploadExternalProgram.includes("{file}")) {
|
|
avatarsUploadExternalProgram = undefined;
|
|
}
|
|
}
|
|
|
|
const avatarsBucket = createBucket('avatars');
|
|
|
|
// Compute storage path:
|
|
// - Docker (WRITABLE_PATH=/data): /data/files/avatars
|
|
// - Snap (WRITABLE_PATH=$SNAP_COMMON/files): $SNAP_COMMON/files/avatars
|
|
const basePath = process.env.WRITABLE_PATH || process.cwd();
|
|
const endsWithFiles = basePath.endsWith('/files') || basePath.endsWith('\\files');
|
|
const storagePath = endsWithFiles
|
|
? path.join(basePath, 'avatars')
|
|
: path.join(basePath, 'files', 'avatars');
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// File store strategy factory
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export const fileStoreStrategyFactory = new FileStoreStrategyFactory(
|
|
FileStoreStrategyFilesystem, storagePath,
|
|
FileStoreStrategyGridFs, avatarsBucket,
|
|
FileStoreStrategyCloud,
|
|
Avatars,
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Assign server-only FilesCollection callbacks
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Avatars.storagePath = function () {
|
|
return fileStoreStrategyFactory.storagePath;
|
|
};
|
|
|
|
Avatars.onAfterUpload = async function (fileObj) {
|
|
// current storage is the filesystem, update object and database
|
|
Object.keys(fileObj.versions).forEach(versionName => {
|
|
fileObj.versions[versionName].storage = STORAGE_NAME_FILESYSTEM;
|
|
});
|
|
|
|
// Sanitize SVG avatars in place (strip scripts, event handlers, javascript:
|
|
// URIs and XML DOCTYPE/ENTITY constructs) before validation so safe SVGs are
|
|
// accepted instead of rejected.
|
|
if (isSvgFile(fileObj.type, fileObj.name)) {
|
|
Object.keys(fileObj.versions).forEach(versionName => {
|
|
const version = fileObj.versions[versionName];
|
|
const newSize = sanitizeSvgFileSync(version && version.path);
|
|
if (newSize !== null) {
|
|
version.size = newSize;
|
|
}
|
|
});
|
|
}
|
|
|
|
await Avatars.updateAsync({ _id: fileObj._id }, { $set: { "versions": fileObj.versions } });
|
|
|
|
const isValid = await isFileValid(fileObj, avatarsUploadMimeTypes, avatarsUploadSize, avatarsUploadExternalProgram);
|
|
|
|
if (isValid) {
|
|
// Set avatar URL using universal URL generator (URL-agnostic)
|
|
const universalUrl = generateUniversalAvatarUrl(fileObj._id);
|
|
|
|
// Check if this is an admin uploading for another user
|
|
let targetUserId = fileObj.userId;
|
|
if (fileObj.meta && fileObj.meta.adminUploadForUserId) {
|
|
// Verify the uploader is an admin
|
|
const uploader = await Meteor.users.findOneAsync(fileObj.userId);
|
|
if (uploader && uploader.isAdmin) {
|
|
targetUserId = fileObj.meta.adminUploadForUserId;
|
|
// Update the file to belong to the target user
|
|
await Avatars.updateAsync({ _id: fileObj._id }, { $set: { userId: targetUserId } });
|
|
}
|
|
}
|
|
|
|
const user = await ReactiveCache.getUser(targetUserId);
|
|
user.setAvatarUrl(universalUrl);
|
|
} else {
|
|
await Avatars.removeAsync(fileObj._id);
|
|
}
|
|
};
|
|
|
|
Avatars.interceptDownload = function (http, fileObj, versionName) {
|
|
const ret = fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).interceptDownload(http, this.cacheControl);
|
|
return ret;
|
|
};
|
|
|
|
Avatars.onBeforeRemove = async function (filesInput) {
|
|
let files;
|
|
try {
|
|
files = normalizeRemovedFiles(filesInput);
|
|
// If normalizeRemovedFiles returns a Promise (from async fetch), await it
|
|
if (files && typeof files.then === 'function') {
|
|
console.warn('normalizeRemovedFiles returned a Promise, awaiting it');
|
|
files = await files;
|
|
}
|
|
} catch (e) {
|
|
console.error('Error normalizing removed files:', e, 'filesInput:', filesInput);
|
|
files = [];
|
|
}
|
|
|
|
// Ensure files is an array
|
|
if (!Array.isArray(files)) {
|
|
console.error('normalizeRemovedFiles did not return an array, got:', typeof files);
|
|
files = [];
|
|
}
|
|
|
|
for (const fileObj of files) {
|
|
if (fileObj && fileObj.userId) {
|
|
const user = await ReactiveCache.getUser(fileObj.userId);
|
|
if (user) {
|
|
await user.setAvatarUrl('');
|
|
}
|
|
}
|
|
}
|
|
|
|
return true;
|
|
};
|
|
|
|
Avatars.onAfterRemove = async function (filesInput) {
|
|
let files;
|
|
try {
|
|
files = normalizeRemovedFiles(filesInput);
|
|
// If normalizeRemovedFiles returns a Promise (from async fetch), await it
|
|
if (files && typeof files.then === 'function') {
|
|
console.warn('normalizeRemovedFiles returned a Promise in onAfterRemove, awaiting it');
|
|
files = await files;
|
|
}
|
|
} catch (e) {
|
|
console.error('Error normalizing removed files in onAfterRemove:', e, 'filesInput:', filesInput);
|
|
files = [];
|
|
}
|
|
|
|
// Ensure files is an array
|
|
if (!Array.isArray(files)) {
|
|
console.error('normalizeRemovedFiles did not return an array in onAfterRemove, got:', typeof files);
|
|
files = [];
|
|
}
|
|
|
|
files.forEach(fileObj => {
|
|
if (!fileObj || !fileObj.versions) {
|
|
return;
|
|
}
|
|
|
|
Object.keys(fileObj.versions).forEach(versionName => {
|
|
fileStoreStrategyFactory.getFileStrategy(fileObj, versionName).onAfterRemove();
|
|
});
|
|
});
|
|
};
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Startup
|
|
// ---------------------------------------------------------------------------
|
|
|
|
Meteor.startup(() => {
|
|
const sp = fileStoreStrategyFactory.storagePath;
|
|
if (!fs.existsSync(sp)) {
|
|
console.log("create storagePath because it doesn't exist: " + sp);
|
|
fs.mkdirSync(sp, { recursive: true });
|
|
}
|
|
});
|