diff --git a/client/components/settings/attachments.css b/client/components/settings/attachments.css index d8b056333..bea618c03 100644 --- a/client/components/settings/attachments.css +++ b/client/components/settings/attachments.css @@ -204,3 +204,39 @@ white-space: nowrap; min-width: 0 !important; } + +/* Cloud storage (S3 / Azure / GCS): example value shown below each input. */ +.attachments-content .cloud-input-example { + display: block; + color: #888; + font-size: 0.82em; + font-family: monospace; + word-break: break-all; + line-height: 1.35; +} + +/* Cloud storage: where-to-find-this-value path shown below the example. */ +.attachments-content .cloud-input-path { + display: block; + color: #6a737d; + font-size: 0.82em; + line-height: 1.4; + margin-top: 2px; +} + +/* Cloud storage: translated label shown under the (English) label. */ +.attachments-content .cloud-input-label-tr { + display: block; + color: #57606a; + font-size: 0.9em; + margin-bottom: 4px; +} + +/* Cloud storage: description text (English literal and translated). */ +.attachments-content .cloud-input-desc { + display: block; + color: #6a737d; + font-size: 0.82em; + line-height: 1.4; + margin-top: 2px; +} diff --git a/client/components/settings/attachments.jade b/client/components/settings/attachments.jade index 49b97c375..d0146af63 100644 --- a/client/components/settings/attachments.jade +++ b/client/components/settings/attachments.jade @@ -228,37 +228,6 @@ template(name="attachments") input#s3-read(type="checkbox" checked=cloudRead.s3) label(for="s3-read") {{_ 'storage-read'}} - .form-group - label {{_ 's3-endpoint'}} - input.wekan-form-control#s3-endpoint(type="text" value=cloudValue.s3.endpoint) - small.form-text.text-muted {{_ 's3-endpoint-description'}} - .form-group - label {{_ 's3-region'}} - input.wekan-form-control#s3-region(type="text" value=cloudValue.s3.region) - .form-group - label {{_ 's3-bucket'}} - input.wekan-form-control#s3-bucket(type="text" value=cloudValue.s3.bucket) - .form-group - label {{_ 's3-access-key'}} - input.wekan-form-control#s3-access-key(type="text" value=cloudValue.s3.accessKeyId) - .form-group - label {{_ 's3-secret-key'}} - input.wekan-form-control#s3-secret-key(type="password" placeholder=cloudSecretPlaceholder.s3.secretAccessKey) - small.form-text.text-muted {{_ 'cloud-secret-keep-blank'}} - .form-group - .storage-rw-item - input#s3-force-path-style(type="checkbox" checked=cloudValue.s3.forcePathStyle) - label(for="s3-force-path-style") {{_ 's3-force-path-style'}} - small.form-text.text-muted {{_ 's3-force-path-style-description'}} - - .form-group - button.js-test-cloud-connection.btn.btn-secondary(data-provider="s3") {{_ 'test-cloud-connection'}} - button.js-save-cloud-settings.btn.btn-primary(data-provider="s3") {{_ 'save'}} - if cloudTestResult.s3 - p.form-text.text-success {{_ 'cloud-connection-success'}} - if cloudTestError.s3 - p.form-text.text-danger {{_ 'cloud-connection-failed'}}: {{cloudTestError.s3}} - .form-group button.js-calculate-s3-stats.btn.btn-primary(type="button") {{_ 'calculate-file-counts'}} if s3StatsLoading @@ -277,6 +246,65 @@ template(name="attachments") td {{_ 'avatars'}} td {{s3Stats.avatars}} + .form-group + label Endpoint + small.form-text.cloud-input-label-tr {{_ 's3-endpoint'}} + input.wekan-form-control#s3-endpoint(type="text" value=cloudValue.s3.endpoint) + small.form-text.cloud-input-example https://s3.eu-west-1.amazonaws.com + small.form-text.cloud-input-desc S3 endpoint URL (e.g., s3.amazonaws.com or minio.example.com) + small.form-text.cloud-input-desc {{_ 's3-endpoint-description'}} + small.form-text.cloud-input-path AWS: leave blank (uses the region endpoint). S3-compatible: the Endpoint URL from your MinIO / Cloudflare R2 / Backblaze B2 / Wasabi / DigitalOcean Spaces console. + small.form-text.cloud-input-path {{_ 's3-endpoint-menu-path'}} + .form-group + label AWS Region + small.form-text.cloud-input-label-tr {{_ 's3-region'}} + input.wekan-form-control#s3-region(type="text" value=cloudValue.s3.region) + small.form-text.cloud-input-example eu-west-1 + small.form-text.cloud-input-desc AWS S3 region (e.g., us-east-1) + small.form-text.cloud-input-desc {{_ 's3-region-description'}} + small.form-text.cloud-input-path AWS Console → S3 → Buckets → your bucket → Properties → AWS Region. + small.form-text.cloud-input-path {{_ 's3-region-menu-path'}} + .form-group + label Bucket name + small.form-text.cloud-input-label-tr {{_ 's3-bucket'}} + input.wekan-form-control#s3-bucket(type="text" value=cloudValue.s3.bucket) + small.form-text.cloud-input-example wekan-s3-test-369341155299-eu-west-1-an + small.form-text.cloud-input-desc S3 bucket name for storing files + small.form-text.cloud-input-desc {{_ 's3-bucket-description'}} + small.form-text.cloud-input-path AWS Console → S3 → Buckets → your bucket name (or Create bucket → Bucket name). + small.form-text.cloud-input-path {{_ 's3-bucket-menu-path'}} + .form-group + label Access key ID + small.form-text.cloud-input-label-tr {{_ 's3-access-key'}} + input.wekan-form-control#s3-access-key(type="text" value=cloudValue.s3.accessKeyId) + small.form-text.cloud-input-example AKIAIOSFODNN7EXAMPLE + small.form-text.cloud-input-desc AWS S3 access key for authentication + small.form-text.cloud-input-desc {{_ 's3-access-key-description'}} + small.form-text.cloud-input-path AWS Console → IAM → Users → your user → Security credentials → Access keys → Create access key → choose use case 'Application running outside AWS' → Create access key. Shown as 'Access key ID'. + small.form-text.cloud-input-path {{_ 's3-access-key-menu-path'}} + .form-group + label Secret access key + small.form-text.cloud-input-label-tr {{_ 's3-secret-key'}} + input.wekan-form-control#s3-secret-key(type="password" placeholder=cloudSecretPlaceholder.s3.secretAccessKey) + small.form-text.cloud-input-example wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY + small.form-text.cloud-input-desc Leave blank to keep the current value. + small.form-text.cloud-input-desc {{_ 'cloud-secret-keep-blank'}} + small.form-text.cloud-input-path Same screen as the Access key ID; 'Secret access key' is shown only once when the key is created — copy it then (or Download .csv). + small.form-text.cloud-input-path {{_ 's3-secret-key-menu-path'}} + .form-group + .storage-rw-item + input#s3-force-path-style(type="checkbox" checked=cloudValue.s3.forcePathStyle) + label(for="s3-force-path-style") {{_ 's3-force-path-style'}} + small.form-text.text-muted {{_ 's3-force-path-style-description'}} + + .form-group + button.js-test-cloud-connection.btn.btn-secondary(data-provider="s3") {{_ 'test-cloud-connection'}} + button.js-save-cloud-settings.btn.btn-primary(data-provider="s3") {{_ 'save'}} + if cloudTestResult.s3 + p.form-text.text-success {{_ 'cloud-connection-success'}} + if cloudTestError.s3 + p.form-text.text-danger {{_ 'cloud-connection-failed'}}: {{cloudTestError.s3}} + else if isAzureActive ul#attachment-azure-setting.setting-detail li @@ -290,19 +318,59 @@ template(name="attachments") label(for="azure-read") {{_ 'storage-read'}} .form-group - label {{_ 'azure-account-name'}} + button.js-calculate-azure-stats.btn.btn-primary(type="button") {{_ 'calculate-file-counts'}} + if azureStatsLoading + p.form-text.text-muted {{_ 'calculating-counts'}} + if azureStatsError + p.form-text.text-danger {{azureStatsError}} + if hasAzureStats + table.gridfs-stats-table + tr + th {{_ 'stats-scope'}} + th {{_ 'stats-count'}} + tr + td {{_ 'attachments'}} + td {{azureStats.attachments}} + tr + td {{_ 'avatars'}} + td {{azureStats.avatars}} + + .form-group + label Storage account name + small.form-text.cloud-input-label-tr {{_ 'azure-account-name'}} input.wekan-form-control#azure-account-name(type="text" value=cloudValue.azure.accountName) + small.form-text.cloud-input-example wekanstorage + small.form-text.cloud-input-desc The name of your Azure Storage account. + small.form-text.cloud-input-desc {{_ 'azure-account-name-description'}} + small.form-text.cloud-input-path Azure Portal → Storage accounts → your account → name shown at the top (also under Security + networking → Access keys → Storage account name). + small.form-text.cloud-input-path {{_ 'azure-account-name-menu-path'}} .form-group - label {{_ 'azure-account-key'}} + label Storage account key + small.form-text.cloud-input-label-tr {{_ 'azure-account-key'}} input.wekan-form-control#azure-account-key(type="password" placeholder=cloudSecretPlaceholder.azure.accountKey) - small.form-text.text-muted {{_ 'cloud-secret-keep-blank'}} + small.form-text.cloud-input-example base64AccountKey== + small.form-text.cloud-input-desc Leave blank to keep the current value. + small.form-text.cloud-input-desc {{_ 'cloud-secret-keep-blank'}} + small.form-text.cloud-input-path Azure Portal → Storage accounts → your account → Security + networking → Access keys → key1 → Show → Key. + small.form-text.cloud-input-path {{_ 'azure-account-key-menu-path'}} .form-group - label {{_ 'azure-connection-string'}} + label Connection string + small.form-text.cloud-input-label-tr {{_ 'azure-connection-string'}} input.wekan-form-control#azure-connection-string(type="password" placeholder=cloudSecretPlaceholder.azure.connectionString) - small.form-text.text-muted {{_ 'azure-connection-string-description'}} + small.form-text.cloud-input-example DefaultEndpointsProtocol=https;AccountName=wekanstorage;AccountKey=...;EndpointSuffix=core.windows.net + small.form-text.cloud-input-desc Optional: use a full connection string instead of account name and key. + small.form-text.cloud-input-desc {{_ 'azure-connection-string-description'}} + small.form-text.cloud-input-path Azure Portal → Storage accounts → your account → Security + networking → Access keys → key1 → Connection string → Show. + small.form-text.cloud-input-path {{_ 'azure-connection-string-menu-path'}} .form-group - label {{_ 'azure-container'}} + label Container name + small.form-text.cloud-input-label-tr {{_ 'azure-container'}} input.wekan-form-control#azure-bucket(type="text" value=cloudValue.azure.bucket) + small.form-text.cloud-input-example wekan-attachments + small.form-text.cloud-input-desc The blob container to store files in. + small.form-text.cloud-input-desc {{_ 'azure-container-description'}} + small.form-text.cloud-input-path Azure Portal → Storage accounts → your account → Data storage → Containers → your container (or + Container to create one). + small.form-text.cloud-input-path {{_ 'azure-container-menu-path'}} .form-group button.js-test-cloud-connection.btn.btn-secondary(data-provider="azure") {{_ 'test-cloud-connection'}} @@ -325,18 +393,59 @@ template(name="attachments") label(for="gcs-read") {{_ 'storage-read'}} .form-group - label {{_ 'gcs-project-id'}} + button.js-calculate-gcs-stats.btn.btn-primary(type="button") {{_ 'calculate-file-counts'}} + if gcsStatsLoading + p.form-text.text-muted {{_ 'calculating-counts'}} + if gcsStatsError + p.form-text.text-danger {{gcsStatsError}} + if hasGcsStats + table.gridfs-stats-table + tr + th {{_ 'stats-scope'}} + th {{_ 'stats-count'}} + tr + td {{_ 'attachments'}} + td {{gcsStats.attachments}} + tr + td {{_ 'avatars'}} + td {{gcsStats.avatars}} + + .form-group + label Project ID + small.form-text.cloud-input-label-tr {{_ 'gcs-project-id'}} input.wekan-form-control#gcs-project-id(type="text" value=cloudValue.gcs.projectId) + small.form-text.cloud-input-example my-gcp-project-123456 + small.form-text.cloud-input-desc Your Google Cloud project id. + small.form-text.cloud-input-desc {{_ 'gcs-project-id-description'}} + small.form-text.cloud-input-path Google Cloud Console → project selector at the top → your project's ID (Cloud overview → Dashboard → Project info → Project ID). + small.form-text.cloud-input-path {{_ 'gcs-project-id-menu-path'}} .form-group - label {{_ 'gcs-bucket'}} + label Bucket name + small.form-text.cloud-input-label-tr {{_ 'gcs-bucket'}} input.wekan-form-control#gcs-bucket(type="text" value=cloudValue.gcs.bucket) + small.form-text.cloud-input-example wekan-attachments + small.form-text.cloud-input-desc The name of the Cloud Storage bucket to store files in. + small.form-text.cloud-input-desc {{_ 'gcs-bucket-description'}} + small.form-text.cloud-input-path Google Cloud Console → Cloud Storage → Buckets → your bucket name (or Create to make one). + small.form-text.cloud-input-path {{_ 'gcs-bucket-menu-path'}} .form-group - label {{_ 'gcs-key-filename'}} + label Service account key file path + small.form-text.cloud-input-label-tr {{_ 'gcs-key-filename'}} input.wekan-form-control#gcs-key-filename(type="text" value=cloudValue.gcs.keyFilename) - small.form-text.text-muted {{_ 'gcs-key-filename-description'}} + small.form-text.cloud-input-example /data/gcs-service-account.json + small.form-text.cloud-input-desc Optional: path to a service account JSON key file on the server. + small.form-text.cloud-input-desc {{_ 'gcs-key-filename-description'}} + small.form-text.cloud-input-path Path on the WeKan server to the downloaded service account JSON key file (created below). Use this OR paste the JSON into the field below. + small.form-text.cloud-input-path {{_ 'gcs-key-filename-menu-path'}} .form-group - label {{_ 'gcs-credentials'}} + label Service account key (JSON) + small.form-text.cloud-input-label-tr {{_ 'gcs-credentials'}} textarea.wekan-form-control#gcs-credentials(rows="4" placeholder=cloudSecretPlaceholder.gcs.credentials) + small.form-text.cloud-input-example { "type": "service_account", "project_id": "my-gcp-project", "client_email": "wekan@my-gcp-project.iam.gserviceaccount.com", "private_key": "-----BEGIN PRIVATE KEY-----..." } + small.form-text.cloud-input-desc Optional: paste the service account JSON key. Leave blank to use the key file or default credentials. + small.form-text.cloud-input-desc {{_ 'gcs-credentials-description'}} + small.form-text.cloud-input-path Google Cloud Console → IAM & Admin → Service accounts → your service account → Keys → Add key → Create new key → JSON → Create (downloads the file). Paste its contents here, or save the file and set its path above. + small.form-text.cloud-input-path {{_ 'gcs-credentials-menu-path'}} small.form-text.text-muted {{_ 'gcs-credentials-description'}} .form-group @@ -377,6 +486,39 @@ template(name="moveAttachments") option(value="collectionfs") {{_ 'move-storage-collectionfs'}} button.js-move-all-attachments.btn.btn-primary {{_ 'move-all-attachments'}} + if lastMove + .move-last-operation + span.emoji-icon + i.fa.fa-info-circle + | {{_ 'attachment-last-move'}}: {{lastMoveText}} + + .move-repair-section + h4 {{_ 'attachment-repair-locations'}} + .description {{_ 'attachment-repair-locations-description'}} + button.js-repair-attachment-locations.btn.btn-secondary(type="button") {{_ 'attachment-repair-locations'}} + if repairLoading + p.form-text.text-muted {{_ 'attachment-repair-running'}} + if repairError + p.form-text.text-danger {{repairError}} + if repairResult + p.form-text.text-success {{_ 'attachment-repair-done'}} + table.gridfs-stats-table + tr + th {{_ 'stats-scope'}} + th {{_ 'attachment-repair-scanned'}} + th {{_ 'attachment-repair-repaired'}} + th {{_ 'attachment-repair-broken'}} + tr + td {{_ 'attachments'}} + td {{repairResult.attachments.scanned}} + td {{repairResult.attachments.repaired}} + td {{repairResult.attachments.broken}} + tr + td {{_ 'avatars'}} + td {{repairResult.avatars.scanned}} + td {{repairResult.avatars.repaired}} + td {{repairResult.avatars.broken}} + if moveProgress .move-progress-section .move-progress-header diff --git a/client/components/settings/attachments.js b/client/components/settings/attachments.js index 8981c7e61..ef025c23f 100644 --- a/client/components/settings/attachments.js +++ b/client/components/settings/attachments.js @@ -250,6 +250,12 @@ Template.attachments.onCreated(function () { this.s3Stats = new ReactiveVar(null); this.s3StatsLoading = new ReactiveVar(false); this.s3StatsError = new ReactiveVar(''); + this.azureStats = new ReactiveVar(null); + this.azureStatsLoading = new ReactiveVar(false); + this.azureStatsError = new ReactiveVar(''); + this.gcsStats = new ReactiveVar(null); + this.gcsStatsLoading = new ReactiveVar(false); + this.gcsStatsError = new ReactiveVar(''); this.compactLoading = new ReactiveVar(false); this.compactResult = new ReactiveVar(null); this.compactError = new ReactiveVar(''); @@ -467,6 +473,30 @@ Template.attachments.helpers({ s3StatsError() { return Template.instance().s3StatsError.get(); }, + hasAzureStats() { + return !!Template.instance().azureStats.get(); + }, + azureStats() { + return Template.instance().azureStats.get(); + }, + azureStatsLoading() { + return Template.instance().azureStatsLoading.get(); + }, + azureStatsError() { + return Template.instance().azureStatsError.get(); + }, + hasGcsStats() { + return !!Template.instance().gcsStats.get(); + }, + gcsStats() { + return Template.instance().gcsStats.get(); + }, + gcsStatsLoading() { + return Template.instance().gcsStatsLoading.get(); + }, + gcsStatsError() { + return Template.instance().gcsStatsError.get(); + }, s3Enabled() { return process.env.S3_ENABLED === 'true'; }, @@ -720,6 +750,34 @@ Template.attachments.events({ tpl.s3Stats.set(result || null); }); }, + 'click button.js-calculate-azure-stats'(event, tpl) { + event.preventDefault(); + tpl.azureStatsLoading.set(true); + tpl.azureStatsError.set(''); + Meteor.call('getAzureStorageStats', (error, result) => { + tpl.azureStatsLoading.set(false); + if (error) { + tpl.azureStats.set(null); + tpl.azureStatsError.set(error.reason || error.message || 'Failed to calculate counts'); + return; + } + tpl.azureStats.set(result || null); + }); + }, + 'click button.js-calculate-gcs-stats'(event, tpl) { + event.preventDefault(); + tpl.gcsStatsLoading.set(true); + tpl.gcsStatsError.set(''); + Meteor.call('getGcsStorageStats', (error, result) => { + tpl.gcsStatsLoading.set(false); + if (error) { + tpl.gcsStats.set(null); + tpl.gcsStatsError.set(error.reason || error.message || 'Failed to calculate counts'); + return; + } + tpl.gcsStats.set(result || null); + }); + }, 'click button.js-compact-mongodb-gridfs'(event, tpl) { event.preventDefault(); runCompact(tpl.compactLoading, tpl.compactResult, tpl.compactError); @@ -731,6 +789,9 @@ Template.moveAttachments.onCreated(function () { // persisted progress so it keeps running (and stays visible) even if the // admin navigates away from or closes this page. this.bulkMoveSubscription = Meteor.subscribe('attachmentBulkMoveStatus'); + this.repairLoading = new ReactiveVar(false); + this.repairResult = new ReactiveVar(null); + this.repairError = new ReactiveVar(''); }); function getBulkMoveProgress() { @@ -738,6 +799,31 @@ function getBulkMoveProgress() { return doc && doc.running ? doc : null; } +function getLastMove() { + const doc = AttachmentBulkMoveStatus.findOne('bulk'); + // Don't show the "last move" line while a move is actively running. + if (!doc || doc.running || !doc.lastMove) return null; + return doc.lastMove; +} + +function storageLabel(value) { + return value ? TAPi18n.__(`move-storage-${value}`) : ''; +} + +function scopeLabel(value) { + return value ? TAPi18n.__(`move-scope-${value}`) : ''; +} + +// Format a Date (or value) as YYYY-MM-DD HH:MM:SS in local time. +function formatDateTime(value) { + if (!value) return ''; + const date = value instanceof Date ? value : new Date(value); + if (isNaN(date.getTime())) return ''; + const p = n => String(n).padStart(2, '0'); + return `${date.getFullYear()}-${p(date.getMonth() + 1)}-${p(date.getDate())} ` + + `${p(date.getHours())}:${p(date.getMinutes())}:${p(date.getSeconds())}`; +} + Template.moveAttachments.helpers({ moveProgress() { return getBulkMoveProgress(); @@ -754,6 +840,25 @@ Template.moveAttachments.helpers({ const p = getBulkMoveProgress(); return p?.size ? filesize(p.size) : ''; }, + lastMove() { + return getLastMove(); + }, + lastMoveText() { + const lm = getLastMove(); + if (!lm) return ''; + const cancelled = lm.cancelled ? ` (${TAPi18n.__('move-progress-cancel')})` : ''; + return `${storageLabel(lm.source)} → ${storageLabel(lm.dest)} ` + + `(${scopeLabel(lm.scope)}) ${formatDateTime(lm.at)}${cancelled}`; + }, + repairLoading() { + return Template.instance().repairLoading.get(); + }, + repairResult() { + return Template.instance().repairResult.get(); + }, + repairError() { + return Template.instance().repairError.get(); + }, }); function runCompact(loadingVar, resultVar, errorVar) { @@ -795,6 +900,20 @@ Template.moveAttachments.events({ } }); }, + 'click button.js-repair-attachment-locations'(event, tpl) { + if (tpl.repairLoading.get()) return; + tpl.repairLoading.set(true); + tpl.repairResult.set(null); + tpl.repairError.set(''); + Meteor.call('repairAttachmentStorageLocations', (error, result) => { + tpl.repairLoading.set(false); + if (error) { + tpl.repairError.set(error.reason || error.message || 'Repair failed'); + return; + } + tpl.repairResult.set(result || {}); + }); + }, 'click button.js-pause-move'() { Meteor.call('pauseBulkAttachmentMove'); }, diff --git a/models/lib/fileStoreStrategy.js b/models/lib/fileStoreStrategy.js index e61c2deea..c0a36fef8 100644 --- a/models/lib/fileStoreStrategy.js +++ b/models/lib/fileStoreStrategy.js @@ -308,6 +308,10 @@ export class FileStoreStrategyGridFs extends FileStoreStrategy { contentType: fileObj.type || 'binary/octet-stream', metadata, }); + // Keep a reference so writeStreamFinished can read the uploaded file's id + // from the stream — the mongodb driver's 'finish' event no longer passes the + // stored file document. + this.gridFsWriteStream = ret; return ret; } @@ -316,9 +320,29 @@ export class FileStoreStrategyGridFs extends FileStoreStrategy { */ writeStreamFinished(finishedData) { const gridFsFileIdName = this.getGridFsFileIdName(); + // Older mongodb drivers passed the stored file document to the 'finish' + // event; current drivers emit it with no argument (so finishedData is + // undefined and `finishedData._id` threw, crashing the server). Resolve the + // GridFS file id from the finish data when present, otherwise from the write + // stream itself (`id` is assigned by openUploadStream; `gridFSFile._id` is + // set once the upload finishes). + const stream = this.gridFsWriteStream; + const gridFsId = + (finishedData && finishedData._id) || + (stream && stream.gridFSFile && stream.gridFSFile._id) || + (stream && stream.id); + if (!gridFsId) { + console.error( + 'GridFS write finished but no file id was available for', + this.fileObj && this.fileObj._id, + ); + return; + } + const hexId = + typeof gridFsId.toHexString === 'function' ? gridFsId.toHexString() : String(gridFsId); this.collection.updateAsync( { _id: this.fileObj._id }, - { $set: { [gridFsFileIdName]: finishedData._id.toHexString() } }, + { $set: { [gridFsFileIdName]: hexId } }, ).catch(error => { console.error('Failed to persist GridFS file id:', error); }); @@ -623,32 +647,52 @@ export class FileStoreStrategyCloud extends FileStoreStrategy { getWriteStream(filePath) { const pass = new PassThrough(); this._key = (typeof filePath === 'string' && filePath) ? filePath : this.getObjectKey(); + this._uploadError = null; const adapter = getCloudAdapter(this.provider); if (!adapter) { - this._uploadPromise = Promise.reject(new Error(`Cloud storage "${this.provider}" is not configured`)); - // Swallow here so the rejection is observed; the error is surfaced via the stream. - this._uploadPromise.catch(() => {}); - process.nextTick(() => pass.destroy(new Error(`Cloud storage "${this.provider}" is not configured`))); + this._uploadError = new Error(`Cloud storage "${this.provider}" is not configured`); + this._uploadPromise = Promise.resolve(); + process.nextTick(() => pass.destroy(this._uploadError)); return pass; } - this._uploadPromise = adapter.storage.addFileFromStream({ - stream: pass, - bucketName: adapter.bucketName, - targetPath: this._key, - }).then(result => { - if (result && result.error) { - throw new Error(result.error); - } - return result; - }); - // Surface upload failures onto the stream so piping callers see the error, - // and avoid unhandled rejection warnings. - this._uploadPromise.catch(error => { - if (!pass.destroyed) { - pass.destroy(error); - } + // Buffer the incoming bytes and upload them as a complete buffer instead of + // streaming a live body to the backend. Streaming an S3 PutObject body whose + // length is unknown fails with + // Invalid value "undefined" for header "x-amz-decoded-content-length" + // and, if the socket drops mid-upload, the AWS SDK's body-stream promise + // rejects UNHANDLED and crashes the server (SyncedCron treats it as fatal). + // A buffer has a known length and no socket-bound stream, so neither + // happens. The bulk move processes one file at a time, so peak memory is one + // file. The upload promise NEVER rejects — any failure is captured in + // this._uploadError (read by waitUntilStored()). + const chunks = []; + this._uploadPromise = new Promise(resolve => { + pass.on('data', chunk => chunks.push(chunk)); + pass.on('error', error => { + if (!this._uploadError) { + this._uploadError = error instanceof Error ? error : new Error(String(error)); + } + resolve(); + }); + pass.on('end', () => { + Promise.resolve() + .then(() => adapter.storage.addFileFromBuffer({ + buffer: Buffer.concat(chunks), + bucketName: adapter.bucketName, + targetPath: this._key, + })) + .then(result => { + if (result && result.error) { + this._uploadError = new Error(result.error); + } + }) + .catch(error => { + this._uploadError = error instanceof Error ? error : new Error(String(error)); + }) + .then(resolve); + }); }); return pass; } @@ -657,7 +701,14 @@ export class FileStoreStrategyCloud extends FileStoreStrategy { * moveToStorage() awaits this before deleting the source file. */ waitUntilStored() { - return this._uploadPromise || Promise.resolve(); + // Wait for the (non-rejecting) upload promise, then re-throw any captured + // error so moveToStorage's try/catch keeps the source file intact instead of + // deleting it after a failed upload. + return Promise.resolve(this._uploadPromise).then(() => { + if (this._uploadError) { + throw this._uploadError; + } + }); } /** writing finished — persist the object key in the version meta */ @@ -793,6 +844,27 @@ export const moveToStorage = function(fileObj, storageDestination, fileStoreStra const filePath = strategyWrite.getNewPath(fileStoreStrategyFactory.storagePath); const writeStream = strategyWrite.getWriteStream(filePath); + // The source binary may be missing at its recorded location (e.g. a file left + // half-moved by an earlier interrupted run: storage says "gridfs" but the + // GridFS id reference is gone, or a filesystem path that no longer exists). In + // that case getReadStream()/getWriteStream() return undefined. Skip this + // version cleanly instead of throwing "Cannot read properties of undefined + // (reading 'on')", and crucially do NOT delete the source — leave it intact so + // no data is lost and the admin can investigate. + if (!readStream || !writeStream) { + console.error( + '[moveToStorage] cannot move attachment', + fileObj._id, + `version "${versionName}" from ${strategyRead.getStorageName()} to ${strategyWrite.getStorageName()}:`, + !readStream + ? 'source file not found at its recorded location' + : 'could not open the destination write stream', + '— skipping this file, source left intact.', + ); + resolve(); + return; + } + let ended = false; let finished = false; let settled = false; @@ -823,7 +895,16 @@ export const moveToStorage = function(fileObj, storageDestination, fileStoreStra // https://forums.meteor.com/t/meteor-code-must-always-run-within-a-fiber-try-wrapping-callbacks-that-you-pass-to-non-meteor-libraries-with-meteor-bindenvironmen/40099/8 const settle = () => { if (ended && finished) { - finalize(); + // finalize() handles its own errors, but guard the floating promise so a + // stray rejection can never become an unhandled rejection (fatal to + // SyncedCron). + finalize().catch(error => { + console.error('[moveToStorage] finalize failed: ', error, fileObj._id); + if (!settled) { + settled = true; + } + resolve(); + }); } }; @@ -839,7 +920,13 @@ export const moveToStorage = function(fileObj, storageDestination, fileStoreStra readStream.on('error', error => fail(error, 'readStream error')); writeStream.on('finish', finishedData => { - strategyWrite.writeStreamFinished(finishedData); + try { + strategyWrite.writeStreamFinished(finishedData); + } catch (error) { + // Persisting the storage id must never crash the process via this + // (synchronous) event callback; the move is finalized below regardless. + console.error('[writeStreamFinished error]: ', error, fileObj._id); + } finished = true; settle(); }); @@ -866,6 +953,17 @@ export const copyFile = async function(fileObj, newCardId, fileStoreStrategyFact const tempPath = path.join(fileStoreStrategyFactory.storagePath, Random.id() + "-" + versionName + "-" + safeName); const writeStream = strategyWrite.getWriteStream(tempPath); + // Source binary missing at its recorded location — skip this version instead + // of throwing "Cannot read properties of undefined (reading 'on')". + if (!readStream || !writeStream) { + console.error( + '[copyFile] cannot copy attachment', + fileObj._id, + `version "${versionName}": source file not found at its recorded location — skipping.`, + ); + return; + } + writeStream.on('error', error => { console.error('[writeStream error]: ', error, fileObj._id); }); diff --git a/models/lib/grid/createObjectId.js b/models/lib/grid/createObjectId.js index 2e6703008..e178f8433 100644 --- a/models/lib/grid/createObjectId.js +++ b/models/lib/grid/createObjectId.js @@ -1,4 +1,5 @@ import { MongoInternals } from 'meteor/mongo'; +// Use ObjectId (ObjectID is a deprecated alias in the bundled mongodb driver). export const createObjectId = ({ gridFsFileId }) => - new MongoInternals.NpmModule.ObjectID(gridFsFileId); + new MongoInternals.NpmModule.ObjectId(gridFsFileId); diff --git a/server/attachmentBulkMove.js b/server/attachmentBulkMove.js index 3e8ffa571..3d6b05a10 100644 --- a/server/attachmentBulkMove.js +++ b/server/attachmentBulkMove.js @@ -79,6 +79,28 @@ async function setStatus(fields) { ); } +// If the server restarted (e.g. crashed) while a bulk move was running, the +// persisted status is left as `running` forever and the admin UI stays stuck on +// the last processed file (e.g. "File 1 / 4"). The in-memory controller always +// starts fresh, so reconcile a stale running status on startup by marking it +// interrupted, freeing the UI and allowing a new move to be started. +Meteor.startup(async () => { + try { + const status = await AttachmentBulkMoveStatus.findOneAsync({ _id: STATUS_ID }); + if (status && status.running) { + await setStatus({ + running: false, + paused: false, + cancelled: false, + interrupted: true, + finishedAt: new Date(), + }); + } + } catch (error) { + console.error('[attachmentBulkMove] Failed to reconcile stale status on startup:', error); + } +}); + async function requireAdmin(userId) { if (!userId) { throw new Meteor.Error('not-authorized', 'Must be logged in'); @@ -200,7 +222,13 @@ async function createMeteorFilesDocFromBuffer(cfg, info) { size, type, name, - meta: { ...(info.meta || {}), source: 'storage-migrate' }, + meta: { + ...(info.meta || {}), + source: 'storage-migrate', + // Exact back-reference to the source file's id, so references (e.g. a + // user's avatarUrl) can be repointed precisely — never to another file. + ...(info.migratedFromId ? { migratedFromId: info.migratedFromId } : {}), + }, ext, extension: ext, extensionWithDot: ext ? `.${ext}` : '', @@ -233,16 +261,29 @@ async function createMeteorFilesDocFromBuffer(cfg, info) { return newId; } -// Repoint references after an attachment's _id changes (cross-format move). +// Repoint references after a file's _id changes (cross-format move). async function remapReferences(cfg, oldId, newId) { - if (cfg.coll !== 'attachments' || !oldId || !newId || oldId === newId) { + if (!oldId || !newId || oldId === newId) { return; } try { const db = getDb(); - await db.collection('cards').updateMany({ coverId: oldId }, { $set: { coverId: newId } }); + if (cfg.coll === 'attachments') { + // Card cover images reference the attachment id. + await db.collection('cards').updateMany({ coverId: oldId }, { $set: { coverId: newId } }); + } else if (cfg.coll === 'avatars') { + // A user's profile.avatarUrl still points at the source avatar id — a + // legacy '/cfs/files/avatars/' or a '/cdn/storage/avatars/' + // URL. After the move that id is gone, so the avatar shows a broken image. + // Repoint avatarUrl to the migrated avatar's new id. + const escaped = String(oldId).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + await db.collection('users').updateMany( + { 'profile.avatarUrl': { $regex: `/avatars/${escaped}([-?/]|$)` } }, + { $set: { 'profile.avatarUrl': `/cdn/storage/avatars/${newId}` } }, + ); + } } catch (error) { - console.error('[attachmentMigration] coverId remap failed', error); + console.error('[attachmentMigration] reference remap failed', error); } } @@ -264,6 +305,7 @@ async function moveItem(item, dest) { meta: item.meta, userId: item.userId, uploadedAt: item.uploadedAt, + migratedFromId: item.sourceId, }); if (dest !== STORAGE_NAME_FILESYSTEM) { const newDoc = await cfg.Collection.collection.findOneAsync({ _id: newId }); @@ -309,7 +351,7 @@ async function moveItem(item, dest) { await moveToStorage(doc, dest, cfg.factory); } -async function runMigration(items, dest) { +async function runMigration(items, dest, source, scope) { const total = items.length; try { for (let i = 0; i < total; i++) { @@ -341,11 +383,22 @@ async function runMigration(items, dest) { controller.running = false; controller.paused = false; controller.cancelled = false; + const finishedAt = new Date(); const finalStatus = { running: false, paused: false, cancelled, - finishedAt: new Date(), + finishedAt, + // Persist a summary of the last completed move so the page can keep showing + // "from to at " after it finishes. + lastMove: { + source, + dest, + scope, + total, + cancelled, + at: finishedAt, + }, }; if (!cancelled) { finalStatus.done = total; @@ -354,7 +407,164 @@ async function runMigration(items, dest) { } } +// --------------------------------------------------------------------------- +// Repair file locations: find attachments/avatars whose recorded storage +// (versions..storage / path / meta.gridFsFileId) no longer matches where the +// binary actually is — left inconsistent by an interrupted/failed move — and fix +// the database to point at the real location. +// --------------------------------------------------------------------------- + +// Look up a Meteor-Files GridFS file by the metadata stamped on upload +// (metadata.fileId = document _id; the GridFS bucket name equals the collection). +async function findGridFsFile(coll, fileId, versionName) { + const db = getDb(); + const files = db.collection(`${coll}.files`); + let f = await files.findOne({ 'metadata.fileId': fileId, 'metadata.versionName': versionName }); + if (!f) { + f = await files.findOne({ 'metadata.fileId': fileId }); + } + return f || null; +} + +// Best-effort check whether a version's binary exists on the filesystem; returns +// the resolved path, or null. +function findFilesystemFile(factory, doc, versionName) { + const v = (doc.versions && doc.versions[versionName]) || {}; + const storagePath = factory.storagePath; + const candidates = []; + if (v.path) { + candidates.push(v.path); + candidates.push(path.join(storagePath, path.basename(v.path))); + } + const prefix = `${doc._id}-${versionName}-`; + try { + const match = fs.readdirSync(storagePath).find(entry => entry.startsWith(prefix)); + if (match) candidates.push(path.join(storagePath, match)); + } catch (e) { + // storage dir may not exist + } + for (const c of candidates) { + try { + if (c && fs.existsSync(c) && fs.statSync(c).isFile()) return c; + } catch (e) { + // ignore unreadable candidate + } + } + return null; +} + +// Authoritative "is this version readable on the filesystem?" check — reuses the +// strategy's own (thorough) path resolution so the repair agrees with what a real +// move/download would find, even when versions..path is stale. +function strategyCanReadFilesystem(factory, doc, versionName) { + try { + const strategy = factory.getFileStrategy(doc, versionName, STORAGE_NAME_FILESYSTEM); + const rs = strategy.getReadStream(); + if (rs) { + try { + if (typeof rs.destroy === 'function') rs.destroy(); + else if (typeof rs.close === 'function') rs.close(); + } catch (e) { + // ignore close errors + } + return true; + } + } catch (e) { + // ignore resolution errors + } + return false; +} + +async function repairCollectionStorage(cfg) { + const result = { scanned: 0, repaired: 0, broken: 0, brokenItems: [] }; + // Index the GridFS files by metadata.fileId so the per-file lookup below is a + // point query rather than a collection scan (idempotent). + try { + await getDb().collection(`${cfg.coll}.files`).createIndex({ 'metadata.fileId': 1 }); + } catch (e) { + // index may already exist or the bucket may not exist yet + } + const cursor = cfg.Collection.collection.find({}, { fields: { versions: 1, name: 1 } }); + await cursor.forEachAsync(async doc => { + const versions = doc.versions || {}; + for (const versionName of Object.keys(versions)) { + result.scanned += 1; + const v = versions[versionName] || {}; + const recorded = v.storage; + const set = {}; + const unset = {}; + + const gfs = await findGridFsFile(cfg.coll, doc._id, versionName); + if (gfs) { + // The binary is in GridFS — make the record say so, with the right id. + const wantId = String(gfs._id); + if (recorded !== 'gridfs') set[`versions.${versionName}.storage`] = 'gridfs'; + if (((v.meta || {}).gridFsFileId) !== wantId) { + set[`versions.${versionName}.meta.gridFsFileId`] = wantId; + } + } else { + const diskPath = findFilesystemFile(cfg.factory, doc, versionName); + const onFilesystem = !!diskPath || strategyCanReadFilesystem(cfg.factory, doc, versionName); + if (onFilesystem) { + // The binary is on the filesystem. + if (recorded !== STORAGE_NAME_FILESYSTEM) set[`versions.${versionName}.storage`] = STORAGE_NAME_FILESYSTEM; + if (diskPath && v.path !== diskPath) set[`versions.${versionName}.path`] = diskPath; + if ((v.meta || {}).gridFsFileId) unset[`versions.${versionName}.meta.gridFsFileId`] = ''; + } else { + // Not found on filesystem or in GridFS. Cloud-stored files are not + // verified here (left untouched); anything else is genuinely broken. + if (!CLOUD_STORAGE_NAMES.includes(recorded)) { + result.broken += 1; + if (result.brokenItems.length < 50) { + result.brokenItems.push({ + id: doc._id, + version: versionName, + recorded: recorded || 'unknown', + name: doc.name || '', + }); + } + } + continue; + } + } + + const modifier = {}; + if (Object.keys(set).length) modifier.$set = set; + if (Object.keys(unset).length) modifier.$unset = unset; + if (Object.keys(modifier).length) { + await cfg.Collection.collection.updateAsync({ _id: doc._id }, modifier); + result.repaired += 1; + } + } + }); + return result; +} + Meteor.methods({ + // Scan attachments and avatars and repair any whose recorded storage location + // is out of sync with where the binary actually lives. + async repairAttachmentStorageLocations() { + await requireAdmin(this.userId); + if (controller.running) { + throw new Meteor.Error( + 'bulk-move-already-running', + 'A bulk attachment move is running; try the repair again after it finishes', + ); + } + const summary = {}; + for (const cfg of collectionConfigs('both')) { + summary[cfg.coll] = await repairCollectionStorage(cfg); + } + await setStatus({ + lastRepair: { + at: new Date(), + attachments: summary.attachments || null, + avatars: summary.avatars || null, + }, + }); + return summary; + }, + async startBulkAttachmentMove(source, dest, scope = 'attachments') { check(source, String); check(dest, String); @@ -395,6 +605,7 @@ Meteor.methods({ ); if (items.length === 0) { + const finishedAt = new Date(); await setStatus({ running: false, paused: false, @@ -404,7 +615,8 @@ Meteor.methods({ source, dest, scope, - finishedAt: new Date(), + finishedAt, + lastMove: { source, dest, scope, total: 0, cancelled: false, at: finishedAt }, }); return { total: 0 }; } @@ -433,7 +645,7 @@ Meteor.methods({ // (SyncedCron treats it as fatal). Meteor.defer(() => { Promise.resolve() - .then(() => runMigration(items, dest)) + .then(() => runMigration(items, dest, source, scope)) .catch(async error => { console.error('[attachmentMigration] Migration aborted:', error); controller.running = false; diff --git a/server/models/attachmentStorageSettings.js b/server/models/attachmentStorageSettings.js index 5c053741b..fdb43d707 100644 --- a/server/models/attachmentStorageSettings.js +++ b/server/models/attachmentStorageSettings.js @@ -385,6 +385,44 @@ Meteor.methods({ return { attachments, avatars, calculatedAt: new Date() }; }, + async getAzureStorageStats() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + const user = await ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + const db = MongoInternals.defaultRemoteCollectionDriver()?.mongo?.db; + if (!db) { + throw new Meteor.Error('mongo-unavailable', 'MongoDB connection is not available'); + } + const [attachments, avatars] = await Promise.all([ + countByStorageSafe(db, 'attachments', 'azure'), + countByStorageSafe(db, 'avatars', 'azure'), + ]); + return { attachments, avatars, calculatedAt: new Date() }; + }, + + async getGcsStorageStats() { + if (!this.userId) { + throw new Meteor.Error('not-authorized', 'Must be logged in'); + } + const user = await ReactiveCache.getUser(this.userId); + if (!user || !user.isAdmin) { + throw new Meteor.Error('not-authorized', 'Admin access required'); + } + const db = MongoInternals.defaultRemoteCollectionDriver()?.mongo?.db; + if (!db) { + throw new Meteor.Error('mongo-unavailable', 'MongoDB connection is not available'); + } + const [attachments, avatars] = await Promise.all([ + countByStorageSafe(db, 'attachments', 'gcs'), + countByStorageSafe(db, 'avatars', 'gcs'), + ]); + return { attachments, avatars, calculatedAt: new Date() }; + }, + async compactMongoGridFs() { if (!this.userId) { throw new Meteor.Error('not-authorized', 'Must be logged in'); diff --git a/server/models/users.js b/server/models/users.js index aacdceccf..982d0a100 100644 --- a/server/models/users.js +++ b/server/models/users.js @@ -13,6 +13,7 @@ import { sendJsonResult } from '/server/apiMiddleware'; import EmailLocalization from '/server/lib/emailLocalization'; import { ensureIndex } from '/server/lib/mongoStartup'; import ImpersonatedUsers from '/models/impersonatedUsers'; +import Avatars from '/models/avatars'; import Boards from '/models/boards'; import InvitationCodes from '/models/invitationCodes'; import InviteToBoardRolesSettings from '/models/inviteToBoardRolesSettings'; @@ -992,16 +993,70 @@ const startNotificationCleanup = debounce( 500, ); +// Repair avatar URLs left pointing at a deleted legacy CollectionFS avatar after +// a CFS -> Meteor-Files migration. The user's profile.avatarUrl is still the +// legacy '/cfs/files/avatars/' form, but that filerecord was deleted by +// the migration and the avatar now lives in Meteor-Files under a new id, so the +// avatar renders broken. Point profile.avatarUrl at the user's migrated +// Meteor-Files avatar. Idempotent: once rewritten the URL no longer matches. +async function repairLegacyAvatarUrls() { + try { + let repaired = 0; + const cursor = Users.find( + { 'profile.avatarUrl': { $regex: '/cfs/files/avatars/' } }, + { fields: { _id: 1, 'profile.avatarUrl': 1 } }, + ); + await cursor.forEachAsync(async user => { + const url = (user.profile && user.profile.avatarUrl) || ''; + const match = url.match(/\/avatars\/([^/?#-]+)/); + const oldId = match && match[1]; + + let avatar = null; + // 1) Precise: the migration stamped the source id it came from, so we can + // repoint to the exact avatar that replaced this one. + if (oldId) { + avatar = await Avatars.collection.findOneAsync( + { 'meta.migratedFromId': oldId }, + { fields: { _id: 1 } }, + ); + } + // 2) Best-effort for avatars migrated before that stamp existed: the user's + // OWN migrated avatar (matched strictly by userId, newest first) — never + // another user's avatar. + if (!avatar) { + avatar = + (await Avatars.collection.findOneAsync( + { userId: user._id, 'meta.source': 'storage-migrate' }, + { sort: { uploadedAtOstrio: -1 }, fields: { _id: 1 } }, + )) || + (await Avatars.collection.findOneAsync( + { userId: user._id }, + { sort: { uploadedAtOstrio: -1 }, fields: { _id: 1 } }, + )); + } + + if (avatar && avatar._id) { + await Users.updateAsync(user._id, { + $set: { 'profile.avatarUrl': `/cdn/storage/avatars/${avatar._id}` }, + }); + repaired += 1; + } + }); + if (repaired > 0) { + console.log(`[users] Repaired ${repaired} legacy avatar URL(s) to migrated Meteor-Files avatars.`); + } + } catch (error) { + console.error('[users] Failed to repair legacy avatar URLs:', error); + } +} + Meteor.startup(async () => { for (const value of allowedSortValues) { await ensureIndex(Lists, value); } await ensureIndex(Users, { modifiedAt: -1 }); - Users.find({ 'profile.avatarUrl': { $regex: '/cfs/files/avatars/' } }).forEach(doc => { - doc.profile.avatarUrl = doc.profile.avatarUrl.replace( - '/cfs/files/avatars/', - '/cdn/storage/avatars/', - ); + Meteor.defer(() => { + repairLegacyAvatarUrls(); }); Meteor.defer(() => { startNotificationCleanup();