Files
wekan/server/methods/activityNotificationRecovery.js
Lauri Ojansivu e0317a9ff3 Drain the activity notification recovery list; index synced-cron's lookup.
#6768: the recovery scan retried every pending activity notification each
round, and one whose delivery fails for good - its activity deleted or
changed since, a recipient who lost access, a plan with nothing or too much
to send - failed the same way every round, forever, so the list only grew.
Such an error is terminal now: after a ten-minute grace (an activity being
written is never touched) the scan cancels that notification through the
same cancellation an administrator uses, as the system, which compacts it
to a cancelled receipt. A pending notification older than
ACTIVITY_NOTIFICATION_MAX_AGE_DAYS (7) is cancelled as expired instead of
being delivered a week late. Network and other transient failures are still
retried. Admin Panel -> Problems gets "Retry all" and "Cancel older..." for
the whole list at once, site administrators only.

#6769: before each run synced-cron asks whether a job of that name is still
running - { name, finishedAt: { $exists: false } } - every minute for every
job on every instance, and only { intendedAt, name } and the TTL on
startedAt were indexed, so it scanned the job history. { name, finishedAt }
is created at startup with WeKan's ensureIndex.

The new strings are translated in every locale.

Thanks to Raymond-Voogt and xet7 !

Fixes #6768,
Fixes #6769.
2026-10-14 14:00:21 +03:00

121 lines
7.2 KiB
JavaScript

import { Meteor } from 'meteor/meteor';
import { check } from 'meteor/check';
import { DDPRateLimiter } from 'meteor/ddp-rate-limiter';
import Activities from '/models/activities';
import { ActivityNotificationIntents } from '/server/notifications/activityIntents';
import { ActivityNotificationPlans, ActivityNotificationLeases, ActivityNotificationControls, resumeActivityNotifications, cleanupCancelledActivityNotifications, giveUpActivityNotification, recoverActivityNotifications } from '/server/notifications/activityPlans';
const { olderThanSelector } = require('/server/lib/activityNotificationExpiry');
const { controlActivityNotification, cancelActivityNotification } = require('/server/lib/activityNotificationControl');
const { activityNotificationReport } = require('/server/lib/activityNotificationReport');
const BULK_LIMIT = 5000;
async function assertAdmin(userId) {
const user = userId && await Meteor.users.findOneAsync(userId, { fields: { isAdmin: 1, loginDisabled: 1 } });
if (!user?.isAdmin || user.loginDisabled) throw new Meteor.Error('not-authorized');
}
Meteor.methods({
// #6768: bulk actions, so a long list is handled in one click, not per row.
// Cancel every pending notification whose activity is older than `days`.
async cancelActivityNotificationsOlderThan(request) {
check(request, { days: Number });
await assertAdmin(this.userId);
if (!Number.isFinite(request.days) || request.days < 0 || request.days > 3650) throw new Meteor.Error('activity-recovery-invalid-request');
const rows = await ActivityNotificationIntents.rawCollection().find(olderThanSelector(request.days), { projection: { _id: 1 } })
.limit(BULK_LIMIT).toArray();
let cancelled = 0;
for (const { _id } of rows) {
await assertAdmin(this.userId);
try { await giveUpActivityNotification(_id, 'admin'); cancelled += 1; } catch (error) { /* busy or already done: the next pass */ }
}
const remaining = await ActivityNotificationIntents.rawCollection().countDocuments(olderThanSelector(request.days));
return { cancelled, remaining };
},
// Retry every pending notification now, a page at a time.
async retryAllActivityNotifications() {
await assertAdmin(this.userId);
const totals = { visited: 0, completed: 0, skipped: 0, failed: 0, busy: 0, cancelled: 0 };
for (let i = 0; i < 50; i += 1) {
await assertAdmin(this.userId);
const result = await recoverActivityNotifications();
for (const key of Object.keys(totals)) totals[key] += result[key] || 0;
if (result.visited < 100) break;
}
return totals;
},
async activityNotificationRecoveryReport(query) {
check(query, { search: String, page: Number });
await assertAdmin(this.userId);
try {
const result = await activityNotificationReport({ intents: ActivityNotificationIntents.rawCollection(),
activities: Activities.rawCollection(), plans: ActivityNotificationPlans.rawCollection(), leases: ActivityNotificationLeases.rawCollection(),
controls: ActivityNotificationControls.rawCollection() }, query);
await assertAdmin(this.userId);
return result;
} catch (error) {
if (error.error === 'not-authorized') throw error;
throw new Meteor.Error('activity-recovery-unavailable');
}
},
async controlActivityNotificationRecovery(request) {
check(request, { intentId: String, paused: Boolean, expectedRevision: Number, requestId: String });
await assertAdmin(this.userId);
try {
const result = await controlActivityNotification({ ...request, actorId: this.userId,
controls: ActivityNotificationControls.rawCollection(), intents: ActivityNotificationIntents.rawCollection(),
leases: ActivityNotificationLeases.rawCollection(), assertAdmin: () => assertAdmin(this.userId) });
await assertAdmin(this.userId);
return result;
} catch (error) {
if (error.error === 'not-authorized') throw error;
if (['sync-busy', 'sync-lease-lost'].includes(error.code)) throw new Meteor.Error('activity-recovery-busy');
if (['activity-notification-control-conflict', 'activity-notification-control-not-pending', 'activity-notification-cancelled'].includes(error.message)) {
throw new Meteor.Error('activity-recovery-control-conflict');
}
throw new Meteor.Error('activity-recovery-control-failed');
}
},
async cancelActivityNotificationRecovery(request) {
check(request, { intentId: String, expectedRevision: Number, requestId: String });
await assertAdmin(this.userId);
try {
const result = await cancelActivityNotification({ ...request, actorId: this.userId,
controls: ActivityNotificationControls.rawCollection(), intents: ActivityNotificationIntents.rawCollection(),
leases: ActivityNotificationLeases.rawCollection(), assertAdmin: () => assertAdmin(this.userId) });
try { await cleanupCancelledActivityNotifications(request.intentId, () => assertAdmin(this.userId)); }
catch (error) { console.error('Cancelled activity payload cleanup incomplete; automatic recovery will retry'); }
await assertAdmin(this.userId);
return result;
} catch (error) {
if (error.error === 'not-authorized') throw error;
if (['sync-busy', 'sync-lease-lost'].includes(error.code)) throw new Meteor.Error('activity-recovery-busy');
if (['activity-notification-control-conflict', 'activity-notification-control-not-pending', 'activity-notification-cancelled'].includes(error.message)) {
throw new Meteor.Error('activity-recovery-control-conflict');
}
throw new Meteor.Error('activity-recovery-control-failed');
}
},
async retryActivityNotification(request) {
check(request, { intentId: String });
await assertAdmin(this.userId);
if (!/^[a-f0-9]{64}$/.test(request.intentId)) throw new Meteor.Error('activity-recovery-invalid-request');
try {
const status = await resumeActivityNotifications(request.intentId, { assertAllowed: () => assertAdmin(this.userId) });
await assertAdmin(this.userId);
return { status };
} catch (error) {
if (error.error === 'not-authorized') throw error;
if (['sync-busy', 'sync-lease-lost'].includes(error.code)) throw new Meteor.Error('activity-recovery-busy');
if (['activity-notification-recipient-denied', 'activity-notification-preference-changed'].includes(error.message)) throw new Meteor.Error('activity-recovery-denied');
if (error.message === 'activity-notification-activity-unconfirmed') throw new Meteor.Error('activity-recovery-source-unavailable');
if (error.message === 'activity-notification-cancelled') throw new Meteor.Error('activity-recovery-status-cancelled');
if (error.message === 'activity-notification-paused') throw new Meteor.Error('activity-recovery-paused');
if (error.message === 'activity-notifications-disabled') throw new Meteor.Error('activity-recovery-disabled');
throw new Meteor.Error('activity-recovery-failed');
}
},
});
for (const name of ['activityNotificationRecoveryReport', 'retryActivityNotification', 'controlActivityNotificationRecovery', 'cancelActivityNotificationRecovery',
'cancelActivityNotificationsOlderThan', 'retryAllActivityNotifications']) {
DDPRateLimiter.addRule({ type: 'method', name, connectionId: () => true }, 30, 10000);
}