Files
wekan/server/lib/listSyncApplication.js
T
Lauri Ojansivu cb69750b17 Let administrators discard List Sync operations that cannot be replayed.
A saved List Sync operation whose list was removed, recreated or
reconfigured, or whose actor lost full-list write access, could never be
replayed. The replay job refused it every minute with only a console line,
and because a new Sync of the list resumes the pending operation first,
that list could not Sync at all.

Such an operation is now marked once, with one Recovery event, and a
blocked manual Sync says why. Admin Panel / Problems / Recovery lists these
operations with what they applied and their live replayability. An
instance administrator can discard one only while the same checks a replay
makes still fail. The discard holds the list's Sync lease, records one
immutable decision, then removes the saved steps and the operation marker,
reading each removal back. It writes no card: changes already applied stay
and the rest are never written. A retry or a second administrator only
finishes the removal, and a replay that finds a decided discard completes it
instead of resuming. The intent stays as evidence and the discard is
audited in Recovery.

Unit, source-wiring and negative tests cover recording once, refusing
non-administrators and replayable operations, and discarding twice; an app
test and a Playwright case cover the Recovery page. New interface text is
in English, pending Transifex.

Thanks to xet7 !
2026-10-08 11:53:06 +03:00

238 lines
13 KiB
JavaScript

// Durable list Sync: the write phase of manual and scheduled Sync through the
// write-ahead journal (server/lib/syncOperationJournal.js), with every card's
// History, activities, rules, notifications and webhooks applied from a saved
// plan and replayed after a restart. When it applies is decided by
// server/lib/listSyncSteps.js; otherwise server/listSync.js writes directly,
// as it always has.
import { Meteor } from 'meteor/meteor';
import { DDP } from 'meteor/ddp';
import Cards from '/models/cards';
import Activities from '/models/activities';
import ChangeHistory from '/models/changeHistory';
import Swimlanes from '/models/swimlanes';
import Boards from '/models/boards';
import Rules from '/models/rules';
import Actions from '/models/actions';
import Triggers from '/models/triggers';
import { getFeatureFlags } from '/models/lib/featureFlags';
import { runStoredSyncActivityDelivery } from '/server/notifications/storedActivityDelivery';
import { withListSyncLease } from '/server/lib/listSyncLease';
import Lists from '/models/lists';
import RecoveryEvents from '/models/recoveryEvents';
import { runStoredListSyncOperation, readPendingListSyncOperation, discardPreparingListSyncOperation,
listPendingListSyncOperations, readListSyncOperationIntent, markStuckListSyncOperation,
completeDecidedListSyncDiscard } from '/server/lib/listSyncOperations';
const { randomUUID } = require('node:crypto');
const { memberCan } = require('/models/lib/boardRoleCapabilities');
const { ruleActionIds } = require('/models/lib/ruleParts');
const { isAssignedOnlyMember } = require('/models/lib/boardCardScope');
const { durableSyncEligibility, durableRuleActionTypes, followableActionIds, buildListSyncSteps } = require('/server/lib/listSyncSteps');
const { createSyncEffectPlanner, validateSyncEffects, applySyncEffectsStep } = require('/server/lib/syncEffects');
const { syncOperationEffectId } = require('/server/lib/syncOperationApply');
const { createSyncHookedCards } = require('/server/lib/syncHookedCards');
const { createSyncHookedActivities } = require('/server/lib/syncHookedActivities');
const { syncEffectPolicy } = require('/server/lib/syncEffectPolicy');
const { syncSourceKey } = require('/models/lib/listSyncSourceIdentity');
const { durableRuleActionType } = require('/server/lib/syncRuleMoveCommand');
const { createSyncOperationScopeGuard } = require('/server/lib/syncOperationScope');
const { stuckReason } = require('/server/lib/listSyncStuck');
const withActor = (userId, work) => DDP._CurrentMethodInvocation.withValue({ userId, isSimulation: false }, work);
const readPolicy = async () => syncEffectPolicy(getFeatureFlags());
export function currentSyncActor() {
return DDP._CurrentMethodInvocation.get()?.userId || null;
}
export async function durableSyncDecision({ list, board, trigger, actorId }) {
// Every action of every rule on the board, extra actions included (#4294).
// A rule whose action is gone cannot be proven durable either.
const readActions = async boardId => {
const rules = await Rules.find({ boardId }, { fields: { actionId: 1, extraActionIds: 1, triggerId: 1 } }).fetchAsync();
const actionIds = [...new Set(rules.flatMap(rule => ruleActionIds(rule)).filter(Boolean))];
const actions = actionIds.length
? await Actions.find({ _id: { $in: actionIds } },
{ fields: { actionType: 1, listName: 1, swimlaneName: 1, boardId: 1 } }).fetchAsync() : [];
if (actions.length !== actionIds.length) return null;
// A move to another board must end every plan it is in (listSyncSteps.js).
const triggerIds = rules.map(rule => rule.triggerId).filter(Boolean);
const triggers = triggerIds.length
? await Triggers.find({ _id: { $in: triggerIds } }, { fields: { activityType: 1 } }).fetchAsync() : [];
const typeOf = new Map(triggers.map(trigger => [trigger._id, trigger.activityType]));
const types = new Map(actions.map(action => [action._id, action.actionType]));
const movable = followableActionIds(rules.map(rule => ({ actionIds: ruleActionIds(rule).filter(Boolean),
activityType: typeOf.get(rule.triggerId) ?? null })), id => types.get(id));
return actions.map(action => ({ ...action, crossBoardMovable: movable.has(action._id) }));
};
// A destination board counts only when the actor may write there, as the
// stored stages of its rules require.
const readBoard = async boardId => {
const destination = typeof boardId === 'string' && boardId ? await Boards.findOneAsync(boardId) : null;
return destination && memberCan(destination.members, actorId, 'write') &&
!isAssignedOnlyMember(destination, actorId) ? destination : null;
};
// Every board a durable cross-board rule action reaches must have opted in
// and have only durable rule actions itself (listSyncSteps.js).
const ruleActionTypes = await durableRuleActionTypes({ boardId: list.boardId, readActions, readBoard,
typeOf: durableRuleActionType });
return durableSyncEligibility({ list, board, trigger, flags: getFeatureFlags(), ruleActionTypes, actorId });
}
// Only a full-list writer runs Sync; checked again on every guard.
async function assertListWriter({ board, userId }) {
const user = await Meteor.users.findOneAsync(userId, { fields: { loginDisabled: 1 } });
return !!user && !user.loginDisabled && !!board && memberCan(board.members, userId, 'write') &&
!isAssignedOnlyMember(board, userId);
}
function scopeOf(list) {
return { listId: list._id, boardId: list.boardId, incarnation: list.syncCredentialIncarnation,
revision: list.syncRevision, sourceKey: syncSourceKey(list.syncSource) };
}
// Everything a saved step needs to be applied - its own actor and trigger,
// never the current request's.
function applier({ actorId, trigger }) {
return (step, context) => {
const effectId = syncOperationEffectId(context.operationId, context.index);
return applySyncEffectsStep({ step, effects: context.effects, operationId: context.operationId,
index: context.index, userId: actorId, assertCurrent: context.assertCurrent, readPolicy,
cards: createSyncHookedCards({ cards: Cards, step, userId: actorId, withActor }),
history: ChangeHistory,
activities: context.effects.activities
? createSyncHookedActivities({ activities: Activities, plan: context.effects, step, effectId, userId: actorId, withActor })
: Activities,
completeDelivery: delivery => runStoredSyncActivityDelivery({ ...delivery, trigger }) });
};
}
const validateEffects = (effects, step, context) =>
validateSyncEffects(effects, step, syncOperationEffectId(context.operationId, context.index));
// Finish the list's unfinished operation first: a new run cannot start while
// one is pending, and it must not be left behind. An operation that cannot be
// replayed any more is recorded once for Problems -> Recovery
// (server/lib/listSyncStuck.js), where an administrator can discard it.
async function resumePending({ listId, lease }) {
const decided = await completeDecidedListSyncDiscard({ listId, assertCurrent: lease.assertCurrent });
if (decided) return { discarded: decided };
let pending = null;
try {
pending = await readPendingListSyncOperation(listId);
if (!pending) return null;
if (pending.state === 'preparing') {
await discardPreparingListSyncOperation({ listId, operationId: pending.operationId, assertCurrent: lease.assertCurrent });
return { discarded: pending.operationId };
}
if (!pending.trigger) throw Object.assign(new Error('sync-operation-trigger-unknown'), { code: 'sync-operation-trigger-unknown' });
return await runStoredListSyncOperation({ scope: pending.scope, actorId: pending.actorId, trigger: pending.trigger,
intentId: pending.intentId, lease,
assertAccess: current => assertListWriter(current),
build: () => { throw new Error('sync-operation-replay-cannot-build'); },
prepareEffects: () => { throw new Error('sync-operation-replay-cannot-build'); },
validateEffects, apply: applier({ actorId: pending.actorId, trigger: pending.trigger }) });
} catch (error) {
const reason = stuckReason(error);
if (reason) error.listSyncStuck = { reason, recorded: await noteStuck({ listId, operationId: pending?.operationId, reason }) };
throw error;
}
}
// One Recovery event when an operation is first found stuck, never one per
// retry. Best effort: recording must not hide the refusal it records.
async function noteStuck({ listId, operationId, reason }) {
let marked;
try { marked = await markStuckListSyncOperation({ listId, operationId, reason }); } catch (_) { return false; }
if (!marked) return false;
const boardId = marked.scope?.boardId;
await RecoveryEvents.record(RecoveryEvents.types.LIST_SYNC_OPERATION_STUCK, {
done: false, severity: 'warning', source: 'server', boardIds: boardId ? [boardId] : undefined,
detail: `List Sync of list ${listId} is blocked: its saved operation ${marked.operationId} cannot be replayed ` +
`(${reason}); ${marked.checkpoint || 0} of ${marked.total || 0} steps were applied. ` +
'Review and discard it under "List Sync operations that cannot be replayed" on this page.',
});
return true;
}
// Can this stored operation be replayed NOW? Returns null when it can, or the
// reason it cannot - the same checks a replay makes before writing anything:
// the stored intent and trigger, the list's exact scope, and the actor's
// full-list write access. Other read failures are thrown, never reported as
// "cannot be replayed", so a transient error cannot unlock a discard.
export async function inspectListSyncOperation(row) {
let intent;
try { intent = await readListSyncOperationIntent(row.intentId); } catch (error) {
const reason = stuckReason(error);
if (reason) return reason;
throw error;
}
if (!intent.trigger) return 'trigger-unknown';
try {
await createSyncOperationScopeGuard({ lists: Lists, boards: Boards, scope: row.scope, assertCurrent: async () => {},
assertAccess: current => assertListWriter({ ...current, userId: intent.actorId }) })();
return null;
} catch (error) {
const reason = stuckReason(error);
if (reason) return reason;
throw error;
}
}
// The durable write phase of one reconcile. `lease` is the list lease the run
// holds, with its mapping checks folded into assertCurrent.
export async function runDurableListSync({ list, trigger, actorId, lease, plan, creations, fetched,
estimateMapping, timeMappings, now = new Date() }) {
await resumePending({ listId: list._id, lease });
const policy = await readPolicy();
const scope = scopeOf(list);
return runStoredListSyncOperation({ scope, actorId, trigger, intentId: randomUUID(), lease,
assertAccess: current => assertListWriter(current),
build: async ({ assertCurrent }) => {
await assertCurrent();
const ids = [...plan.toUpdate.map(row => row.cardId), ...plan.toArchive];
const cards = ids.length ? await Cards.find({ _id: { $in: ids }, boardId: list.boardId, listId: list._id },
{ transform: null }).fetchAsync() : [];
const steps = buildListSyncSteps({ creations, updates: plan.toUpdate, archives: plan.toArchive, fetched,
current: new Map(cards.map(card => [card._id, card])), estimateMapping, timeMappings, now });
await assertCurrent();
return steps;
},
prepareEffects: await (async () => {
const user = await Meteor.users.findOneAsync(actorId, { fields: { username: 1 } });
const swimlanes = await Swimlanes.find({ boardId: list.boardId }, { fields: { _id: 1, boardId: 1, title: 1 },
transform: null }).fetchAsync();
const redoRows = await ChangeHistory.find({ boardId: list.boardId, userId: actorId, undone: true,
superseded: { $ne: true } }, { transform: null, limit: 10000 }).fetchAsync();
return createSyncEffectPlanner({ userId: actorId, username: user?.username || '', createdAt: now,
list: { _id: list._id, boardId: list.boardId, title: list.title || '' },
swimlanes: swimlanes.map(lane => ({ _id: lane._id, boardId: lane.boardId, title: lane.title || '' })),
redoRows, policy });
})(),
validateEffects, apply: applier({ actorId, trigger }) });
}
// Replay after a restart: every unfinished operation, under its list's lease.
export async function replayStoredListSync() {
const result = { resumed: 0, discarded: 0, busy: 0, failed: 0, stuck: 0 };
for (const { _id: listId } of await listPendingListSyncOperations()) {
try {
const outcome = await withListSyncLease(listId, lease => resumePending({ listId, lease }));
if (outcome?.discarded) result.discarded++; else if (outcome) result.resumed++;
} catch (error) {
if (error?.error === 'sync-busy') result.busy++;
else if (error?.listSyncStuck) {
// Waits for an administrator in Problems -> Recovery; said once.
result.stuck++;
if (error.listSyncStuck.recorded) {
console.error(`List Sync replay of list ${listId} cannot continue (${error.listSyncStuck.reason}); ` +
'it is listed in Admin Panel -> Problems -> Recovery.');
}
} else {
result.failed++;
// Kept for the next pass; say why, or a stuck replay is invisible.
console.error('List Sync replay failed; it is retried on the next pass:', error?.message || error);
}
}
}
return result;
}