Continuous backup: files are watched before the first scan, and a macOS start catches up once.

The file engine scanned its sources first and started watching them
after, so a change made between the two - after the scan had passed a
file, before the watcher began - waited for the next scan, five minutes
by default. And macOS reports file events through FSEvents, which
delivers nothing for a moment after a watcher is created: an EVERYTHING
run on a loaded Mac lost a log line appended right after the start
(tests/integration/continuousBackup.test.cjs, "the appended log line was
streamed by watching", 4 !== 5).

The watchers start first now; a change seen by the scan and a watcher
too is recorded once, as they share one queue. On macOS one catch-up
scan runs two seconds after the start. Not elsewhere: inotify is
watching when fs.watch returns, and a second full scan of a large store
would double the start-up cost.

A new test, on macOS and Linux, with a watcher that never reports:
watching starts before the first scan; on macOS the catch-up scan
records a change made right after the start; on Linux no second scan
runs. It fails on the code before this change.

Thanks to xet7 !

server/lib/continuousBackup/files.js

@@ -43,6 +43,9 @@ async function walk(root) {
 // single-file source is the file itself.
 const sourceFile = (source, relative) => (source.file ? source.path : path.join(source.path, ...relative.split('/')));
 
+// How long after start the macOS catch-up scan runs (see start()).
+const CATCH_UP_SCAN_MS = 2000;
+
 class FileEngine {
   // `sources`: [{ area, key, path, file }] - `key` names the source within
   // its area ('attachments', 'logs', 'recovery-events.jsonl', ...).
@@ -57,14 +60,25 @@ class FileEngine {
 
   async start() {
     this.log = await new SegmentLog(path.join(this.generation.dir, 'files'), 'ndjson', this.cipher).open();
+    // Watch BEFORE the first scan: a change made after the scan had passed a
+    // file but before watching began was missed until the next scan, five
+    // minutes later by default. A change seen by both is recorded once - the
+    // scan and the watcher share one queue.
+    if (this.watchEnabled) for (const source of this.sources) this.watch(source);
     await this.scan();
     this.scanTimer = setInterval(() => { this.scan().catch(this.onError); }, this.scanMs);
     this.sealTimer = setInterval(() => { this.flushDirty().catch(this.onError); }, this.sealMs);
-    if (this.watchEnabled) for (const source of this.sources) this.watch(source);
+    // macOS reports file events through FSEvents, which delivers nothing for a
+    // moment after a watcher is created: one more scan catches what changed
+    // then. Not elsewhere - inotify is watching when fs.watch returns, and a
+    // second full scan of a large store would double the start-up cost.
+    if (this.watchEnabled && process.platform === 'darwin') {
+      this.catchUpTimer = setTimeout(() => { this.scan().catch(this.onError); }, CATCH_UP_SCAN_MS);
+    }
     return this;
   }
   async stop() {
-    clearInterval(this.scanTimer); clearInterval(this.sealTimer);
+    clearInterval(this.scanTimer); clearInterval(this.sealTimer); clearTimeout(this.catchUpTimer);
     for (const watcher of this.watchers) watcher.close();
     await this.busy?.catch(() => {});
     await this.flushDirty();

tests/integration/continuousBackup.test.cjs

@@ -126,6 +126,43 @@ test('SQLite page engine: a WAL database written meanwhile, rebuilt at a moment
   assert.equal(fs.existsSync(path.join(root, 'restore', 'tampered.sqlite')), false);
 });
 
+// The engine scanned FIRST and started watching after, so a change in between
+// waited for the next scan (five minutes by default); and macOS's FSEvents
+// delivers nothing for a moment after a watcher is created - an EVERYTHING
+// run on a loaded Mac lost the appended log line below that way.
+for (const platform of ['darwin', 'linux']) {
+  test(`watching starts before the first scan, and on macOS a catch-up scan follows (${platform})`, { timeout: 30000 }, async t => {
+    const root = tmpRoot();
+    const watch = fs.watch;
+    const real = Object.getOwnPropertyDescriptor(process, 'platform');
+    const calls = [];
+    // A watcher that never reports anything: FSEvents in its first moments.
+    fs.watch = () => { calls.push('watch'); const w = new (require('node:events'))(); w.close = () => {}; return w; };
+    Object.defineProperty(process, 'platform', { value: platform });
+    t.after(() => { fs.watch = watch; Object.defineProperty(process, 'platform', real); fs.rmSync(root, { recursive: true, force: true }); });
+    const live = path.join(root, 'live');
+    fs.mkdirSync(live, { recursive: true });
+    fs.writeFileSync(path.join(live, 'one.txt'), 'one');
+    const target = await store.openTarget(path.join(root, 'target'));
+    const generation = await store.createGeneration(target, {});
+    const engine = new FileEngine({ sources: [{ area: 'attachments', key: 'attachments', path: live }], target, generation,
+      scanMs: 3600000, sealMs: 100, onError: error => { throw error; } });
+    const scan = engine.scan.bind(engine);
+    engine.scan = () => { calls.push('scan'); return scan(); };
+    await engine.start();
+    t.after(() => engine.stop());
+    assert.deepEqual(calls.slice(0, 2), ['watch', 'scan'], 'the watcher is up before the first scan');
+    fs.writeFileSync(path.join(live, 'two.txt'), 'two');
+    for (let i = 0; i < 40 && engine.status().puts < 2; i += 1) await sleep(100);
+    if (platform === 'darwin') {
+      assert.equal(engine.status().puts, 2, 'the catch-up scan recorded what the silent watcher missed');
+    } else {
+      assert.equal(engine.status().puts, 1, 'negative: no second start-up scan where watching is live at once');
+      assert.equal(calls.filter(c => c === 'scan').length, 1);
+    }
+  });
+}
+
 test('file streams: attachments, avatars and logs, with watching, restored to a moment', { timeout: 60000 }, async t => {
   const root = tmpRoot();
   t.after(() => fs.rmSync(root, { recursive: true, force: true }));