wekan/models/lib/apiUsage.js

'use strict';

// What the REST API is being used for: who called what, and how often.
//
// THE QUESTION IT ANSWERS. Admin Panel → Problems shows what went wrong; this
// shows what is being DONE. An admin wants to know that `POST /api/boards` was
// called 34 times by alice last month and 812 times by nobody-in-particular
// yesterday - which endpoints are used, by whom, from where, and whether that
// changed. Without it the only record of API use is whatever the reverse proxy
// happens to keep, and most instances have none.
//
// A SUMMARY, like everything else on that page: one row per (account, endpoint),
// with a count and the window it covers, never a row per request. The identity
// here DOES include the account, which is the one deliberate exception to the
// rule in models/lib/eventLogSummary.js - for every other stream the question is
// "what is happening" and the actor would multiply the rows, but here "who
// called what" IS the report, and the cardinality is bounded by real accounts
// times real endpoints rather than by whoever is sending.
//
// THE NAME IS THE ROUTE PATTERN, NOT THE PATH. `/api/boards/:boardId/lists` is
// one endpoint; `/api/boards/abc123/lists` and ten thousand of its siblings are
// one endpoint being used. Recording the concrete path would put a row per board
// in the collection, which is the cost this whole design removes - so a request
// that matched no route at all is recorded under a single `(no route)` name
// rather than under the path it invented.
//
// This module is the part with no Meteor in it: naming, and the accumulator that
// keeps writes bounded. server/lib/apiUsageLog.js is the plumbing.

// How many distinct (account, endpoint) pairs are tracked before the rest are
// counted together. Real use is far under this - an instance has tens of
// endpoints and, in the API's case, few accounts using them - so reaching it
// means something unusual, which is what the overflow row is for.
const MAX_TRACKED = 500;

// The name for a request that matched no route. ONE name for all of them: a 404
// sweep is an attacker walking a wordlist, and a row per guessed path would be
// the log-per-event bug arriving through the back door. The count still says it
// happened, and how much.
const UNMATCHED = '(no route)';

// The name of the endpoint a request reached.
//
// `routePath` is what the router matched - Express fills `req.route.path` once a
// handler runs, and that is the pattern with its `:params` intact, which is
// exactly the name wanted. Everything else is a fallback.
function apiName(method, routePath) {
  const verb = String(method || 'GET').toUpperCase();
  // No route matched: one name for all of them, never the path that was tried.
  return `${verb} ${routePath || UNMATCHED}`;
}

// Is this a request the report is about? Only /api - the client's own DDP
// traffic, the static assets and the health check are not API use, and counting
// them would bury the thing being looked for.
function isApiRequest(url) {
  const path = String(url || '').split('?')[0];
  return path === '/api' || path.startsWith('/api/');
}

// The key one row accumulates under: the account and the endpoint, and nothing
// else. The ADDRESS is deliberately not part of it - it is tallied inside the
// row like every other actor, so one account calling from four addresses is one
// row that names four addresses rather than four rows.
function usageKey(name, username) {
  return `${username || ''}${name}`;
}

// An accumulator, so a busy API does not become a database write per request.
//
// Every other stream on this page records something RARE - a guard firing, a
// slow query, a failing test - and writes it as it happens. Ordinary API use is
// not rare; it is the traffic. So calls are counted in memory and folded into
// their rows on a timer, which turns a thousand requests into one write instead
// of a thousand. What is lost if the process dies before a flush is up to a few
// seconds of counts, and this is a usage report: it does not decide anything,
// and nothing is protected by it.
class UsageAccumulator {
  constructor({ maxTracked = MAX_TRACKED } = {}) {
    this.maxTracked = maxTracked;
    this.pending = new Map();
    this.overflow = 0;
  }

  // Count one call. `at` is when it happened, so a flush that arrives later
  // still writes the right window.
  add({ name, userId, ip, location, at }) {
    if (!name) return false;
    const key = usageKey(name, userId);
    const existing = this.pending.get(key);
    if (existing) {
      existing.count += 1;
      if (at > existing.at) {
        existing.at = at;
        // The LATEST caller's address, like every other summary's latest fields.
        existing.ip = ip || existing.ip;
        existing.location = location || existing.location;
      }
      return true;
    }
    if (this.pending.size >= this.maxTracked) {
      // Not dropped - counted. "And 12,000 calls to something else" is itself
      // the signal that something unusual is happening.
      this.overflow += 1;
      return false;
    }
    this.pending.set(key, {
      name, userId: userId || '', ip: ip || '', location: location || null,
      count: 1, firstAt: at, at,
    });
    return true;
  }

  get size() { return this.pending.size; }

  // Take everything counted so far and reset. Returns the rows to fold, each
  // carrying its own count - the fold multiplies by it rather than being called
  // once per call.
  drain() {
    const rows = [...this.pending.values()];
    if (this.overflow > 0) {
      rows.push({
        name: `${UNMATCHED} (over ${this.maxTracked} tracked)`,
        userId: '', ip: '', location: null,
        count: this.overflow,
        firstAt: rows.length ? rows[0].firstAt : new Date(),
        at: new Date(),
      });
    }
    this.pending = new Map();
    this.overflow = 0;
    return rows;
  }
}

module.exports = { MAX_TRACKED, UNMATCHED, apiName, isApiRequest, usageKey, UsageAccumulator };