Files
wekan/server/lib/listSyncFetch.js
T
Lauri Ojansivu 1dc5b82369 Sandstorm: Jira List Sync works from a grain.
It was recorded as impossible: Jira signs in only with an Authorization
header, and the bridge proxy takes that header for the grain's grant. But
Sandstorm's HTTP driver can keep the sign-in in the grant itself
(shell/imports/server/drivers/external-ui-view.js in the Sandstorm
source): the member connects the Powerbox to "any address" and enters the
Jira address with the credentials in it - https://EMAIL:API-TOKEN@site
(the e-mail's @ as %40), or https://jira.example.org#TOKEN for a Server /
Data Center token - and Sandstorm moves them into the grant and adds
Authorization: Basic (or Bearer) to every request the grain sends through
it. WeKan never sees them.

- egress.mjs syncApiRoot grants a Jira list at its own address, and the
  Powerbox request asks for Basic authentication
  (ApiSession.PowerboxTag.authentication; apiSessionDescriptor takes it).
- In a grain the Jira fetcher sends no Authorization of its own, and a
  Jira source needs no WeKan token there: a marker credential says the
  grant holds the sign-in. Only for Jira - GitLab sends its token in the
  query, where a marker would reach GitLab.
- The List Sync popup tells a Jira list in a grain what to enter in the
  Powerbox, translated in all locales; the Sandstorm docs say how, and
  that the same #token form may let a private GitHub, Gitea or Forgejo
  through (not tried yet).
- TODO Later drops "Not possible from a Sandstorm grain", its last
  category naming an issue; changelogFormat no longer needs one.

Not run in a real grain here. Tests: sandstormSyncEgress.test.mjs (the
root and the Basic descriptor, and an address with credentials in it
still refused), sandstormJiraGrant.test.cjs (the server, the fetcher,
the marker only for Jira, the popup and the docs).

Thanks to xet7 !
2026-10-15 00:33:30 +03:00

361 lines
20 KiB
JavaScript

// Fetchers for the periodic list-sync job (server/listSync.js). Each one
// calls the external tracker's REST API and returns the RAW JSON in exactly
// the shape models/lib/externalParsers.js already parses for one-time import
// (Jira's `{ issues: [...] }` search result, a GitHub/Gitea issues array, a
// GitLab issues array) - so the sync job hands that JSON to the SAME parser
// the import path uses (parseJira/parseGithub/parseGitlab/parseGitea) instead
// of a second, sync-only parsing implementation.
//
// SyncBleed (GHSA-5q84-p3vr-f3xv): the server URL is chosen by any board
// member with write access, so every request goes through the SSRF guard like
// every other outbound request WeKan makes for a user: private, loopback and
// link-local addresses are refused, DNS is resolved once and pinned, and a
// redirect is refused. What a member sees of a failure is the origin and the
// HTTP status, never the response, because the error text is returned by the
// preview and stored in syncSource.lastSyncError for the board to read.
import { fetchSafe } from '/server/lib/ssrfGuard';
// In a Sandstorm grain the bridge proxy is the only way out: through the
// Powerbox grant a board member gave the list's tracker (server/sandstormEgress.js).
import { isSandstormServer, syncFetch } from '/server/sandstormEgress';
import { validateAttachmentUrl } from '/models/lib/attachmentUrlValidation';
import { attachRepositoryComments, rateLimitWait } from '/models/lib/listSyncIssueDetails';
// Jira's Sprint field schema, as models/lib/jiraScrumPlanning.js
// JIRA_PLANNING_SCHEMA.sprint names it (tests/listSyncPlanning.test.cjs keeps
// the two the same).
const JIRA_SPRINT_SCHEMA = 'com.pyxis.greenhopper.jira:gh-sprint';
const FETCH_TIMEOUT_MS = 20000;
const MAX_RESPONSE_BYTES = 32 * 1024 * 1024;
const MAX_SYNC_PAGES = 1000;
const MAX_SYNC_ITEMS = 100000;
// typeof: the source-loading tests strip the import, which reads as "not a grain".
const inGrain = () => typeof isSandstormServer === 'function' && isSandstormServer();
export const SYNC_URL_BLOCKED_MESSAGE =
'The Sync server address is not allowed: private, loopback and link-local addresses are refused.';
// What a caller may show for a request that failed. A refusal by the guard is
// one message whatever it named - which address a name resolved to is itself
// internal information - and is marked so the caller can record the attempt.
export function syncFetchError(error) {
if (/^SSRF_GUARD:/.test(error && error.message)) {
const blocked = new Error(SYNC_URL_BLOCKED_MESSAGE);
blocked.ssrfBlocked = true;
blocked.ssrfDetail = String(error.message).slice(0, 200);
return blocked;
}
return error;
}
// Record a refused Sync address in Admin Panel -> Problems. Nobody can save
// or sync one in the ordinary course, so each refusal is an attempt.
export function recordSyncUrlBlocked({ source, userId, detail }) {
try {
// eslint-disable-next-line global-require
require('/server/lib/securityLog').record({
key: 'ssrf.list-sync', action: 'blocked', source, userId, detail: String(detail || '').slice(0, 200),
});
} catch (e) { /* logging must never break the guard */ }
}
// The same check when Sync settings are SAVED, so a refused address is never
// stored. An address that does not resolve now is left to the fetch, which
// checks again on every request (a name can change what it resolves to).
const BLOCKING_REASONS = new Set(['Localhost is not allowed', 'IP address is not allowed',
'Resolved IP address is not allowed', 'Only HTTP and HTTPS protocols are allowed']);
export async function assertSyncUrlAllowed(url, { userId } = {}) {
if (operatorAllowedHost(url)) return;
const verdict = await validateAttachmentUrl(url);
if (verdict.valid || !BLOCKING_REASONS.has(verdict.reason)) return;
recordSyncUrlBlocked({ source: 'setListSyncSource', userId, detail: `${new URL(url).host}: ${verdict.reason}` });
const error = new Error(SYNC_URL_BLOCKED_MESSAGE);
error.ssrfBlocked = true;
throw error;
}
// A self-hosted tracker on the server's own network is a legitimate source,
// and only the server's ADMINISTRATOR may say so: LIST_SYNC_ALLOWED_PRIVATE_HOSTS
// is a comma-separated list of exact host names or addresses, compared with
// the URL's host as written. No board member can change it, so the guard still
// holds against them; an empty or unset value allows nothing.
export function operatorAllowedHost(url, env = process.env) {
const allowed = String(env.LIST_SYNC_ALLOWED_PRIVATE_HOSTS || '').split(',')
.map(host => host.trim().toLowerCase()).filter(Boolean);
return allowed.length > 0 && allowed.includes(new URL(url).hostname.toLowerCase());
}
// The administrator's own host is fetched directly, still without redirects,
// bounded in time and size like any other request.
async function fetchOperatorHost(url, headers) {
const response = await fetch(url, { headers, redirect: 'error', signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
const length = Number(response.headers.get('content-length'));
if (Number.isFinite(length) && length > MAX_RESPONSE_BYTES) throw new Error(`${new URL(url).origin} answer is too large`);
return response;
}
// A rate-limited answer is waited out once when the API says it frees up
// within this long (GitHub's secondary limit asks for about a minute at most);
// a longer wait ends the run with the time to try again, and the next
// scheduled run picks it up - git-bug's and Gitea's migrations do the same.
const RATE_LIMIT_WAIT_MS = 60 * 1000;
async function fetchJson(url, headers, retried = false) {
const origin = new URL(url).origin;
let response;
try {
response = inGrain() ? await syncFetch(url, { headers, timeoutMs: FETCH_TIMEOUT_MS, maxResponseBytes: MAX_RESPONSE_BYTES })
: operatorAllowedHost(url) ? await fetchOperatorHost(url, headers) : await fetchSafe(url, {
headers, timeoutMs: FETCH_TIMEOUT_MS, totalTimeoutMs: FETCH_TIMEOUT_MS,
maxRedirects: 0, maxResponseBytes: MAX_RESPONSE_BYTES,
// server/listSync.js records a refusal as SyncBleed; a second DnsBleed
// row for the same attempt would block a member previewing a source
// somebody saved before the fix.
recordBlocked: false,
});
} catch (error) {
throw syncFetchError(error);
}
if (!response.ok) {
const wait = rateLimitWait(response.status, response.headers);
if (wait !== null) {
if (!retried && wait <= RATE_LIMIT_WAIT_MS) {
await new Promise(resolve => setTimeout(resolve, wait + 1000));
return fetchJson(url, headers, true);
}
const error = new Error(`${origin} rate limit reached; try again after ${new Date(Date.now() + wait).toISOString()}`);
error.status = response.status;
error.rateLimited = true;
throw error;
}
const error = new Error(`${origin} responded with HTTP ${response.status}`);
error.status = response.status;
throw error;
}
let body;
// A parse error quotes the start of the response; say only that it was not JSON.
try { body = await response.json(); } catch (error) { throw new Error(`${origin} did not answer with JSON`); }
return { body, headers: response.headers };
}
// Follow provider pagination only within the configured origin. No partial
// collection escapes: failures or bounds abort before reconciliation starts.
async function fetchArrayPages(initial, headers) {
const rows = [], seen = new Set();
let url = initial, total;
while (url) {
if (seen.has(url) || seen.size >= MAX_SYNC_PAGES) throw new Error('Sync pagination did not finish');
seen.add(url);
const { body, headers: responseHeaders } = await fetchJson(url, headers);
if (!Array.isArray(body)) throw new Error('Invalid sync issue page');
const advertised = responseHeaders.get('x-total-count') ?? responseHeaders.get('x-total');
if (advertised !== null) {
const count = Number(advertised);
if (!/^[0-9]+$/.test(advertised) || !Number.isSafeInteger(count) || (total !== undefined && total !== count)) throw new Error('Invalid or changing sync total');
total = count;
if (total > MAX_SYNC_ITEMS) throw new Error('Sync item limit exceeded');
}
rows.push(...body);
if (rows.length > MAX_SYNC_ITEMS) throw new Error('Sync item limit exceeded');
const link = responseHeaders.get('link') || '';
const next = link.match(/<([^>]+)>;\s*rel="next"/);
const nextPage = responseHeaders.get('x-next-page');
if (next) {
const target = new URL(next[1], url);
if (target.origin !== new URL(initial).origin || target.username || target.password) throw new Error('Invalid sync pagination origin');
url = target.href;
} else if (nextPage) {
if (!/^[1-9][0-9]*$/.test(nextPage)) throw new Error('Invalid sync next page');
const target = new URL(url); target.searchParams.set('page', nextPage); url = target.href;
} else url = null;
}
if (total !== undefined && rows.length !== total) throw new Error('Incomplete sync pagination');
return rows;
}
// Jira Cloud/Server REST search API. `credential.username` + `.token` are
// Basic-auth'd (Jira Cloud: account email + API token). `list.syncSource.url`
// is the Jira base URL (e.g. https://org.atlassian.net),
// `list.syncSource.projectKey` the project key.
// Planning Sync (models/lib/listSyncPlanning.js): Jira's Sprint field is a
// custom field whose id differs per site, so it is found the way the Jira
// import finds it - by its schema - in Jira's own field list. Its id must be
// a custom field's, and exactly one field may say it is the Sprint.
async function fetchJiraSprintField(base, headers) {
const { body } = await fetchJson(`${base}/rest/api/2/field`, headers);
if (!Array.isArray(body)) throw new Error('Invalid Jira field list');
const ids = body.filter(field => field && field.schema && field.schema.custom === JIRA_SPRINT_SCHEMA)
.map(field => field.id);
if (ids.length !== 1 || !/^customfield_\d{1,20}$/.test(ids[0])) throw new Error('Jira has no single Sprint field to sync.');
return ids[0];
}
// The Sprint field's id in the form Jira's own `expand=schema` gives it, which
// the planning reader (models/lib/jiraScrumPlanning.js jiraPlanningFields) reads.
const sprintSchema = id => ({ [id]: { type: 'array', items: 'json', custom: JIRA_SPRINT_SCHEMA } });
async function fetchJiraCloudIssues(base, jql, headers, estimateFieldId, planning = {}) {
const issues = [], tokens = new Set();
const url = new URL(`${base}/rest/api/3/search/jql`);
url.searchParams.set('jql', jql);
url.searchParams.set('maxResults', '200');
// Enhanced search otherwise returns only issue IDs. Request the fields the
// shared parser uses so synchronization cannot replace titles with defaults.
const fields = ['summary', 'description', 'status', 'duedate', 'assignee', 'reporter', 'labels', 'timetracking', 'timespent', 'timeoriginalestimate', 'timeestimate'];
if (estimateFieldId !== undefined) {
if (!/^customfield_\d{1,20}$/.test(estimateFieldId)) throw new Error('Invalid Jira estimate field');
fields.push(estimateFieldId);
}
if (planning.sprintField) fields.push(planning.sprintField);
if (planning.releases) fields.push('fixVersions');
url.searchParams.set('fields', fields.join(','));
for (let page = 0; page < MAX_SYNC_PAGES; page += 1) {
const { body } = await fetchJson(url.href, headers);
if (!body || !Array.isArray(body.issues) || typeof body.isLast !== 'boolean') throw new Error('Invalid Jira Cloud pagination');
issues.push(...body.issues);
if (issues.length > MAX_SYNC_ITEMS) throw new Error('Sync item limit exceeded');
if (body.isLast) return planning.sprintField ? { issues, schema: sprintSchema(planning.sprintField) } : { issues };
const token = body.nextPageToken;
if (typeof token !== 'string' || !token || token.length > 10000 || tokens.has(token)) throw new Error('Incomplete or repeating Jira Cloud pagination');
tokens.add(token);
url.searchParams.set('nextPageToken', token);
}
throw new Error('Sync pagination did not finish');
}
export async function fetchJiraIssues(syncSource, credential) {
const base = String(syncSource.url || '').replace(/\/+$/, '');
const query = `project=${syncSource.projectKey}`;
const jql = encodeURIComponent(query);
const url = `${base}/rest/api/2/search?jql=${jql}&maxResults=200`;
const auth = Buffer.from(`${credential.username || ''}:${credential.token}`).toString('base64');
// In a grain the Powerbox grant carries the Jira sign-in (egress.mjs
// syncApiRoot): the bridge proxy owns Authorization, so none is sent.
const headers = inGrain() ? { Accept: 'application/json' } : { Authorization: `Basic ${auth}`, Accept: 'application/json' };
const fields = Array.isArray(syncSource.fields) ? syncSource.fields : [];
const planning = { releases: fields.includes('releases'),
sprintField: fields.includes('sprint') ? await fetchJiraSprintField(base, headers) : null };
if (new URL(base).hostname.endsWith('.atlassian.net')) {
return fetchJiraCloudIssues(base, query, headers, syncSource.estimateFieldId, planning);
}
const issues = []; let total;
for (let page = 0; page < MAX_SYNC_PAGES; page += 1) {
const { body } = await fetchJson(`${url}&startAt=${issues.length}`, headers);
if (!body || !Array.isArray(body.issues) || !Number.isSafeInteger(body.total) || body.total < 0 || body.startAt !== issues.length) throw new Error('Invalid Jira pagination');
if (total !== undefined && total !== body.total) throw new Error('Jira results changed during pagination; retry sync');
total = body.total;
if (total > MAX_SYNC_ITEMS) throw new Error('Sync item limit exceeded');
if (!body.issues.length && issues.length < total) throw new Error('Incomplete Jira pagination');
issues.push(...body.issues);
if (issues.length > total) throw new Error('Invalid Jira result count');
// Jira Server/Data Center returns every navigable field, the Sprint field
// and fixVersions included; the field list names which one is the Sprint.
if (issues.length === total) {
return { ...body, startAt: 0, issues, ...(planning.sprintField ? { schema: sprintSchema(planning.sprintField) } : {}) };
}
}
throw new Error('Sync pagination did not finish');
}
// GET /repos/{owner}/{repo}/issues?state=all
// A public repository or organization needs no token: none is sent then.
function githubHeaders(token) {
return { Accept: 'application/vnd.github+json', ...(token ? { Authorization: `token ${token}` } : {}) };
}
// The GitHub REST API for the GitHub organization import
// (server/githubOrgImport.js): one answer, or every page of a list, through
// the same SSRF guard, pagination checks and rate-limit handling as sync.
export async function fetchGithubApi(pathAndQuery, token) {
return (await fetchJson(`https://api.github.com${pathAndQuery}`, githubHeaders(token))).body;
}
export async function fetchGithubApiPages(pathAndQuery, token) {
return fetchArrayPages(`https://api.github.com${pathAndQuery}`, githubHeaders(token));
}
// With "Comments, labels, state and files" (syncSource.issueDetails), also
// GET /repos/{owner}/{repo}/issues/comments - every comment of the repository
// in one paged list, each naming its issue - rather than one call per issue
// (models/lib/listSyncIssueDetails.js).
export async function fetchGithubIssues(syncSource, credential) {
const url = `https://api.github.com/repos/${syncSource.projectKey}/issues?state=all&per_page=100`;
const headers = githubHeaders(credential.token);
const issues = await fetchArrayPages(url, headers);
if (!syncSource.issueDetails) return issues;
const comments = await fetchArrayPages(`https://api.github.com/repos/${syncSource.projectKey}/issues/comments?per_page=100`, headers);
return attachRepositoryComments(issues, comments);
}
// GET /projects/{id-or-path}/issues?state=all - Gitea shares GitHub's issue shape.
// type=issues: without it Gitea lists pull requests too, which the parser
// would only drop again. Comments as for GitHub: GET .../issues/comments.
export async function fetchGiteaIssues(syncSource, credential) {
const base = String(syncSource.url || '').replace(/\/+$/, '');
const url = `${base}/api/v1/repos/${syncSource.projectKey}/issues?state=all&type=issues&limit=100`;
const headers = { Authorization: `token ${credential.token}` };
const issues = await fetchArrayPages(url, headers);
if (!syncSource.issueDetails) return issues;
const comments = await fetchArrayPages(`${base}/api/v1/repos/${syncSource.projectKey}/issues/comments?limit=100`, headers);
return attachRepositoryComments(issues, comments);
}
// GET /projects/{id}/issues - GitLab, project id or URL-encoded path.
export async function fetchGitlabIssues(syncSource, credential) {
const base = String(syncSource.url || 'https://gitlab.com').replace(/\/+$/, '');
const project = encodeURIComponent(syncSource.projectKey);
// In a grain the bridge forwards only whitelisted headers, so PRIVATE-TOKEN
// would be dropped; GitLab also takes the token as private_token.
const grain = inGrain();
const query = grain && credential.token ? `&private_token=${encodeURIComponent(credential.token)}` : '';
const headers = grain ? {} : { 'PRIVATE-TOKEN': credential.token };
const issues = await fetchArrayPages(`${base}/api/v4/projects/${project}/issues?per_page=100${query}`, headers);
if (!syncSource.issueDetails) return issues;
// GitLab has no project-wide notes list: GET .../issues/{iid}/notes, only
// for the issues that have comments, as Gitea's migration does.
const withNotes = [];
for (const issue of issues) {
if (!issue || !(Number(issue.user_notes_count) > 0) || !Number.isSafeInteger(Number(issue.iid))) {
withNotes.push(issue);
continue;
}
// eslint-disable-next-line no-await-in-loop
const notes = await fetchArrayPages(`${base}/api/v4/projects/${project}/issues/${Number(issue.iid)}/notes?per_page=100&sort=asc${query}`, headers);
withNotes.push({ ...issue, notes });
}
return withNotes;
}
// #2698 write-back: one JSON request to a GitLab project's API, PUT or POST,
// through the same address checks as the reads (no redirects, bounded in time
// and size). Not from a Sandstorm grain: its egress bridge only reads.
export async function gitlabWrite(syncSource, credential, method, path, body) {
if (inGrain()) throw new Error('Writing back to GitLab is not available in Sandstorm.');
if (!['PUT', 'POST'].includes(method) || !/^issues\/[1-9][0-9]{0,15}(?:\/notes)?$/.test(path)) throw new Error('Invalid write-back request');
const base = String(syncSource.url || 'https://gitlab.com').replace(/\/+$/, '');
const url = `${base}/api/v4/projects/${encodeURIComponent(syncSource.projectKey)}/${path}`;
const origin = new URL(url).origin;
const headers = { 'PRIVATE-TOKEN': credential.token, 'Content-Type': 'application/json', Accept: 'application/json' };
const payload = JSON.stringify(body);
let response;
try {
response = operatorAllowedHost(url)
? await fetch(url, { method, headers, body: payload, redirect: 'error', signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) })
: await fetchSafe(url, { method, headers: { ...headers, 'Content-Length': String(Buffer.byteLength(payload)) }, body: payload,
timeoutMs: FETCH_TIMEOUT_MS, totalTimeoutMs: FETCH_TIMEOUT_MS, maxRedirects: 0, maxResponseBytes: MAX_RESPONSE_BYTES,
recordBlocked: false });
} catch (error) {
throw syncFetchError(error);
}
if (!response.ok) {
const error = new Error(`${origin} refused the write-back with HTTP ${response.status}`);
error.status = response.status;
throw error;
}
try { return await response.json(); } catch (error) { throw new Error(`${origin} did not answer with JSON`); }
}
export const LIST_SYNC_FETCHERS = {
jira: fetchJiraIssues,
github: fetchGithubIssues,
gitea: fetchGiteaIssues,
forgejo: fetchGiteaIssues,
gitlab: fetchGitlabIssues,
};