Files
wekan/tools/mirror-github.mjs
T

111 lines
8.4 KiB
JavaScript

// GitHub HTTP transport for human-run mirrors: serial, bounded retries, no token
// required for public REST reads. State contains timestamps only, never tokens.
import fs from 'node:fs';
import path from 'node:path';
import { setTimeout as pause } from 'node:timers/promises';
import { spawnSync, execFile } from 'node:child_process';
import { promisify } from 'node:util';
import { parseCliResponse } from './mirror-rate-limits.mjs';
import { fileURLToPath } from 'node:url';
const root=path.resolve(path.dirname(fileURLToPath(import.meta.url)),'..');
export function retryDelay(headers, now, attempt=0) {
const retry=headers.get('retry-after'),reset=Number(headers.get('x-ratelimit-reset'))*1000;
const retryMs=retry===null?0:/^\d+(?:\.\d+)?$/.test(retry)?Number(retry)*1000:Math.max(0,Date.parse(retry)-now);
const primary=headers.get('x-ratelimit-remaining')==='0'&&Number.isFinite(reset)?Math.max(0,reset-now)+1000:0;
return Math.max(Number.isFinite(retryMs)?retryMs:0,primary,60000*2**attempt);
}
function tokenFromCli() {
const result=spawnSync('gh',['auth','token','--hostname','github.com'],{encoding:'utf8',shell:false,windowsHide:true,timeout:10000,stdio:['ignore','pipe','pipe']});
return result.status===0?result.stdout.trim():'';
}
export function githubCliAvailable() {
return spawnSync('gh',['--version'],{shell:false,windowsHide:true,timeout:10000,stdio:'ignore'}).status===0;
}
export async function githubCliRead(url, headers, credential, execute=promisify(execFile)) {
const args=['api',url,'--method','GET','--include'];
for(const [name,value] of Object.entries(headers)) if(name.toLowerCase()!=='authorization') args.push('--header',`${name}: ${value}`);
const options={encoding:'utf8',shell:false,windowsHide:true,timeout:120000,maxBuffer:16*1024*1024,
env:{...process.env,GH_TOKEN:credential,GH_PROMPT_DISABLED:'1'}};
let result;
try { result={...await execute('gh',args,options),status:0}; }
catch(error) {
// gh exits nonzero for HTTP errors; preserve their rate-limit headers.
if(!/^HTTP\/\S+\s+\d{3}/m.test(error.stdout||'')) throw Error('GitHub CLI request failed or timed out');
result={stdout:error.stdout,stderr:error.stderr,status:1};
}
const parsed=parseCliResponse(result);
if(result.status!==0 && parsed.response.status<400) throw Error('GitHub CLI request failed');
return new Response([204,205,304].includes(parsed.response.status)?null:parsed.body,
{status:parsed.response.status,headers:parsed.response.headers});
}
export function createGithubClient({fetcher=fetch,sleep=pause,now=Date.now,log=console.log,tokenLoader=tokenFromCli,token,
cliAvailable=token===undefined&&tokenLoader===tokenFromCli?githubCliAvailable:()=>false,
cliRead=githubCliRead,stateFile=path.join(root,'.tools/mirror/github-rate-limit.json')}={}) {
let queue=Promise.resolve(),credential=token,useCli;
const load=()=>{if(!fs.existsSync(stateFile))return {};const state=JSON.parse(fs.readFileSync(stateFile,'utf8'));if(state.version!==1||!state.until||Object.values(state.until).some(n=>!Number.isFinite(n)))throw Error('Invalid GitHub rate-limit state');return state.until;};
const save=(resource,until)=>{const state=load();state[resource]=Math.max(state[resource]||0,until);fs.mkdirSync(path.dirname(stateFile),{recursive:true});const incoming=stateFile+`.incoming-${process.pid}`;fs.writeFileSync(incoming,JSON.stringify({version:1,until:state})+'\n');fs.renameSync(incoming,stateFile);};
async function wait(until) {let remaining=until-now();if(remaining<=0)return;log(`[github] rate limit: waiting ${Math.ceil(remaining/1000)} seconds`);while(remaining>0){await sleep(Math.min(remaining,60000));remaining=until-now();}}
async function request(endpoint,{method='GET',data,headers:extra={}}={}) {
let url=new URL(endpoint.startsWith('https://')?endpoint:`https://api.github.com/${endpoint.replace(/^\//,'')}`);
if(!['api.github.com','uploads.github.com'].includes(url.hostname)||url.protocol!=='https:'||url.username||url.password)throw Error('Unsupported GitHub API URL');
if(useCli===undefined) {
const available=cliAvailable();
if(credential===undefined)credential=process.env.GH_TOKEN||process.env.GITHUB_TOKEN||
(available||tokenLoader!==tokenFromCli?tokenLoader():'')||'';
useCli=available&&Boolean(credential);
log(useCli?'[github] Using authenticated gh api for source metadata':
`[github] Using ${credential?'authenticated':'public unauthenticated'} HTTP API; gh ${available?'has no authentication':'is unavailable'}`);
}
const graphql=url.pathname==='/graphql';
const query=graphql&&method==='POST'&&typeof data?.query==='string'&&/^\s*(?:query\b|\{)/.test(data.query);
if(!credential && method!=='GET' && method!=='HEAD' && !query)throw Error('GitHub token is required for remote writes');
if(!credential && graphql)throw Error('Public Projects GraphQL inventory requires a GitHub token; public REST mirroring remains available');
const resource=graphql?'graphql':'core';
const headers={Accept:'application/vnd.github+json','X-GitHub-Api-Version':'2022-11-28','User-Agent':'WeKan-mirror',...extra,...(credential?{Authorization:`Bearer ${credential}`}:{})};
const body=data===undefined?undefined:data instanceof Blob?data:JSON.stringify(data);
if(data!==undefined && !(data instanceof Blob))headers['Content-Type']='application/json';
let redirects=0;
for(let attempt=0;attempt<6;attempt++) {
await wait(load()[resource]||0);
log(`[github] ${method} ${url.hostname}${url.pathname}${url.searchParams.has('page') ? ` page ${url.searchParams.get('page')}` : ''} (attempt ${attempt + 1}/6)`);
const response=useCli&&method==='GET'&&url.hostname==='api.github.com'&&headers.Accept==='application/vnd.github+json'
? await cliRead(url.href,headers,credential)
: await fetcher(url.href,{method,headers,body,redirect:'manual',signal:AbortSignal.timeout(3600000)});
if(response.headers.get('x-ratelimit-remaining')==='0') {
const reset=Number(response.headers.get('x-ratelimit-reset'))*1000;
if(Number.isFinite(reset)&&reset>now())save(resource,reset+1000);
}
if([301,302,303,307,308].includes(response.status)) {
const next=new URL(response.headers.get('location')||'',url);await response.body?.cancel();
const fileRedirect=method==='GET'&&extra.Accept==='application/octet-stream'&&(next.hostname.endsWith('.githubusercontent.com')||/^github-production-[a-z0-9-]+\.s3(?:[.-][a-z0-9-]+)?\.amazonaws\.com$/.test(next.hostname));
if(++redirects>10 || (!fileRedirect && next.hostname!==url.hostname) || next.protocol!=='https:' || next.username || next.password)throw Error('Unsupported GitHub API redirect');
if(next.hostname!==url.hostname)delete headers.Authorization;
url=next;attempt--;continue;
}
if(response.status===401 && credential && (method==='GET'||query)) {
await response.body?.cancel();credential='';useCli=false;delete headers.Authorization;
log('[github] token rejected; retrying public REST access without a token');
if(graphql)throw Error('GitHub token rejected for Projects GraphQL inventory');
continue;
}
if([403,429,503].includes(response.status)) {
const message=await response.text();
if(response.status===429 || response.headers.has('retry-after') || response.headers.get('x-ratelimit-remaining')==='0' || /(?:secondary |API )?rate limit|abuse detection/i.test(message)) {
const until=now()+retryDelay(response.headers,now(),attempt);save(resource,until);
if(attempt===5)throw Error('GitHub rate limit persisted after six attempts; cooldown saved for the next run');
await wait(until);continue;
}
throw Error(`GitHub HTTP ${response.status}: ${message.slice(0,500)}`);
}
if(!response.ok)throw Error(`GitHub HTTP ${response.status}: ${(await response.text()).slice(0,500)}`);
// One-second spacing also obeys guidance for mutative request bursts.
save(resource,Math.max(load()[resource]||0,now()+1000));
return response;
}
throw Error('GitHub request did not complete');
}
return (endpoint,options)=>{const result=queue.then(()=>request(endpoint,options));queue=result.then(()=>{},()=>{});return result;};
}
export const githubRequest=createGithubClient();
export async function githubJson(endpoint,method='GET',data) {return (await githubRequest(endpoint,{method,data})).json();}