Source previews
bundle.js
43292 bytes · SHA-256 a12e6c7dbc44b8a940d6aa633cc9d849fa8494ec725d0e0f946a507605b202e1
var __create = Object.create;
var __defProp = Object.defineProperty;
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
var __getOwnPropNames = Object.getOwnPropertyNames;
var __getProtoOf = Object.getPrototypeOf;
var __hasOwnProp = Object.prototype.hasOwnProperty;
var __copyProps = (to, from, except, desc) => {
if (from && typeof from === "object" || typeof from === "function") {
for (let key of __getOwnPropNames(from))
if (!__hasOwnProp.call(to, key) && key !== except)
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
}
return to;
};
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
// If the importer is in node compatibility mode or this is not an ESM
// file that has been converted to a CommonJS file using a Babel-
// compatible transform (i.e. "__esModule" has not been set), then set
// "default" to the CommonJS "module.exports" for node compatibility.
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
mod
));
// cert-fields.js
var COMPONENT_OIDS = {
sct: "1.3.6.1.4.1.11129.2.4.2",
ski: "2.5.29.14",
aki: "2.5.29.35"
};
var ALIASES = { spki: "publicKey", pubkey: "publicKey", publickey: "publicKey" };
var DEFAULT_DROP = ["publicKey", "sct", "ski", "aki"];
function parseDropSpec(spec = DEFAULT_DROP) {
const tokens = Array.isArray(spec) ? spec : String(spec).split(",");
const drop = { serial: false, publicKey: false, oids: /* @__PURE__ */ new Set() };
for (const raw of tokens) {
const token = raw.trim();
if (!token) continue;
const name = ALIASES[token.toLowerCase()] ?? token;
if (name === "serial") drop.serial = true;
else if (name === "publicKey") drop.publicKey = true;
else if (COMPONENT_OIDS[name.toLowerCase()]) drop.oids.add(COMPONENT_OIDS[name.toLowerCase()]);
else if (/^\d+(\.\d+)+$/.test(name)) drop.oids.add(name);
else {
throw new Error(
`cert-fields: unknown component "${token}" (names: serial, publicKey, ${Object.keys(COMPONENT_OIDS).join(", ")}, or a dotted OID)`
);
}
}
return drop;
}
// scraper/config.js
var env = (name) => typeof process !== "undefined" ? process.env?.[name] : void 0;
var str = (baked, name, dflt) => baked ?? env(name) ?? dflt;
var num = (baked, name, dflt) => Number(baked ?? env(name) ?? dflt);
var ORCH_URL = str(
true ? "https://crtl.papers.tech/" : void 0,
"ORCH_URL",
"http://127.0.0.1:8080"
);
var FLEET_ID = str(
typeof SCRAPER_FLEET_ID !== "undefined" ? SCRAPER_FLEET_ID : void 0,
"FLEET_ID",
"dev-fleet"
);
var JOB_TAG = str(
typeof SCRAPER_JOB_TAG !== "undefined" ? SCRAPER_JOB_TAG : void 0,
"JOB_TAG",
"dev"
);
var CHUNK_ENTRIES = Math.min(
num(typeof SCRAPER_CHUNK_ENTRIES !== "undefined" ? SCRAPER_CHUNK_ENTRIES : void 0, "CHUNK_ENTRIES", 1e4),
5e4
);
var FETCH_INITIAL_WINDOW = num(
typeof SCRAPER_FETCH_INITIAL_WINDOW !== "undefined" ? SCRAPER_FETCH_INITIAL_WINDOW : void 0,
"FETCH_INITIAL_WINDOW",
256
);
var FETCH_MAX_ATTEMPTS = num(
true ? 30 : void 0,
"FETCH_MAX_ATTEMPTS",
8
);
var FETCH_TIMEOUT_MS = num(
true ? 6e4 : void 0,
"FETCH_TIMEOUT_MS",
3e4
);
var STOP_MARGIN_MS = num(
true ? 18e4 : void 0,
"STOP_MARGIN_MS",
45e3
);
var FETCH_CONCURRENCY = num(
true ? 8 : void 0,
"FETCH_CONCURRENCY",
8
);
var CHUNK_CONCURRENCY = num(
true ? 6 : void 0,
"CHUNK_CONCURRENCY",
4
);
var HOST_RATE_LIMITS = str(
typeof SCRAPER_HOST_RATE_LIMITS !== "undefined" ? SCRAPER_HOST_RATE_LIMITS : void 0,
"HOST_RATE_LIMITS",
"ct.cloudflare.com=3,skylight.geomys.org=0.5"
);
var EXECUTION_TIME_MS = num(
true ? 36e6 : void 0,
"EXECUTION_TIME_MS",
9e5
);
var CERT_DROP = str(
typeof SCRAPER_CERT_DROP !== "undefined" ? SCRAPER_CERT_DROP : void 0,
"CERT_DROP",
DEFAULT_DROP.join(",")
);
// scraper/fetcher.js
var import_node_zlib = require("node:zlib");
// scraper/transport.js
var import_node_http = __toESM(require("node:http"), 1);
var import_node_https = __toESM(require("node:https"), 1);
var agents = {
"http:": new import_node_http.default.Agent({ keepAlive: true, maxSockets: 128 }),
"https:": new import_node_https.default.Agent({ keepAlive: true, maxSockets: 128 })
};
var transportStats = { statuses: /* @__PURE__ */ new Map() };
function httpRequest(url, { method = "GET", headers = {}, body, timeoutMs = 3e4 } = {}) {
return new Promise((resolve, reject) => {
const u = new URL(url);
const mod = u.protocol === "http:" ? import_node_http.default : import_node_https.default;
const req = mod.request(
u,
{ method, headers, agent: agents[u.protocol], timeout: timeoutMs },
(res) => {
transportStats.statuses.set(res.statusCode, (transportStats.statuses.get(res.statusCode) ?? 0) + 1);
const chunks = [];
res.on("data", (c) => chunks.push(c));
res.on(
"end",
() => resolve({ status: res.statusCode, headers: res.headers, body: Buffer.concat(chunks) })
);
res.on("error", reject);
}
);
req.on("timeout", () => req.destroy(new Error(`request to ${u.host} timed out after ${timeoutMs}ms`)));
req.on("error", reject);
if (body !== void 0) req.write(body);
req.end();
});
}
// scraper/fetcher.js
async function httpsTransport(url, timeoutMs) {
const res = await httpRequest(url, { timeoutMs });
return {
status: res.status,
retryAfter: res.headers["retry-after"] ?? null,
encoding: res.headers["content-encoding"] ?? null,
body: res.body
};
}
var newFetchStats = () => ({ requests: 0, retries: 0, status429: 0, netMs: 0, pacedMs: 0, backoffMs: 0 });
var DEFAULTS = {
initialWindow: 256,
minWindow: 1,
maxAttempts: 8,
baseDelayMs: 500,
maxDelayMs: 3e4,
timeoutMs: 3e4,
transport: httpsTransport,
tileSize: 256,
// requests never cross a tile boundary; 0 disables
regrowAfter: 8,
// full responses in a row before the window doubles back
hostRateLimits: {}
// { 'ct.cloudflare.com': 3 } — requests/s per host suffix, device-wide
};
var HostLimiter = class {
constructor(rps) {
this.intervalMs = 1e3 / rps;
this.nextAt = 0;
}
async acquire() {
const now = Date.now();
const at = Math.max(now, this.nextAt);
this.nextAt = at + this.intervalMs;
if (at > now) await sleep(at - now);
}
defer(ms) {
this.nextAt = Math.max(this.nextAt, Date.now() + ms);
}
};
var limiters = /* @__PURE__ */ new Map();
function limiterFor(host, limits) {
for (const [suffix, rps] of Object.entries(limits ?? {})) {
if (!(rps > 0)) continue;
if (host === suffix || host.endsWith(`.${suffix}`)) {
const key = `${suffix}@${rps}`;
if (!limiters.has(key)) limiters.set(key, new HostLimiter(rps));
return limiters.get(key);
}
}
return null;
}
function parseHostRateLimits(spec) {
const out = {};
for (const part of String(spec ?? "").split(",")) {
const [host, rps] = part.split("=").map((x) => x.trim());
if (host && Number(rps) > 0) out[host.toLowerCase()] = Number(rps);
}
return out;
}
var FetchExhaustedError = class extends Error {
constructor(message, opts) {
super(message, opts);
this.name = "FetchExhaustedError";
}
};
var HttpError = class extends Error {
constructor(status, url) {
super(`HTTP ${status} for ${url}`);
this.name = "HttpError";
this.status = status;
}
};
function parseRetryAfter(header) {
if (!header) return null;
const secs = Number(header);
if (Number.isFinite(secs)) return Math.max(0, secs);
const at = Date.parse(header);
return Number.isNaN(at) ? null : Math.max(0, Math.ceil((at - Date.now()) / 1e3));
}
var RetryableHttpError = class extends HttpError {
constructor(status, url, retryAfter) {
super(status, url);
this.name = "RetryableHttpError";
this.retryAfter = parseRetryAfter(retryAfter);
}
};
var AdaptiveFetcher = class {
#window;
#opts;
#ceiling;
// largest window the server has not rejected with 400
#fullRun = 0;
// consecutive full responses at the current window
#limiter;
Preview truncated.
manifest.json
86 bytes · SHA-256 346de335a6301b627fca8c492c58599822962f981bf5f64cf5705c43a390626f
Preview truncated.