const { EventEmitter } = require("node:events"); const { fetch: undiciFetch, Agent } = require("undici"); const Log = require("logger"); const { getUserAgent } = require("#server_functions"); const FIFTEEN_MINUTES = 15 * 60 * 1000; const THIRTY_MINUTES = 30 * 60 * 1000; const MAX_SERVER_BACKOFF = 3; const DEFAULT_TIMEOUT = 30000; // 30 seconds /** * Maps errorType to MagicMirror translation keys. * This allows HTTPFetcher to provide ready-to-use translation keys, * eliminating the need to call NodeHelper.checkFetchError(). */ const ERROR_TYPE_TO_TRANSLATION = { AUTH_FAILURE: "MODULE_ERROR_UNAUTHORIZED", RATE_LIMITED: "MODULE_ERROR_RATE_LIMITED", SERVER_ERROR: "MODULE_ERROR_SERVER_ERROR", CLIENT_ERROR: "MODULE_ERROR_CLIENT_ERROR", NETWORK_ERROR: "MODULE_ERROR_NO_CONNECTION", UNKNOWN_ERROR: "MODULE_ERROR_UNSPECIFIED" }; /** * HTTPFetcher - Centralized HTTP fetching with intelligent error handling * * Features: * - Automatic retry strategies based on HTTP status codes * - Exponential backoff for server errors * - Retry-After header parsing for rate limiting * - Authentication support (Basic, Bearer) * - Self-signed certificate support * @augments EventEmitter * @fires HTTPFetcher#response - When fetch succeeds with ok response * @fires HTTPFetcher#error - When fetch fails or returns non-ok response * @example * const fetcher = new HTTPFetcher(url, { reloadInterval: 60000 }); * fetcher.on('response', (response) => { ... }); * fetcher.on('error', (errorInfo) => { ... }); * fetcher.startPeriodicFetch(); */ class HTTPFetcher extends EventEmitter { /** * Calculates exponential backoff delay for retries * @param {number} attempt - Attempt number (1-based) * @param {object} options - Configuration options * @param {number} [options.baseDelay] - Initial delay in ms (default: 15s) * @param {number} [options.maxDelay] - Maximum delay in ms (default: 5min) * @returns {number} Delay in milliseconds * @example * HTTPFetcher.calculateBackoffDelay(1) // 15000 (15s) * HTTPFetcher.calculateBackoffDelay(2) // 30000 (30s) * HTTPFetcher.calculateBackoffDelay(3) // 60000 (60s) * HTTPFetcher.calculateBackoffDelay(6) // 300000 (5min, capped) */ static calculateBackoffDelay (attempt, { baseDelay = 15000, maxDelay = 300000 } = {}) { return Math.min(baseDelay * Math.pow(2, attempt - 1), maxDelay); } /** * Creates a new HTTPFetcher instance * @param {string} url - The URL to fetch * @param {object} options - Configuration options * @param {number} [options.reloadInterval] - Time in ms between fetches (default: 5 min) * @param {object} [options.auth] - Authentication options * @param {string} [options.auth.method] - 'basic' or 'bearer' * @param {string} [options.auth.user] - Username for basic auth * @param {string} [options.auth.pass] - Password or token * @param {boolean} [options.selfSignedCert] - Accept self-signed certificates * @param {object} [options.headers] - Additional headers to send * @param {number} [options.maxRetries] - Max retries for 5xx errors (default: 3) * @param {number} [options.timeout] - Request timeout in ms (default: 30000) * @param {string} [options.logContext] - Optional context for log messages (e.g., provider name) */ constructor (url, options = {}) { super(); this.url = url; this.reloadInterval = options.reloadInterval || 5 * 60 * 1000; this.auth = options.auth || null; this.selfSignedCert = options.selfSignedCert || false; this.customHeaders = options.headers || {}; this.maxRetries = options.maxRetries || MAX_SERVER_BACKOFF; this.timeout = options.timeout || DEFAULT_TIMEOUT; this.logContext = options.logContext ? `[${options.logContext}] ` : ""; this.reloadTimer = null; this.serverErrorCount = 0; this.networkErrorCount = 0; } /** * Clears any pending reload timer */ clearTimer () { if (this.reloadTimer) { clearTimeout(this.reloadTimer); this.reloadTimer = null; } } /** * Schedules the next fetch. * If no delay is provided, uses reloadInterval. * If delay is provided but very short (< 1 second), clamps to reloadInterval * to prevent hammering servers. * @param {number} [delay] - Delay in milliseconds */ scheduleNextFetch (delay) { let nextDelay = delay ?? this.reloadInterval; // Only clamp if delay is unreasonably short (< 1 second) // This allows respecting Retry-After headers while preventing abuse if (nextDelay < 1000) { nextDelay = this.reloadInterval; } // Don't schedule in test mode if (process.env.mmTestMode === "true") { return; } this.reloadTimer = setTimeout(() => this.fetch(), nextDelay); } /** * Starts periodic fetching */ startPeriodicFetch () { this.fetch(); } /** * Builds the options object for fetch * @returns {object} Options object containing headers (and dispatcher if needed) */ getRequestOptions () { const headers = { "User-Agent": getUserAgent(), ...this.customHeaders }; const options = { headers }; if (this.selfSignedCert) { options.dispatcher = new Agent({ connect: { rejectUnauthorized: false } }); } if (this.auth) { if (this.auth.method === "bearer") { headers.Authorization = `Bearer ${this.auth.pass}`; } else { headers.Authorization = `Basic ${Buffer.from(`${this.auth.user}:${this.auth.pass}`).toString("base64")}`; } } return options; } /** * Parses the Retry-After header value * @param {string} retryAfter - The Retry-After header value * @returns {number|null} Milliseconds to wait or null if parsing failed */ #parseRetryAfter (retryAfter) { // Try parsing as seconds const seconds = Number(retryAfter); if (!Number.isNaN(seconds) && seconds >= 0) { return seconds * 1000; } // Try parsing as HTTP-date const retryDate = Date.parse(retryAfter); if (!Number.isNaN(retryDate)) { return Math.max(0, retryDate - Date.now()); } return null; } /** * Determines the retry delay for a non-ok response * @param {Response} response - The fetch Response object * @returns {{delay: number, errorInfo: object}} Computed retry delay and error info */ #getDelayForResponse (response) { const { status } = response; let delay = this.reloadInterval; let message; let errorType = "UNKNOWN_ERROR"; if (status === 401 || status === 403) { errorType = "AUTH_FAILURE"; delay = Math.max(this.reloadInterval * 5, THIRTY_MINUTES); message = `Authentication failed (${status}). Check your API key. Waiting ${Math.round(delay / 60000)} minutes before retry.`; Log.error(`${this.logContext}${this.url} - ${message}`); } else if (status === 429) { errorType = "RATE_LIMITED"; const retryAfter = response.headers.get("retry-after"); const parsed = retryAfter ? this.#parseRetryAfter(retryAfter) : null; delay = parsed !== null ? Math.max(parsed, this.reloadInterval) : Math.max(this.reloadInterval * 2, FIFTEEN_MINUTES); message = `Rate limited (429). Retrying in ${Math.round(delay / 60000)} minutes.`; Log.warn(`${this.logContext}${this.url} - ${message}`); } else if (status >= 500) { errorType = "SERVER_ERROR"; this.serverErrorCount = Math.min(this.serverErrorCount + 1, this.maxRetries); delay = this.reloadInterval * Math.pow(2, this.serverErrorCount); message = `Server error (${status}). Retry #${this.serverErrorCount} in ${Math.round(delay / 60000)} minutes.`; Log.error(`${this.logContext}${this.url} - ${message}`); } else if (status >= 400) { errorType = "CLIENT_ERROR"; delay = Math.max(this.reloadInterval * 2, FIFTEEN_MINUTES); message = `Client error (${status}). Retrying in ${Math.round(delay / 60000)} minutes.`; Log.error(`${this.logContext}${this.url} - ${message}`); } else { message = `Unexpected HTTP status ${status}.`; Log.error(`${this.logContext}${this.url} - ${message}`); } return { delay, errorInfo: this.#createErrorInfo(message, status, errorType, delay) }; } /** * Creates a standardized error info object * @param {string} message - Error message * @param {number|null} status - HTTP status code or null for network errors * @param {string} errorType - Error type: AUTH_FAILURE, RATE_LIMITED, SERVER_ERROR, CLIENT_ERROR, NETWORK_ERROR * @param {number} retryAfter - Delay until next retry in ms * @param {Error} [originalError] - The original error if any * @returns {object} Error info object with translationKey for direct use */ #createErrorInfo (message, status, errorType, retryAfter, originalError = null) { return { message, status, errorType, translationKey: ERROR_TYPE_TO_TRANSLATION[errorType] || "MODULE_ERROR_UNSPECIFIED", retryAfter, retryCount: errorType === "NETWORK_ERROR" ? this.networkErrorCount : this.serverErrorCount, url: this.url, originalError }; } /** * Performs the HTTP fetch and emits appropriate events * @fires HTTPFetcher#response * @fires HTTPFetcher#error */ async fetch () { this.clearTimer(); let nextDelay = this.reloadInterval; const controller = new AbortController(); const timeoutId = setTimeout(() => controller.abort(), this.timeout); try { const requestOptions = this.getRequestOptions(); // Use undici.fetch when a custom dispatcher is present (e.g. selfSignedCert), // because Node's global fetch and npm undici@8 Agents are incompatible. // For regular requests, use globalThis.fetch so MSW and other interceptors work. const fetchFn = requestOptions.dispatcher ? undiciFetch : globalThis.fetch; const response = await fetchFn(this.url, { ...requestOptions, signal: controller.signal }); if (!response.ok) { const { delay, errorInfo } = this.#getDelayForResponse(response); nextDelay = delay; this.emit("error", errorInfo); } else { // Reset error counts on success this.serverErrorCount = 0; this.networkErrorCount = 0; /** * Response event - fired when fetch succeeds * @event HTTPFetcher#response * @type {Response} */ this.emit("response", response); } } catch (error) { const isTimeout = error.name === "AbortError"; const message = isTimeout ? `Request timeout after ${this.timeout}ms` : `Network error: ${error.message}`; // Apply exponential backoff for network errors this.networkErrorCount = Math.min(this.networkErrorCount + 1, this.maxRetries); const backoffDelay = HTTPFetcher.calculateBackoffDelay(this.networkErrorCount, { maxDelay: this.reloadInterval }); nextDelay = backoffDelay; // Truncate URL for cleaner logs let shortUrl = this.url; try { const urlObj = new URL(this.url); shortUrl = `${urlObj.origin}${urlObj.pathname}${urlObj.search.length > 50 ? "?..." : urlObj.search}`; } catch { // If URL parsing fails, use original URL } // Gradual log-level escalation: WARN for first 2 attempts, ERROR after const retryMessage = `Retry #${this.networkErrorCount} in ${Math.round(nextDelay / 1000)}s.`; if (this.networkErrorCount <= 2) { Log.warn(`${this.logContext}${shortUrl} - ${message} ${retryMessage}`); } else { Log.error(`${this.logContext}${shortUrl} - ${message} ${retryMessage}`); } const errorInfo = this.#createErrorInfo( message, null, "NETWORK_ERROR", nextDelay, error ); /** * Error event - fired when fetch fails * @event HTTPFetcher#error * @type {object} * @property {string} message - Error description * @property {number|null} statusCode - HTTP status or null for network errors * @property {number} retryDelay - Ms until next retry * @property {number} retryCount - Number of consecutive server errors * @property {string} url - The URL that was fetched * @property {Error|null} originalError - The original error */ this.emit("error", errorInfo); } finally { clearTimeout(timeoutId); } this.scheduleNextFetch(nextDelay); } } module.exports = HTTPFetcher;