Feature: allow disable ipv6 in proxy, refactor cacheFetch to use proxy (#5011)

src/pages/api/releases.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import createLogger from "utils/logger";
 
 const logger = createLogger("releases");
@@ -6,7 +6,7 @@ const logger = createLogger("releases");
 export default async function handler(req, res) {
   const releasesURL = "https://api.github.com/repos/gethomepage/homepage/releases";
   try {
-    return res.send(await cachedFetch(releasesURL, 5));
+    return res.send(await cachedRequest(releasesURL, 5));
   } catch (e) {
     logger.error(`Error checking GitHub releases: ${e}`);
     return res.send([]);

src/pages/api/search/searchSuggestion.js

@@ -1,7 +1,7 @@
 import { searchProviders } from "components/widgets/search/search";
 
 import { getSettings } from "utils/config/config";
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { widgetsFromConfig } from "utils/config/widget-helpers";
 
 export default async function handler(req, res) {
@@ -29,5 +29,5 @@ export default async function handler(req, res) {
     return res.json([query, []]); // Responde with the same array format but with no suggestions.
   }
 
-  return res.send(await cachedFetch(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0"));
+  return res.send(await cachedRequest(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0"));
 }

src/pages/api/widgets/openmeteo.js

@@ -1,9 +1,9 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 
 export default async function handler(req, res) {
   const { latitude, longitude, units, cache, timezone } = req.query;
   const degrees = units === "metric" ? "celsius" : "fahrenheit";
   const timezeone = timezone ?? "auto";
   const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&daily=sunrise,sunset&current_weather=true&temperature_unit=${degrees}&timezone=${timezeone}`;
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/pages/api/widgets/openweathermap.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import { getPrivateWidgetOptions } from "utils/config/widget-helpers";
 
@@ -26,5 +26,5 @@ export default async function handler(req, res) {
 
   const apiUrl = `https://api.openweathermap.org/data/2.5/weather?lat=${latitude}&lon=${longitude}&appid=${apiKey}&units=${units}&lang=${lang}`;
 
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/pages/api/widgets/stocks.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import createLogger from "utils/logger";
 
@@ -60,7 +60,7 @@ export default async function handler(req, res) {
         const apiUrl = `https://finnhub.io/api/v1/quote?symbol=${ticker}&token=${apiKey}`;
         // Finnhub free accounts allow up to 60 calls/minute
         // https://finnhub.io/pricing
-        const { c, dp } = await cachedFetch(apiUrl, cache || 1);
+        const { c, dp } = await cachedRequest(apiUrl, cache || 1);
         logger.debug("Finnhub API response for %s: %o", ticker, { c, dp });
 
         // API sometimes returns 200, but values returned are `null`

src/pages/api/widgets/weather.js

@@ -1,4 +1,4 @@
-import cachedFetch from "utils/proxy/cached-fetch";
+import { cachedRequest } from "utils/proxy/http";
 import { getSettings } from "utils/config/config";
 import { getPrivateWidgetOptions } from "utils/config/widget-helpers";
 
@@ -26,5 +26,5 @@ export default async function handler(req, res) {
 
   const apiUrl = `http://api.weatherapi.com/v1/current.json?q=${latitude},${longitude}&key=${apiKey}&lang=${lang}`;
 
-  return res.send(await cachedFetch(apiUrl, cache));
+  return res.send(await cachedRequest(apiUrl, cache));
 }

src/utils/config/widget-helpers.js

@@ -56,21 +56,22 @@ export async function cleanWidgetGroups(widgets) {
 export async function getPrivateWidgetOptions(type, widgetIndex) {
   const widgets = await widgetsFromConfig();
 
-  const privateOptions = widgets.map((widget) => {
-    const { index, url, username, password, key, apiKey } = widget.options;
+  const privateOptions =
+    widgets.map((widget) => {
+      const { index, url, username, password, key, apiKey } = widget.options;
 
-    return {
-      type: widget.type,
-      options: {
-        index,
-        url,
-        username,
-        password,
-        key,
-        apiKey,
-      },
-    };
-  });
+      return {
+        type: widget.type,
+        options: {
+          index,
+          url,
+          username,
+          password,
+          key,
+          apiKey,
+        },
+      };
+    }) || {};
 
   return type !== undefined && widgetIndex !== undefined
     ? privateOptions.find((o) => o.type === type && o.options.index === parseInt(widgetIndex, 10))?.options

src/utils/proxy/cached-fetch.js

@@ -1,25 +0,0 @@
-import cache from "memory-cache";
-
-const defaultDuration = 5;
-
-export default async function cachedFetch(url, duration, ua) {
-  const cached = cache.get(url);
-
-  // eslint-disable-next-line no-param-reassign
-  duration = duration || defaultDuration;
-
-  if (cached) {
-    return cached;
-  }
-
-  // wrapping text in JSON.parse to handle utf-8 issues
-  const options = {};
-  if (ua) {
-    options.headers = {
-      "User-Agent": ua,
-    };
-  }
-  const data = await fetch(url, options).then((res) => res.json());
-  cache.put(url, data, duration * 1000 * 60);
-  return data;
-}

src/utils/proxy/http.js

@@ -3,6 +3,7 @@
 import { createUnzip, constants as zlibConstants } from "node:zlib";
 
 import { http, https } from "follow-redirects";
+import cache from "memory-cache";
 
 import { addCookieToJar, setCookieHeader } from "./cookie-jar";
 import { sanitizeErrorURL } from "./api-helpers";
@@ -81,20 +82,46 @@ export function httpRequest(url, params) {
   return handleRequest(http, url, params);
 }
 
+export async function cachedRequest(url, duration = 5, ua = "homepage") {
+  const cached = cache.get(url);
+
+  if (cached) {
+    return cached;
+  }
+
+  const options = {
+    headers: {
+      "User-Agent": ua,
+      Accept: "application/json",
+    },
+  };
+  let [, , data] = await httpProxy(url, options);
+  if (Buffer.isBuffer(data)) {
+    try {
+      data = JSON.parse(Buffer.from(data).toString());
+    } catch (e) {
+      logger.debug("Error parsing cachedRequest data for %s: %s %s", url, Buffer.from(data).toString(), e);
+      data = Buffer.from(data).toString();
+    }
+  }
+  cache.put(url, data, duration * 1000 * 60);
+  return data;
+}
+
 export async function httpProxy(url, params = {}) {
   const constructedUrl = new URL(url);
+  const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true";
+  const agentOptions = disableIpv6 ? { family: 4, autoSelectFamily: false } : {};
 
   let request = null;
   if (constructedUrl.protocol === "https:") {
     request = httpsRequest(constructedUrl, {
-      agent: new https.Agent({
-        rejectUnauthorized: false,
-      }),
+      agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }),
       ...params,
     });
   } else {
     request = httpRequest(constructedUrl, {
-      agent: new http.Agent(),
+      agent: new http.Agent(agentOptions),
       ...params,
     });
   }