Enhancement: Allow Disabling Indexing (#5954)

Co-authored-by: shamoon <4887959+shamoon@users.noreply.github.com>

docs/configs/settings.md

@@ -571,3 +571,18 @@ or per service widget (`services.yaml`) with:
 ```
 
 If either value is set to true, the error message will be hidden.
+
+## Disable Search Engine Indexing
+
+You can request that search engines not to index your Homepage instance by enabling the `disableIndexing` setting.
+
+```yaml
+disableIndexing: true
+```
+
+When enabled, this will:
+
+- Disallow all crawlers in `robots.txt`
+- Add `<meta name="robots" content="noindex, nofollow">` tags to prevent indexing
+
+By default this feature is disabled.

src/pages/index.jsx

@@ -400,6 +400,7 @@ function Home({ initialSettings }) {
             "A highly customizable homepage (or startpage / application dashboard) with Docker and service API integrations."
           }
         />
+        {settings.disableIndexing && <meta name="robots" content="noindex, nofollow" />}
         {settings.base && <base href={settings.base} />}
         {settings.favicon ? (
           <>

src/pages/robots.txt.js

@@ -0,0 +1,19 @@
+import { getSettings } from "utils/config/config";
+
+export async function getServerSideProps({ res }) {
+  const settings = getSettings();
+  const content = ["User-agent: *", !!settings.disableIndexing ? "Disallow: /" : "Allow: /"].join("\n");
+
+  res.setHeader("Content-Type", "text/plain");
+  res.write(content);
+  res.end();
+
+  return {
+    props: {},
+  };
+}
+
+export default function RobotsTxt() {
+  // placeholder component
+  return null;
+}