chore: block search engine indexing with noindex signals

.github/workflows/build-and-lint.yml

@@ -100,7 +100,7 @@ jobs:
       - name: Test
         run: npm test
 
-      - name: Smoke test - server renders dynamic routes
+      - name: Smoke test - server renders dynamic routes and serves noindex
         env:
           NEXT_PUBLIC_HASURA_GRAPHQL_URL: "https://unreachable.invalid/v1/graphql"
           NEXT_PUBLIC_POLL_INTERVAL: "30000"
@@ -127,12 +127,51 @@ jobs:
 
           ROOT_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/")
           REQUEST_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/request/0x1234567890abcdef1234567890abcdef12345678-0")
+          XROBOTS_HEADER=$(curl -sI "http://localhost:3000/" | tr -d '\r' | grep -i '^x-robots-tag:' || true)
+          ROBOTS_TXT_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/robots.txt")
+          ROBOTS_TXT_BODY=$(curl -s "http://localhost:3000/robots.txt" | tr -d '\r')
 
-          echo "Observed status codes: / -> $ROOT_STATUS, /request/<id> -> $REQUEST_STATUS"
+          echo "Observed status codes: / -> $ROOT_STATUS, /request/<id> -> $REQUEST_STATUS, /robots.txt -> $ROBOTS_TXT_STATUS"
+          echo "Observed header: $XROBOTS_HEADER"
+          echo "Observed /robots.txt body:"
+          echo "$ROBOTS_TXT_BODY"
 
           kill "$SERVER_PID" 2>/dev/null || true
 
           if [ "$ROOT_STATUS" -ge 500 ] || [ "$REQUEST_STATUS" -ge 500 ]; then
             echo "Smoke test failed: server returned a 5xx status code"
             exit 1
           fi
+
+          # The full policy must be present: dropping either token weakens de-indexing.
+          if ! echo "$XROBOTS_HEADER" | grep -qi "noindex"; then
+            echo "Smoke test failed: X-Robots-Tag is missing 'noindex' on /"
+            exit 1
+          fi
+
+          if ! echo "$XROBOTS_HEADER" | grep -qi "nofollow"; then
+            echo "Smoke test failed: X-Robots-Tag is missing 'nofollow' on /"
+            exit 1
+          fi
+
+          if [ "$ROBOTS_TXT_STATUS" -ne 200 ]; then
+            echo "Smoke test failed: /robots.txt was not served (status $ROBOTS_TXT_STATUS)"
+            exit 1
+          fi
+
+          # /robots.txt must stay allow-all. Blocking crawlers would hide the noindex
+          # signals above and leave already-indexed pages stuck in search results.
+          if ! echo "$ROBOTS_TXT_BODY" | grep -qiE '^user-agent:[[:space:]]*\*[[:space:]]*$'; then
+            echo "Smoke test failed: /robots.txt has no 'User-Agent: *' group"
+            exit 1
+          fi
+
+          if ! echo "$ROBOTS_TXT_BODY" | grep -qiE '^allow:[[:space:]]*/[[:space:]]*$'; then
+            echo "Smoke test failed: /robots.txt is missing the required 'Allow: /' rule"
+            exit 1
+          fi
+
+          if echo "$ROBOTS_TXT_BODY" | grep -qiE '^disallow:[[:space:]]*/'; then
+            echo "Smoke test failed: /robots.txt contains a Disallow rule, which would hide the noindex signals from crawlers"
+            exit 1
+          fi

next.config.mjs

@@ -1,4 +1,14 @@
 /** @type {import('next').NextConfig} */
-const nextConfig = {};
+const nextConfig = {
+  async headers() {
+    return [
+      {
+        // All routes: pages, API and asset responses included
+        source: "/:path*",
+        headers: [{ key: "X-Robots-Tag", value: "noindex, nofollow" }],
+      },
+    ];
+  },
+};
 
 export default nextConfig;

package.json

@@ -1,6 +1,6 @@
 {
   "name": "request-scan",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "private": true,
   "scripts": {
     "dev": "next dev",

src/app/layout.tsx

@@ -16,6 +16,14 @@ export const metadata: Metadata = {
   title: "Request Scan (The Request Network Explorer)",
   description:
     "Request Scan allows you to explore and search the Request Network for requests, payments, addresses, other activities taking place on Request Network Protocol.",
+  robots: {
+    index: false,
+    follow: false,
+    googleBot: {
+      index: false,
+      follow: false,
+    },
+  },
 };
 
 export default function RootLayout({

src/app/robots.ts

@@ -0,0 +1,15 @@
+/** @format */
+
+import type { MetadataRoute } from "next";
+
+// Deliberately allows crawling: pages must remain fetchable so crawlers can SEE
+// the noindex signals (X-Robots-Tag header + robots meta tag). A `Disallow: /`
+// here would hide the noindex and leave already-indexed pages stuck in Google.
+export default function robots(): MetadataRoute.Robots {
+  return {
+    rules: {
+      userAgent: "*",
+      allow: "/",
+    },
+  };
+}