diff --git a/.github/workflows/build-and-lint.yml b/.github/workflows/build-and-lint.yml index 8c15e81..e657850 100644 --- a/.github/workflows/build-and-lint.yml +++ b/.github/workflows/build-and-lint.yml @@ -100,7 +100,7 @@ jobs: - name: Test run: npm test - - name: Smoke test - server renders dynamic routes + - name: Smoke test - server renders dynamic routes and serves noindex env: NEXT_PUBLIC_HASURA_GRAPHQL_URL: "https://unreachable.invalid/v1/graphql" NEXT_PUBLIC_POLL_INTERVAL: "30000" @@ -127,8 +127,14 @@ jobs: ROOT_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/") REQUEST_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/request/0x1234567890abcdef1234567890abcdef12345678-0") + XROBOTS_HEADER=$(curl -sI "http://localhost:3000/" | tr -d '\r' | grep -i '^x-robots-tag:' || true) + ROBOTS_TXT_STATUS=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:3000/robots.txt") + ROBOTS_TXT_BODY=$(curl -s "http://localhost:3000/robots.txt" | tr -d '\r') - echo "Observed status codes: / -> $ROOT_STATUS, /request/ -> $REQUEST_STATUS" + echo "Observed status codes: / -> $ROOT_STATUS, /request/ -> $REQUEST_STATUS, /robots.txt -> $ROBOTS_TXT_STATUS" + echo "Observed header: $XROBOTS_HEADER" + echo "Observed /robots.txt body:" + echo "$ROBOTS_TXT_BODY" kill "$SERVER_PID" 2>/dev/null || true @@ -136,3 +142,36 @@ jobs: echo "Smoke test failed: server returned a 5xx status code" exit 1 fi + + # The full policy must be present: dropping either token weakens de-indexing. + if ! echo "$XROBOTS_HEADER" | grep -qi "noindex"; then + echo "Smoke test failed: X-Robots-Tag is missing 'noindex' on /" + exit 1 + fi + + if ! echo "$XROBOTS_HEADER" | grep -qi "nofollow"; then + echo "Smoke test failed: X-Robots-Tag is missing 'nofollow' on /" + exit 1 + fi + + if [ "$ROBOTS_TXT_STATUS" -ne 200 ]; then + echo "Smoke test failed: /robots.txt was not served (status $ROBOTS_TXT_STATUS)" + exit 1 + fi + + # /robots.txt must stay allow-all. Blocking crawlers would hide the noindex + # signals above and leave already-indexed pages stuck in search results. + if ! echo "$ROBOTS_TXT_BODY" | grep -qiE '^user-agent:[[:space:]]*\*[[:space:]]*$'; then + echo "Smoke test failed: /robots.txt has no 'User-Agent: *' group" + exit 1 + fi + + if ! echo "$ROBOTS_TXT_BODY" | grep -qiE '^allow:[[:space:]]*/[[:space:]]*$'; then + echo "Smoke test failed: /robots.txt is missing the required 'Allow: /' rule" + exit 1 + fi + + if echo "$ROBOTS_TXT_BODY" | grep -qiE '^disallow:[[:space:]]*/'; then + echo "Smoke test failed: /robots.txt contains a Disallow rule, which would hide the noindex signals from crawlers" + exit 1 + fi diff --git a/next.config.mjs b/next.config.mjs index 4678774..733bebb 100644 --- a/next.config.mjs +++ b/next.config.mjs @@ -1,4 +1,14 @@ /** @type {import('next').NextConfig} */ -const nextConfig = {}; +const nextConfig = { + async headers() { + return [ + { + // All routes: pages, API and asset responses included + source: "/:path*", + headers: [{ key: "X-Robots-Tag", value: "noindex, nofollow" }], + }, + ]; + }, +}; export default nextConfig; diff --git a/package-lock.json b/package-lock.json index 1089f50..abfe99f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "request-scan", - "version": "0.4.0", + "version": "0.4.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "request-scan", - "version": "0.4.0", + "version": "0.4.1", "dependencies": { "@next/third-parties": "^15.0.3", "@radix-ui/react-avatar": "^1.1.0", diff --git a/package.json b/package.json index 1e688a6..2c959c0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "request-scan", - "version": "0.4.0", + "version": "0.4.1", "private": true, "scripts": { "dev": "next dev", diff --git a/src/app/layout.tsx b/src/app/layout.tsx index 27fe66a..52f4ce3 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -16,6 +16,14 @@ export const metadata: Metadata = { title: "Request Scan (The Request Network Explorer)", description: "Request Scan allows you to explore and search the Request Network for requests, payments, addresses, other activities taking place on Request Network Protocol.", + robots: { + index: false, + follow: false, + googleBot: { + index: false, + follow: false, + }, + }, }; export default function RootLayout({ diff --git a/src/app/robots.ts b/src/app/robots.ts new file mode 100644 index 0000000..47c829e --- /dev/null +++ b/src/app/robots.ts @@ -0,0 +1,15 @@ +/** @format */ + +import type { MetadataRoute } from "next"; + +// Deliberately allows crawling: pages must remain fetchable so crawlers can SEE +// the noindex signals (X-Robots-Tag header + robots meta tag). A `Disallow: /` +// here would hide the noindex and leave already-indexed pages stuck in Google. +export default function robots(): MetadataRoute.Robots { + return { + rules: { + userAgent: "*", + allow: "/", + }, + }; +}