From 63554bc323d5ef9884c29299349e4fce5f93a677 Mon Sep 17 00:00:00 2001 From: Jatin K Malik Date: Mon, 27 Jul 2026 22:44:28 -0700 Subject: [PATCH] fix(web): stop robots.txt from blocking indexable pages (#610) Remove Disallow for query-string URLs and /_next/, which caused Google Search Console "Indexed, though blocked by robots.txt" for legitimate pages discovered with UTM/tracking params. Canonical tags already handle duplicate param URLs. Co-authored-by: Cursor Agent Co-authored-by: Jatin K Malik --- web/public/robots.txt | 8 +++++--- web/src/app/__tests__/seo-assets.test.ts | 14 +++++++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/web/public/robots.txt b/web/public/robots.txt index 7419fb1..069df73 100644 --- a/web/public/robots.txt +++ b/web/public/robots.txt @@ -1,13 +1,15 @@ # Robots.txt for Vocalinux # https://vocalinux.com +# +# Intentionally permissive for marketing pages. Do not blanket-block +# query-string URLs or /_next/static — those caused Google Search Console +# "Indexed, though blocked by robots.txt" for otherwise valid pages. +# Duplicate/param URLs are handled via canonical tags instead. User-agent: * Allow: / Disallow: /api/ -Disallow: /_next/ Disallow: /out/ -Disallow: /*?* # Sitemap location Sitemap: https://vocalinux.com/sitemap.xml -Host: vocalinux.com diff --git a/web/src/app/__tests__/seo-assets.test.ts b/web/src/app/__tests__/seo-assets.test.ts index 37e1462..7cfb864 100644 --- a/web/src/app/__tests__/seo-assets.test.ts +++ b/web/src/app/__tests__/seo-assets.test.ts @@ -21,10 +21,22 @@ describe("SEO Assets", () => { it("blocks crawler access to technical paths", () => { expect(robotsContent).toContain("Disallow: /api/"); - expect(robotsContent).toContain("Disallow: /_next/"); expect(robotsContent).toContain("Disallow: /out/"); }); + it("does not block Next.js static assets needed for rendering", () => { + expect(robotsContent).not.toContain("Disallow: /_next/"); + }); + + it("does not blanket-block query-string URLs", () => { + // Blocking all query strings causes GSC "Indexed, though blocked by robots.txt" + // for pages discovered with UTM/tracking params. Use canonicals instead. + const disallowRules = robotsContent + .split("\n") + .filter((line) => line.trimStart().startsWith("Disallow:")); + expect(disallowRules.some((line) => /\/\*\?\*/.test(line))).toBe(false); + }); + it("includes core landing pages in sitemap", () => { const expectedUrls = [ "https://vocalinux.com/",