diff --git a/web/public/robots.txt b/web/public/robots.txt index 7419fb1..069df73 100644 --- a/web/public/robots.txt +++ b/web/public/robots.txt @@ -1,13 +1,15 @@ # Robots.txt for Vocalinux # https://vocalinux.com +# +# Intentionally permissive for marketing pages. Do not blanket-block +# query-string URLs or /_next/static — those caused Google Search Console +# "Indexed, though blocked by robots.txt" for otherwise valid pages. +# Duplicate/param URLs are handled via canonical tags instead. User-agent: * Allow: / Disallow: /api/ -Disallow: /_next/ Disallow: /out/ -Disallow: /*?* # Sitemap location Sitemap: https://vocalinux.com/sitemap.xml -Host: vocalinux.com diff --git a/web/src/app/__tests__/seo-assets.test.ts b/web/src/app/__tests__/seo-assets.test.ts index 37e1462..7cfb864 100644 --- a/web/src/app/__tests__/seo-assets.test.ts +++ b/web/src/app/__tests__/seo-assets.test.ts @@ -21,10 +21,22 @@ describe("SEO Assets", () => { it("blocks crawler access to technical paths", () => { expect(robotsContent).toContain("Disallow: /api/"); - expect(robotsContent).toContain("Disallow: /_next/"); expect(robotsContent).toContain("Disallow: /out/"); }); + it("does not block Next.js static assets needed for rendering", () => { + expect(robotsContent).not.toContain("Disallow: /_next/"); + }); + + it("does not blanket-block query-string URLs", () => { + // Blocking all query strings causes GSC "Indexed, though blocked by robots.txt" + // for pages discovered with UTM/tracking params. Use canonicals instead. + const disallowRules = robotsContent + .split("\n") + .filter((line) => line.trimStart().startsWith("Disallow:")); + expect(disallowRules.some((line) => /\/\*\?\*/.test(line))).toBe(false); + }); + it("includes core landing pages in sitemap", () => { const expectedUrls = [ "https://vocalinux.com/",