fix(web): stop robots.txt from blocking indexable pages (#610)

Remove Disallow for query-string URLs and /_next/, which caused Google
Search Console "Indexed, though blocked by robots.txt" for legitimate
pages discovered with UTM/tracking params. Canonical tags already handle
duplicate param URLs.

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Jatin K Malik <jatinkrmalik@users.noreply.github.com>

web/public/robots.txt

@@ -1,13 +1,15 @@
 # Robots.txt for Vocalinux
 # https://vocalinux.com
+#
+# Intentionally permissive for marketing pages. Do not blanket-block
+# query-string URLs or /_next/static — those caused Google Search Console
+# "Indexed, though blocked by robots.txt" for otherwise valid pages.
+# Duplicate/param URLs are handled via canonical tags instead.
 
 User-agent: *
 Allow: /
 Disallow: /api/
-Disallow: /_next/
 Disallow: /out/
-Disallow: /*?*
 
 # Sitemap location
 Sitemap: https://vocalinux.com/sitemap.xml
-Host: vocalinux.com

web/src/app/__tests__/seo-assets.test.ts

@@ -21,10 +21,22 @@ describe("SEO Assets", () => {
 
   it("blocks crawler access to technical paths", () => {
     expect(robotsContent).toContain("Disallow: /api/");
-    expect(robotsContent).toContain("Disallow: /_next/");
     expect(robotsContent).toContain("Disallow: /out/");
   });
 
+  it("does not block Next.js static assets needed for rendering", () => {
+    expect(robotsContent).not.toContain("Disallow: /_next/");
+  });
+
+  it("does not blanket-block query-string URLs", () => {
+    // Blocking all query strings causes GSC "Indexed, though blocked by robots.txt"
+    // for pages discovered with UTM/tracking params. Use canonicals instead.
+    const disallowRules = robotsContent
+      .split("\n")
+      .filter((line) => line.trimStart().startsWith("Disallow:"));
+    expect(disallowRules.some((line) => /\/\*\?\*/.test(line))).toBe(false);
+  });
+
   it("includes core landing pages in sitemap", () => {
     const expectedUrls = [
       "https://vocalinux.com/",