Repository: e2b-dev/dashboard
Pull request #52: Remove ai-agents middleware rewrite
Author: login: ben-fornefeld
Labels: name: rewrites
Created: 2025-04-24T18:35:20Z | Merged: 2025-04-25T15:46:41Z

This pr removes references to our  framer landing page by removing `LANDING_PAGE_FRAMER_DOMAIN` from global/rewrite config and sitemap generation.

Additionally:
- it adds a `seo` config for marking path globs as deprecated routes to inform seo crawlers about deprecation to get them removed from indexing faster by returning `410` (no content) status codes instead of `404` in catch-all route handler
- configures `/ai-agents/**` paths to return `410`


Diff:
diff --git a/bun.lock b/bun.lock
index 40e8f68d7..12f15f4c3 100644
--- a/bun.lock
+++ b/bun.lock
@@ -35,6 +35,7 @@
         "@tanstack/react-virtual": "^3.11.3",
         "@theguild/remark-mermaid": "^0.2.0",
         "@types/mdx": "^2.0.13",
+        "@types/micromatch": "^4.0.9",
         "@vercel/analytics": "^1.5.0",
         "@vercel/kv": "^3.0.0",
         "@worker-tools/html-rewriter": "^0.1.0-pre.19",
@@ -51,6 +52,7 @@
         "immer": "^10.1.1",
         "jose": "^6.0.4",
         "lucide-react": "^0.487.0",
+        "micromatch": "^4.0.8",
         "motion": "^12.0.6",
         "nanoid": "^5.0.9",
         "next": "^15.3.0-canary.23",
@@ -1039,6 +1041,8 @@
 
     "@types/babel__traverse": ["@types/babel__traverse@7.20.6", "", { "dependencies": { "@babel/types": "^7.20.7" } }, "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg=="],
 
+    "@types/braces": ["@types/braces@3.0.5", "", {}, "sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w=="],
+
     "@types/bun": ["@types/bun@1.2.5", "", { "dependencies": { "bun-types": "1.2.5" } }, "sha512-w2OZTzrZTVtbnJew1pdFmgV99H0/L+Pvw+z1P67HaR18MHOzYnTYOi6qzErhK8HyT+DB782ADVPPE92Xu2/Opg=="],
 
     "@types/caseless": ["@types/caseless@0.12.5", "", {}, "sha512-hWtVTC2q7hc7xZ/RLbxapMvDMgUnDvKvMOpKal4DrMyfGBUfB1oKaZlIRr6mJL+If3bAP6sV/QneGzF6tJjZDg=="],
@@ -1145,6 +1149,8 @@
 
     "@types/mdx": ["@types/mdx@2.0.13", "", {}, "sha512-+OWZQfAYyio6YkJb3HLxDrvnx6SWWDbC0zVPfBRzUk0/nqoDyf6dNxQi3eArPe8rJ473nobTMQ/8Zk+LxJ+Yuw=="],
 
+    "@types/micromatch": ["@types/micromatch@4.0.9", "", { "dependencies": { "@types/braces": "*" } }, "sha512-7V+8ncr22h4UoYRLnLXSpTxjQrNUXtWHGeMPRJt1nULXI57G9bIcpyrHlmrQ7QK24EyyuXvYcSSWAM8GA9nqCg=="],
+
     "@types/ms": ["@types/ms@2.1.0", "", {}, "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA=="],
 
     "@types/mysql": ["@types/mysql@2.15.26", "", { "dependencies": { "@types/node": "*" } }, "sha512-DSLCOXhkvfS5WNNPbfn2KdICAmk8lLc+/PNvnPnF7gOdMZCxopXduqv0OQ13y/yA/zXTSikZZqVgybUxOEg6YQ=="],
@@ -2641,7 +2647,7 @@
 
     "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
 
-    "picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="],
+    "picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
 
     "pify": ["pify@2.3.0", "", {}, "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog=="],
 
@@ -3543,8 +3549,12 @@
 
     "@rollup/plugin-commonjs/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="],
 
+    "@rollup/plugin-commonjs/picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="],
+
     "@rollup/pluginutils/estree-walker": ["estree-walker@2.0.2", "", {}, "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w=="],
 
+    "@rollup/pluginutils/picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="],
+
     "@sentry/bundler-plugin-core/glob": ["glob@9.3.5", "", { "dependencies": { "fs.realpath": "^1.0.0", "minimatch": "^8.0.2", "minipass": "^4.2.4", "path-scurry": "^1.6.1" } }, "sha512-e1LleDykUz2Iu+MTYdkSsuWX8lvAjAcs0Xef0lNIu0S2wOAzuTxCJtcd9S3cijlwYF18EsU3rzb8jPVobxDh9Q=="],
 
     "@sentry/bundler-plugin-core/magic-string": ["magic-string@0.30.8", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.4.15" } }, "sha512-ISQTe55T2ao7XtlAStud6qwYPZjE4GK1S/BeVPus4jrq6JuOnQ00YKQC581RWhR122W7msZV263KzVeLoqidyQ=="],
@@ -3607,8 +3617,6 @@
 
     "ajv-keywords/ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="],
 
-    "anymatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
-
     "asn1.js/bn.js": ["bn.js@4.12.1", "", {}, "sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg=="],
 
     "babel-plugin-polyfill-corejs2/semver": ["semver@6.3.1", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA=="],
@@ -3675,6 +3683,8 @@
 
     "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="],
 
+    "fdir/picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="],
+
     "fork-ts-checker-webpack-plugin/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
 
     "fork-ts-checker-webpack-plugin/chokidar": ["chokidar@3.6.0", "", { "dependencies": { "anymatch": "~3.1.2", "braces": "~3.0.2", "glob-parent": "~5.1.2", "is-binary-path": "~2.1.0", "is-glob": "~4.0.1", "normalize-path": "~3.0.0", "readdirp": "~3.6.0" }, "optionalDependencies": { "fsevents": "~2.3.2" } }, "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw=="],
@@ -3735,8 +3745,6 @@
 
     "jest-util/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
 
-    "jest-util/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
-
     "jest-worker/supports-color": ["supports-color@8.1.1", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q=="],
 
     "mdast-util-find-and-replace/escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="],
@@ -3747,8 +3755,6 @@
 
     "mermaid.cli/commander": ["commander@2.20.3", "", {}, "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ=="],
 
-    "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
-
     "miller-rabin/bn.js": ["bn.js@4.12.1", "", {}, "sha512-k8TVBiPkPJT9uHLdOKfFpqcfprwBFOAAXXozRubr7R7PfIuKvQlzcI4M0pALeqXN09vdaMbUdUj+pass+uULAg=="],
 
     "mlly/pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="],
@@ -3907,6 +3913,8 @@
 
     "test-exclude/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="],
 
+    "tinyglobby/picomatch": ["picomatch@4.0.2", "", {}, "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg=="],
+
     "tsconfig-paths-webpack-plugin/chalk": ["chalk@4.1.2", "", { "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" } }, "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA=="],
 
     "tsx/esbuild": ["esbuild@0.25.1", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.1", "@esbuild/android-arm": "0.25.1", "@esbuild/android-arm64": "0.25.1", "@esbuild/android-x64": "0.25.1", "@esbuild/darwin-arm64": "0.25.1", "@esbuild/darwin-x64": "0.25.1", "@esbuild/freebsd-arm64": "0.25.1", "@esbuild/freebsd-x64": "0.25.1", "@esbuild/linux-arm": "0.25.1", "@esbuild/linux-arm64": "0.25.1", "@esbuild/linux-ia32": "0.25.1", "@esbuild/linux-loong64": "0.25.1", "@esbuild/linux-mips64el": "0.25.1", "@esbuild/linux-ppc64": "0.25.1", "@esbuild/linux-riscv64": "0.25.1", "@esbuild/linux-s390x": "0.25.1", "@esbuild/linux-x64": "0.25.1", "@esbuild/netbsd-arm64": "0.25.1", "@esbuild/netbsd-x64": "0.25.1", "@esbuild/openbsd-arm64": "0.25.1", "@esbuild/openbsd-x64": "0.25.1", "@esbuild/sunos-x64": "0.25.1", "@esbuild/win32-arm64": "0.25.1", "@esbuild/win32-ia32": "0.25.1", "@esbuild/win32-x64": "0.25.1" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-BGO5LtrGC7vxnqucAe/rmvKdJllfGaYWdyABvyMoXQlfYMb2bbRuReWR5tEGE//4LcNJj9XrkovTqNYRFZHAMQ=="],
@@ -4733,8 +4741,6 @@
 
     "css-select/domutils/dom-serializer/entities": ["entities@2.2.0", "", {}, "sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A=="],
 
-    "fork-ts-checker-webpack-plugin/chokidar/readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
-
     "fumadocs-ui/@radix-ui/react-dialog/@radix-ui/react-dismissable-layer/@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.0", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw=="],
 
     "fumadocs-ui/@radix-ui/react-dialog/@radix-ui/react-dismissable-layer/@radix-ui/react-use-escape-keydown": ["@radix-ui/react-use-escape-keydown@1.1.0", "", { "dependencies": { "@radix-ui/react-use-callback-ref": "1.1.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw=="],
@@ -4799,8 +4805,6 @@
 
     "renderkid/htmlparser2/domutils/dom-serializer": ["dom-serializer@1.4.1", "", { "dependencies": { "domelementtype": "^2.0.1", "domhandler": "^4.2.0", "entities": "^2.0.0" } }, "sha512-VHwB3KfrcOOkelEG2ZOfxqLZdfkil8PtJi4P8N2MMXucZq2yLp75ClViUlOVwyoHEDjYU433Aq+5zWP61+RGag=="],
 
-    "unplugin/chokidar/readdirp/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="],
-
     "vaul/@radix-ui/react-dialog/@radix-ui/react-dismissable-layer/@radix-ui/react-use-callback-ref": ["@radix-ui/react-use-callback-ref@1.1.0", "", { "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw=="],
 
     "vaul/@radix-ui/react-dialog/@radix-ui/react-dismissable-layer/@radix-ui/react-use-escape-keydown": ["@radix-ui/react-use-escape-keydown@1.1.0", "", { "dependencies": { "@radix-ui/react-use-callback-ref": "1.1.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-L7vwWlR1kTTQ3oh7g1O0CBF3YCyyTj8NmhLR+phShpyA50HCfBFKVJTpshm9PzLiKmehsrQzTYTpX9HvmC9rhw=="],
diff --git a/package.json b/package.json
index ae1578322..7294fed28 100644
--- a/package.json
+++ b/package.json
@@ -68,6 +68,7 @@
     "@tanstack/react-virtual": "^3.11.3",
     "@theguild/remark-mermaid": "^0.2.0",
     "@types/mdx": "^2.0.13",
+    "@types/micromatch": "^4.0.9",
     "@vercel/analytics": "^1.5.0",
     "@vercel/kv": "^3.0.0",
     "@worker-tools/html-rewriter": "^0.1.0-pre.19",
@@ -84,6 +85,7 @@
     "immer": "^10.1.1",
     "jose": "^6.0.4",
     "lucide-react": "^0.487.0",
+    "micromatch": "^4.0.8",
     "motion": "^12.0.6",
     "nanoid": "^5.0.9",
     "next": "^15.3.0-canary.23",
diff --git a/src/app/(rewrites)/[[...slug]]/route.ts b/src/app/(rewrites)/[[...slug]]/route.ts
index 900199aec..c8ae7386c 100644
--- a/src/app/(rewrites)/[[...slug]]/route.ts
+++ b/src/app/(rewrites)/[[...slug]]/route.ts
@@ -7,6 +7,7 @@ import { NO_INDEX } from '@/lib/utils/flags'
 import { logError } from '@/lib/clients/logger'
 import { HTMLRewriter } from '@worker-tools/html-rewriter/base64'
 import { ROUTE_REWRITE_CONFIG } from '@/configs/rewrites'
+import { checkSEODeprecation } from '@/lib/utils/seo'
 
 export const revalidate = 900
 export const dynamic = 'force-static'
@@ -15,6 +16,11 @@ const REVALIDATE_TIME = 900 // 15 minutes ttl
 
 export async function GET(request: NextRequest): Promise<Response> {
   const url = new URL(request.url)
+  const pathname = url.pathname
+
+  const deprecationResponse = checkSEODeprecation(pathname)
+  if (deprecationResponse) return deprecationResponse
+
   const requestHostname = url.hostname
 
   const updateUrlHostname = (newHostname: string) => {
diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts
index b43c20d03..dd0556785 100644
--- a/src/app/sitemap.ts
+++ b/src/app/sitemap.ts
@@ -12,7 +12,6 @@ import { XMLParser } from 'fast-xml-parser'
 import {
   DOCS_NEXT_DOMAIN,
   LANDING_PAGE_DOMAIN,
-  LANDING_PAGE_FRAMER_DOMAIN,
   ROUTE_REWRITE_CONFIG,
 } from '@/configs/rewrites'
 import { DomainConfig } from '@/types/rewrites.types'
@@ -62,12 +61,6 @@ const sites: Site[] = [
     changeFrequency: 'weekly',
     baseUrl: 'https://e2b.dev',
   },
-  {
-    sitemapUrl: `https://${LANDING_PAGE_FRAMER_DOMAIN}/sitemap.xml`,
-    priority: 0.7,
-    changeFrequency: 'weekly',
-    baseUrl: 'https://e2b.dev',
-  },
 ]
 
 /**
diff --git a/src/configs/logs.ts b/src/configs/logs.ts
index e7060eb9f..009a0afaf 100644
--- a/src/configs/logs.ts
+++ b/src/configs/logs.ts
@@ -7,6 +7,11 @@ export const ERROR_CODES = {
   GUARD: 'GUARD_ERROR',
   EMAIL_VALIDATION: 'EMAIL_VALIDATION_ERROR',
   SUPABASE: 'SUPABASE_ERROR',
+  INVALID_REGEX: 'INVALID_REGEX_ERROR',
+} as const
+
+export const WARNING_CODES = {
+  SEO: 'SEO',
 } as const
 
 export const INFO_CODES = {
diff --git a/src/configs/rewrites.ts b/src/configs/rewrites.ts
index 71ae7c6df..54757c5f3 100644
--- a/src/configs/rewrites.ts
+++ b/src/configs/rewrites.ts
@@ -1,7 +1,6 @@
 import { DomainConfig } from '@/types/rewrites.types'
 
 export const LANDING_PAGE_DOMAIN = 'www.e2b-landing-page.com'
-export const LANDING_PAGE_FRAMER_DOMAIN = 'e2b-landing-page.framer.website'
 export const DOCS_NEXT_DOMAIN = 'e2b-docs.vercel.app'
 
 // Currently we have two locations for rewrites to happen.
@@ -42,8 +41,4 @@ export const MIDDLEWARE_REWRITE_CONFIG: DomainConfig[] = [
     domain: DOCS_NEXT_DOMAIN,
     rules: [{ path: '/docs' }],
   },
-  {
-    domain: LANDING_PAGE_FRAMER_DOMAIN,
-    rules: [{ path: '/ai-agents' }],
-  },
 ]
diff --git a/src/configs/seo.ts b/src/configs/seo.ts
new file mode 100644
index 000000000..63924a911
--- /dev/null
+++ b/src/configs/seo.ts
@@ -0,0 +1,17 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of glob patterns for deprecated routes that will return a 410 (Gone) status.
+ *
+ * Pattern syntax examples:
+ * - '/ai-agents' - Exact match for /ai-agents
+ * - '/ai-agents/*' - Matches a single segment (e.g., /ai-agents/foo)
+ * - '/ai-agents/**' - Matches multiple segments (e.g., /ai-agents/foo/bar)
+ * - '/ai-agents/{foo,bar}' - Matches /ai-agents/foo or /ai-agents/bar
+ * - '/ai-agents/!(foo|bar)' - Matches anything except /ai-agents/foo or /ai-agents/bar
+ */
+export const DEPRECATED_PATH_PATTERNS = [
+  '/ai-agents/**', // Match any path under /ai-agents
+]
diff --git a/src/lib/utils/seo.ts b/src/lib/utils/seo.ts
new file mode 100644
index 000000000..ee6de5ffa
--- /dev/null
+++ b/src/lib/utils/seo.ts
@@ -0,0 +1,43 @@
+import { logWarning } from '@/lib/clients/logger'
+import { WARNING_CODES } from '@/configs/logs'
+import micromatch from 'micromatch'
+import { DEPRECATED_PATH_PATTERNS } from '@/configs/seo'
+
+/**
+ * Checks if a path is deprecated based on the configured glob patterns.
+ *
+ * @param pathname URL pathname to check against deprecated patterns
+ * @returns boolean indicating if the path is deprecated
+ */
+export function isPathDeprecated(pathname: string): boolean {
+  // Normalize the pathname to ensure consistent matching
+  const normalizedPath =
+    pathname.endsWith('/') && pathname !== '/'
+      ? pathname.slice(0, -1) // Remove trailing slash except for root path
+      : pathname
+
+  return micromatch.isMatch(normalizedPath, DEPRECATED_PATH_PATTERNS, {
+    dot: true, // Match dot files
+    nocase: true, // Case insensitive matching
+    noglobstar: false, // Enable ** for matching across path segments
+    posix: true, // Use forward slashes for path separators
+  })
+}
+
+/**
+ * Checks if a pathname matches any of the deprecated path patterns.
+ * Returns a 410 Gone response if the path is deprecated.
+ *
+ * @param pathname The URL pathname to check
+ * @returns A 410 Response if path is deprecated, undefined otherwise
+ */
+export function checkSEODeprecation(pathname: string): Response | void {
+  try {
+    if (isPathDeprecated(pathname)) {
+      return new Response(null, { status: 410 })
+    }
+  } catch (e) {
+    console.error(e)
+    logWarning(WARNING_CODES.SEO, 'Error checking path deprecation:', pathname)
+  }
+}


Reviews:
user: login: mishushakov
state: CHANGES_REQUESTED
author_association: MEMBER
submitted_at: 2025-04-25T11:42:30Z
user: login: ben-fornefeld
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T12:00:24Z
user: login: ben-fornefeld
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T12:47:02Z
user: login: mishushakov
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T13:58:23Z
user: login: mishushakov
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T13:59:15Z
user: login: mishushakov
body: Small nits, but otherwise good to go
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-25T14:00:08Z
user: login: ben-fornefeld
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-25T14:01:43Z

Review comments:
diff_hunk: @@ -15,6 +16,22 @@ const REVALIDATE_TIME = 900 // 15 minutes ttl
 
 export async function GET(request: NextRequest): Promise<Response> {
   const url = new URL(request.url)
+  const pathname = url.pathname
+
+  // Check if the path matches any deprecated regex pattern
+  for (const pattern of DEPRECATED_PATH_REGEXPS) {
+    try {
+      const regex = new RegExp(pattern)
+      if (regex.test(pathname)) {
+        return new Response(null, { status: 410 })
+      }
+    } catch (error) {
+      logWarning(WARNING_CODES.SEO, pattern, error)
+
+      continue
+    }
+  }
path: src/app/(rewrites)/[[...slug]]/route.ts
user: login: mishushakov
body: can't you just

```
DEPRECATED_PATH_REGEXPS.some(regex => regex.test(pathname))
```
created_at: 2025-04-25T11:37:44Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,14 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of regular expression strings. Paths matching any of these regexps
+ * will return a 410 (Gone) status.
+ * Ensure these are valid JavaScript RegExp patterns.
+ * Example: ['^/old-blog(/.*)?$', '^/features/removed-feature$']
+ */
+export const DEPRECATED_PATH_REGEXPS: string[] = [
path: src/configs/seo.ts
user: login: mishushakov
body: deprecatedPaths
created_at: 2025-04-25T11:38:03Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,14 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of regular expression strings. Paths matching any of these regexps
+ * will return a 410 (Gone) status.
+ * Ensure these are valid JavaScript RegExp patterns.
+ * Example: ['^/old-blog(/.*)?$', '^/features/removed-feature$']
+ */
+export const DEPRECATED_PATH_REGEXPS: string[] = [
+  '^/ai-agents(/.*)?$', // Matches /ai-agents and all subpaths
path: src/configs/seo.ts
user: login: mishushakov
body: /^\/ai-agents(\/.*)?$/
created_at: 2025-04-25T11:40:49Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,14 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of regular expression strings. Paths matching any of these regexps
+ * will return a 410 (Gone) status.
+ * Ensure these are valid JavaScript RegExp patterns.
+ * Example: ['^/old-blog(/.*)?$', '^/features/removed-feature$']
+ */
+export const DEPRECATED_PATH_REGEXPS: string[] = [
path: src/configs/seo.ts
user: login: mishushakov
body: string[] > RegExp[]
created_at: 2025-04-25T11:41:14Z
author_association: MEMBER
diff_hunk: @@ -15,6 +16,22 @@ const REVALIDATE_TIME = 900 // 15 minutes ttl
 
 export async function GET(request: NextRequest): Promise<Response> {
   const url = new URL(request.url)
+  const pathname = url.pathname
+
+  // Check if the path matches any deprecated regex pattern
+  for (const pattern of DEPRECATED_PATH_REGEXPS) {
+    try {
+      const regex = new RegExp(pattern)
+      if (regex.test(pathname)) {
+        return new Response(null, { status: 410 })
+      }
+    } catch (error) {
+      logWarning(WARNING_CODES.SEO, pattern, error)
+
+      continue
+    }
+  }
path: src/app/(rewrites)/[[...slug]]/route.ts
user: login: ben-fornefeld
body: i didn't use native `RegExp` for configuration to make this easier/more readable when configuring
created_at: 2025-04-25T12:00:24Z
author_association: MEMBER
diff_hunk: @@ -15,6 +16,22 @@ const REVALIDATE_TIME = 900 // 15 minutes ttl
 
 export async function GET(request: NextRequest): Promise<Response> {
   const url = new URL(request.url)
+  const pathname = url.pathname
+
+  // Check if the path matches any deprecated regex pattern
+  for (const pattern of DEPRECATED_PATH_REGEXPS) {
+    try {
+      const regex = new RegExp(pattern)
+      if (regex.test(pathname)) {
+        return new Response(null, { status: 410 })
+      }
+    } catch (error) {
+      logWarning(WARNING_CODES.SEO, pattern, error)
+
+      continue
+    }
+  }
path: src/app/(rewrites)/[[...slug]]/route.ts
user: login: ben-fornefeld
body: this way we do not get invalid regex patterns logged.
created_at: 2025-04-25T12:47:01Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,17 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of glob patterns for deprecated routes that will return a 410 (Gone) status.
+ *
+ * Pattern syntax examples:
+ * - '/ai-agents' - Exact match for /ai-agents
+ * - '/ai-agents/*' - Matches a single segment (e.g., /ai-agents/foo)
+ * - '/ai-agents/**' - Matches multiple segments (e.g., /ai-agents/foo/bar)
+ * - '/ai-agents/{foo,bar}' - Matches /ai-agents/foo or /ai-agents/bar
+ * - '/ai-agents/!(foo|bar)' - Matches anything except /ai-agents/foo or /ai-agents/bar
+ */
path: src/configs/seo.ts
user: login: mishushakov
body: remove comments, I think the code is self-explanatory
created_at: 2025-04-25T13:58:23Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,43 @@
+import { logWarning } from '@/lib/clients/logger'
+import { WARNING_CODES } from '@/configs/logs'
+import micromatch from 'micromatch'
+import { DEPRECATED_PATH_PATTERNS } from '@/configs/seo'
+
+/**
+ * Checks if a path is deprecated based on the configured glob patterns.
+ *
+ * @param pathname URL pathname to check against deprecated patterns
+ * @returns boolean indicating if the path is deprecated
+ */
+export function isPathDeprecated(pathname: string): boolean {
+  // Normalize the pathname to ensure consistent matching
+  const normalizedPath =
+    pathname.endsWith('/') && pathname !== '/'
+      ? pathname.slice(0, -1) // Remove trailing slash except for root path
+      : pathname
path: src/lib/utils/seo.ts
user: login: mishushakov
body: do you actually need to normalize it?
created_at: 2025-04-25T13:59:15Z
author_association: MEMBER
diff_hunk: @@ -0,0 +1,17 @@
+/**
+ * Configuration for SEO-related settings.
+ */
+
+/**
+ * An array of glob patterns for deprecated routes that will return a 410 (Gone) status.
+ *
+ * Pattern syntax examples:
+ * - '/ai-agents' - Exact match for /ai-agents
+ * - '/ai-agents/*' - Matches a single segment (e.g., /ai-agents/foo)
+ * - '/ai-agents/**' - Matches multiple segments (e.g., /ai-agents/foo/bar)
+ * - '/ai-agents/{foo,bar}' - Matches /ai-agents/foo or /ai-agents/bar
+ * - '/ai-agents/!(foo|bar)' - Matches anything except /ai-agents/foo or /ai-agents/bar
+ */
path: src/configs/seo.ts
user: login: ben-fornefeld
body: it's not. if i just want to configure new paths and don't know anything about the repo this is helpful
created_at: 2025-04-25T14:01:43Z
author_association: MEMBER

PR comments:
user: login: vercel[bot]
created_at: 2025-04-24T18:35:25Z
body: [vc]: #1NqMaznHtH4NlJHMRMT6G6Tlu8H3b+qfYl0Dg8yZnoU=:eyJpc01vbm9yZXBvIjp0cnVlLCJ0eXBlIjoiZ2l0aHViIiwicHJvamVjdHMiOlt7Im5hbWUiOiJ3ZWIiLCJyb290RGlyZWN0b3J5IjpudWxsLCJpbnNwZWN0b3JVcmwiOiJodHRwczovL3ZlcmNlbC5jb20vZTJiL3dlYi82QUxRZVVZWXQyYWRSVGhZM3lYVkRXTnptR0x5IiwicHJldmlld1VybCI6IndlYi1naXQtcmVtb3ZlLWFpLWFnZW50cy1yZXdyaXRlLmUyYi1wcmV2aWV3LmRldiIsIm5leHRDb21taXRTdGF0dXMiOiJERVBMT1lFRCIsImxpdmVGZWVkYmFjayI6eyJyZXNvbHZlZCI6MCwidW5yZXNvbHZlZCI6MCwidG90YWwiOjAsImxpbmsiOiJ3ZWItZ2l0LXJlbW92ZS1haS1hZ2VudHMtcmV3cml0ZS5lMmItcHJldmlldy5kZXYifX0seyJuYW1lIjoid2ViLWp1bGlldHQiLCJyb290RGlyZWN0b3J5IjpudWxsLCJpbnNwZWN0b3JVcmwiOiJodHRwczovL3ZlcmNlbC5jb20vZTJiL3dlYi1qdWxpZXR0LzcyVHI4aXF5Q3lwTE5yNkN0MUQ1cVVWb3VOZGUiLCJwcmV2aWV3VXJsIjoid2ViLWp1bGlldHQtZ2l0LXJlbW92ZS1haS1hZ2VudHMtcmV3cml0ZS5lMmItcHJldmlldy5kZXYiLCJuZXh0Q29tbWl0U3RhdHVzIjoiREVQTE9ZRUQiLCJsaXZlRmVlZGJhY2siOnsicmVzb2x2ZWQiOjAsInVucmVzb2x2ZWQiOjAsInRvdGFsIjowLCJsaW5rIjoid2ViLWp1bGlldHQtZ2l0LXJlbW92ZS1haS1hZ2VudHMtcmV3cml0ZS5lMmItcHJldmlldy5kZXYifX1dfQ==
**The latest updates on your projects**. Learn more about [Vercel for Git ↗︎](https://vercel.link/github-learn-more)

| Name | Status | Preview | Comments | Updated (UTC) |
| :--- | :----- | :------ | :------- | :------ |
| **web** | ✅ Ready ([Inspect](https://vercel.com/e2b/web/6ALQeUYYt2adRThY3yXVDWNzmGLy)) | [Visit Preview](https://web-git-remove-ai-agents-rewrite.e2b-preview.dev) | 💬 [**Add feedback**](https://vercel.live/open-feedback/web-git-remove-ai-agents-rewrite.e2b-preview.dev?via=pr-comment-feedback-link) | Apr 25, 2025 1:03pm |
| **web-juliett** | ✅ Ready ([Inspect](https://vercel.com/e2b/web-juliett/72Tr8iqyCypLNr6Ct1D5qUVouNde)) | [Visit Preview](https://web-juliett-git-remove-ai-agents-rewrite.e2b-preview.dev) | 💬 [**Add feedback**](https://vercel.live/open-feedback/web-juliett-git-remove-ai-agents-rewrite.e2b-preview.dev?via=pr-comment-feedback-link) | Apr 25, 2025 1:03pm |



author_association: NONE
user: login: ben-fornefeld
created_at: 2025-04-25T12:46:09Z
body: @mishushakov the way you suggested makes it slightly annoying to configure. remember when using js native regexp, for every literal `/` match you have to prefix with a `\`. since we try to match paths here, that can get quite cumbersome.


author_association: MEMBER
user: login: ben-fornefeld
created_at: 2025-04-25T13:02:32Z
body: refactored to glob pattern matching for easier configurability 
author_association: MEMBER