From 5dca32ecefd8a5731db4335b33b255ff4957d4e8 Mon Sep 17 00:00:00 2001 From: Nikil Kuruvilla Date: Sat, 27 Jun 2026 19:14:54 +0100 Subject: [PATCH 1/2] feat: add rules to block out most bot traffic --- apps/web/public/robots.txt | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt index ea2db3d2..cb3515d9 100644 --- a/apps/web/public/robots.txt +++ b/apps/web/public/robots.txt @@ -1,4 +1,35 @@ User-agent: * Allow: / +# Block Semrush crawlers — https://www.semrush.com/bot/ +User-agent: SemrushBot +Disallow: / + +User-agent: SiteAuditBot +Disallow: / + +User-agent: SemrushBot-BA +Disallow: / + +User-agent: SemrushBot-SI +Disallow: / + +User-agent: SemrushBot-SWA +Disallow: / + +User-agent: SemrushBot-OCOB +Disallow: / + +User-agent: SemrushBot-FT +Disallow: / + +User-agent: SemrushBot-ESI +Disallow: / + +User-agent: SplitSignalBot +Disallow: / + +User-agent: RyteBot +Disallow: / + Sitemap: https://sponsorsearch.co.uk/sitemap.xml From a605a4d30247bcf0f70d891740d3f5eea5773904 Mon Sep 17 00:00:00 2001 From: Nikil Kuruvilla Date: Sat, 27 Jun 2026 19:18:12 +0100 Subject: [PATCH 2/2] fix: update remarks --- apps/web/public/robots.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/web/public/robots.txt b/apps/web/public/robots.txt index cb3515d9..97f74ffe 100644 --- a/apps/web/public/robots.txt +++ b/apps/web/public/robots.txt @@ -1,7 +1,7 @@ User-agent: * Allow: / -# Block Semrush crawlers — https://www.semrush.com/bot/ +# Block SEO/backlink-analysis crawlers (Semrush family, Ryte, et al.) — https://www.semrush.com/bot/ User-agent: SemrushBot Disallow: /