1use std::sync::LazyLock;
4
5use regex::Regex;
6
7use crate::{FilterConfig, FilterStatKey, Filterable};
8
9static WEB_CRAWLERS: LazyLock<Regex> = LazyLock::new(|| {
10 Regex::new(
11 r"(?ix)
12 Mediapartners-Google|
13 AdsBot-Google|
14 Googlebot|
15 FeedFetcher-Google|
16 BingBot| # Bing search
17 BingPreview|
18 Baiduspider| # Baidu search
19 Slurp| # Yahoo
20 Sogou| # Sogou
21 facebook| # facebook
22 meta-| # meta/facebook
23 ia_archiver| # Alexa
24 bots?([/\s\);]|$)| # Generic bot
25 spider([/\s\);]|$)| # Generic spider
26 Slack| # Slack - see https://api.slack.com/robots
27 Calypso\sAppCrawler| # Google indexing bot
28 pingdom| # Pingdom
29 lyticsbot| # Lytics
30 AWS\sSecurity\sScanner| # AWS Security Scanner causing DisallowedHost errors in Django, see
31 # https://forums.aws.amazon.com/thread.jspa?messageID=932404
32 # and https://github.com/getsentry/sentry-python/issues/641
33 HubSpot\sCrawler| # HubSpot web crawler (web-crawlers@hubspot.com)
34 Bytespider| # Bytedance
35 Better\sUptime| # Better Uptime
36 Cloudflare-Healthchecks| # Cloudflare Health Checks
37 GTmetrix| # GTmetrix
38 BrightEdgeOnCrawl| # BrightEdge - see https://www.brightedge.com/news/press-releases/brightedge-acquires-oncrawl-future-proof-web-30-strategies
39 ELB-HealthChecker| # AWS Elastic Load Balancing Health Checks
40 naver.me/spd| # Yeti/1.1 - naver.me
41 ClaudeBot| # Anthropic - see https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
42 CCBot| # CCBot - see https://commoncrawl.org/ccbot
43 OAI-SearchBot| # OpenAI - see https://platform.openai.com/docs/bots
44 GPTBot| # OpenAI - see https://platform.openai.com/docs/bots
45 PerplexityBot| # Perplexity - see https://docs.perplexity.ai/guides/bots
46 Applebot| # Apple - see https://support.apple.com/en-us/119829
47 DuckDuckBot # DuckDuckGo - see https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
48 "
49 )
50 .expect("Invalid web crawlers filter Regex")
51});
52
53static ALLOWED_WEB_CRAWLERS: LazyLock<Regex> = LazyLock::new(|| {
54 Regex::new(
55 r"(?ix)
56 Slackbot\s1\.\d+| # Slack - see https://api.slack.com/robots
57 SentryUptimeBot| # Uptime Checker https://docs.sentry.io/product/alerts/uptime-monitoring/
58 ChatGPT-User # ChatGPT user prompted requests
59 ",
60 )
61 .expect("Invalid allowed web crawlers filter Regex")
62});
63
64fn matches(user_agent: &str) -> bool {
66 WEB_CRAWLERS.is_match(user_agent) && !ALLOWED_WEB_CRAWLERS.is_match(user_agent)
67}
68
69pub fn should_filter<F: Filterable>(item: &F, config: &FilterConfig) -> Result<(), FilterStatKey> {
71 if !config.is_enabled {
72 return Ok(());
73 }
74
75 let user_agent = item.user_agent();
76 let family = user_agent.parsed.as_ref().map(|ua| ua.family.as_ref());
77
78 if user_agent.raw.or(family).is_some_and(matches) {
83 return Err(FilterStatKey::WebCrawlers);
84 }
85
86 Ok(())
87}
88
89#[cfg(test)]
90mod tests {
91 use super::*;
92 use crate::{UserAgent, testutils};
93
94 #[derive(Debug)]
95 struct TestFilterable<'a>(UserAgent<'a>);
96
97 impl<'a> Filterable for TestFilterable<'a> {
98 fn user_agent(&self) -> UserAgent<'_> {
99 self.0.clone()
100 }
101 }
102
103 #[test]
104 fn test_filter_when_disabled() {
105 let evt = testutils::get_event_with_user_agent("Googlebot");
106 let filter_result = should_filter(&evt, &FilterConfig { is_enabled: false });
107 assert_eq!(
108 filter_result,
109 Ok(()),
110 "Event filtered although filter should have been disabled"
111 )
112 }
113
114 #[test]
115 fn test_filter_banned_user_agents() {
116 let user_agents = [
117 "Mediapartners-Google",
118 "AdsBot-Google",
119 "Googlebot",
120 "FeedFetcher-Google",
121 "BingBot",
122 "BingPreview",
123 "Baiduspider",
124 "Slurp",
125 "Sogou",
126 "facebook",
127 "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
128 "facebookcatalog/1.0",
129 "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
130 "meta-externalfetcher/1.1",
131 "ia_archiver",
132 "bots ",
133 "bots;",
134 "bots)",
135 "spider ",
136 "spider;",
137 "spider)",
138 "Calypso AppCrawler",
139 "pingdom",
140 "lyticsbot",
141 "AWS Security Scanner",
142 "Mozilla/5.0 (Linux; Android 6.0.1; Calypso AppCrawler Build/MMB30Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.124 Mobile Safari/537.36",
143 "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
144 "Slack-ImgProxy 0.19 (+https://api.slack.com/robots)",
145 "Twitterbot/1.0",
146 "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
147 "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
148 "AdsBot-Google (+http://www.google.com/adsbot.html)",
149 "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)",
150 "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
151 "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
152 "Mozilla/5.0 (compatible;Cloudflare-Healthchecks/1.0;+https://www.cloudflare.com/; healthcheck-id: 0d1ca23e292c8c14)",
153 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 GTmetrix",
154 "Mozilla/5.0 (compatible; BrightEdgeOnCrawl/1.0; +http://www.oncrawl.com)",
155 "ELB-HealthChecker/2.0",
156 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Yeti/1.1; +https://naver.me/spd) Chrome/127.0.0.0 Safari/537.36",
157 "Mozilla/5.0; ClaudeBot",
158 "Mozilla/5.0; CCBot",
159 "; OAI-SearchBot/1.0; +https://openai.com/searchbot",
160 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot",
161 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
162 "Mozilla/5.0 (Device; OS_version) AppleWebKit/WebKit_version (KHTML, like Gecko)Version/Safari_version [Mobile/Mobile_version] Safari/WebKit_version (Applebot/Applebot_version; +http://www.apple.com/go/applebot)",
163 "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
164 ];
165
166 for banned_user_agent in &user_agents {
167 let event = testutils::get_event_with_user_agent(banned_user_agent);
168 let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
169 assert_ne!(
170 filter_result,
171 Ok(()),
172 "Failed to filter web crawler with user agent '{banned_user_agent}'"
173 );
174 }
175 }
176
177 #[test]
178 fn test_dont_filter_normal_user_agents() {
179 let normal_user_agents = [
180 "some user agent",
181 "IE",
182 "ie",
183 "opera",
184 "safari",
185 "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
186 "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
187 "Slackbot 1.0(+https://api.slack.com/robots)",
188 "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)",
189 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
190 ];
191 for user_agent in &normal_user_agents {
192 let event = testutils::get_event_with_user_agent(user_agent);
193 let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
194 assert_eq!(
195 filter_result,
196 Ok(()),
197 "Failed benign user agent '{user_agent}'"
198 );
199 }
200 }
201
202 #[test]
203 fn test_filter_parsed_ua_only() {
204 let ua = UserAgent {
205 raw: None,
206 parsed: Some(relay_ua::UserAgent {
207 family: "Twitterbot".into(),
208 ..Default::default()
209 }),
210 };
211
212 let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
213 assert_ne!(filter_result, Ok(()));
214 }
215
216 #[test]
217 fn test_filter_parsed_ua_does_not_filter_default() {
218 let ua = UserAgent {
219 raw: None,
220 parsed: Some(Default::default()),
223 };
224
225 let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
226 assert_eq!(filter_result, Ok(()));
227 }
228}