1use once_cell::sync::Lazy;
4use regex::Regex;
5
6use crate::{FilterConfig, FilterStatKey, Filterable};
7
8static WEB_CRAWLERS: Lazy<Regex> = Lazy::new(|| {
9 Regex::new(
10 r"(?ix)
11 Mediapartners-Google|
12 AdsBot-Google|
13 Googlebot|
14 FeedFetcher-Google|
15 BingBot| # Bing search
16 BingPreview|
17 Baiduspider| # Baidu search
18 Slurp| # Yahoo
19 Sogou| # Sogou
20 facebook| # facebook
21 meta-| # meta/facebook
22 ia_archiver| # Alexa
23 bots?([/\s\);]|$)| # Generic bot
24 spider([/\s\);]|$)| # Generic spider
25 Slack| # Slack - see https://api.slack.com/robots
26 Calypso\sAppCrawler| # Google indexing bot
27 pingdom| # Pingdom
28 lyticsbot| # Lytics
29 AWS\sSecurity\sScanner| # AWS Security Scanner causing DisallowedHost errors in Django, see
30 # https://forums.aws.amazon.com/thread.jspa?messageID=932404
31 # and https://github.com/getsentry/sentry-python/issues/641
32 HubSpot\sCrawler| # HubSpot web crawler (web-crawlers@hubspot.com)
33 Bytespider| # Bytedance
34 Better\sUptime| # Better Uptime
35 Cloudflare-Healthchecks| # Cloudflare Health Checks
36 GTmetrix| # GTmetrix
37 BrightEdgeOnCrawl| # BrightEdge - see https://www.brightedge.com/news/press-releases/brightedge-acquires-oncrawl-future-proof-web-30-strategies
38 ELB-HealthChecker| # AWS Elastic Load Balancing Health Checks
39 naver.me/spd| # Yeti/1.1 - naver.me
40 ClaudeBot| # Anthropic - see https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
41 CCBot| # CCBot - see https://commoncrawl.org/ccbot
42 OAI-SearchBot| # OpenAI - see https://platform.openai.com/docs/bots
43 GPTBot| # OpenAI - see https://platform.openai.com/docs/bots
44 PerplexityBot| # Perplexity - see https://docs.perplexity.ai/guides/bots
45 Applebot| # Apple - see https://support.apple.com/en-us/119829
46 DuckDuckBot # DuckDuckGo - see https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
47 "
48 )
49 .expect("Invalid web crawlers filter Regex")
50});
51
52static ALLOWED_WEB_CRAWLERS: Lazy<Regex> = Lazy::new(|| {
53 Regex::new(
54 r"(?ix)
55 Slackbot\s1\.\d+| # Slack - see https://api.slack.com/robots
56 SentryUptimeBot| # Uptime Checker https://docs.sentry.io/product/alerts/uptime-monitoring/
57 ChatGPT-User # ChatGPT user prompted requests
58 ",
59 )
60 .expect("Invalid allowed web crawlers filter Regex")
61});
62
63fn matches(user_agent: &str) -> bool {
65 WEB_CRAWLERS.is_match(user_agent) && !ALLOWED_WEB_CRAWLERS.is_match(user_agent)
66}
67
68pub fn should_filter<F: Filterable>(item: &F, config: &FilterConfig) -> Result<(), FilterStatKey> {
70 if !config.is_enabled {
71 return Ok(());
72 }
73
74 let user_agent = item.user_agent();
75 let family = user_agent.parsed.as_ref().map(|ua| ua.family.as_ref());
76
77 if user_agent.raw.or(family).is_some_and(matches) {
82 return Err(FilterStatKey::WebCrawlers);
83 }
84
85 Ok(())
86}
87
88#[cfg(test)]
89mod tests {
90 use super::*;
91 use crate::{UserAgent, testutils};
92
93 #[derive(Debug)]
94 struct TestFilterable<'a>(UserAgent<'a>);
95
96 impl<'a> Filterable for TestFilterable<'a> {
97 fn user_agent(&self) -> UserAgent<'_> {
98 self.0.clone()
99 }
100 }
101
102 #[test]
103 fn test_filter_when_disabled() {
104 let evt = testutils::get_event_with_user_agent("Googlebot");
105 let filter_result = should_filter(&evt, &FilterConfig { is_enabled: false });
106 assert_eq!(
107 filter_result,
108 Ok(()),
109 "Event filtered although filter should have been disabled"
110 )
111 }
112
113 #[test]
114 fn test_filter_banned_user_agents() {
115 let user_agents = [
116 "Mediapartners-Google",
117 "AdsBot-Google",
118 "Googlebot",
119 "FeedFetcher-Google",
120 "BingBot",
121 "BingPreview",
122 "Baiduspider",
123 "Slurp",
124 "Sogou",
125 "facebook",
126 "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
127 "facebookcatalog/1.0",
128 "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
129 "meta-externalfetcher/1.1",
130 "ia_archiver",
131 "bots ",
132 "bots;",
133 "bots)",
134 "spider ",
135 "spider;",
136 "spider)",
137 "Calypso AppCrawler",
138 "pingdom",
139 "lyticsbot",
140 "AWS Security Scanner",
141 "Mozilla/5.0 (Linux; Android 6.0.1; Calypso AppCrawler Build/MMB30Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.124 Mobile Safari/537.36",
142 "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
143 "Slack-ImgProxy 0.19 (+https://api.slack.com/robots)",
144 "Twitterbot/1.0",
145 "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
146 "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
147 "AdsBot-Google (+http://www.google.com/adsbot.html)",
148 "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)",
149 "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
150 "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
151 "Mozilla/5.0 (compatible;Cloudflare-Healthchecks/1.0;+https://www.cloudflare.com/; healthcheck-id: 0d1ca23e292c8c14)",
152 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 GTmetrix",
153 "Mozilla/5.0 (compatible; BrightEdgeOnCrawl/1.0; +http://www.oncrawl.com)",
154 "ELB-HealthChecker/2.0",
155 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Yeti/1.1; +https://naver.me/spd) Chrome/127.0.0.0 Safari/537.36",
156 "Mozilla/5.0; ClaudeBot",
157 "Mozilla/5.0; CCBot",
158 "; OAI-SearchBot/1.0; +https://openai.com/searchbot",
159 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot",
160 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
161 "Mozilla/5.0 (Device; OS_version) AppleWebKit/WebKit_version (KHTML, like Gecko)Version/Safari_version [Mobile/Mobile_version] Safari/WebKit_version (Applebot/Applebot_version; +http://www.apple.com/go/applebot)",
162 "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
163 ];
164
165 for banned_user_agent in &user_agents {
166 let event = testutils::get_event_with_user_agent(banned_user_agent);
167 let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
168 assert_ne!(
169 filter_result,
170 Ok(()),
171 "Failed to filter web crawler with user agent '{banned_user_agent}'"
172 );
173 }
174 }
175
176 #[test]
177 fn test_dont_filter_normal_user_agents() {
178 let normal_user_agents = [
179 "some user agent",
180 "IE",
181 "ie",
182 "opera",
183 "safari",
184 "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
185 "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
186 "Slackbot 1.0(+https://api.slack.com/robots)",
187 "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)",
188 "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
189 ];
190 for user_agent in &normal_user_agents {
191 let event = testutils::get_event_with_user_agent(user_agent);
192 let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
193 assert_eq!(
194 filter_result,
195 Ok(()),
196 "Failed benign user agent '{user_agent}'"
197 );
198 }
199 }
200
201 #[test]
202 fn test_filter_parsed_ua_only() {
203 let ua = UserAgent {
204 raw: None,
205 parsed: Some(relay_ua::UserAgent {
206 family: "Twitterbot".into(),
207 ..Default::default()
208 }),
209 };
210
211 let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
212 assert_ne!(filter_result, Ok(()));
213 }
214
215 #[test]
216 fn test_filter_parsed_ua_does_not_filter_default() {
217 let ua = UserAgent {
218 raw: None,
219 parsed: Some(Default::default()),
222 };
223
224 let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
225 assert_eq!(filter_result, Ok(()));
226 }
227}