relay_filter/
web_crawlers.rs

1//! Filters events coming from user agents known to be web crawlers.
2
3use std::sync::LazyLock;
4
5use regex::Regex;
6
7use crate::{FilterConfig, FilterStatKey, Filterable};
8
9static WEB_CRAWLERS: LazyLock<Regex> = LazyLock::new(|| {
10    Regex::new(
11        r"(?ix)
12        Mediapartners-Google|
13        AdsBot-Google|
14        Googlebot|
15        FeedFetcher-Google|
16        Storebot-Google|
17        BingBot|                    # Bing search
18        BingPreview|
19        Baiduspider|                # Baidu search
20        Slurp|                      # Yahoo
21        Sogou|                      # Sogou
22        facebook|                   # facebook
23        meta-|                      # meta/facebook
24        ia_archiver|                # Alexa
25        bots?([/\s\);]|$)|          # Generic bot
26        spider([/\s\);]|$)|         # Generic spider
27        Slack|                      # Slack - see https://api.slack.com/robots
28        Calypso\sAppCrawler|        # Google indexing bot
29        pingdom|                    # Pingdom
30        lyticsbot|                  # Lytics
31        AWS\sSecurity\sScanner|     # AWS Security Scanner causing DisallowedHost errors in Django, see
32                                    # https://forums.aws.amazon.com/thread.jspa?messageID=932404
33                                    # and https://github.com/getsentry/sentry-python/issues/641
34        HubSpot\sCrawler|           # HubSpot web crawler (web-crawlers@hubspot.com)
35        Bytespider|                 # Bytedance
36        Better\sUptime|             # Better Uptime
37        Cloudflare-Healthchecks|    # Cloudflare Health Checks
38        GTmetrix|                   # GTmetrix
39        BrightEdgeOnCrawl|          # BrightEdge - see https://www.brightedge.com/news/press-releases/brightedge-acquires-oncrawl-future-proof-web-30-strategies
40        ELB-HealthChecker|          # AWS Elastic Load Balancing Health Checks
41        naver.me/spd|               # Yeti/1.1 - naver.me
42        ClaudeBot|                  # Anthropic - see https://support.anthropic.com/en/articles/8896518-does-anthropic-crawl-data-from-the-web-and-how-can-site-owners-block-the-crawler
43        CCBot|                      # CCBot - see https://commoncrawl.org/ccbot
44        OAI-SearchBot|              # OpenAI - see https://platform.openai.com/docs/bots
45        GPTBot|                     # OpenAI - see https://platform.openai.com/docs/bots
46        PerplexityBot|              # Perplexity - see https://docs.perplexity.ai/guides/bots
47        Applebot|                   # Apple - see https://support.apple.com/en-us/119829
48        DuckDuckBot|                # DuckDuckGo - see https://duckduckgo.com/duckduckgo-help-pages/results/duckduckbot
49        Lightpanda                  # Lightpanda - see https://lightpanda.io/
50    "
51    )
52    .expect("Invalid web crawlers filter Regex")
53});
54
55static ALLOWED_WEB_CRAWLERS: LazyLock<Regex> = LazyLock::new(|| {
56    Regex::new(
57        r"(?ix)
58        Slackbot\s1\.\d+|            # Slack - see https://api.slack.com/robots
59        SentryUptimeBot|             # Uptime Checker https://docs.sentry.io/product/alerts/uptime-monitoring/
60        ChatGPT-User                 # ChatGPT user prompted requests
61    ",
62    )
63    .expect("Invalid allowed web crawlers filter Regex")
64});
65
66/// Checks if the event originates from a known web crawler.
67fn matches(user_agent: &str) -> bool {
68    WEB_CRAWLERS.is_match(user_agent) && !ALLOWED_WEB_CRAWLERS.is_match(user_agent)
69}
70
71/// Filters events originating from a known web crawler.
72pub fn should_filter<F: Filterable>(item: &F, config: &FilterConfig) -> Result<(), FilterStatKey> {
73    if !config.is_enabled {
74        return Ok(());
75    }
76
77    let user_agent = item.user_agent();
78    let family = user_agent.parsed.as_ref().map(|ua| ua.family.as_ref());
79
80    // Use the raw user agent if it is available, as it is higher quality. For example some user
81    // agents may be parsed as `Other` while the raw user agent would be filtered.
82    //
83    // Fallback to the parsed user agent as under circumstances only that may be available.
84    if user_agent.raw.or(family).is_some_and(matches) {
85        return Err(FilterStatKey::WebCrawlers);
86    }
87
88    Ok(())
89}
90
91#[cfg(test)]
92mod tests {
93    use super::*;
94    use crate::{UserAgent, testutils};
95
96    #[derive(Debug)]
97    struct TestFilterable<'a>(UserAgent<'a>);
98
99    impl<'a> Filterable for TestFilterable<'a> {
100        fn user_agent(&self) -> UserAgent<'_> {
101            self.0.clone()
102        }
103    }
104
105    #[test]
106    fn test_filter_when_disabled() {
107        let evt = testutils::get_event_with_user_agent("Googlebot");
108        let filter_result = should_filter(&evt, &FilterConfig { is_enabled: false });
109        assert_eq!(
110            filter_result,
111            Ok(()),
112            "Event filtered although filter should have been disabled"
113        )
114    }
115
116    #[test]
117    fn test_filter_banned_user_agents() {
118        let user_agents = [
119            "Mediapartners-Google",
120            "AdsBot-Google",
121            "Googlebot",
122            "FeedFetcher-Google",
123            "Storebot-Google",
124            "Mozilla/5.0 (X11; Linux x86_64; Storebot-Google/1.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
125            "BingBot",
126            "BingPreview",
127            "Baiduspider",
128            "Slurp",
129            "Sogou",
130            "facebook",
131            "facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)",
132            "facebookcatalog/1.0",
133            "meta-externalagent/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
134            "meta-externalfetcher/1.1",
135            "ia_archiver",
136            "bots ",
137            "bots;",
138            "bots)",
139            "spider ",
140            "spider;",
141            "spider)",
142            "Calypso AppCrawler",
143            "pingdom",
144            "lyticsbot",
145            "AWS Security Scanner",
146            "Mozilla/5.0 (Linux; Android 6.0.1; Calypso AppCrawler Build/MMB30Y; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/53.0.2785.124 Mobile Safari/537.36",
147            "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
148            "Slack-ImgProxy 0.19 (+https://api.slack.com/robots)",
149            "Twitterbot/1.0",
150            "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
151            "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)",
152            "AdsBot-Google (+http://www.google.com/adsbot.html)",
153            "Mozilla/5.0 (compatible; HubSpot Crawler; web-crawlers@hubspot.com)",
154            "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; Bytespider; spider-feedback@bytedance.com)",
155            "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36",
156            "Mozilla/5.0 (compatible;Cloudflare-Healthchecks/1.0;+https://www.cloudflare.com/; healthcheck-id: 0d1ca23e292c8c14)",
157            "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36 GTmetrix",
158            "Mozilla/5.0 (compatible; BrightEdgeOnCrawl/1.0; +http://www.oncrawl.com)",
159            "ELB-HealthChecker/2.0",
160            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko; compatible; Yeti/1.1; +https://naver.me/spd) Chrome/127.0.0.0 Safari/537.36",
161            "Mozilla/5.0; ClaudeBot",
162            "Mozilla/5.0; CCBot",
163            "; OAI-SearchBot/1.0; +https://openai.com/searchbot",
164            "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; GPTBot/1.1; +https://openai.com/gptbot",
165            "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)",
166            "Mozilla/5.0 (Device; OS_version) AppleWebKit/WebKit_version (KHTML, like Gecko)Version/Safari_version [Mobile/Mobile_version] Safari/WebKit_version (Applebot/Applebot_version; +http://www.apple.com/go/applebot)",
167            "DuckDuckBot/1.1; (+http://duckduckgo.com/duckduckbot.html)",
168            "Lightpanda/1.0",
169        ];
170
171        for banned_user_agent in &user_agents {
172            let event = testutils::get_event_with_user_agent(banned_user_agent);
173            let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
174            assert_ne!(
175                filter_result,
176                Ok(()),
177                "Failed to filter web crawler with user agent '{banned_user_agent}'"
178            );
179        }
180    }
181
182    #[test]
183    fn test_dont_filter_normal_user_agents() {
184        let normal_user_agents = [
185            "some user agent",
186            "IE",
187            "ie",
188            "opera",
189            "safari",
190            "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
191            "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
192            "Slackbot 1.0(+https://api.slack.com/robots)",
193            "SentryUptimeBot/1.0 (+http://docs.sentry.io/product/alerts/uptime-monitoring/)",
194            "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",
195        ];
196        for user_agent in &normal_user_agents {
197            let event = testutils::get_event_with_user_agent(user_agent);
198            let filter_result = should_filter(&event, &FilterConfig { is_enabled: true });
199            assert_eq!(
200                filter_result,
201                Ok(()),
202                "Failed benign user agent '{user_agent}'"
203            );
204        }
205    }
206
207    #[test]
208    fn test_filter_parsed_ua_only() {
209        let ua = UserAgent {
210            raw: None,
211            parsed: Some(relay_ua::UserAgent {
212                family: "Twitterbot".into(),
213                ..Default::default()
214            }),
215        };
216
217        let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
218        assert_ne!(filter_result, Ok(()));
219    }
220
221    #[test]
222    fn test_filter_parsed_ua_does_not_filter_default() {
223        let ua = UserAgent {
224            raw: None,
225            // This may happen if the raw user agent cannot be parsed or is not available,
226            // in this case the filter should not accidentally remove the event.
227            parsed: Some(Default::default()),
228        };
229
230        let filter_result = should_filter(&TestFilterable(ua), &FilterConfig { is_enabled: true });
231        assert_eq!(filter_result, Ok(()));
232    }
233}
relay_filter/web_crawlers.rs

relay_filter/
web_crawlers.rs
