objectstore_server/
killswitches.rs

1//! Runtime killswitches for disabling access to specific object contexts.
2//!
3//! A [`Killswitch`] matches requests by usecase, scope values, and optionally a downstream
4//! service glob pattern. When any configured killswitch matches an incoming request, the server
5//! rejects it immediately without forwarding to the storage backend.
6//!
7//! Killswitches are part of [`crate::config::Config`] and take effect on the next request after
8//! a configuration reload — no server restart is required.
9
10use std::cell::RefCell;
11use std::num::NonZeroUsize;
12
13use globset::{Glob, GlobMatcher};
14use lru::LruCache;
15use objectstore_options::Options;
16use objectstore_service::id::ObjectContext;
17use thread_local::ThreadLocal;
18
19pub use objectstore_options::Killswitch;
20
21/// A list of killswitches that may disable access to certain object contexts.
22///
23/// This serializes and deserializes directly from a list of killswitches.
24#[derive(Debug, Default)]
25pub struct Killswitches {
26    /// The actual list of killswitches.
27    switches: Vec<Killswitch>,
28
29    /// Glob cache for fast matching of killswitches.
30    cache: ThreadLocal<RefCell<LruCache<String, Option<GlobMatcher>>>>,
31}
32
33impl Killswitches {
34    /// Creates a new `Killswitches` instance with the given killswitches.
35    pub fn new(killswitches: Vec<Killswitch>) -> Self {
36        Self {
37            switches: killswitches,
38            cache: ThreadLocal::new(),
39        }
40    }
41
42    /// Returns `true` if any of the contained killswitches matches the given context.
43    ///
44    /// On match, emits a `server.request.killswitched` metric counter and a `warn!` log.
45    pub fn matches(&self, context: &ObjectContext, service: Option<&str>) -> bool {
46        let options = Options::get();
47
48        let mut cache = self
49            .cache
50            .get_or(|| RefCell::new(LruCache::new(NonZeroUsize::MIN)))
51            .borrow_mut();
52
53        let total_count = self.switches.len() + options.killswitches().len();
54        cache.resize(total_count.try_into().unwrap_or(NonZeroUsize::MIN));
55
56        let Some(killswitch) = self
57            .switches
58            .iter()
59            .chain(options.killswitches())
60            .find(|s| matches(s, context, service, &mut cache))
61        else {
62            return false;
63        };
64
65        objectstore_metrics::count!("server.request.killswitched");
66        objectstore_log::warn!(?killswitch, "Request rejected: killswitch active");
67        true
68    }
69
70    /// Returns a slice of the contained killswitches.
71    pub fn as_slice(&self) -> &[Killswitch] {
72        &self.switches
73    }
74}
75
76impl serde::Serialize for Killswitches {
77    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
78    where
79        S: serde::Serializer,
80    {
81        self.switches.serialize(serializer)
82    }
83}
84
85impl<'de> serde::Deserialize<'de> for Killswitches {
86    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
87    where
88        D: serde::Deserializer<'de>,
89    {
90        Ok(Self::new(Vec::deserialize(deserializer)?))
91    }
92}
93
94/// Returns `true` if this killswitch matches the given context and service.
95fn matches(
96    switch: &Killswitch,
97    context: &ObjectContext,
98    service: Option<&str>,
99    cache: &mut LruCache<String, Option<GlobMatcher>>,
100) -> bool {
101    if let Some(ref switch_usecase) = switch.usecase
102        && switch_usecase != &context.usecase
103    {
104        return false;
105    }
106
107    for (scope_name, scope_value) in &switch.scopes {
108        match context.scopes.get_value(scope_name) {
109            Some(value) if value == scope_value => (),
110            _ => return false,
111        }
112    }
113
114    if let Some(ref pattern) = switch.service {
115        // If pattern is specified but no service header present, don't match
116        let Some(service_value) = service else {
117            return false;
118        };
119
120        let lookup = cache.get_or_insert_ref(pattern, || {
121            Glob::new(pattern).ok().map(|g| g.compile_matcher())
122        });
123
124        match lookup {
125            Some(m) if m.is_match(service_value) => (),
126            _ => return false,
127        }
128    }
129
130    true
131}
132
133#[cfg(test)]
134mod tests {
135    use std::collections::BTreeMap;
136
137    use objectstore_types::scope::{Scope, Scopes};
138
139    use super::*;
140
141    fn cache() -> LruCache<String, Option<GlobMatcher>> {
142        LruCache::new(NonZeroUsize::MIN)
143    }
144
145    #[test]
146    fn test_matches_empty() {
147        let switch = Killswitch {
148            usecase: None,
149            scopes: BTreeMap::new(),
150            service: None,
151        };
152
153        let context = ObjectContext {
154            usecase: "any".to_string(),
155            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
156        };
157
158        assert!(matches(&switch, &context, None, &mut cache()));
159    }
160
161    #[test]
162    fn test_matches_usecase() {
163        let switch = Killswitch {
164            usecase: Some("test".to_string()),
165            scopes: BTreeMap::new(),
166            service: None,
167        };
168
169        let context = ObjectContext {
170            usecase: "test".to_string(),
171            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
172        };
173        assert!(matches(&switch, &context, Some("anyservice"), &mut cache()));
174
175        // usecase differs
176        let context = ObjectContext {
177            usecase: "other".to_string(),
178            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
179        };
180        assert!(!matches(
181            &switch,
182            &context,
183            Some("anyservice"),
184            &mut cache()
185        ));
186    }
187
188    #[test]
189    fn test_matches_scopes() {
190        let switch = Killswitch {
191            usecase: None,
192            scopes: BTreeMap::from([
193                ("org".to_string(), "123".to_string()),
194                ("project".to_string(), "456".to_string()),
195            ]),
196            service: None,
197        };
198
199        // match, ignoring extra scope
200        let context = ObjectContext {
201            usecase: "any".to_string(),
202            scopes: Scopes::from_iter([
203                Scope::create("org", "123").unwrap(),
204                Scope::create("project", "456").unwrap(),
205                Scope::create("extra", "789").unwrap(),
206            ]),
207        };
208        assert!(matches(&switch, &context, Some("anyservice"), &mut cache()));
209
210        // project differs
211        let context = ObjectContext {
212            usecase: "any".to_string(),
213            scopes: Scopes::from_iter([
214                Scope::create("org", "123").unwrap(),
215                Scope::create("project", "999").unwrap(),
216            ]),
217        };
218        assert!(!matches(
219            &switch,
220            &context,
221            Some("anyservice"),
222            &mut cache()
223        ));
224
225        // missing project
226        let context = ObjectContext {
227            usecase: "any".to_string(),
228            scopes: Scopes::from_iter([Scope::create("org", "123").unwrap()]),
229        };
230        assert!(!matches(
231            &switch,
232            &context,
233            Some("anyservice"),
234            &mut cache()
235        ));
236    }
237
238    #[test]
239    fn test_matches_full() {
240        let switch = Killswitch {
241            usecase: Some("test".to_string()),
242            scopes: BTreeMap::from([("org".to_string(), "123".to_string())]),
243            service: Some("myservice-*".to_string()),
244        };
245
246        // match with all filters
247        let context = ObjectContext {
248            usecase: "test".to_string(),
249            scopes: Scopes::from_iter([Scope::create("org", "123").unwrap()]),
250        };
251        assert!(matches(
252            &switch,
253            &context,
254            Some("myservice-prod"),
255            &mut cache()
256        ));
257
258        // usecase differs
259        let context = ObjectContext {
260            usecase: "other".to_string(),
261            scopes: Scopes::from_iter([Scope::create("org", "123").unwrap()]),
262        };
263        assert!(!matches(
264            &switch,
265            &context,
266            Some("myservice-prod"),
267            &mut cache()
268        ));
269
270        // scope differs
271        let context = ObjectContext {
272            usecase: "test".to_string(),
273            scopes: Scopes::from_iter([Scope::create("org", "999").unwrap()]),
274        };
275        assert!(!matches(
276            &switch,
277            &context,
278            Some("myservice-prod"),
279            &mut cache()
280        ));
281
282        // service differs
283        let context = ObjectContext {
284            usecase: "test".to_string(),
285            scopes: Scopes::from_iter([Scope::create("org", "123").unwrap()]),
286        };
287        assert!(!matches(
288            &switch,
289            &context,
290            Some("otherservice"),
291            &mut cache()
292        ));
293
294        // missing service header
295        let context = ObjectContext {
296            usecase: "test".to_string(),
297            scopes: Scopes::from_iter([Scope::create("org", "123").unwrap()]),
298        };
299        assert!(!matches(&switch, &context, None, &mut cache()));
300    }
301
302    #[test]
303    fn test_matches_service_exact() {
304        let switch = Killswitch {
305            usecase: None,
306            scopes: BTreeMap::new(),
307            service: Some("myservice".to_string()),
308        };
309
310        let context = ObjectContext {
311            usecase: "any".to_string(),
312            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
313        };
314
315        assert!(matches(&switch, &context, Some("myservice"), &mut cache()));
316        assert!(!matches(
317            &switch,
318            &context,
319            Some("otherservice"),
320            &mut cache()
321        ));
322        assert!(!matches(&switch, &context, None, &mut cache()));
323    }
324
325    #[test]
326    fn test_matches_service_glob() {
327        let switch = Killswitch {
328            usecase: None,
329            scopes: BTreeMap::new(),
330            service: Some("myservice-*".to_string()),
331        };
332
333        let context = ObjectContext {
334            usecase: "any".to_string(),
335            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
336        };
337
338        // Matches with glob pattern
339        assert!(matches(
340            &switch,
341            &context,
342            Some("myservice-prod"),
343            &mut cache()
344        ));
345        assert!(matches(
346            &switch,
347            &context,
348            Some("myservice-dev"),
349            &mut cache()
350        ));
351        assert!(matches(
352            &switch,
353            &context,
354            Some("myservice-staging"),
355            &mut cache()
356        ));
357
358        // Doesn't match different service
359        assert!(!matches(
360            &switch,
361            &context,
362            Some("otherservice"),
363            &mut cache()
364        ));
365        assert!(!matches(
366            &switch,
367            &context,
368            Some("otherservice-prod"),
369            &mut cache()
370        ));
371
372        // Doesn't match prefix without separator
373        assert!(!matches(&switch, &context, Some("myservice"), &mut cache()));
374    }
375
376    #[test]
377    fn test_matches_service_invalid_glob() {
378        let switch = Killswitch {
379            usecase: None,
380            scopes: BTreeMap::new(),
381            service: Some("[invalid".to_string()), // Invalid glob pattern
382        };
383
384        let context = ObjectContext {
385            usecase: "any".to_string(),
386            scopes: Scopes::from_iter([Scope::create("any", "value").unwrap()]),
387        };
388
389        // Invalid pattern should not match anything
390        assert!(!matches(
391            &switch,
392            &context,
393            Some("anyservice"),
394            &mut cache()
395        ));
396        assert!(!matches(&switch, &context, Some("[invalid"), &mut cache()));
397    }
398}
objectstore_server/killswitches.rs

objectstore_server/
killswitches.rs
