Skip to main content

relay_pii/
processor.rs

1use std::borrow::Cow;
2use std::collections::{BTreeMap, BTreeSet};
3use std::mem;
4use std::sync::OnceLock;
5
6use regex::Regex;
7use relay_event_schema::processor::{
8    self, Chunk, FieldAttrs, Pii, ProcessValue, ProcessingAction, ProcessingResult,
9    ProcessingState, Processor, ValueType, enum_set, process_value,
10};
11use relay_event_schema::protocol::{
12    AsPair, Event, IpAddr, NativeImagePath, PairList, Replay, ResponseContext, User,
13};
14use relay_protocol::{Annotated, Array, Meta, Remark, RemarkType, Value};
15
16use crate::compiledconfig::{CompiledPiiConfig, RuleRef};
17use crate::config::RuleType;
18use crate::redactions::Redaction;
19use crate::regex::ScrubbingRegexRef;
20use crate::regexes::{self, ANYTHING_REGEX, PatternType, ReplaceBehavior};
21use crate::utils;
22
23/// Controls how scrubbing rules are applied to attributes.
24#[derive(Debug, Clone, Copy)]
25pub enum AttributeMode {
26    /// Treat the attribute as an object and allow referring
27    /// to individual fields.
28    Object,
29    /// Identify the attribute with its value and apply all
30    /// rules there directly.
31    ValueOnly,
32}
33
34/// A processor that performs PII stripping.
35pub struct PiiProcessor<'a> {
36    /// Controls how rules are applied to attributes.
37    attribute_mode: AttributeMode,
38    compiled_config: &'a CompiledPiiConfig,
39}
40
41impl<'a> PiiProcessor<'a> {
42    /// Creates a new processor based on a config.
43    pub fn new(compiled_config: &'a CompiledPiiConfig) -> PiiProcessor<'a> {
44        // this constructor needs to be cheap... a new PiiProcessor is created for each event. Move
45        // any init logic into CompiledPiiConfig::new.
46        PiiProcessor {
47            compiled_config,
48            attribute_mode: AttributeMode::Object,
49        }
50    }
51
52    /// Sets an `AttributeMode` on this processor.
53    pub fn attribute_mode(mut self, attribute_mode: AttributeMode) -> Self {
54        self.attribute_mode = attribute_mode;
55        self
56    }
57
58    fn apply_all_rules(
59        &self,
60        meta: &mut Meta,
61        state: &ProcessingState<'_>,
62        mut value: Option<&mut String>,
63    ) -> ProcessingResult {
64        let pii = state.pii();
65        if pii == Pii::False {
66            return Ok(());
67        }
68
69        for (selector, rules) in self.compiled_config.applications.iter() {
70            if selector.matches_path(&state.path()) {
71                #[allow(clippy::needless_option_as_deref)]
72                for rule in rules {
73                    let reborrowed_value = value.as_deref_mut();
74                    apply_rule_to_value(meta, rule, state.path().key(), reborrowed_value)?;
75                }
76            }
77        }
78
79        Ok(())
80    }
81}
82
83impl Processor for PiiProcessor<'_> {
84    fn before_process<T: ProcessValue>(
85        &mut self,
86        value: Option<&T>,
87        meta: &mut Meta,
88        state: &ProcessingState<'_>,
89    ) -> ProcessingResult {
90        if let Some(Value::String(original_value)) = meta.original_value_as_mut() {
91            // Also apply pii scrubbing to the original value (set by normalization or other processors),
92            // such that we do not leak sensitive data through meta. Deletes `original_value` if an Error
93            // value is returned.
94            if let Some(parent) = state.iter().next() {
95                let path = state.path();
96                let new_state = parent.enter_borrowed(
97                    path.key().unwrap_or(""),
98                    Some(Cow::Borrowed(state.attrs())),
99                    enum_set!(ValueType::String),
100                );
101
102                if self
103                    .apply_all_rules(&mut Meta::default(), &new_state, Some(original_value))
104                    .is_err()
105                {
106                    // `apply_all_rules` returned `DeleteValueHard` or `DeleteValueSoft`, so delete the original as well.
107                    meta.set_original_value(Option::<String>::None);
108                }
109            }
110        }
111
112        // booleans cannot be PII, and strings are handled in process_string
113        if state.value_type().contains(ValueType::Boolean)
114            || state.value_type().contains(ValueType::String)
115        {
116            return Ok(());
117        }
118
119        if value.is_none() {
120            return Ok(());
121        }
122
123        // apply rules based on key/path
124        self.apply_all_rules(meta, state, None)
125    }
126
127    fn process_array<T>(
128        &mut self,
129        array: &mut Array<T>,
130        _meta: &mut Meta,
131        state: &ProcessingState<'_>,
132    ) -> ProcessingResult
133    where
134        T: ProcessValue,
135    {
136        if is_pairlist(array) {
137            for annotated in array {
138                let mut mapped = mem::take(annotated).map_value(T::into_value);
139
140                if let Some(Value::Array(pair)) = mapped.value_mut() {
141                    let mut value = mem::take(&mut pair[1]);
142                    let value_type = ValueType::for_field(&value);
143
144                    if let Some(key_name) = &pair[0].as_str() {
145                        // We enter the key of the first element of the array, since we treat it
146                        // as a pair.
147                        let key_state =
148                            state.enter_borrowed(key_name, state.inner_attrs(), value_type);
149                        // We process the value with a state that "simulates" the first value of the
150                        // array as if it was the key of a dictionary.
151                        process_value(&mut value, self, &key_state)?;
152                    }
153
154                    // Put value back into pair.
155                    pair[1] = value;
156                }
157
158                // Put pair back into array.
159                *annotated = T::from_value(mapped);
160            }
161
162            Ok(())
163        } else {
164            // If we didn't find a pairlist, we can process child values as normal.
165            array.process_child_values(self, state)
166        }
167    }
168
169    fn process_string(
170        &mut self,
171        value: &mut String,
172        meta: &mut Meta,
173        state: &ProcessingState<'_>,
174    ) -> ProcessingResult {
175        if let "" | "true" | "false" | "null" | "undefined" = value.as_str() {
176            return Ok(());
177        }
178
179        // same as before_process. duplicated here because we can only check for "true",
180        // "false" etc in process_string.
181        self.apply_all_rules(meta, state, Some(value))
182    }
183
184    fn process_native_image_path(
185        &mut self,
186        NativeImagePath(value): &mut NativeImagePath,
187        meta: &mut Meta,
188        state: &ProcessingState<'_>,
189    ) -> ProcessingResult {
190        // In NativeImagePath we must not strip the file's basename because that would break
191        // processing.
192        //
193        // We pop the basename from the end of the string, call process_string and push the
194        // basename again.
195        //
196        // The ranges in Meta should still be right as long as we only pop/push from the end of the
197        // string. If we decide that we need to preserve anything other than suffixes all PII
198        // tooltips/annotations are potentially wrong.
199
200        if let Some(index) = value.rfind(['/', '\\']) {
201            let basename = value.split_off(index);
202            match self.process_string(value, meta, state) {
203                Ok(()) => value.push_str(&basename),
204                Err(
205                    ProcessingAction::DeleteValueHard
206                    | ProcessingAction::DeleteValueWithRemark(_)
207                    | ProcessingAction::DeleteValueSoft,
208                ) => {
209                    basename[1..].clone_into(value);
210                }
211                Err(ProcessingAction::InvalidTransaction(x)) => {
212                    return Err(ProcessingAction::InvalidTransaction(x));
213                }
214            }
215        }
216
217        Ok(())
218    }
219
220    fn process_pairlist<T: ProcessValue + AsPair>(
221        &mut self,
222        value: &mut PairList<T>,
223        _meta: &mut Meta,
224        state: &ProcessingState,
225    ) -> ProcessingResult {
226        utils::process_pairlist(self, value, state)
227    }
228
229    fn process_attributes(
230        &mut self,
231        value: &mut relay_event_schema::protocol::Attributes,
232        _meta: &mut Meta,
233        state: &ProcessingState,
234    ) -> ProcessingResult {
235        match self.attribute_mode {
236            // Treat each attribute as an object and just process them field by field.
237            AttributeMode::Object => value.process_child_values(self, state),
238            // Identify each attribute with its `value` and only process that.
239            AttributeMode::ValueOnly => {
240                for (key, attribute) in value.0.iter_mut() {
241                    let Some(attribute) = attribute.value_mut() else {
242                        continue;
243                    };
244
245                    // We need some manual state management here because we're bypassing all the
246                    // intermediate structures and pointing at the value directly. This essentially
247                    // mimics the attributes and value type that the metastructure derivation would
248                    // produce for the attribute vaue.
249                    let attrs = FieldAttrs::new()
250                        .pii_dynamic(relay_event_schema::protocol::attribute_pii_from_conventions);
251                    let inner_value = &mut attribute.value.value;
252                    let inner_value_type = ValueType::for_field(inner_value);
253                    let entered =
254                        state.enter_borrowed(key, Some(Cow::Borrowed(&attrs)), inner_value_type);
255
256                    processor::process_value(inner_value, self, &entered)?;
257                    self.process_other(&mut attribute.other, state)?;
258                }
259                Ok(())
260            }
261        }
262    }
263
264    fn process_user(
265        &mut self,
266        user: &mut User,
267        _meta: &mut Meta,
268        state: &ProcessingState<'_>,
269    ) -> ProcessingResult {
270        let ip_was_valid = user.ip_address.value().is_none_or(IpAddr::is_valid);
271
272        // Recurse into the user and does PII processing on fields.
273        user.process_child_values(self, state)?;
274
275        let has_other_fields = user.id.value().is_some()
276            || user.username.value().is_some()
277            || user.email.value().is_some();
278
279        let ip_is_still_valid = user.ip_address.value().is_none_or(IpAddr::is_valid);
280
281        // If the IP address has become invalid as part of PII processing, we move it into the user
282        // ID. That ensures people can do IP hashing and still have a correct users-affected count.
283        //
284        // Right now both Snuba and EventUser discard unparseable IPs for indexing, and we assume
285        // we want to keep it that way.
286        //
287        // If there are any other fields set that take priority over the IP for uniquely
288        // identifying a user (has_other_fields), we do not want to do anything. The value will be
289        // wiped out in renormalization anyway.
290        if ip_was_valid && !has_other_fields && !ip_is_still_valid {
291            user.id = mem::take(&mut user.ip_address).map_value(|ip| ip.into_inner().into());
292            user.ip_address.meta_mut().add_remark(Remark::new(
293                RemarkType::Removed,
294                "pii:ip_address".to_owned(),
295            ));
296        }
297
298        Ok(())
299    }
300
301    // Replay PII processor entry point.
302    fn process_replay(
303        &mut self,
304        replay: &mut Replay,
305        _meta: &mut Meta,
306        state: &ProcessingState<'_>,
307    ) -> ProcessingResult {
308        replay.process_child_values(self, state)?;
309        Ok(())
310    }
311}
312
313#[derive(Default)]
314struct PairListProcessor {
315    is_pair: bool,
316    has_string_key: bool,
317}
318
319impl PairListProcessor {
320    /// Returns true if the processor identified the supplied data as an array composed of
321    /// a key (string) and a value.
322    fn is_pair_array(&self) -> bool {
323        self.is_pair && self.has_string_key
324    }
325}
326
327impl Processor for PairListProcessor {
328    fn process_array<T>(
329        &mut self,
330        value: &mut Array<T>,
331        _meta: &mut Meta,
332        state: &ProcessingState<'_>,
333    ) -> ProcessingResult
334    where
335        T: ProcessValue,
336    {
337        self.is_pair = state.depth() == 0 && value.len() == 2;
338        if self.is_pair {
339            let key_type = ValueType::for_field(&value[0]);
340            process_value(
341                &mut value[0],
342                self,
343                &state.enter_index(0, state.inner_attrs(), key_type),
344            )?;
345        }
346
347        Ok(())
348    }
349
350    fn process_string(
351        &mut self,
352        _value: &mut String,
353        _meta: &mut Meta,
354        state: &ProcessingState<'_>,
355    ) -> ProcessingResult where {
356        if state.depth() == 1 && state.path().index() == Some(0) {
357            self.has_string_key = true;
358        }
359
360        Ok(())
361    }
362}
363
364fn is_pairlist<T: ProcessValue>(array: &mut Array<T>) -> bool {
365    for element in array.iter_mut() {
366        let mut visitor = PairListProcessor::default();
367        process_value(element, &mut visitor, ProcessingState::root()).ok();
368        if !visitor.is_pair_array() {
369            return false;
370        }
371    }
372
373    !array.is_empty()
374}
375
376/// Scrubs GraphQL variables from the event.
377pub fn scrub_graphql(event: &mut Event) {
378    let mut keys: BTreeSet<&str> = BTreeSet::new();
379
380    let mut is_graphql = false;
381
382    // Collect the variables keys and scrub them out.
383    if let Some(request) = event.request.value_mut()
384        && let Some(Value::Object(data)) = request.data.value_mut()
385    {
386        if let Some(api_target) = request.api_target.value()
387            && api_target.eq_ignore_ascii_case("graphql")
388        {
389            is_graphql = true;
390        }
391
392        if is_graphql
393            && let Some(Annotated(Some(Value::Object(variables)), _)) = data.get_mut("variables")
394        {
395            for (key, value) in variables.iter_mut() {
396                keys.insert(key);
397                value.set_value(Some(Value::String("[Filtered]".to_owned())));
398            }
399        }
400    }
401
402    if !is_graphql {
403        return;
404    }
405
406    // Scrub PII from the data object if they match the variables keys.
407    if let Some(contexts) = event.contexts.value_mut()
408        && let Some(response) = contexts.get_mut::<ResponseContext>()
409        && let Some(Value::Object(data)) = response.data.value_mut()
410        && let Some(Annotated(Some(Value::Object(graphql_data)), _)) = data.get_mut("data")
411    {
412        if !keys.is_empty() {
413            scrub_graphql_data(&keys, graphql_data);
414        } else {
415            // If we don't have the variable keys, we scrub the whole data object
416            // because the query or mutation weren't parameterized.
417            data.remove("data");
418        }
419    }
420}
421
422/// Scrubs values from the data object to `[Filtered]`.
423fn scrub_graphql_data(keys: &BTreeSet<&str>, data: &mut BTreeMap<String, Annotated<Value>>) {
424    for (key, value) in data.iter_mut() {
425        match value.value_mut() {
426            Some(Value::Object(item_data)) => {
427                scrub_graphql_data(keys, item_data);
428            }
429            _ => {
430                if keys.contains(key.as_str()) {
431                    value.set_value(Some(Value::String("[Filtered]".to_owned())));
432                }
433            }
434        }
435    }
436}
437
438fn apply_rule_to_value(
439    meta: &mut Meta,
440    rule: &RuleRef,
441    key: Option<&str>,
442    mut value: Option<&mut String>,
443) -> ProcessingResult {
444    // The rule might specify to remove or to redact. If redaction is chosen, we need to
445    // chunk up the value, otherwise we need to simply mark the value for deletion.
446    let should_redact_chunks = !matches!(rule.redaction, Redaction::Default | Redaction::Remove);
447
448    // In case the value is not a string (but a container, bool or number) and the rule matches on
449    // anything, we can only remove the value (not replace, hash, etc).
450    if rule.ty == RuleType::Anything && (value.is_none() || !should_redact_chunks) {
451        // The value is a container, @anything on a container can do nothing but delete.
452        meta.add_remark(Remark::new(RemarkType::Removed, rule.origin.clone()));
453        return Err(ProcessingAction::DeleteValueHard);
454    }
455
456    macro_rules! apply_regex {
457        ($regex:expr, $replace_behavior:expr) => {
458            if let Some(ref mut value) = value {
459                processor::process_chunked_value(value, meta, |chunks| {
460                    apply_regex_to_chunks(chunks, rule, $regex, $replace_behavior)
461                });
462            }
463        };
464    }
465
466    for (pattern_type, regex, replace_behavior) in regexes::get_regex_for_rule_type(&rule.ty) {
467        if matches!(pattern_type, PatternType::Key | PatternType::KeyValue)
468            && key.is_some_and(|key| regex.is_match(key))
469        {
470            if value.is_some() && should_redact_chunks {
471                // If we're given a string value here, redact the value like we would with
472                // @anything.
473                apply_regex!(ANYTHING_REGEX.as_ref(), replace_behavior);
474            } else {
475                meta.add_remark(Remark::new(RemarkType::Removed, rule.origin.clone()));
476                return Err(ProcessingAction::DeleteValueHard);
477            }
478        } else if matches!(pattern_type, PatternType::Value | PatternType::KeyValue) {
479            apply_regex!(regex, replace_behavior);
480        }
481    }
482
483    Ok(())
484}
485
486fn apply_regex_to_chunks<'a>(
487    chunks: Vec<Chunk<'a>>,
488    rule: &RuleRef,
489    regex: ScrubbingRegexRef<'_>,
490    replace_behavior: ReplaceBehavior,
491) -> Vec<Chunk<'a>> {
492    // NB: This function allocates the entire string and all chunks a second time. This means it
493    // cannot reuse chunks and reallocates them. Ideally, we would be able to run the regex directly
494    // on the chunks, but the `regex` crate does not support that.
495
496    let mut search_string = String::new();
497    let mut has_text = false;
498    for chunk in &chunks {
499        match chunk {
500            Chunk::Text { text } => {
501                has_text = true;
502                search_string.push_str(&text.replace('\x00', ""));
503            }
504            Chunk::Redaction { .. } => search_string.push('\x00'),
505        }
506    }
507
508    if !has_text {
509        // Nothing to replace.
510        return chunks;
511    }
512
513    // Early exit if this regex does not match and return the original chunks.
514    let mut captures_iter = regex.captures_iter(&search_string).peekable();
515    if captures_iter.peek().is_none() {
516        return chunks;
517    }
518
519    let mut replacement_chunks = vec![];
520    for chunk in chunks {
521        if let Chunk::Redaction { .. } = chunk {
522            replacement_chunks.push(chunk);
523        }
524    }
525    replacement_chunks.reverse();
526
527    fn process_text<'a>(
528        text: &str,
529        rv: &mut Vec<Chunk<'a>>,
530        replacement_chunks: &mut Vec<Chunk<'a>>,
531    ) {
532        if text.is_empty() {
533            return;
534        }
535
536        // ALERT: This logic assumes that `regex` doesn't match a capture
537        // group starting on a null byte. If you get an error in debug mode
538        // about `replacement_chunks` not being empty, check the regex.
539        static NULL_SPLIT_RE: OnceLock<Regex> = OnceLock::new();
540        let regex = NULL_SPLIT_RE.get_or_init(|| {
541            #[allow(clippy::trivial_regex)]
542            Regex::new("\x00").unwrap()
543        });
544
545        let mut pos = 0;
546        for piece in regex.find_iter(text) {
547            rv.push(Chunk::Text {
548                text: Cow::Owned(text[pos..piece.start()].to_string()),
549            });
550            rv.push(replacement_chunks.pop().unwrap());
551            pos = piece.end();
552        }
553
554        rv.push(Chunk::Text {
555            text: Cow::Owned(text[pos..].to_string()),
556        });
557    }
558
559    let mut pos = 0;
560    let mut rv = Vec::with_capacity(replacement_chunks.len());
561
562    match replace_behavior {
563        ReplaceBehavior::Groups(ref groups) => {
564            for m in captures_iter {
565                for (idx, g) in m.iter().enumerate() {
566                    if let Some(g) = g
567                        && groups.contains(&(idx as u8))
568                    {
569                        process_text(
570                            &search_string[pos..g.start()],
571                            &mut rv,
572                            &mut replacement_chunks,
573                        );
574                        insert_replacement_chunks(rule, g.as_str(), &mut rv);
575                        pos = g.end();
576                    }
577                }
578            }
579            process_text(&search_string[pos..], &mut rv, &mut replacement_chunks);
580            debug_assert!(replacement_chunks.is_empty());
581        }
582        ReplaceBehavior::Value => {
583            // We only want to replace a string value, and the replacement chunk for that is
584            // inserted by insert_replacement_chunks. Adding chunks from replacement_chunks
585            // results in the incorrect behavior of a total of more chunks than the input.
586            insert_replacement_chunks(rule, &search_string, &mut rv);
587        }
588    }
589    rv
590}
591
592fn insert_replacement_chunks(rule: &RuleRef, text: &str, output: &mut Vec<Chunk<'_>>) {
593    match &rule.redaction {
594        Redaction::Default | Redaction::Remove => {
595            output.push(Chunk::Redaction {
596                text: Cow::Borrowed(""),
597                rule_id: Cow::Owned(rule.origin.to_string()),
598                ty: RemarkType::Removed,
599            });
600        }
601        Redaction::Mask => {
602            let buf = vec!['*'; text.chars().count()];
603
604            output.push(Chunk::Redaction {
605                ty: RemarkType::Masked,
606                rule_id: Cow::Owned(rule.origin.to_string()),
607                text: buf.into_iter().collect(),
608            })
609        }
610        Redaction::Hash => {
611            output.push(Chunk::Redaction {
612                ty: RemarkType::Pseudonymized,
613                rule_id: Cow::Owned(rule.origin.to_string()),
614                text: Cow::Owned(utils::hash_value(text.as_bytes())),
615            });
616        }
617        Redaction::Replace(replace) => {
618            output.push(Chunk::Redaction {
619                ty: RemarkType::Substituted,
620                rule_id: Cow::Owned(rule.origin.to_string()),
621                text: Cow::Owned(replace.text.clone()),
622            });
623        }
624        Redaction::Other => relay_log::debug!("Incoming redaction is not supported"),
625    }
626}
627
628#[cfg(test)]
629mod tests {
630    use insta::{allow_duplicates, assert_debug_snapshot};
631    use relay_event_schema::processor::process_value;
632    use relay_event_schema::protocol::{
633        Addr, Breadcrumb, DebugImage, DebugMeta, ExtraValue, Headers, LogEntry, Message,
634        NativeDebugImage, Request, Span, TagEntry, Tags, TraceContext,
635    };
636    use relay_protocol::{FromValue, Object, assert_annotated_snapshot, get_value};
637    use serde_json::json;
638
639    use super::*;
640    use crate::{DataScrubbingConfig, PiiConfig, ReplaceRedaction};
641
642    fn to_pii_config(datascrubbing_config: &DataScrubbingConfig) -> Option<PiiConfig> {
643        use crate::convert::to_pii_config as to_pii_config_impl;
644        let rv = to_pii_config_impl(datascrubbing_config);
645        if let Some(ref config) = rv {
646            let roundtrip: PiiConfig =
647                serde_json::from_value(serde_json::to_value(config).unwrap()).unwrap();
648            assert_eq!(&roundtrip, config);
649        }
650        rv
651    }
652
653    #[test]
654    fn test_scrub_original_value() {
655        let mut data = Event::from_value(
656            json!({
657                "user": {
658                    "username": "hey  man 73.133.27.120", // should be stripped despite not being "known ip field"
659                    "ip_address": "is this an ip address? 73.133.27.120", //  <--------
660                },
661                "extra":"invalid data my ip address is  74.133.27.120 and my credit card number is  4571234567890112 ",
662            })
663            .into(),
664        );
665
666        let scrubbing_config = DataScrubbingConfig {
667            scrub_data: true,
668            scrub_ip_addresses: true,
669            scrub_defaults: true,
670            ..Default::default()
671        };
672
673        let pii_config = to_pii_config(&scrubbing_config).unwrap();
674        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
675
676        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
677
678        assert_debug_snapshot!(&data);
679    }
680
681    #[test]
682    fn test_remark_overlap() {
683        let mut data = Annotated::<Event>::from_json_bytes(
684            br#"{
685                "extra": {"foo": "bar"},
686                "_meta":{
687                    "extra": {
688                        "foo":{
689                            "":{
690                                "rem":[["some_rule","s",0,3],["some_rule","s",0,3]]
691                            }
692                        }
693                    }
694                }
695            }"#,
696        )
697        .unwrap();
698
699        let scrubbing_config = DataScrubbingConfig {
700            scrub_data: true,
701            scrub_ip_addresses: true,
702            scrub_defaults: true,
703            ..Default::default()
704        };
705
706        let pii_config = to_pii_config(&scrubbing_config).unwrap();
707        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
708
709        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
710
711        // Verify that overlapping remarks do not make the string longer:
712        assert_eq!(get_value!(data.extra["foo"]!).0.as_str(), Some("bar"));
713    }
714
715    #[test]
716    fn test_resume_after_gap() {
717        let mut data = Annotated::<Event>::from_json_bytes(
718            br#"{
719                "extra": {"foo": "abcdefghijklmnopqrstuvwxyz"},
720                "_meta":{
721                    "extra": {
722                        "foo":{
723                            "":{
724                                "rem":[["some_rule","s",0,3],["some_rule","s",6,5]]
725                            }
726                        }
727                    }
728                }
729            }"#,
730        )
731        .unwrap();
732
733        let scrubbing_config = DataScrubbingConfig {
734            scrub_data: true,
735            scrub_ip_addresses: true,
736            scrub_defaults: true,
737            ..Default::default()
738        };
739
740        let pii_config = to_pii_config(&scrubbing_config).unwrap();
741        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
742
743        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
744
745        // Verify that an invalid remark after a gap does not repeat the gap:
746        assert_eq!(
747            get_value!(data.extra["foo"]!).0.as_str(),
748            Some("abcdefghijklmnopqrstuvwxyz")
749        );
750    }
751
752    #[test]
753    fn test_basic_stripping() {
754        let config = serde_json::from_str::<PiiConfig>(
755            r#"
756            {
757                "rules": {
758                    "remove_bad_headers": {
759                        "type": "redact_pair",
760                        "keyPattern": "(?i)cookie|secret[-_]?key"
761                    }
762                },
763                "applications": {
764                    "$string": ["@ip"],
765                    "$object.**": ["remove_bad_headers"]
766                }
767            }
768            "#,
769        )
770        .unwrap();
771
772        let mut event = Annotated::new(Event {
773            logentry: Annotated::new(LogEntry {
774                formatted: Annotated::new("Hello world!".to_owned().into()),
775                ..Default::default()
776            }),
777            request: Annotated::new(Request {
778                env: {
779                    let mut rv = Object::new();
780                    rv.insert(
781                        "SECRET_KEY".to_owned(),
782                        Annotated::new(Value::String("134141231231231231231312".into())),
783                    );
784                    Annotated::new(rv)
785                },
786                headers: {
787                    let rv = vec![
788                        Annotated::new((
789                            Annotated::new("Cookie".to_owned().into()),
790                            Annotated::new("super secret".to_owned().into()),
791                        )),
792                        Annotated::new((
793                            Annotated::new("X-Forwarded-For".to_owned().into()),
794                            Annotated::new("127.0.0.1".to_owned().into()),
795                        )),
796                    ];
797                    Annotated::new(Headers(PairList(rv)))
798                },
799                ..Default::default()
800            }),
801            tags: Annotated::new(Tags(
802                vec![Annotated::new(TagEntry(
803                    Annotated::new("forwarded_for".to_owned()),
804                    Annotated::new("127.0.0.1".to_owned()),
805                ))]
806                .into(),
807            )),
808            ..Default::default()
809        });
810
811        let mut processor = PiiProcessor::new(config.compiled());
812        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
813        assert_annotated_snapshot!(event);
814    }
815
816    #[test]
817    fn test_redact_containers() {
818        let config = serde_json::from_str::<PiiConfig>(
819            r#"
820            {
821                "applications": {
822                    "$object": ["@anything"]
823                }
824            }
825            "#,
826        )
827        .unwrap();
828
829        let mut event = Annotated::new(Event {
830            extra: {
831                let mut map = Object::new();
832                map.insert(
833                    "foo".to_owned(),
834                    Annotated::new(ExtraValue(Value::String("bar".to_owned()))),
835                );
836                Annotated::new(map)
837            },
838            ..Default::default()
839        });
840
841        let mut processor = PiiProcessor::new(config.compiled());
842        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
843        assert_annotated_snapshot!(event);
844    }
845
846    #[test]
847    fn test_redact_custom_pattern() {
848        let config = serde_json::from_str::<PiiConfig>(
849            r#"
850            {
851                "applications": {
852                    "$string": ["myrule"]
853                },
854                "rules": {
855                    "myrule": {
856                        "type": "pattern",
857                        "pattern": "foo",
858                        "redaction": {
859                            "method": "replace",
860                            "text": "asd"
861                        }
862                    }
863                }
864            }
865            "#,
866        )
867        .unwrap();
868
869        let mut event = Annotated::new(Event {
870            extra: {
871                let mut map = Object::new();
872                map.insert(
873                    "myvalue".to_owned(),
874                    Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
875                );
876                Annotated::new(map)
877            },
878            ..Default::default()
879        });
880
881        let mut processor = PiiProcessor::new(config.compiled());
882        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
883        assert_annotated_snapshot!(event);
884    }
885
886    #[test]
887    fn test_redact_custom_negative_pattern() {
888        let config = serde_json::from_str::<PiiConfig>(
889            r#"
890            {
891                "applications": {
892                    "$string": ["myrule"]
893                },
894                "rules": {
895                    "myrule": {
896                        "type": "pattern",
897                        "pattern": "the good string|.*OK.*|(.*)",
898                        "replaceGroups": [1],
899                        "redaction": {
900                            "method": "mask"
901                        }
902                    }
903                }
904            }
905            "#,
906        )
907        .unwrap();
908
909        let mut event = Annotated::<Event>::from_json(
910            r#"{
911            "extra": {
912                "1": "the good string",
913                "2": "a bad string",
914                "3": "another OK string",
915                "4": "another bad one"
916            }
917        }"#,
918        )
919        .unwrap();
920
921        let mut processor = PiiProcessor::new(config.compiled());
922        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
923        assert_annotated_snapshot!(event.value().unwrap().extra, @r#"
924        {
925          "1": "the good string",
926          "2": "************",
927          "3": "another OK string",
928          "4": "***************",
929          "_meta": {
930            "2": {
931              "": {
932                "rem": [
933                  [
934                    "myrule",
935                    "m",
936                    0,
937                    12
938                  ]
939                ],
940                "len": 12
941              }
942            },
943            "4": {
944              "": {
945                "rem": [
946                  [
947                    "myrule",
948                    "m",
949                    0,
950                    15
951                  ]
952                ],
953                "len": 15
954              }
955            }
956          }
957        }
958        "#);
959    }
960
961    #[test]
962    fn test_no_field_upsert() {
963        let config = serde_json::from_str::<PiiConfig>(
964            r#"
965            {
966                "applications": {
967                    "**": ["@anything:remove"]
968                }
969            }
970            "#,
971        )
972        .unwrap();
973
974        let mut event = Annotated::new(Event {
975            extra: {
976                let mut map = Object::new();
977                map.insert(
978                    "myvalue".to_owned(),
979                    Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
980                );
981                Annotated::new(map)
982            },
983            ..Default::default()
984        });
985
986        let mut processor = PiiProcessor::new(config.compiled());
987        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
988        assert_annotated_snapshot!(event);
989    }
990
991    #[test]
992    fn test_anything_hash_on_string() {
993        let config = serde_json::from_str::<PiiConfig>(
994            r#"
995            {
996                "applications": {
997                    "$string": ["@anything:hash"]
998                }
999            }
1000            "#,
1001        )
1002        .unwrap();
1003
1004        let mut event = Annotated::new(Event {
1005            extra: {
1006                let mut map = Object::new();
1007                map.insert(
1008                    "myvalue".to_owned(),
1009                    Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
1010                );
1011                Annotated::new(map)
1012            },
1013            ..Default::default()
1014        });
1015
1016        let mut processor = PiiProcessor::new(config.compiled());
1017        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1018        assert_annotated_snapshot!(event);
1019    }
1020
1021    #[test]
1022    fn test_anything_hash_on_container() {
1023        let config = serde_json::from_str::<PiiConfig>(
1024            r#"
1025            {
1026                "applications": {
1027                    "$object": ["@anything:hash"]
1028                }
1029            }
1030            "#,
1031        )
1032        .unwrap();
1033
1034        let mut event = Annotated::new(Event {
1035            extra: {
1036                let mut map = Object::new();
1037                map.insert(
1038                    "myvalue".to_owned(),
1039                    Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
1040                );
1041                Annotated::new(map)
1042            },
1043            ..Default::default()
1044        });
1045
1046        let mut processor = PiiProcessor::new(config.compiled());
1047        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1048        assert_annotated_snapshot!(event);
1049    }
1050
1051    #[test]
1052    fn test_only_match_token_on_keys() {
1053        let mut data = Event::from_value(
1054            json!({
1055                "request": {
1056                    "headers": [
1057                        ["X-Token", "oof this is very sensitive"],
1058                        ["Token", "also bad"],
1059                    ]
1060                },
1061                "extra": {
1062                    "url": "foo.bar/endpoint?token=sensitive",
1063                    "url2": "foo.bar/endpoint?token_foobar=sensitive",
1064                    "aaa": "token:12345",
1065                    "foo-token-bar": "sensitive",
1066                    "llm": "token count",
1067                },
1068            })
1069            .into(),
1070        );
1071
1072        let scrubbing_config = DataScrubbingConfig {
1073            scrub_data: true,
1074            scrub_ip_addresses: true,
1075            scrub_defaults: true,
1076            ..Default::default()
1077        };
1078
1079        let pii_config = to_pii_config(&scrubbing_config).unwrap();
1080        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1081
1082        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
1083
1084        assert_annotated_snapshot!(&data);
1085    }
1086
1087    #[test]
1088    fn test_ignore_user_agent_ip_scrubbing() {
1089        let mut data = Event::from_value(
1090            json!({
1091                "request": {
1092                    "headers": [
1093                        ["User-Agent", "127.0.0.1"],
1094                        ["X-Client-Ip", "10.0.0.1"]
1095                    ]
1096                },
1097            })
1098            .into(),
1099        );
1100
1101        let scrubbing_config = DataScrubbingConfig {
1102            scrub_data: true,
1103            scrub_ip_addresses: true,
1104            scrub_defaults: true,
1105            ..Default::default()
1106        };
1107
1108        let pii_config = to_pii_config(&scrubbing_config).unwrap();
1109        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1110
1111        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
1112
1113        assert_annotated_snapshot!(&data);
1114    }
1115
1116    #[test]
1117    fn test_remove_debugmeta_path() {
1118        let config = serde_json::from_str::<PiiConfig>(
1119            r#"
1120            {
1121                "applications": {
1122                    "debug_meta.images.*.code_file": ["@anything:remove"],
1123                    "debug_meta.images.*.debug_file": ["@anything:remove"]
1124                }
1125            }
1126            "#,
1127        )
1128        .unwrap();
1129
1130        let mut event = Annotated::new(Event {
1131            debug_meta: Annotated::new(DebugMeta {
1132                images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1133                    NativeDebugImage {
1134                        code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1135                        code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1136                        debug_id: Annotated::new(
1137                            "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1138                        ),
1139                        debug_file: Annotated::new("wntdll.pdb".into()),
1140                        debug_checksum: Annotated::empty(),
1141                        arch: Annotated::new("arm64".to_owned()),
1142                        image_addr: Annotated::new(Addr(0)),
1143                        image_size: Annotated::new(4096),
1144                        image_vmaddr: Annotated::new(Addr(32768)),
1145                        other: {
1146                            let mut map = Object::new();
1147                            map.insert(
1148                                "other".to_owned(),
1149                                Annotated::new(Value::String("value".to_owned())),
1150                            );
1151                            map
1152                        },
1153                    },
1154                )))]),
1155                ..Default::default()
1156            }),
1157            ..Default::default()
1158        });
1159
1160        let mut processor = PiiProcessor::new(config.compiled());
1161        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1162        assert_annotated_snapshot!(event);
1163    }
1164
1165    #[test]
1166    fn test_replace_debugmeta_path() {
1167        let config = serde_json::from_str::<PiiConfig>(
1168            r#"
1169            {
1170                "applications": {
1171                    "debug_meta.images.*.code_file": ["@anything:replace"],
1172                    "debug_meta.images.*.debug_file": ["@anything:replace"]
1173                }
1174            }
1175            "#,
1176        )
1177        .unwrap();
1178
1179        let mut event = Annotated::new(Event {
1180            debug_meta: Annotated::new(DebugMeta {
1181                images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1182                    NativeDebugImage {
1183                        code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1184                        code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1185                        debug_id: Annotated::new(
1186                            "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1187                        ),
1188                        debug_file: Annotated::new("wntdll.pdb".into()),
1189                        debug_checksum: Annotated::empty(),
1190                        arch: Annotated::new("arm64".to_owned()),
1191                        image_addr: Annotated::new(Addr(0)),
1192                        image_size: Annotated::new(4096),
1193                        image_vmaddr: Annotated::new(Addr(32768)),
1194                        other: {
1195                            let mut map = Object::new();
1196                            map.insert(
1197                                "other".to_owned(),
1198                                Annotated::new(Value::String("value".to_owned())),
1199                            );
1200                            map
1201                        },
1202                    },
1203                )))]),
1204                ..Default::default()
1205            }),
1206            ..Default::default()
1207        });
1208
1209        let mut processor = PiiProcessor::new(config.compiled());
1210        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1211        assert_annotated_snapshot!(event);
1212    }
1213
1214    #[test]
1215    fn test_hash_debugmeta_path() {
1216        let config = serde_json::from_str::<PiiConfig>(
1217            r#"
1218            {
1219                "applications": {
1220                    "debug_meta.images.*.code_file": ["@anything:hash"],
1221                    "debug_meta.images.*.debug_file": ["@anything:hash"]
1222                }
1223            }
1224            "#,
1225        )
1226        .unwrap();
1227
1228        let mut event = Annotated::new(Event {
1229            debug_meta: Annotated::new(DebugMeta {
1230                images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1231                    NativeDebugImage {
1232                        code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1233                        code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1234                        debug_id: Annotated::new(
1235                            "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1236                        ),
1237                        debug_file: Annotated::new("wntdll.pdb".into()),
1238                        debug_checksum: Annotated::empty(),
1239                        arch: Annotated::new("arm64".to_owned()),
1240                        image_addr: Annotated::new(Addr(0)),
1241                        image_size: Annotated::new(4096),
1242                        image_vmaddr: Annotated::new(Addr(32768)),
1243                        other: {
1244                            let mut map = Object::new();
1245                            map.insert(
1246                                "other".to_owned(),
1247                                Annotated::new(Value::String("value".to_owned())),
1248                            );
1249                            map
1250                        },
1251                    },
1252                )))]),
1253                ..Default::default()
1254            }),
1255            ..Default::default()
1256        });
1257
1258        let mut processor = PiiProcessor::new(config.compiled());
1259        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1260        assert_annotated_snapshot!(event);
1261    }
1262
1263    #[test]
1264    fn test_debugmeta_path_not_addressible_with_wildcard_selector() {
1265        let config = serde_json::from_str::<PiiConfig>(
1266            r#"
1267            {
1268                "applications": {
1269                    "$string": ["@anything:remove"],
1270                    "**": ["@anything:remove"],
1271                    "debug_meta.**": ["@anything:remove"],
1272                    "(debug_meta.images.**.code_file & $string)": ["@anything:remove"]
1273                }
1274            }
1275            "#,
1276        )
1277        .unwrap();
1278
1279        let mut event = Annotated::new(Event {
1280            debug_meta: Annotated::new(DebugMeta {
1281                images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1282                    NativeDebugImage {
1283                        code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1284                        code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1285                        debug_id: Annotated::new(
1286                            "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1287                        ),
1288                        debug_file: Annotated::new("wntdll.pdb".into()),
1289                        debug_checksum: Annotated::empty(),
1290                        arch: Annotated::new("arm64".to_owned()),
1291                        image_addr: Annotated::new(Addr(0)),
1292                        image_size: Annotated::new(4096),
1293                        image_vmaddr: Annotated::new(Addr(32768)),
1294                        other: {
1295                            let mut map = Object::new();
1296                            map.insert(
1297                                "other".to_owned(),
1298                                Annotated::new(Value::String("value".to_owned())),
1299                            );
1300                            map
1301                        },
1302                    },
1303                )))]),
1304                ..Default::default()
1305            }),
1306            ..Default::default()
1307        });
1308
1309        let mut processor = PiiProcessor::new(config.compiled());
1310        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1311        assert_annotated_snapshot!(event);
1312    }
1313
1314    #[test]
1315    fn test_quoted_keys() {
1316        let config = serde_json::from_str::<PiiConfig>(
1317            r#"
1318            {
1319                "applications": {
1320                    "extra.'special ,./<>?!@#$%^&*())''gärbage'''": ["@anything:remove"]
1321                }
1322            }
1323            "#,
1324        )
1325        .unwrap();
1326
1327        let mut event = Annotated::new(Event {
1328            extra: {
1329                let mut map = Object::new();
1330                map.insert(
1331                    "do not ,./<>?!@#$%^&*())'ßtrip'".to_owned(),
1332                    Annotated::new(ExtraValue(Value::String("foo".to_owned()))),
1333                );
1334                map.insert(
1335                    "special ,./<>?!@#$%^&*())'gärbage'".to_owned(),
1336                    Annotated::new(ExtraValue(Value::String("bar".to_owned()))),
1337                );
1338                Annotated::new(map)
1339            },
1340            ..Default::default()
1341        });
1342
1343        let mut processor = PiiProcessor::new(config.compiled());
1344        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1345        assert_annotated_snapshot!(event);
1346    }
1347
1348    #[test]
1349    fn test_logentry_value_types() {
1350        // Assert that logentry.formatted is addressable as $string, $message and $logentry.formatted.
1351        for formatted_selector in &[
1352            "$logentry.formatted",
1353            "$message",
1354            "$logentry.formatted && $message",
1355            "$string",
1356        ] {
1357            let config = serde_json::from_str::<PiiConfig>(&format!(
1358                r##"
1359                {{
1360                    "applications": {{
1361                        "{formatted_selector}": ["@anything:remove"]
1362                    }}
1363                }}
1364                "##
1365            ))
1366            .unwrap();
1367
1368            let mut event = Annotated::new(Event {
1369                logentry: Annotated::new(LogEntry {
1370                    formatted: Annotated::new("Hello world!".to_owned().into()),
1371                    ..Default::default()
1372                }),
1373                ..Default::default()
1374            });
1375
1376            let mut processor = PiiProcessor::new(config.compiled());
1377            process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1378            assert!(
1379                event
1380                    .value()
1381                    .unwrap()
1382                    .logentry
1383                    .value()
1384                    .unwrap()
1385                    .formatted
1386                    .value()
1387                    .is_none()
1388            );
1389        }
1390    }
1391
1392    #[test]
1393    fn test_logentry_formatted_never_fully_filtered() {
1394        // Test that logentry.formatted gets smart PII scrubbing via to_pii_config
1395        // and is never completely filtered even with aggressive PII rules
1396        let config = crate::convert::to_pii_config(&crate::DataScrubbingConfig {
1397            scrub_data: true,
1398            scrub_defaults: true,
1399            scrub_ip_addresses: true,
1400            ..Default::default()
1401        })
1402        .unwrap();
1403
1404        let mut event = Annotated::new(Event {
1405            logentry: Annotated::new(LogEntry {
1406                formatted: Annotated::new(
1407                    "User john.doe@company.com failed login with card 4111-1111-1111-1111"
1408                        .to_owned()
1409                        .into(),
1410                ),
1411                ..Default::default()
1412            }),
1413            ..Default::default()
1414        });
1415
1416        let mut processor = PiiProcessor::new(config.compiled());
1417        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1418        assert_annotated_snapshot!(event, @r#"
1419        {
1420          "logentry": {
1421            "formatted": "User [email] failed login with card [creditcard]"
1422          },
1423          "_meta": {
1424            "logentry": {
1425              "formatted": {
1426                "": {
1427                  "rem": [
1428                    [
1429                      "@email:replace",
1430                      "s",
1431                      5,
1432                      12
1433                    ],
1434                    [
1435                      "@creditcard:replace",
1436                      "s",
1437                      36,
1438                      48
1439                    ]
1440                  ],
1441                  "len": 68
1442                }
1443              }
1444            }
1445          }
1446        }
1447        "#);
1448    }
1449
1450    #[test]
1451    fn test_logentry_formatted_bearer_token_scrubbing() {
1452        // Test that bearer tokens are properly scrubbed in logentry.formatted
1453        let config = crate::convert::to_pii_config(&crate::DataScrubbingConfig {
1454            scrub_data: true,
1455            scrub_defaults: true,
1456            ..Default::default()
1457        })
1458        .unwrap();
1459
1460        let mut event = Annotated::new(Event {
1461            logentry: Annotated::new(LogEntry {
1462                formatted: Annotated::new(
1463                    "API request failed with Bearer ABC123XYZ789TOKEN and other data"
1464                        .to_owned()
1465                        .into(),
1466                ),
1467                ..Default::default()
1468            }),
1469            ..Default::default()
1470        });
1471
1472        let mut processor = PiiProcessor::new(config.compiled());
1473        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1474        assert_annotated_snapshot!(event, @r#"
1475        {
1476          "logentry": {
1477            "formatted": "API request failed with Bearer [token] and other data"
1478          },
1479          "_meta": {
1480            "logentry": {
1481              "formatted": {
1482                "": {
1483                  "rem": [
1484                    [
1485                      "@bearer:replace",
1486                      "s",
1487                      24,
1488                      38
1489                    ]
1490                  ],
1491                  "len": 63
1492                }
1493              }
1494            }
1495          }
1496        }
1497        "#);
1498    }
1499
1500    #[test]
1501    fn test_logentry_formatted_password_word_not_scrubbed() {
1502        let config = PiiConfig::default();
1503        let mut event = Annotated::new(Event {
1504            logentry: Annotated::new(LogEntry {
1505                formatted: Annotated::new(
1506                    "User password is secret123 for authentication"
1507                        .to_owned()
1508                        .into(),
1509                ),
1510                ..Default::default()
1511            }),
1512            ..Default::default()
1513        });
1514
1515        let mut processor = PiiProcessor::new(config.compiled());
1516        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1517        assert_annotated_snapshot!(event, @r#"
1518        {
1519          "logentry": {
1520            "formatted": "User password is secret123 for authentication"
1521          }
1522        }
1523        "#);
1524    }
1525
1526    #[test]
1527    fn test_ip_address_hashing() {
1528        let config = serde_json::from_str::<PiiConfig>(
1529            r#"
1530            {
1531                "applications": {
1532                    "$user.ip_address": ["@ip:hash"]
1533                }
1534            }
1535            "#,
1536        )
1537        .unwrap();
1538
1539        let mut event = Annotated::new(Event {
1540            user: Annotated::new(User {
1541                ip_address: Annotated::new(IpAddr("127.0.0.1".to_owned())),
1542                ..Default::default()
1543            }),
1544            ..Default::default()
1545        });
1546
1547        let mut processor = PiiProcessor::new(config.compiled());
1548        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1549
1550        let user = event.value().unwrap().user.value().unwrap();
1551
1552        assert!(user.ip_address.value().is_none());
1553
1554        assert_eq!(
1555            user.id.value().unwrap().as_str(),
1556            "AE12FE3B5F129B5CC4CDD2B136B7B7947C4D2741"
1557        );
1558    }
1559
1560    #[test]
1561    fn test_ip_address_hashing_does_not_overwrite_id() {
1562        let config = serde_json::from_str::<PiiConfig>(
1563            r#"
1564            {
1565                "applications": {
1566                    "$user.ip_address": ["@ip:hash"]
1567                }
1568            }
1569            "#,
1570        )
1571        .unwrap();
1572
1573        let mut event = Annotated::new(Event {
1574            user: Annotated::new(User {
1575                id: Annotated::new("123".to_owned().into()),
1576                ip_address: Annotated::new(IpAddr("127.0.0.1".to_owned())),
1577                ..Default::default()
1578            }),
1579            ..Default::default()
1580        });
1581
1582        let mut processor = PiiProcessor::new(config.compiled());
1583        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1584
1585        let user = event.value().unwrap().user.value().unwrap();
1586
1587        // This will get wiped out in renormalization though
1588        assert_eq!(
1589            user.ip_address.value().unwrap().as_str(),
1590            "AE12FE3B5F129B5CC4CDD2B136B7B7947C4D2741"
1591        );
1592
1593        assert_eq!(user.id.value().unwrap().as_str(), "123");
1594    }
1595
1596    #[test]
1597    fn test_replace_replaced_text() {
1598        let chunks = vec![Chunk::Redaction {
1599            text: "[ip]".into(),
1600            rule_id: "@ip".into(),
1601            ty: RemarkType::Substituted,
1602        }];
1603        let rule = RuleRef {
1604            id: "@ip:replace".into(),
1605            origin: "@ip".into(),
1606            ty: RuleType::Ip,
1607            redaction: Redaction::Replace(ReplaceRedaction {
1608                text: "[ip]".into(),
1609            }),
1610        };
1611        let res = apply_regex_to_chunks(
1612            chunks.clone(),
1613            &rule,
1614            ANYTHING_REGEX.as_ref(),
1615            ReplaceBehavior::Value,
1616        );
1617        assert_eq!(chunks, res);
1618    }
1619
1620    #[test]
1621    fn test_replace_replaced_text_anything() {
1622        let chunks = vec![Chunk::Redaction {
1623            text: "[Filtered]".into(),
1624            rule_id: "@password:filter".into(),
1625            ty: RemarkType::Substituted,
1626        }];
1627        let rule = RuleRef {
1628            id: "@anything:filter".into(),
1629            origin: "@anything:filter".into(),
1630            ty: RuleType::Anything,
1631            redaction: Redaction::Replace(ReplaceRedaction {
1632                text: "[Filtered]".into(),
1633            }),
1634        };
1635        let res = apply_regex_to_chunks(
1636            chunks.clone(),
1637            &rule,
1638            ANYTHING_REGEX.as_ref(),
1639            ReplaceBehavior::Groups(smallvec::smallvec![0]),
1640        );
1641        assert_eq!(chunks, res);
1642    }
1643
1644    #[test]
1645    fn test_trace_route_params_scrubbed() {
1646        let mut trace_context: Annotated<TraceContext> = Annotated::from_json(
1647            r#"
1648            {
1649                "type": "trace",
1650                "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1651                "span_id": "fa90fdead5f74052",
1652                "data": {
1653                    "previousRoute": {
1654                        "params": {
1655                            "password": "test"
1656                        }
1657                    }
1658                }
1659            }
1660            "#,
1661        )
1662        .unwrap();
1663
1664        let ds_config = DataScrubbingConfig {
1665            scrub_data: true,
1666            scrub_defaults: true,
1667            ..Default::default()
1668        };
1669        let pii_config = ds_config.pii_config().as_ref().unwrap();
1670        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1671
1672        process_value(
1673            &mut trace_context,
1674            &mut pii_processor,
1675            ProcessingState::root(),
1676        )
1677        .unwrap();
1678        assert_annotated_snapshot!(trace_context);
1679    }
1680
1681    #[test]
1682    fn test_scrub_span_data_http_not_scrubbed() {
1683        let mut span: Annotated<Span> = Annotated::from_json(
1684            r#"{
1685                "data": {
1686                    "http": {
1687                        "query": "dance=true"
1688                    }
1689                }
1690            }"#,
1691        )
1692        .unwrap();
1693
1694        let ds_config = DataScrubbingConfig {
1695            scrub_data: true,
1696            scrub_defaults: true,
1697            ..Default::default()
1698        };
1699        let pii_config = ds_config.pii_config().as_ref().unwrap();
1700        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1701
1702        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1703        assert_annotated_snapshot!(span);
1704    }
1705
1706    #[test]
1707    fn test_scrub_span_data_http_strings_are_scrubbed() {
1708        let mut span: Annotated<Span> = Annotated::from_json(
1709            r#"{
1710                "data": {
1711                    "http": {
1712                        "query": "ccnumber=5105105105105100&process_id=123",
1713                        "fragment": "ccnumber=5105105105105100,process_id=123"
1714                    }
1715                }
1716            }"#,
1717        )
1718        .unwrap();
1719
1720        let ds_config = DataScrubbingConfig {
1721            scrub_data: true,
1722            scrub_defaults: true,
1723            ..Default::default()
1724        };
1725        let pii_config = ds_config.pii_config().as_ref().unwrap();
1726        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1727
1728        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1729        assert_annotated_snapshot!(span);
1730    }
1731
1732    #[test]
1733    fn test_scrub_span_data_http_objects_are_scrubbed() {
1734        let mut span: Annotated<Span> = Annotated::from_json(
1735            r#"{
1736                "data": {
1737                    "http": {
1738                        "query": {
1739                            "ccnumber": "5105105105105100",
1740                            "process_id": "123"
1741                        },
1742                        "fragment": {
1743                            "ccnumber": "5105105105105100",
1744                            "process_id": "123"
1745                        }
1746                    }
1747                }
1748            }"#,
1749        )
1750        .unwrap();
1751
1752        let ds_config = DataScrubbingConfig {
1753            scrub_data: true,
1754            scrub_defaults: true,
1755            ..Default::default()
1756        };
1757        let pii_config = ds_config.pii_config().as_ref().unwrap();
1758        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1759
1760        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1761        assert_annotated_snapshot!(span);
1762    }
1763
1764    #[test]
1765    fn test_scrub_span_data_untyped_props_are_scrubbed() {
1766        let mut span: Annotated<Span> = Annotated::from_json(
1767            r#"{
1768                "data": {
1769                    "untyped": "ccnumber=5105105105105100",
1770                    "more_untyped": {
1771                        "typed": "no",
1772                        "scrubbed": "yes",
1773                        "ccnumber": "5105105105105100"
1774                    }
1775                }
1776            }"#,
1777        )
1778        .unwrap();
1779
1780        let ds_config = DataScrubbingConfig {
1781            scrub_data: true,
1782            scrub_defaults: true,
1783            ..Default::default()
1784        };
1785        let pii_config = ds_config.pii_config().as_ref().unwrap();
1786        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1787
1788        process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1789        assert_annotated_snapshot!(span);
1790    }
1791
1792    #[test]
1793    fn test_span_data_pii() {
1794        let mut span = Span::from_value(
1795            json!({
1796                "data": {
1797                    "code.filepath": "src/sentry/api/authentication.py",
1798                }
1799            })
1800            .into(),
1801        );
1802
1803        let ds_config = DataScrubbingConfig {
1804            scrub_data: true,
1805            scrub_defaults: true,
1806            ..Default::default()
1807        };
1808        let pii_config = ds_config.pii_config().as_ref().unwrap();
1809
1810        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1811        processor::process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1812        assert_eq!(
1813            span.0.unwrap().data.0.unwrap().other["code.filepath"].as_str(),
1814            Some("src/sentry/api/authentication.py")
1815        );
1816    }
1817
1818    #[test]
1819    fn test_csp_source_file_pii() {
1820        let mut event = Event::from_value(
1821            json!({
1822                "csp": {
1823                    "source_file": "authentication.js",
1824                }
1825            })
1826            .into(),
1827        );
1828
1829        let config = serde_json::from_str::<PiiConfig>(
1830            r#"
1831            {
1832                "applications": {
1833                    "csp.source_file": ["@anything:filter"]
1834                }
1835            }
1836            "#,
1837        )
1838        .unwrap();
1839
1840        let mut pii_processor = PiiProcessor::new(config.compiled());
1841        processor::process_value(&mut event, &mut pii_processor, ProcessingState::root()).unwrap();
1842        assert_eq!(get_value!(event.csp.source_file!).as_str(), "[Filtered]");
1843    }
1844
1845    #[test]
1846    fn test_csp_script_sample_pii() {
1847        let mut event = Event::from_value(
1848            json!({
1849                "csp": {
1850                    "script_sample": "var user = {\"email\":\"jane@example.com\"",
1851                }
1852            })
1853            .into(),
1854        );
1855
1856        let config = serde_json::from_str::<PiiConfig>(
1857            r#"
1858            {
1859                "applications": {
1860                    "csp.script_sample": ["@anything:filter"]
1861                }
1862            }
1863            "#,
1864        )
1865        .unwrap();
1866
1867        let mut pii_processor = PiiProcessor::new(config.compiled());
1868        processor::process_value(&mut event, &mut pii_processor, ProcessingState::root()).unwrap();
1869        assert_eq!(get_value!(event.csp.script_sample!).as_str(), "[Filtered]");
1870    }
1871
1872    #[test]
1873    fn test_scrub_breadcrumb_data_http_not_scrubbed() {
1874        let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1875            r#"{
1876                "data": {
1877                    "http": {
1878                        "query": "dance=true"
1879                    }
1880                }
1881            }"#,
1882        )
1883        .unwrap();
1884
1885        let ds_config = DataScrubbingConfig {
1886            scrub_data: true,
1887            scrub_defaults: true,
1888            ..Default::default()
1889        };
1890        let pii_config = ds_config.pii_config().as_ref().unwrap();
1891        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1892        process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1893        assert_annotated_snapshot!(breadcrumb);
1894    }
1895
1896    #[test]
1897    fn test_scrub_breadcrumb_data_http_strings_are_scrubbed() {
1898        let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1899            r#"{
1900                "data": {
1901                    "http": {
1902                        "query": "ccnumber=5105105105105100&process_id=123",
1903                        "fragment": "ccnumber=5105105105105100,process_id=123"
1904                    }
1905                }
1906            }"#,
1907        )
1908        .unwrap();
1909
1910        let ds_config = DataScrubbingConfig {
1911            scrub_data: true,
1912            scrub_defaults: true,
1913            ..Default::default()
1914        };
1915        let pii_config = ds_config.pii_config().as_ref().unwrap();
1916        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1917        process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1918        assert_annotated_snapshot!(breadcrumb);
1919    }
1920
1921    #[test]
1922    fn test_scrub_breadcrumb_data_http_objects_are_scrubbed() {
1923        let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1924            r#"{
1925                "data": {
1926                    "http": {
1927                        "query": {
1928                            "ccnumber": "5105105105105100",
1929                            "process_id": "123"
1930                        },
1931                        "fragment": {
1932                            "ccnumber": "5105105105105100",
1933                            "process_id": "123"
1934                        }
1935                    }
1936                }
1937            }"#,
1938        )
1939        .unwrap();
1940
1941        let ds_config = DataScrubbingConfig {
1942            scrub_data: true,
1943            scrub_defaults: true,
1944            ..Default::default()
1945        };
1946        let pii_config = ds_config.pii_config().as_ref().unwrap();
1947        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1948
1949        process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1950        assert_annotated_snapshot!(breadcrumb);
1951    }
1952
1953    #[test]
1954    fn test_scrub_breadcrumb_data_untyped_props_are_scrubbed() {
1955        let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1956            r#"{
1957                "data": {
1958                    "untyped": "ccnumber=5105105105105100",
1959                    "more_untyped": {
1960                        "typed": "no",
1961                        "scrubbed": "yes",
1962                        "ccnumber": "5105105105105100"
1963                    }
1964                }
1965            }"#,
1966        )
1967        .unwrap();
1968
1969        let ds_config = DataScrubbingConfig {
1970            scrub_data: true,
1971            scrub_defaults: true,
1972            ..Default::default()
1973        };
1974        let pii_config = ds_config.pii_config().as_ref().unwrap();
1975        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1976        process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1977        assert_annotated_snapshot!(breadcrumb);
1978    }
1979
1980    #[test]
1981    fn test_scrub_graphql_response_data_with_variables() {
1982        let mut data = Event::from_value(
1983            json!({
1984              "request": {
1985                "data": {
1986                  "query": "{\n  viewer {\n    login\n  }\n}",
1987                  "variables": {
1988                    "login": "foo"
1989                  }
1990                },
1991                "api_target": "graphql"
1992              },
1993              "contexts": {
1994                "response": {
1995                  "type": "response",
1996                  "data": {
1997                    "data": {
1998                      "viewer": {
1999                        "login": "foo"
2000                      }
2001                    }
2002                  }
2003                }
2004              }
2005            })
2006            .into(),
2007        );
2008
2009        scrub_graphql(data.value_mut().as_mut().unwrap());
2010
2011        assert_debug_snapshot!(&data);
2012    }
2013
2014    #[test]
2015    fn test_scrub_graphql_response_data_without_variables() {
2016        let mut data = Event::from_value(
2017            json!({
2018              "request": {
2019                "data": {
2020                  "query": "{\n  viewer {\n    login\n  }\n}"
2021                },
2022                "api_target": "graphql"
2023              },
2024              "contexts": {
2025                "response": {
2026                  "type": "response",
2027                  "data": {
2028                    "data": {
2029                      "viewer": {
2030                        "login": "foo"
2031                      }
2032                    }
2033                  }
2034                }
2035              }
2036            })
2037            .into(),
2038        );
2039
2040        scrub_graphql(data.value_mut().as_mut().unwrap());
2041        assert_debug_snapshot!(&data);
2042    }
2043
2044    #[test]
2045    fn test_does_not_scrub_if_no_graphql() {
2046        let mut data = Event::from_value(
2047            json!({
2048              "request": {
2049                "data": {
2050                  "query": "{\n  viewer {\n    login\n  }\n}",
2051                  "variables": {
2052                    "login": "foo"
2053                  }
2054                },
2055              },
2056              "contexts": {
2057                "response": {
2058                  "type": "response",
2059                  "data": {
2060                    "data": {
2061                      "viewer": {
2062                        "login": "foo"
2063                      }
2064                    }
2065                  }
2066                }
2067              }
2068            })
2069            .into(),
2070        );
2071
2072        let scrubbing_config = DataScrubbingConfig {
2073            scrub_data: true,
2074            scrub_ip_addresses: true,
2075            scrub_defaults: true,
2076            ..Default::default()
2077        };
2078
2079        let pii_config = to_pii_config(&scrubbing_config).unwrap();
2080        let mut pii_processor = PiiProcessor::new(pii_config.compiled());
2081
2082        process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
2083
2084        assert_debug_snapshot!(&data);
2085    }
2086
2087    #[test]
2088    fn test_logentry_params_scrubbed() {
2089        let config = serde_json::from_str::<PiiConfig>(
2090            r##"
2091                {
2092                    "applications": {
2093                        "$string": ["@anything:remove"]
2094                    }
2095                }
2096                "##,
2097        )
2098        .unwrap();
2099
2100        let mut event = Annotated::new(Event {
2101            logentry: Annotated::new(LogEntry {
2102                message: Annotated::new(Message::from("failed to parse report id=%s".to_owned())),
2103                formatted: Annotated::new("failed to parse report id=1".to_owned().into()),
2104                params: Annotated::new(Value::Array(vec![Annotated::new(Value::String(
2105                    "12345".to_owned(),
2106                ))])),
2107                ..Default::default()
2108            }),
2109            ..Default::default()
2110        });
2111
2112        let mut processor = PiiProcessor::new(config.compiled());
2113        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2114
2115        let params = get_value!(event.logentry.params!);
2116        assert_debug_snapshot!(params, @r###"
2117        Array(
2118            [
2119                Meta {
2120                    remarks: [
2121                        Remark {
2122                            ty: Removed,
2123                            rule_id: "@anything:remove",
2124                            range: None,
2125                        },
2126                    ],
2127                    errors: [],
2128                    original_length: None,
2129                    original_value: None,
2130                },
2131            ],
2132        )
2133        "###);
2134    }
2135
2136    #[test]
2137    fn test_is_pairlist() {
2138        for (case, expected) in [
2139            (r#"[]"#, false),
2140            (r#"["foo"]"#, false),
2141            (r#"["foo", 123]"#, false),
2142            (r#"[[1, "foo"]]"#, false),
2143            (r#"[[["too_nested", 123]]]"#, false),
2144            (r#"[["foo", "bar"], [1, "foo"]]"#, false),
2145            (r#"[["foo", "bar"], ["foo", "bar", "baz"]]"#, false),
2146            (r#"[["foo", "bar", "baz"], ["foo", "bar"]]"#, false),
2147            (r#"["foo", ["bar", "baz"], ["foo", "bar"]]"#, false),
2148            (r#"[["foo", "bar"], [["too_nested", 123]]]"#, false),
2149            (r#"[["foo", 123]]"#, true),
2150            (r#"[["foo", "bar"]]"#, true),
2151            (
2152                r#"[["foo", "bar"], ["foo", {"nested": {"something": 1}}]]"#,
2153                true,
2154            ),
2155        ] {
2156            let v = Annotated::<Value>::from_json(case).unwrap();
2157            let Annotated(Some(Value::Array(mut a)), _) = v else {
2158                panic!()
2159            };
2160            assert_eq!(is_pairlist(&mut a), expected, "{case}");
2161        }
2162    }
2163
2164    #[test]
2165    fn test_tuple_array_scrubbed_with_path_selector() {
2166        // We expect that both of these configs express the same semantics.
2167        let configs = vec![
2168            // This configuration matches on the authorization element (the 1st element of the array
2169            // represents the key).
2170            r##"
2171                {
2172                    "applications": {
2173                        "exception.values.0.stacktrace.frames.0.vars.headers.authorization": ["@anything:replace"]
2174                    }
2175                }
2176                "##,
2177            // This configuration matches on the 2nd element of the array.
2178            r##"
2179                {
2180                    "applications": {
2181                        "exception.values.0.stacktrace.frames.0.vars.headers.0.1": ["@anything:replace"]
2182                    }
2183                }
2184                "##,
2185        ];
2186
2187        let mut event = Event::from_value(
2188            serde_json::json!(
2189            {
2190              "message": "hi",
2191              "exception": {
2192                "values": [
2193                  {
2194                    "type": "BrokenException",
2195                    "value": "Something failed",
2196                    "stacktrace": {
2197                      "frames": [
2198                        {
2199                            "vars": {
2200                                "headers": [
2201                                    ["authorization", "Bearer abc123"]
2202                                ]
2203                            }
2204                        }
2205                      ]
2206                    }
2207                  }
2208                ]
2209              }
2210            })
2211            .into(),
2212        );
2213
2214        for config in configs {
2215            let config = serde_json::from_str::<PiiConfig>(config).unwrap();
2216            let mut processor = PiiProcessor::new(config.compiled());
2217            process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2218
2219            let vars = get_value!(event.exceptions.values[0].stacktrace.frames[0].vars).unwrap();
2220
2221            allow_duplicates!(assert_debug_snapshot!(vars, @r###"
2222                              FrameVars(
2223                                  {
2224                                      "headers": Array(
2225                                          [
2226                                              Array(
2227                                                  [
2228                                                      String(
2229                                                          "authorization",
2230                                                      ),
2231                                                      Annotated(
2232                                                          String(
2233                                                              "[Filtered]",
2234                                                          ),
2235                                                          Meta {
2236                                                              remarks: [
2237                                                                  Remark {
2238                                                                      ty: Substituted,
2239                                                                      rule_id: "@anything:replace",
2240                                                                      range: Some(
2241                                                                          (
2242                                                                              0,
2243                                                                              10,
2244                                                                          ),
2245                                                                      ),
2246                                                                  },
2247                                                              ],
2248                                                              errors: [],
2249                                                              original_length: Some(
2250                                                                  13,
2251                                                              ),
2252                                                              original_value: None,
2253                                                          },
2254                                                      ),
2255                                                  ],
2256                                              ),
2257                                          ],
2258                                      ),
2259                                  },
2260                              )
2261                              "###));
2262        }
2263    }
2264
2265    #[test]
2266    fn test_tuple_array_scrubbed_with_string_selector_and_password_matcher() {
2267        let config = serde_json::from_str::<PiiConfig>(
2268            r##"
2269                {
2270                    "applications": {
2271                        "$string": ["@password:remove"]
2272                    }
2273                }
2274                "##,
2275        )
2276        .unwrap();
2277
2278        let mut event = Event::from_value(
2279            serde_json::json!(
2280            {
2281              "message": "hi",
2282              "exception": {
2283                "values": [
2284                  {
2285                    "type": "BrokenException",
2286                    "value": "Something failed",
2287                    "stacktrace": {
2288                      "frames": [
2289                        {
2290                            "vars": {
2291                                "headers": [
2292                                    ["authorization", "abc123"]
2293                                ]
2294                            }
2295                        }
2296                      ]
2297                    }
2298                  }
2299                ]
2300              }
2301            })
2302            .into(),
2303        );
2304
2305        let mut processor = PiiProcessor::new(config.compiled());
2306        process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2307
2308        let vars = get_value!(event.exceptions.values[0].stacktrace.frames[0].vars).unwrap();
2309
2310        assert_debug_snapshot!(vars, @r###"
2311        FrameVars(
2312            {
2313                "headers": Array(
2314                    [
2315                        Array(
2316                            [
2317                                String(
2318                                    "authorization",
2319                                ),
2320                                Meta {
2321                                    remarks: [
2322                                        Remark {
2323                                            ty: Removed,
2324                                            rule_id: "@password:remove",
2325                                            range: None,
2326                                        },
2327                                    ],
2328                                    errors: [],
2329                                    original_length: None,
2330                                    original_value: None,
2331                                },
2332                            ],
2333                        ),
2334                    ],
2335                ),
2336            },
2337        )
2338        "###);
2339    }
2340}