1use std::borrow::Cow;
2use std::collections::{BTreeMap, BTreeSet};
3use std::mem;
4use std::sync::OnceLock;
5
6use regex::Regex;
7use relay_event_schema::processor::{
8 self, Chunk, FieldAttrs, Pii, ProcessValue, ProcessingAction, ProcessingResult,
9 ProcessingState, Processor, ValueType, enum_set, process_value,
10};
11use relay_event_schema::protocol::{
12 AsPair, Event, IpAddr, NativeImagePath, PairList, Replay, ResponseContext, User,
13};
14use relay_protocol::{Annotated, Array, Meta, Remark, RemarkType, Value};
15
16use crate::compiledconfig::{CompiledPiiConfig, RuleRef};
17use crate::config::RuleType;
18use crate::redactions::Redaction;
19use crate::regex::ScrubbingRegexRef;
20use crate::regexes::{self, ANYTHING_REGEX, PatternType, ReplaceBehavior};
21use crate::utils;
22
23#[derive(Debug, Clone, Copy)]
25pub enum AttributeMode {
26 Object,
29 ValueOnly,
32}
33
34pub struct PiiProcessor<'a> {
36 attribute_mode: AttributeMode,
38 compiled_config: &'a CompiledPiiConfig,
39}
40
41impl<'a> PiiProcessor<'a> {
42 pub fn new(compiled_config: &'a CompiledPiiConfig) -> PiiProcessor<'a> {
44 PiiProcessor {
47 compiled_config,
48 attribute_mode: AttributeMode::Object,
49 }
50 }
51
52 pub fn attribute_mode(mut self, attribute_mode: AttributeMode) -> Self {
54 self.attribute_mode = attribute_mode;
55 self
56 }
57
58 fn apply_all_rules(
59 &self,
60 meta: &mut Meta,
61 state: &ProcessingState<'_>,
62 mut value: Option<&mut String>,
63 ) -> ProcessingResult {
64 let pii = state.pii();
65 if pii == Pii::False {
66 return Ok(());
67 }
68
69 for (selector, rules) in self.compiled_config.applications.iter() {
70 if selector.matches_path(&state.path()) {
71 #[allow(clippy::needless_option_as_deref)]
72 for rule in rules {
73 let reborrowed_value = value.as_deref_mut();
74 apply_rule_to_value(meta, rule, state.path().key(), reborrowed_value)?;
75 }
76 }
77 }
78
79 Ok(())
80 }
81}
82
83impl Processor for PiiProcessor<'_> {
84 fn before_process<T: ProcessValue>(
85 &mut self,
86 value: Option<&T>,
87 meta: &mut Meta,
88 state: &ProcessingState<'_>,
89 ) -> ProcessingResult {
90 if let Some(Value::String(original_value)) = meta.original_value_as_mut() {
91 if let Some(parent) = state.iter().next() {
95 let path = state.path();
96 let new_state = parent.enter_borrowed(
97 path.key().unwrap_or(""),
98 Some(Cow::Borrowed(state.attrs())),
99 enum_set!(ValueType::String),
100 );
101
102 if self
103 .apply_all_rules(&mut Meta::default(), &new_state, Some(original_value))
104 .is_err()
105 {
106 meta.set_original_value(Option::<String>::None);
108 }
109 }
110 }
111
112 if state.value_type().contains(ValueType::Boolean)
114 || state.value_type().contains(ValueType::String)
115 {
116 return Ok(());
117 }
118
119 if value.is_none() {
120 return Ok(());
121 }
122
123 self.apply_all_rules(meta, state, None)
125 }
126
127 fn process_array<T>(
128 &mut self,
129 array: &mut Array<T>,
130 _meta: &mut Meta,
131 state: &ProcessingState<'_>,
132 ) -> ProcessingResult
133 where
134 T: ProcessValue,
135 {
136 if is_pairlist(array) {
137 for annotated in array {
138 let mut mapped = mem::take(annotated).map_value(T::into_value);
139
140 if let Some(Value::Array(pair)) = mapped.value_mut() {
141 let mut value = mem::take(&mut pair[1]);
142 let value_type = ValueType::for_field(&value);
143
144 if let Some(key_name) = &pair[0].as_str() {
145 let key_state =
148 state.enter_borrowed(key_name, state.inner_attrs(), value_type);
149 process_value(&mut value, self, &key_state)?;
152 }
153
154 pair[1] = value;
156 }
157
158 *annotated = T::from_value(mapped);
160 }
161
162 Ok(())
163 } else {
164 array.process_child_values(self, state)
166 }
167 }
168
169 fn process_string(
170 &mut self,
171 value: &mut String,
172 meta: &mut Meta,
173 state: &ProcessingState<'_>,
174 ) -> ProcessingResult {
175 if let "" | "true" | "false" | "null" | "undefined" = value.as_str() {
176 return Ok(());
177 }
178
179 self.apply_all_rules(meta, state, Some(value))
182 }
183
184 fn process_native_image_path(
185 &mut self,
186 NativeImagePath(value): &mut NativeImagePath,
187 meta: &mut Meta,
188 state: &ProcessingState<'_>,
189 ) -> ProcessingResult {
190 if let Some(index) = value.rfind(['/', '\\']) {
201 let basename = value.split_off(index);
202 match self.process_string(value, meta, state) {
203 Ok(()) => value.push_str(&basename),
204 Err(
205 ProcessingAction::DeleteValueHard
206 | ProcessingAction::DeleteValueWithRemark(_)
207 | ProcessingAction::DeleteValueSoft,
208 ) => {
209 basename[1..].clone_into(value);
210 }
211 Err(ProcessingAction::InvalidTransaction(x)) => {
212 return Err(ProcessingAction::InvalidTransaction(x));
213 }
214 }
215 }
216
217 Ok(())
218 }
219
220 fn process_pairlist<T: ProcessValue + AsPair>(
221 &mut self,
222 value: &mut PairList<T>,
223 _meta: &mut Meta,
224 state: &ProcessingState,
225 ) -> ProcessingResult {
226 utils::process_pairlist(self, value, state)
227 }
228
229 fn process_attributes(
230 &mut self,
231 value: &mut relay_event_schema::protocol::Attributes,
232 _meta: &mut Meta,
233 state: &ProcessingState,
234 ) -> ProcessingResult {
235 match self.attribute_mode {
236 AttributeMode::Object => value.process_child_values(self, state),
238 AttributeMode::ValueOnly => {
240 for (key, attribute) in value.0.iter_mut() {
241 let Some(attribute) = attribute.value_mut() else {
242 continue;
243 };
244
245 let attrs = FieldAttrs::new()
250 .pii_dynamic(relay_event_schema::protocol::attribute_pii_from_conventions);
251 let inner_value = &mut attribute.value.value;
252 let inner_value_type = ValueType::for_field(inner_value);
253 let entered =
254 state.enter_borrowed(key, Some(Cow::Borrowed(&attrs)), inner_value_type);
255
256 processor::process_value(inner_value, self, &entered)?;
257 self.process_other(&mut attribute.other, state)?;
258 }
259 Ok(())
260 }
261 }
262 }
263
264 fn process_user(
265 &mut self,
266 user: &mut User,
267 _meta: &mut Meta,
268 state: &ProcessingState<'_>,
269 ) -> ProcessingResult {
270 let ip_was_valid = user.ip_address.value().is_none_or(IpAddr::is_valid);
271
272 user.process_child_values(self, state)?;
274
275 let has_other_fields = user.id.value().is_some()
276 || user.username.value().is_some()
277 || user.email.value().is_some();
278
279 let ip_is_still_valid = user.ip_address.value().is_none_or(IpAddr::is_valid);
280
281 if ip_was_valid && !has_other_fields && !ip_is_still_valid {
291 user.id = mem::take(&mut user.ip_address).map_value(|ip| ip.into_inner().into());
292 user.ip_address.meta_mut().add_remark(Remark::new(
293 RemarkType::Removed,
294 "pii:ip_address".to_owned(),
295 ));
296 }
297
298 Ok(())
299 }
300
301 fn process_replay(
303 &mut self,
304 replay: &mut Replay,
305 _meta: &mut Meta,
306 state: &ProcessingState<'_>,
307 ) -> ProcessingResult {
308 replay.process_child_values(self, state)?;
309 Ok(())
310 }
311}
312
313#[derive(Default)]
314struct PairListProcessor {
315 is_pair: bool,
316 has_string_key: bool,
317}
318
319impl PairListProcessor {
320 fn is_pair_array(&self) -> bool {
323 self.is_pair && self.has_string_key
324 }
325}
326
327impl Processor for PairListProcessor {
328 fn process_array<T>(
329 &mut self,
330 value: &mut Array<T>,
331 _meta: &mut Meta,
332 state: &ProcessingState<'_>,
333 ) -> ProcessingResult
334 where
335 T: ProcessValue,
336 {
337 self.is_pair = state.depth() == 0 && value.len() == 2;
338 if self.is_pair {
339 let key_type = ValueType::for_field(&value[0]);
340 process_value(
341 &mut value[0],
342 self,
343 &state.enter_index(0, state.inner_attrs(), key_type),
344 )?;
345 }
346
347 Ok(())
348 }
349
350 fn process_string(
351 &mut self,
352 _value: &mut String,
353 _meta: &mut Meta,
354 state: &ProcessingState<'_>,
355 ) -> ProcessingResult where {
356 if state.depth() == 1 && state.path().index() == Some(0) {
357 self.has_string_key = true;
358 }
359
360 Ok(())
361 }
362}
363
364fn is_pairlist<T: ProcessValue>(array: &mut Array<T>) -> bool {
365 for element in array.iter_mut() {
366 let mut visitor = PairListProcessor::default();
367 process_value(element, &mut visitor, ProcessingState::root()).ok();
368 if !visitor.is_pair_array() {
369 return false;
370 }
371 }
372
373 !array.is_empty()
374}
375
376pub fn scrub_graphql(event: &mut Event) {
378 let mut keys: BTreeSet<&str> = BTreeSet::new();
379
380 let mut is_graphql = false;
381
382 if let Some(request) = event.request.value_mut()
384 && let Some(Value::Object(data)) = request.data.value_mut()
385 {
386 if let Some(api_target) = request.api_target.value()
387 && api_target.eq_ignore_ascii_case("graphql")
388 {
389 is_graphql = true;
390 }
391
392 if is_graphql
393 && let Some(Annotated(Some(Value::Object(variables)), _)) = data.get_mut("variables")
394 {
395 for (key, value) in variables.iter_mut() {
396 keys.insert(key);
397 value.set_value(Some(Value::String("[Filtered]".to_owned())));
398 }
399 }
400 }
401
402 if !is_graphql {
403 return;
404 }
405
406 if let Some(contexts) = event.contexts.value_mut()
408 && let Some(response) = contexts.get_mut::<ResponseContext>()
409 && let Some(Value::Object(data)) = response.data.value_mut()
410 && let Some(Annotated(Some(Value::Object(graphql_data)), _)) = data.get_mut("data")
411 {
412 if !keys.is_empty() {
413 scrub_graphql_data(&keys, graphql_data);
414 } else {
415 data.remove("data");
418 }
419 }
420}
421
422fn scrub_graphql_data(keys: &BTreeSet<&str>, data: &mut BTreeMap<String, Annotated<Value>>) {
424 for (key, value) in data.iter_mut() {
425 match value.value_mut() {
426 Some(Value::Object(item_data)) => {
427 scrub_graphql_data(keys, item_data);
428 }
429 _ => {
430 if keys.contains(key.as_str()) {
431 value.set_value(Some(Value::String("[Filtered]".to_owned())));
432 }
433 }
434 }
435 }
436}
437
438fn apply_rule_to_value(
439 meta: &mut Meta,
440 rule: &RuleRef,
441 key: Option<&str>,
442 mut value: Option<&mut String>,
443) -> ProcessingResult {
444 let should_redact_chunks = !matches!(rule.redaction, Redaction::Default | Redaction::Remove);
447
448 if rule.ty == RuleType::Anything && (value.is_none() || !should_redact_chunks) {
451 meta.add_remark(Remark::new(RemarkType::Removed, rule.origin.clone()));
453 return Err(ProcessingAction::DeleteValueHard);
454 }
455
456 macro_rules! apply_regex {
457 ($regex:expr, $replace_behavior:expr) => {
458 if let Some(ref mut value) = value {
459 processor::process_chunked_value(value, meta, |chunks| {
460 apply_regex_to_chunks(chunks, rule, $regex, $replace_behavior)
461 });
462 }
463 };
464 }
465
466 for (pattern_type, regex, replace_behavior) in regexes::get_regex_for_rule_type(&rule.ty) {
467 if matches!(pattern_type, PatternType::Key | PatternType::KeyValue)
468 && key.is_some_and(|key| regex.is_match(key))
469 {
470 if value.is_some() && should_redact_chunks {
471 apply_regex!(ANYTHING_REGEX.as_ref(), replace_behavior);
474 } else {
475 meta.add_remark(Remark::new(RemarkType::Removed, rule.origin.clone()));
476 return Err(ProcessingAction::DeleteValueHard);
477 }
478 } else if matches!(pattern_type, PatternType::Value | PatternType::KeyValue) {
479 apply_regex!(regex, replace_behavior);
480 }
481 }
482
483 Ok(())
484}
485
486fn apply_regex_to_chunks<'a>(
487 chunks: Vec<Chunk<'a>>,
488 rule: &RuleRef,
489 regex: ScrubbingRegexRef<'_>,
490 replace_behavior: ReplaceBehavior,
491) -> Vec<Chunk<'a>> {
492 let mut search_string = String::new();
497 let mut has_text = false;
498 for chunk in &chunks {
499 match chunk {
500 Chunk::Text { text } => {
501 has_text = true;
502 search_string.push_str(&text.replace('\x00', ""));
503 }
504 Chunk::Redaction { .. } => search_string.push('\x00'),
505 }
506 }
507
508 if !has_text {
509 return chunks;
511 }
512
513 let mut captures_iter = regex.captures_iter(&search_string).peekable();
515 if captures_iter.peek().is_none() {
516 return chunks;
517 }
518
519 let mut replacement_chunks = vec![];
520 for chunk in chunks {
521 if let Chunk::Redaction { .. } = chunk {
522 replacement_chunks.push(chunk);
523 }
524 }
525 replacement_chunks.reverse();
526
527 fn process_text<'a>(
528 text: &str,
529 rv: &mut Vec<Chunk<'a>>,
530 replacement_chunks: &mut Vec<Chunk<'a>>,
531 ) {
532 if text.is_empty() {
533 return;
534 }
535
536 static NULL_SPLIT_RE: OnceLock<Regex> = OnceLock::new();
540 let regex = NULL_SPLIT_RE.get_or_init(|| {
541 #[allow(clippy::trivial_regex)]
542 Regex::new("\x00").unwrap()
543 });
544
545 let mut pos = 0;
546 for piece in regex.find_iter(text) {
547 rv.push(Chunk::Text {
548 text: Cow::Owned(text[pos..piece.start()].to_string()),
549 });
550 rv.push(replacement_chunks.pop().unwrap());
551 pos = piece.end();
552 }
553
554 rv.push(Chunk::Text {
555 text: Cow::Owned(text[pos..].to_string()),
556 });
557 }
558
559 let mut pos = 0;
560 let mut rv = Vec::with_capacity(replacement_chunks.len());
561
562 match replace_behavior {
563 ReplaceBehavior::Groups(ref groups) => {
564 for m in captures_iter {
565 for (idx, g) in m.iter().enumerate() {
566 if let Some(g) = g
567 && groups.contains(&(idx as u8))
568 {
569 process_text(
570 &search_string[pos..g.start()],
571 &mut rv,
572 &mut replacement_chunks,
573 );
574 insert_replacement_chunks(rule, g.as_str(), &mut rv);
575 pos = g.end();
576 }
577 }
578 }
579 process_text(&search_string[pos..], &mut rv, &mut replacement_chunks);
580 debug_assert!(replacement_chunks.is_empty());
581 }
582 ReplaceBehavior::Value => {
583 insert_replacement_chunks(rule, &search_string, &mut rv);
587 }
588 }
589 rv
590}
591
592fn insert_replacement_chunks(rule: &RuleRef, text: &str, output: &mut Vec<Chunk<'_>>) {
593 match &rule.redaction {
594 Redaction::Default | Redaction::Remove => {
595 output.push(Chunk::Redaction {
596 text: Cow::Borrowed(""),
597 rule_id: Cow::Owned(rule.origin.to_string()),
598 ty: RemarkType::Removed,
599 });
600 }
601 Redaction::Mask => {
602 let buf = vec!['*'; text.chars().count()];
603
604 output.push(Chunk::Redaction {
605 ty: RemarkType::Masked,
606 rule_id: Cow::Owned(rule.origin.to_string()),
607 text: buf.into_iter().collect(),
608 })
609 }
610 Redaction::Hash => {
611 output.push(Chunk::Redaction {
612 ty: RemarkType::Pseudonymized,
613 rule_id: Cow::Owned(rule.origin.to_string()),
614 text: Cow::Owned(utils::hash_value(text.as_bytes())),
615 });
616 }
617 Redaction::Replace(replace) => {
618 output.push(Chunk::Redaction {
619 ty: RemarkType::Substituted,
620 rule_id: Cow::Owned(rule.origin.to_string()),
621 text: Cow::Owned(replace.text.clone()),
622 });
623 }
624 Redaction::Other => relay_log::debug!("Incoming redaction is not supported"),
625 }
626}
627
628#[cfg(test)]
629mod tests {
630 use insta::{allow_duplicates, assert_debug_snapshot};
631 use relay_event_schema::processor::process_value;
632 use relay_event_schema::protocol::{
633 Addr, Breadcrumb, DebugImage, DebugMeta, ExtraValue, Headers, LogEntry, Message,
634 NativeDebugImage, Request, Span, TagEntry, Tags, TraceContext,
635 };
636 use relay_protocol::{FromValue, Object, assert_annotated_snapshot, get_value};
637 use serde_json::json;
638
639 use super::*;
640 use crate::{DataScrubbingConfig, PiiConfig, ReplaceRedaction};
641
642 fn to_pii_config(datascrubbing_config: &DataScrubbingConfig) -> Option<PiiConfig> {
643 use crate::convert::to_pii_config as to_pii_config_impl;
644 let rv = to_pii_config_impl(datascrubbing_config);
645 if let Some(ref config) = rv {
646 let roundtrip: PiiConfig =
647 serde_json::from_value(serde_json::to_value(config).unwrap()).unwrap();
648 assert_eq!(&roundtrip, config);
649 }
650 rv
651 }
652
653 #[test]
654 fn test_scrub_original_value() {
655 let mut data = Event::from_value(
656 json!({
657 "user": {
658 "username": "hey man 73.133.27.120", "ip_address": "is this an ip address? 73.133.27.120", },
661 "extra":"invalid data my ip address is 74.133.27.120 and my credit card number is 4571234567890112 ",
662 })
663 .into(),
664 );
665
666 let scrubbing_config = DataScrubbingConfig {
667 scrub_data: true,
668 scrub_ip_addresses: true,
669 scrub_defaults: true,
670 ..Default::default()
671 };
672
673 let pii_config = to_pii_config(&scrubbing_config).unwrap();
674 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
675
676 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
677
678 assert_debug_snapshot!(&data);
679 }
680
681 #[test]
682 fn test_remark_overlap() {
683 let mut data = Annotated::<Event>::from_json_bytes(
684 br#"{
685 "extra": {"foo": "bar"},
686 "_meta":{
687 "extra": {
688 "foo":{
689 "":{
690 "rem":[["some_rule","s",0,3],["some_rule","s",0,3]]
691 }
692 }
693 }
694 }
695 }"#,
696 )
697 .unwrap();
698
699 let scrubbing_config = DataScrubbingConfig {
700 scrub_data: true,
701 scrub_ip_addresses: true,
702 scrub_defaults: true,
703 ..Default::default()
704 };
705
706 let pii_config = to_pii_config(&scrubbing_config).unwrap();
707 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
708
709 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
710
711 assert_eq!(get_value!(data.extra["foo"]!).0.as_str(), Some("bar"));
713 }
714
715 #[test]
716 fn test_resume_after_gap() {
717 let mut data = Annotated::<Event>::from_json_bytes(
718 br#"{
719 "extra": {"foo": "abcdefghijklmnopqrstuvwxyz"},
720 "_meta":{
721 "extra": {
722 "foo":{
723 "":{
724 "rem":[["some_rule","s",0,3],["some_rule","s",6,5]]
725 }
726 }
727 }
728 }
729 }"#,
730 )
731 .unwrap();
732
733 let scrubbing_config = DataScrubbingConfig {
734 scrub_data: true,
735 scrub_ip_addresses: true,
736 scrub_defaults: true,
737 ..Default::default()
738 };
739
740 let pii_config = to_pii_config(&scrubbing_config).unwrap();
741 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
742
743 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
744
745 assert_eq!(
747 get_value!(data.extra["foo"]!).0.as_str(),
748 Some("abcdefghijklmnopqrstuvwxyz")
749 );
750 }
751
752 #[test]
753 fn test_basic_stripping() {
754 let config = serde_json::from_str::<PiiConfig>(
755 r#"
756 {
757 "rules": {
758 "remove_bad_headers": {
759 "type": "redact_pair",
760 "keyPattern": "(?i)cookie|secret[-_]?key"
761 }
762 },
763 "applications": {
764 "$string": ["@ip"],
765 "$object.**": ["remove_bad_headers"]
766 }
767 }
768 "#,
769 )
770 .unwrap();
771
772 let mut event = Annotated::new(Event {
773 logentry: Annotated::new(LogEntry {
774 formatted: Annotated::new("Hello world!".to_owned().into()),
775 ..Default::default()
776 }),
777 request: Annotated::new(Request {
778 env: {
779 let mut rv = Object::new();
780 rv.insert(
781 "SECRET_KEY".to_owned(),
782 Annotated::new(Value::String("134141231231231231231312".into())),
783 );
784 Annotated::new(rv)
785 },
786 headers: {
787 let rv = vec![
788 Annotated::new((
789 Annotated::new("Cookie".to_owned().into()),
790 Annotated::new("super secret".to_owned().into()),
791 )),
792 Annotated::new((
793 Annotated::new("X-Forwarded-For".to_owned().into()),
794 Annotated::new("127.0.0.1".to_owned().into()),
795 )),
796 ];
797 Annotated::new(Headers(PairList(rv)))
798 },
799 ..Default::default()
800 }),
801 tags: Annotated::new(Tags(
802 vec![Annotated::new(TagEntry(
803 Annotated::new("forwarded_for".to_owned()),
804 Annotated::new("127.0.0.1".to_owned()),
805 ))]
806 .into(),
807 )),
808 ..Default::default()
809 });
810
811 let mut processor = PiiProcessor::new(config.compiled());
812 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
813 assert_annotated_snapshot!(event);
814 }
815
816 #[test]
817 fn test_redact_containers() {
818 let config = serde_json::from_str::<PiiConfig>(
819 r#"
820 {
821 "applications": {
822 "$object": ["@anything"]
823 }
824 }
825 "#,
826 )
827 .unwrap();
828
829 let mut event = Annotated::new(Event {
830 extra: {
831 let mut map = Object::new();
832 map.insert(
833 "foo".to_owned(),
834 Annotated::new(ExtraValue(Value::String("bar".to_owned()))),
835 );
836 Annotated::new(map)
837 },
838 ..Default::default()
839 });
840
841 let mut processor = PiiProcessor::new(config.compiled());
842 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
843 assert_annotated_snapshot!(event);
844 }
845
846 #[test]
847 fn test_redact_custom_pattern() {
848 let config = serde_json::from_str::<PiiConfig>(
849 r#"
850 {
851 "applications": {
852 "$string": ["myrule"]
853 },
854 "rules": {
855 "myrule": {
856 "type": "pattern",
857 "pattern": "foo",
858 "redaction": {
859 "method": "replace",
860 "text": "asd"
861 }
862 }
863 }
864 }
865 "#,
866 )
867 .unwrap();
868
869 let mut event = Annotated::new(Event {
870 extra: {
871 let mut map = Object::new();
872 map.insert(
873 "myvalue".to_owned(),
874 Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
875 );
876 Annotated::new(map)
877 },
878 ..Default::default()
879 });
880
881 let mut processor = PiiProcessor::new(config.compiled());
882 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
883 assert_annotated_snapshot!(event);
884 }
885
886 #[test]
887 fn test_redact_custom_negative_pattern() {
888 let config = serde_json::from_str::<PiiConfig>(
889 r#"
890 {
891 "applications": {
892 "$string": ["myrule"]
893 },
894 "rules": {
895 "myrule": {
896 "type": "pattern",
897 "pattern": "the good string|.*OK.*|(.*)",
898 "replaceGroups": [1],
899 "redaction": {
900 "method": "mask"
901 }
902 }
903 }
904 }
905 "#,
906 )
907 .unwrap();
908
909 let mut event = Annotated::<Event>::from_json(
910 r#"{
911 "extra": {
912 "1": "the good string",
913 "2": "a bad string",
914 "3": "another OK string",
915 "4": "another bad one"
916 }
917 }"#,
918 )
919 .unwrap();
920
921 let mut processor = PiiProcessor::new(config.compiled());
922 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
923 assert_annotated_snapshot!(event.value().unwrap().extra, @r#"
924 {
925 "1": "the good string",
926 "2": "************",
927 "3": "another OK string",
928 "4": "***************",
929 "_meta": {
930 "2": {
931 "": {
932 "rem": [
933 [
934 "myrule",
935 "m",
936 0,
937 12
938 ]
939 ],
940 "len": 12
941 }
942 },
943 "4": {
944 "": {
945 "rem": [
946 [
947 "myrule",
948 "m",
949 0,
950 15
951 ]
952 ],
953 "len": 15
954 }
955 }
956 }
957 }
958 "#);
959 }
960
961 #[test]
962 fn test_no_field_upsert() {
963 let config = serde_json::from_str::<PiiConfig>(
964 r#"
965 {
966 "applications": {
967 "**": ["@anything:remove"]
968 }
969 }
970 "#,
971 )
972 .unwrap();
973
974 let mut event = Annotated::new(Event {
975 extra: {
976 let mut map = Object::new();
977 map.insert(
978 "myvalue".to_owned(),
979 Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
980 );
981 Annotated::new(map)
982 },
983 ..Default::default()
984 });
985
986 let mut processor = PiiProcessor::new(config.compiled());
987 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
988 assert_annotated_snapshot!(event);
989 }
990
991 #[test]
992 fn test_anything_hash_on_string() {
993 let config = serde_json::from_str::<PiiConfig>(
994 r#"
995 {
996 "applications": {
997 "$string": ["@anything:hash"]
998 }
999 }
1000 "#,
1001 )
1002 .unwrap();
1003
1004 let mut event = Annotated::new(Event {
1005 extra: {
1006 let mut map = Object::new();
1007 map.insert(
1008 "myvalue".to_owned(),
1009 Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
1010 );
1011 Annotated::new(map)
1012 },
1013 ..Default::default()
1014 });
1015
1016 let mut processor = PiiProcessor::new(config.compiled());
1017 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1018 assert_annotated_snapshot!(event);
1019 }
1020
1021 #[test]
1022 fn test_anything_hash_on_container() {
1023 let config = serde_json::from_str::<PiiConfig>(
1024 r#"
1025 {
1026 "applications": {
1027 "$object": ["@anything:hash"]
1028 }
1029 }
1030 "#,
1031 )
1032 .unwrap();
1033
1034 let mut event = Annotated::new(Event {
1035 extra: {
1036 let mut map = Object::new();
1037 map.insert(
1038 "myvalue".to_owned(),
1039 Annotated::new(ExtraValue(Value::String("foobar".to_owned()))),
1040 );
1041 Annotated::new(map)
1042 },
1043 ..Default::default()
1044 });
1045
1046 let mut processor = PiiProcessor::new(config.compiled());
1047 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1048 assert_annotated_snapshot!(event);
1049 }
1050
1051 #[test]
1052 fn test_only_match_token_on_keys() {
1053 let mut data = Event::from_value(
1054 json!({
1055 "request": {
1056 "headers": [
1057 ["X-Token", "oof this is very sensitive"],
1058 ["Token", "also bad"],
1059 ]
1060 },
1061 "extra": {
1062 "url": "foo.bar/endpoint?token=sensitive",
1063 "url2": "foo.bar/endpoint?token_foobar=sensitive",
1064 "aaa": "token:12345",
1065 "foo-token-bar": "sensitive",
1066 "llm": "token count",
1067 },
1068 })
1069 .into(),
1070 );
1071
1072 let scrubbing_config = DataScrubbingConfig {
1073 scrub_data: true,
1074 scrub_ip_addresses: true,
1075 scrub_defaults: true,
1076 ..Default::default()
1077 };
1078
1079 let pii_config = to_pii_config(&scrubbing_config).unwrap();
1080 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1081
1082 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
1083
1084 assert_annotated_snapshot!(&data);
1085 }
1086
1087 #[test]
1088 fn test_ignore_user_agent_ip_scrubbing() {
1089 let mut data = Event::from_value(
1090 json!({
1091 "request": {
1092 "headers": [
1093 ["User-Agent", "127.0.0.1"],
1094 ["X-Client-Ip", "10.0.0.1"]
1095 ]
1096 },
1097 })
1098 .into(),
1099 );
1100
1101 let scrubbing_config = DataScrubbingConfig {
1102 scrub_data: true,
1103 scrub_ip_addresses: true,
1104 scrub_defaults: true,
1105 ..Default::default()
1106 };
1107
1108 let pii_config = to_pii_config(&scrubbing_config).unwrap();
1109 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1110
1111 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
1112
1113 assert_annotated_snapshot!(&data);
1114 }
1115
1116 #[test]
1117 fn test_remove_debugmeta_path() {
1118 let config = serde_json::from_str::<PiiConfig>(
1119 r#"
1120 {
1121 "applications": {
1122 "debug_meta.images.*.code_file": ["@anything:remove"],
1123 "debug_meta.images.*.debug_file": ["@anything:remove"]
1124 }
1125 }
1126 "#,
1127 )
1128 .unwrap();
1129
1130 let mut event = Annotated::new(Event {
1131 debug_meta: Annotated::new(DebugMeta {
1132 images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1133 NativeDebugImage {
1134 code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1135 code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1136 debug_id: Annotated::new(
1137 "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1138 ),
1139 debug_file: Annotated::new("wntdll.pdb".into()),
1140 debug_checksum: Annotated::empty(),
1141 arch: Annotated::new("arm64".to_owned()),
1142 image_addr: Annotated::new(Addr(0)),
1143 image_size: Annotated::new(4096),
1144 image_vmaddr: Annotated::new(Addr(32768)),
1145 other: {
1146 let mut map = Object::new();
1147 map.insert(
1148 "other".to_owned(),
1149 Annotated::new(Value::String("value".to_owned())),
1150 );
1151 map
1152 },
1153 },
1154 )))]),
1155 ..Default::default()
1156 }),
1157 ..Default::default()
1158 });
1159
1160 let mut processor = PiiProcessor::new(config.compiled());
1161 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1162 assert_annotated_snapshot!(event);
1163 }
1164
1165 #[test]
1166 fn test_replace_debugmeta_path() {
1167 let config = serde_json::from_str::<PiiConfig>(
1168 r#"
1169 {
1170 "applications": {
1171 "debug_meta.images.*.code_file": ["@anything:replace"],
1172 "debug_meta.images.*.debug_file": ["@anything:replace"]
1173 }
1174 }
1175 "#,
1176 )
1177 .unwrap();
1178
1179 let mut event = Annotated::new(Event {
1180 debug_meta: Annotated::new(DebugMeta {
1181 images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1182 NativeDebugImage {
1183 code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1184 code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1185 debug_id: Annotated::new(
1186 "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1187 ),
1188 debug_file: Annotated::new("wntdll.pdb".into()),
1189 debug_checksum: Annotated::empty(),
1190 arch: Annotated::new("arm64".to_owned()),
1191 image_addr: Annotated::new(Addr(0)),
1192 image_size: Annotated::new(4096),
1193 image_vmaddr: Annotated::new(Addr(32768)),
1194 other: {
1195 let mut map = Object::new();
1196 map.insert(
1197 "other".to_owned(),
1198 Annotated::new(Value::String("value".to_owned())),
1199 );
1200 map
1201 },
1202 },
1203 )))]),
1204 ..Default::default()
1205 }),
1206 ..Default::default()
1207 });
1208
1209 let mut processor = PiiProcessor::new(config.compiled());
1210 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1211 assert_annotated_snapshot!(event);
1212 }
1213
1214 #[test]
1215 fn test_hash_debugmeta_path() {
1216 let config = serde_json::from_str::<PiiConfig>(
1217 r#"
1218 {
1219 "applications": {
1220 "debug_meta.images.*.code_file": ["@anything:hash"],
1221 "debug_meta.images.*.debug_file": ["@anything:hash"]
1222 }
1223 }
1224 "#,
1225 )
1226 .unwrap();
1227
1228 let mut event = Annotated::new(Event {
1229 debug_meta: Annotated::new(DebugMeta {
1230 images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1231 NativeDebugImage {
1232 code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1233 code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1234 debug_id: Annotated::new(
1235 "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1236 ),
1237 debug_file: Annotated::new("wntdll.pdb".into()),
1238 debug_checksum: Annotated::empty(),
1239 arch: Annotated::new("arm64".to_owned()),
1240 image_addr: Annotated::new(Addr(0)),
1241 image_size: Annotated::new(4096),
1242 image_vmaddr: Annotated::new(Addr(32768)),
1243 other: {
1244 let mut map = Object::new();
1245 map.insert(
1246 "other".to_owned(),
1247 Annotated::new(Value::String("value".to_owned())),
1248 );
1249 map
1250 },
1251 },
1252 )))]),
1253 ..Default::default()
1254 }),
1255 ..Default::default()
1256 });
1257
1258 let mut processor = PiiProcessor::new(config.compiled());
1259 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1260 assert_annotated_snapshot!(event);
1261 }
1262
1263 #[test]
1264 fn test_debugmeta_path_not_addressible_with_wildcard_selector() {
1265 let config = serde_json::from_str::<PiiConfig>(
1266 r#"
1267 {
1268 "applications": {
1269 "$string": ["@anything:remove"],
1270 "**": ["@anything:remove"],
1271 "debug_meta.**": ["@anything:remove"],
1272 "(debug_meta.images.**.code_file & $string)": ["@anything:remove"]
1273 }
1274 }
1275 "#,
1276 )
1277 .unwrap();
1278
1279 let mut event = Annotated::new(Event {
1280 debug_meta: Annotated::new(DebugMeta {
1281 images: Annotated::new(vec![Annotated::new(DebugImage::Symbolic(Box::new(
1282 NativeDebugImage {
1283 code_id: Annotated::new("59b0d8f3183000".parse().unwrap()),
1284 code_file: Annotated::new("C:\\Windows\\System32\\ntdll.dll".into()),
1285 debug_id: Annotated::new(
1286 "971f98e5-ce60-41ff-b2d7-235bbeb34578-1".parse().unwrap(),
1287 ),
1288 debug_file: Annotated::new("wntdll.pdb".into()),
1289 debug_checksum: Annotated::empty(),
1290 arch: Annotated::new("arm64".to_owned()),
1291 image_addr: Annotated::new(Addr(0)),
1292 image_size: Annotated::new(4096),
1293 image_vmaddr: Annotated::new(Addr(32768)),
1294 other: {
1295 let mut map = Object::new();
1296 map.insert(
1297 "other".to_owned(),
1298 Annotated::new(Value::String("value".to_owned())),
1299 );
1300 map
1301 },
1302 },
1303 )))]),
1304 ..Default::default()
1305 }),
1306 ..Default::default()
1307 });
1308
1309 let mut processor = PiiProcessor::new(config.compiled());
1310 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1311 assert_annotated_snapshot!(event);
1312 }
1313
1314 #[test]
1315 fn test_quoted_keys() {
1316 let config = serde_json::from_str::<PiiConfig>(
1317 r#"
1318 {
1319 "applications": {
1320 "extra.'special ,./<>?!@#$%^&*())''gärbage'''": ["@anything:remove"]
1321 }
1322 }
1323 "#,
1324 )
1325 .unwrap();
1326
1327 let mut event = Annotated::new(Event {
1328 extra: {
1329 let mut map = Object::new();
1330 map.insert(
1331 "do not ,./<>?!@#$%^&*())'ßtrip'".to_owned(),
1332 Annotated::new(ExtraValue(Value::String("foo".to_owned()))),
1333 );
1334 map.insert(
1335 "special ,./<>?!@#$%^&*())'gärbage'".to_owned(),
1336 Annotated::new(ExtraValue(Value::String("bar".to_owned()))),
1337 );
1338 Annotated::new(map)
1339 },
1340 ..Default::default()
1341 });
1342
1343 let mut processor = PiiProcessor::new(config.compiled());
1344 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1345 assert_annotated_snapshot!(event);
1346 }
1347
1348 #[test]
1349 fn test_logentry_value_types() {
1350 for formatted_selector in &[
1352 "$logentry.formatted",
1353 "$message",
1354 "$logentry.formatted && $message",
1355 "$string",
1356 ] {
1357 let config = serde_json::from_str::<PiiConfig>(&format!(
1358 r##"
1359 {{
1360 "applications": {{
1361 "{formatted_selector}": ["@anything:remove"]
1362 }}
1363 }}
1364 "##
1365 ))
1366 .unwrap();
1367
1368 let mut event = Annotated::new(Event {
1369 logentry: Annotated::new(LogEntry {
1370 formatted: Annotated::new("Hello world!".to_owned().into()),
1371 ..Default::default()
1372 }),
1373 ..Default::default()
1374 });
1375
1376 let mut processor = PiiProcessor::new(config.compiled());
1377 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1378 assert!(
1379 event
1380 .value()
1381 .unwrap()
1382 .logentry
1383 .value()
1384 .unwrap()
1385 .formatted
1386 .value()
1387 .is_none()
1388 );
1389 }
1390 }
1391
1392 #[test]
1393 fn test_logentry_formatted_never_fully_filtered() {
1394 let config = crate::convert::to_pii_config(&crate::DataScrubbingConfig {
1397 scrub_data: true,
1398 scrub_defaults: true,
1399 scrub_ip_addresses: true,
1400 ..Default::default()
1401 })
1402 .unwrap();
1403
1404 let mut event = Annotated::new(Event {
1405 logentry: Annotated::new(LogEntry {
1406 formatted: Annotated::new(
1407 "User john.doe@company.com failed login with card 4111-1111-1111-1111"
1408 .to_owned()
1409 .into(),
1410 ),
1411 ..Default::default()
1412 }),
1413 ..Default::default()
1414 });
1415
1416 let mut processor = PiiProcessor::new(config.compiled());
1417 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1418 assert_annotated_snapshot!(event, @r#"
1419 {
1420 "logentry": {
1421 "formatted": "User [email] failed login with card [creditcard]"
1422 },
1423 "_meta": {
1424 "logentry": {
1425 "formatted": {
1426 "": {
1427 "rem": [
1428 [
1429 "@email:replace",
1430 "s",
1431 5,
1432 12
1433 ],
1434 [
1435 "@creditcard:replace",
1436 "s",
1437 36,
1438 48
1439 ]
1440 ],
1441 "len": 68
1442 }
1443 }
1444 }
1445 }
1446 }
1447 "#);
1448 }
1449
1450 #[test]
1451 fn test_logentry_formatted_bearer_token_scrubbing() {
1452 let config = crate::convert::to_pii_config(&crate::DataScrubbingConfig {
1454 scrub_data: true,
1455 scrub_defaults: true,
1456 ..Default::default()
1457 })
1458 .unwrap();
1459
1460 let mut event = Annotated::new(Event {
1461 logentry: Annotated::new(LogEntry {
1462 formatted: Annotated::new(
1463 "API request failed with Bearer ABC123XYZ789TOKEN and other data"
1464 .to_owned()
1465 .into(),
1466 ),
1467 ..Default::default()
1468 }),
1469 ..Default::default()
1470 });
1471
1472 let mut processor = PiiProcessor::new(config.compiled());
1473 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1474 assert_annotated_snapshot!(event, @r#"
1475 {
1476 "logentry": {
1477 "formatted": "API request failed with Bearer [token] and other data"
1478 },
1479 "_meta": {
1480 "logentry": {
1481 "formatted": {
1482 "": {
1483 "rem": [
1484 [
1485 "@bearer:replace",
1486 "s",
1487 24,
1488 38
1489 ]
1490 ],
1491 "len": 63
1492 }
1493 }
1494 }
1495 }
1496 }
1497 "#);
1498 }
1499
1500 #[test]
1501 fn test_logentry_formatted_password_word_not_scrubbed() {
1502 let config = PiiConfig::default();
1503 let mut event = Annotated::new(Event {
1504 logentry: Annotated::new(LogEntry {
1505 formatted: Annotated::new(
1506 "User password is secret123 for authentication"
1507 .to_owned()
1508 .into(),
1509 ),
1510 ..Default::default()
1511 }),
1512 ..Default::default()
1513 });
1514
1515 let mut processor = PiiProcessor::new(config.compiled());
1516 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1517 assert_annotated_snapshot!(event, @r#"
1518 {
1519 "logentry": {
1520 "formatted": "User password is secret123 for authentication"
1521 }
1522 }
1523 "#);
1524 }
1525
1526 #[test]
1527 fn test_ip_address_hashing() {
1528 let config = serde_json::from_str::<PiiConfig>(
1529 r#"
1530 {
1531 "applications": {
1532 "$user.ip_address": ["@ip:hash"]
1533 }
1534 }
1535 "#,
1536 )
1537 .unwrap();
1538
1539 let mut event = Annotated::new(Event {
1540 user: Annotated::new(User {
1541 ip_address: Annotated::new(IpAddr("127.0.0.1".to_owned())),
1542 ..Default::default()
1543 }),
1544 ..Default::default()
1545 });
1546
1547 let mut processor = PiiProcessor::new(config.compiled());
1548 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1549
1550 let user = event.value().unwrap().user.value().unwrap();
1551
1552 assert!(user.ip_address.value().is_none());
1553
1554 assert_eq!(
1555 user.id.value().unwrap().as_str(),
1556 "AE12FE3B5F129B5CC4CDD2B136B7B7947C4D2741"
1557 );
1558 }
1559
1560 #[test]
1561 fn test_ip_address_hashing_does_not_overwrite_id() {
1562 let config = serde_json::from_str::<PiiConfig>(
1563 r#"
1564 {
1565 "applications": {
1566 "$user.ip_address": ["@ip:hash"]
1567 }
1568 }
1569 "#,
1570 )
1571 .unwrap();
1572
1573 let mut event = Annotated::new(Event {
1574 user: Annotated::new(User {
1575 id: Annotated::new("123".to_owned().into()),
1576 ip_address: Annotated::new(IpAddr("127.0.0.1".to_owned())),
1577 ..Default::default()
1578 }),
1579 ..Default::default()
1580 });
1581
1582 let mut processor = PiiProcessor::new(config.compiled());
1583 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1584
1585 let user = event.value().unwrap().user.value().unwrap();
1586
1587 assert_eq!(
1589 user.ip_address.value().unwrap().as_str(),
1590 "AE12FE3B5F129B5CC4CDD2B136B7B7947C4D2741"
1591 );
1592
1593 assert_eq!(user.id.value().unwrap().as_str(), "123");
1594 }
1595
1596 #[test]
1597 fn test_replace_replaced_text() {
1598 let chunks = vec![Chunk::Redaction {
1599 text: "[ip]".into(),
1600 rule_id: "@ip".into(),
1601 ty: RemarkType::Substituted,
1602 }];
1603 let rule = RuleRef {
1604 id: "@ip:replace".into(),
1605 origin: "@ip".into(),
1606 ty: RuleType::Ip,
1607 redaction: Redaction::Replace(ReplaceRedaction {
1608 text: "[ip]".into(),
1609 }),
1610 };
1611 let res = apply_regex_to_chunks(
1612 chunks.clone(),
1613 &rule,
1614 ANYTHING_REGEX.as_ref(),
1615 ReplaceBehavior::Value,
1616 );
1617 assert_eq!(chunks, res);
1618 }
1619
1620 #[test]
1621 fn test_replace_replaced_text_anything() {
1622 let chunks = vec![Chunk::Redaction {
1623 text: "[Filtered]".into(),
1624 rule_id: "@password:filter".into(),
1625 ty: RemarkType::Substituted,
1626 }];
1627 let rule = RuleRef {
1628 id: "@anything:filter".into(),
1629 origin: "@anything:filter".into(),
1630 ty: RuleType::Anything,
1631 redaction: Redaction::Replace(ReplaceRedaction {
1632 text: "[Filtered]".into(),
1633 }),
1634 };
1635 let res = apply_regex_to_chunks(
1636 chunks.clone(),
1637 &rule,
1638 ANYTHING_REGEX.as_ref(),
1639 ReplaceBehavior::Groups(smallvec::smallvec![0]),
1640 );
1641 assert_eq!(chunks, res);
1642 }
1643
1644 #[test]
1645 fn test_trace_route_params_scrubbed() {
1646 let mut trace_context: Annotated<TraceContext> = Annotated::from_json(
1647 r#"
1648 {
1649 "type": "trace",
1650 "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1651 "span_id": "fa90fdead5f74052",
1652 "data": {
1653 "previousRoute": {
1654 "params": {
1655 "password": "test"
1656 }
1657 }
1658 }
1659 }
1660 "#,
1661 )
1662 .unwrap();
1663
1664 let ds_config = DataScrubbingConfig {
1665 scrub_data: true,
1666 scrub_defaults: true,
1667 ..Default::default()
1668 };
1669 let pii_config = ds_config.pii_config().as_ref().unwrap();
1670 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1671
1672 process_value(
1673 &mut trace_context,
1674 &mut pii_processor,
1675 ProcessingState::root(),
1676 )
1677 .unwrap();
1678 assert_annotated_snapshot!(trace_context);
1679 }
1680
1681 #[test]
1682 fn test_scrub_span_data_http_not_scrubbed() {
1683 let mut span: Annotated<Span> = Annotated::from_json(
1684 r#"{
1685 "data": {
1686 "http": {
1687 "query": "dance=true"
1688 }
1689 }
1690 }"#,
1691 )
1692 .unwrap();
1693
1694 let ds_config = DataScrubbingConfig {
1695 scrub_data: true,
1696 scrub_defaults: true,
1697 ..Default::default()
1698 };
1699 let pii_config = ds_config.pii_config().as_ref().unwrap();
1700 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1701
1702 process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1703 assert_annotated_snapshot!(span);
1704 }
1705
1706 #[test]
1707 fn test_scrub_span_data_http_strings_are_scrubbed() {
1708 let mut span: Annotated<Span> = Annotated::from_json(
1709 r#"{
1710 "data": {
1711 "http": {
1712 "query": "ccnumber=5105105105105100&process_id=123",
1713 "fragment": "ccnumber=5105105105105100,process_id=123"
1714 }
1715 }
1716 }"#,
1717 )
1718 .unwrap();
1719
1720 let ds_config = DataScrubbingConfig {
1721 scrub_data: true,
1722 scrub_defaults: true,
1723 ..Default::default()
1724 };
1725 let pii_config = ds_config.pii_config().as_ref().unwrap();
1726 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1727
1728 process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1729 assert_annotated_snapshot!(span);
1730 }
1731
1732 #[test]
1733 fn test_scrub_span_data_http_objects_are_scrubbed() {
1734 let mut span: Annotated<Span> = Annotated::from_json(
1735 r#"{
1736 "data": {
1737 "http": {
1738 "query": {
1739 "ccnumber": "5105105105105100",
1740 "process_id": "123"
1741 },
1742 "fragment": {
1743 "ccnumber": "5105105105105100",
1744 "process_id": "123"
1745 }
1746 }
1747 }
1748 }"#,
1749 )
1750 .unwrap();
1751
1752 let ds_config = DataScrubbingConfig {
1753 scrub_data: true,
1754 scrub_defaults: true,
1755 ..Default::default()
1756 };
1757 let pii_config = ds_config.pii_config().as_ref().unwrap();
1758 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1759
1760 process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1761 assert_annotated_snapshot!(span);
1762 }
1763
1764 #[test]
1765 fn test_scrub_span_data_untyped_props_are_scrubbed() {
1766 let mut span: Annotated<Span> = Annotated::from_json(
1767 r#"{
1768 "data": {
1769 "untyped": "ccnumber=5105105105105100",
1770 "more_untyped": {
1771 "typed": "no",
1772 "scrubbed": "yes",
1773 "ccnumber": "5105105105105100"
1774 }
1775 }
1776 }"#,
1777 )
1778 .unwrap();
1779
1780 let ds_config = DataScrubbingConfig {
1781 scrub_data: true,
1782 scrub_defaults: true,
1783 ..Default::default()
1784 };
1785 let pii_config = ds_config.pii_config().as_ref().unwrap();
1786 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1787
1788 process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1789 assert_annotated_snapshot!(span);
1790 }
1791
1792 #[test]
1793 fn test_span_data_pii() {
1794 let mut span = Span::from_value(
1795 json!({
1796 "data": {
1797 "code.filepath": "src/sentry/api/authentication.py",
1798 }
1799 })
1800 .into(),
1801 );
1802
1803 let ds_config = DataScrubbingConfig {
1804 scrub_data: true,
1805 scrub_defaults: true,
1806 ..Default::default()
1807 };
1808 let pii_config = ds_config.pii_config().as_ref().unwrap();
1809
1810 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1811 processor::process_value(&mut span, &mut pii_processor, ProcessingState::root()).unwrap();
1812 assert_eq!(
1813 span.0.unwrap().data.0.unwrap().other["code.filepath"].as_str(),
1814 Some("src/sentry/api/authentication.py")
1815 );
1816 }
1817
1818 #[test]
1819 fn test_csp_source_file_pii() {
1820 let mut event = Event::from_value(
1821 json!({
1822 "csp": {
1823 "source_file": "authentication.js",
1824 }
1825 })
1826 .into(),
1827 );
1828
1829 let config = serde_json::from_str::<PiiConfig>(
1830 r#"
1831 {
1832 "applications": {
1833 "csp.source_file": ["@anything:filter"]
1834 }
1835 }
1836 "#,
1837 )
1838 .unwrap();
1839
1840 let mut pii_processor = PiiProcessor::new(config.compiled());
1841 processor::process_value(&mut event, &mut pii_processor, ProcessingState::root()).unwrap();
1842 assert_eq!(get_value!(event.csp.source_file!).as_str(), "[Filtered]");
1843 }
1844
1845 #[test]
1846 fn test_csp_script_sample_pii() {
1847 let mut event = Event::from_value(
1848 json!({
1849 "csp": {
1850 "script_sample": "var user = {\"email\":\"jane@example.com\"",
1851 }
1852 })
1853 .into(),
1854 );
1855
1856 let config = serde_json::from_str::<PiiConfig>(
1857 r#"
1858 {
1859 "applications": {
1860 "csp.script_sample": ["@anything:filter"]
1861 }
1862 }
1863 "#,
1864 )
1865 .unwrap();
1866
1867 let mut pii_processor = PiiProcessor::new(config.compiled());
1868 processor::process_value(&mut event, &mut pii_processor, ProcessingState::root()).unwrap();
1869 assert_eq!(get_value!(event.csp.script_sample!).as_str(), "[Filtered]");
1870 }
1871
1872 #[test]
1873 fn test_scrub_breadcrumb_data_http_not_scrubbed() {
1874 let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1875 r#"{
1876 "data": {
1877 "http": {
1878 "query": "dance=true"
1879 }
1880 }
1881 }"#,
1882 )
1883 .unwrap();
1884
1885 let ds_config = DataScrubbingConfig {
1886 scrub_data: true,
1887 scrub_defaults: true,
1888 ..Default::default()
1889 };
1890 let pii_config = ds_config.pii_config().as_ref().unwrap();
1891 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1892 process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1893 assert_annotated_snapshot!(breadcrumb);
1894 }
1895
1896 #[test]
1897 fn test_scrub_breadcrumb_data_http_strings_are_scrubbed() {
1898 let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1899 r#"{
1900 "data": {
1901 "http": {
1902 "query": "ccnumber=5105105105105100&process_id=123",
1903 "fragment": "ccnumber=5105105105105100,process_id=123"
1904 }
1905 }
1906 }"#,
1907 )
1908 .unwrap();
1909
1910 let ds_config = DataScrubbingConfig {
1911 scrub_data: true,
1912 scrub_defaults: true,
1913 ..Default::default()
1914 };
1915 let pii_config = ds_config.pii_config().as_ref().unwrap();
1916 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1917 process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1918 assert_annotated_snapshot!(breadcrumb);
1919 }
1920
1921 #[test]
1922 fn test_scrub_breadcrumb_data_http_objects_are_scrubbed() {
1923 let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1924 r#"{
1925 "data": {
1926 "http": {
1927 "query": {
1928 "ccnumber": "5105105105105100",
1929 "process_id": "123"
1930 },
1931 "fragment": {
1932 "ccnumber": "5105105105105100",
1933 "process_id": "123"
1934 }
1935 }
1936 }
1937 }"#,
1938 )
1939 .unwrap();
1940
1941 let ds_config = DataScrubbingConfig {
1942 scrub_data: true,
1943 scrub_defaults: true,
1944 ..Default::default()
1945 };
1946 let pii_config = ds_config.pii_config().as_ref().unwrap();
1947 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1948
1949 process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1950 assert_annotated_snapshot!(breadcrumb);
1951 }
1952
1953 #[test]
1954 fn test_scrub_breadcrumb_data_untyped_props_are_scrubbed() {
1955 let mut breadcrumb: Annotated<Breadcrumb> = Annotated::from_json(
1956 r#"{
1957 "data": {
1958 "untyped": "ccnumber=5105105105105100",
1959 "more_untyped": {
1960 "typed": "no",
1961 "scrubbed": "yes",
1962 "ccnumber": "5105105105105100"
1963 }
1964 }
1965 }"#,
1966 )
1967 .unwrap();
1968
1969 let ds_config = DataScrubbingConfig {
1970 scrub_data: true,
1971 scrub_defaults: true,
1972 ..Default::default()
1973 };
1974 let pii_config = ds_config.pii_config().as_ref().unwrap();
1975 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
1976 process_value(&mut breadcrumb, &mut pii_processor, ProcessingState::root()).unwrap();
1977 assert_annotated_snapshot!(breadcrumb);
1978 }
1979
1980 #[test]
1981 fn test_scrub_graphql_response_data_with_variables() {
1982 let mut data = Event::from_value(
1983 json!({
1984 "request": {
1985 "data": {
1986 "query": "{\n viewer {\n login\n }\n}",
1987 "variables": {
1988 "login": "foo"
1989 }
1990 },
1991 "api_target": "graphql"
1992 },
1993 "contexts": {
1994 "response": {
1995 "type": "response",
1996 "data": {
1997 "data": {
1998 "viewer": {
1999 "login": "foo"
2000 }
2001 }
2002 }
2003 }
2004 }
2005 })
2006 .into(),
2007 );
2008
2009 scrub_graphql(data.value_mut().as_mut().unwrap());
2010
2011 assert_debug_snapshot!(&data);
2012 }
2013
2014 #[test]
2015 fn test_scrub_graphql_response_data_without_variables() {
2016 let mut data = Event::from_value(
2017 json!({
2018 "request": {
2019 "data": {
2020 "query": "{\n viewer {\n login\n }\n}"
2021 },
2022 "api_target": "graphql"
2023 },
2024 "contexts": {
2025 "response": {
2026 "type": "response",
2027 "data": {
2028 "data": {
2029 "viewer": {
2030 "login": "foo"
2031 }
2032 }
2033 }
2034 }
2035 }
2036 })
2037 .into(),
2038 );
2039
2040 scrub_graphql(data.value_mut().as_mut().unwrap());
2041 assert_debug_snapshot!(&data);
2042 }
2043
2044 #[test]
2045 fn test_does_not_scrub_if_no_graphql() {
2046 let mut data = Event::from_value(
2047 json!({
2048 "request": {
2049 "data": {
2050 "query": "{\n viewer {\n login\n }\n}",
2051 "variables": {
2052 "login": "foo"
2053 }
2054 },
2055 },
2056 "contexts": {
2057 "response": {
2058 "type": "response",
2059 "data": {
2060 "data": {
2061 "viewer": {
2062 "login": "foo"
2063 }
2064 }
2065 }
2066 }
2067 }
2068 })
2069 .into(),
2070 );
2071
2072 let scrubbing_config = DataScrubbingConfig {
2073 scrub_data: true,
2074 scrub_ip_addresses: true,
2075 scrub_defaults: true,
2076 ..Default::default()
2077 };
2078
2079 let pii_config = to_pii_config(&scrubbing_config).unwrap();
2080 let mut pii_processor = PiiProcessor::new(pii_config.compiled());
2081
2082 process_value(&mut data, &mut pii_processor, ProcessingState::root()).unwrap();
2083
2084 assert_debug_snapshot!(&data);
2085 }
2086
2087 #[test]
2088 fn test_logentry_params_scrubbed() {
2089 let config = serde_json::from_str::<PiiConfig>(
2090 r##"
2091 {
2092 "applications": {
2093 "$string": ["@anything:remove"]
2094 }
2095 }
2096 "##,
2097 )
2098 .unwrap();
2099
2100 let mut event = Annotated::new(Event {
2101 logentry: Annotated::new(LogEntry {
2102 message: Annotated::new(Message::from("failed to parse report id=%s".to_owned())),
2103 formatted: Annotated::new("failed to parse report id=1".to_owned().into()),
2104 params: Annotated::new(Value::Array(vec![Annotated::new(Value::String(
2105 "12345".to_owned(),
2106 ))])),
2107 ..Default::default()
2108 }),
2109 ..Default::default()
2110 });
2111
2112 let mut processor = PiiProcessor::new(config.compiled());
2113 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2114
2115 let params = get_value!(event.logentry.params!);
2116 assert_debug_snapshot!(params, @r###"
2117 Array(
2118 [
2119 Meta {
2120 remarks: [
2121 Remark {
2122 ty: Removed,
2123 rule_id: "@anything:remove",
2124 range: None,
2125 },
2126 ],
2127 errors: [],
2128 original_length: None,
2129 original_value: None,
2130 },
2131 ],
2132 )
2133 "###);
2134 }
2135
2136 #[test]
2137 fn test_is_pairlist() {
2138 for (case, expected) in [
2139 (r#"[]"#, false),
2140 (r#"["foo"]"#, false),
2141 (r#"["foo", 123]"#, false),
2142 (r#"[[1, "foo"]]"#, false),
2143 (r#"[[["too_nested", 123]]]"#, false),
2144 (r#"[["foo", "bar"], [1, "foo"]]"#, false),
2145 (r#"[["foo", "bar"], ["foo", "bar", "baz"]]"#, false),
2146 (r#"[["foo", "bar", "baz"], ["foo", "bar"]]"#, false),
2147 (r#"["foo", ["bar", "baz"], ["foo", "bar"]]"#, false),
2148 (r#"[["foo", "bar"], [["too_nested", 123]]]"#, false),
2149 (r#"[["foo", 123]]"#, true),
2150 (r#"[["foo", "bar"]]"#, true),
2151 (
2152 r#"[["foo", "bar"], ["foo", {"nested": {"something": 1}}]]"#,
2153 true,
2154 ),
2155 ] {
2156 let v = Annotated::<Value>::from_json(case).unwrap();
2157 let Annotated(Some(Value::Array(mut a)), _) = v else {
2158 panic!()
2159 };
2160 assert_eq!(is_pairlist(&mut a), expected, "{case}");
2161 }
2162 }
2163
2164 #[test]
2165 fn test_tuple_array_scrubbed_with_path_selector() {
2166 let configs = vec![
2168 r##"
2171 {
2172 "applications": {
2173 "exception.values.0.stacktrace.frames.0.vars.headers.authorization": ["@anything:replace"]
2174 }
2175 }
2176 "##,
2177 r##"
2179 {
2180 "applications": {
2181 "exception.values.0.stacktrace.frames.0.vars.headers.0.1": ["@anything:replace"]
2182 }
2183 }
2184 "##,
2185 ];
2186
2187 let mut event = Event::from_value(
2188 serde_json::json!(
2189 {
2190 "message": "hi",
2191 "exception": {
2192 "values": [
2193 {
2194 "type": "BrokenException",
2195 "value": "Something failed",
2196 "stacktrace": {
2197 "frames": [
2198 {
2199 "vars": {
2200 "headers": [
2201 ["authorization", "Bearer abc123"]
2202 ]
2203 }
2204 }
2205 ]
2206 }
2207 }
2208 ]
2209 }
2210 })
2211 .into(),
2212 );
2213
2214 for config in configs {
2215 let config = serde_json::from_str::<PiiConfig>(config).unwrap();
2216 let mut processor = PiiProcessor::new(config.compiled());
2217 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2218
2219 let vars = get_value!(event.exceptions.values[0].stacktrace.frames[0].vars).unwrap();
2220
2221 allow_duplicates!(assert_debug_snapshot!(vars, @r###"
2222 FrameVars(
2223 {
2224 "headers": Array(
2225 [
2226 Array(
2227 [
2228 String(
2229 "authorization",
2230 ),
2231 Annotated(
2232 String(
2233 "[Filtered]",
2234 ),
2235 Meta {
2236 remarks: [
2237 Remark {
2238 ty: Substituted,
2239 rule_id: "@anything:replace",
2240 range: Some(
2241 (
2242 0,
2243 10,
2244 ),
2245 ),
2246 },
2247 ],
2248 errors: [],
2249 original_length: Some(
2250 13,
2251 ),
2252 original_value: None,
2253 },
2254 ),
2255 ],
2256 ),
2257 ],
2258 ),
2259 },
2260 )
2261 "###));
2262 }
2263 }
2264
2265 #[test]
2266 fn test_tuple_array_scrubbed_with_string_selector_and_password_matcher() {
2267 let config = serde_json::from_str::<PiiConfig>(
2268 r##"
2269 {
2270 "applications": {
2271 "$string": ["@password:remove"]
2272 }
2273 }
2274 "##,
2275 )
2276 .unwrap();
2277
2278 let mut event = Event::from_value(
2279 serde_json::json!(
2280 {
2281 "message": "hi",
2282 "exception": {
2283 "values": [
2284 {
2285 "type": "BrokenException",
2286 "value": "Something failed",
2287 "stacktrace": {
2288 "frames": [
2289 {
2290 "vars": {
2291 "headers": [
2292 ["authorization", "abc123"]
2293 ]
2294 }
2295 }
2296 ]
2297 }
2298 }
2299 ]
2300 }
2301 })
2302 .into(),
2303 );
2304
2305 let mut processor = PiiProcessor::new(config.compiled());
2306 process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
2307
2308 let vars = get_value!(event.exceptions.values[0].stacktrace.frames[0].vars).unwrap();
2309
2310 assert_debug_snapshot!(vars, @r###"
2311 FrameVars(
2312 {
2313 "headers": Array(
2314 [
2315 Array(
2316 [
2317 String(
2318 "authorization",
2319 ),
2320 Meta {
2321 remarks: [
2322 Remark {
2323 ty: Removed,
2324 rule_id: "@password:remove",
2325 range: None,
2326 },
2327 ],
2328 errors: [],
2329 original_length: None,
2330 original_value: None,
2331 },
2332 ],
2333 ),
2334 ],
2335 ),
2336 },
2337 )
2338 "###);
2339 }
2340}