Skip to main content

relay_pii/
eap.rs

1//! Provides the `scrub` function for scrubbing EAP items.
2//!
3//! This module also contains tests for scrubbing attributes.
4
5use relay_event_schema::processor::{
6    ProcessValue, ProcessingResult, ProcessingState, ValueType, process_value,
7};
8use relay_protocol::Annotated;
9
10use crate::{AttributeMode, PiiConfig, PiiProcessor};
11
12/// Scrubs an EAP item (such as an `OurLog` or `SpanV2`) with the given PII configs.
13pub fn scrub<T: ProcessValue>(
14    value_type: ValueType,
15    item: &mut Annotated<T>,
16    advanced_rules: Option<&PiiConfig>,
17    legacy_rule: Option<&PiiConfig>,
18) -> ProcessingResult {
19    let state = ProcessingState::root().enter_borrowed("", None, [value_type]);
20
21    if let Some(config) = advanced_rules {
22        let mut processor = PiiProcessor::new(config.compiled())
23            // For advanced rules we want to treat attributes as objects.
24            .attribute_mode(AttributeMode::Object);
25        process_value(item, &mut processor, &state)?;
26    }
27
28    if let Some(config) = legacy_rule {
29        let mut processor = PiiProcessor::new(config.compiled())
30            // For "legacy" rules we want to identify attributes with their values.
31            .attribute_mode(AttributeMode::ValueOnly);
32        process_value(item, &mut processor, &state)?;
33    }
34
35    Ok(())
36}
37
38#[cfg(test)]
39mod tests {
40    use relay_event_schema::processor::ValueType;
41    use relay_event_schema::protocol::{Attributes, OurLog, SpanV2, TraceMetric};
42    use relay_protocol::{Annotated, SerializableAnnotated};
43
44    use crate::{DataScrubbingConfig, PiiConfig, eap};
45
46    #[test]
47    fn test_scrub_ai_attributes_with_default_rules() {
48        // AI attributes are marked as `manual` in `sentry-conventions` and are important to the
49        // product, so default rules only apply the replace-only rules from `REPLACE_ONLY_SELECTOR`.
50        // `gen_ai.system_instructions` contains a match for every replace-only rule.
51        let json = r#"{
52              "mcp.tool.result.content": {
53                  "type": "string",
54                  "value": "Contact user@example.com"
55              },
56              "gen_ai.tool.call.result": {
57                  "type": "string",
58                  "value": "{\"content\":[{\"type\":\"text\",\"text\":\"Contact user@example.com\"}]}"
59              },
60              "gen_ai.system_instructions": {
61                  "type": "string",
62                  "value": "[{\"type\":\"text\",\"content\":\"Auth with Bearer sk-1234567890abcdef and charge 4111-1111-1111-1111 or IBAN DE89370400440532013000, SSN 078-05-1120, escalate to user@example.com\"}]"
63              },
64              "ai.preamble": {
65                  "type": "string",
66                  "value": "Escalate to user@example.com"
67              },
68              "gen_ai.system.message": {
69                  "type": "string",
70                  "value": "Escalate to user@example.com"
71              }
72        }"#;
73
74        let mut span = Annotated::new(SpanV2 {
75            attributes: Annotated::<Attributes>::from_json(json).unwrap(),
76            ..Default::default()
77        });
78
79        let data_scrubbing_config = DataScrubbingConfig {
80            scrub_data: true,
81            scrub_defaults: true,
82            ..Default::default()
83        };
84        let scrubbing_config = data_scrubbing_config.pii_config();
85
86        eap::scrub(ValueType::Span, &mut span, None, scrubbing_config.as_ref()).unwrap();
87
88        insta::assert_json_snapshot!(SerializableAnnotated(&span.value().unwrap().attributes), @r###"
89        {
90          "ai.preamble": {
91            "type": "string",
92            "value": "Escalate to [email]"
93          },
94          "gen_ai.system.message": {
95            "type": "string",
96            "value": "Escalate to [email]"
97          },
98          "gen_ai.system_instructions": {
99            "type": "string",
100            "value": "[{\"type\":\"text\",\"content\":\"Auth with Bearer [token] and charge [creditcard] or IBAN [iban], SSN [us-ssn], escalate to [email]\"}]"
101          },
102          "gen_ai.tool.call.result": {
103            "type": "string",
104            "value": "{\"content\":[{\"type\":\"text\",\"text\":\"Contact [email]\"}]}"
105          },
106          "mcp.tool.result.content": {
107            "type": "string",
108            "value": "Contact [email]"
109          },
110          "_meta": {
111            "ai.preamble": {
112              "value": {
113                "": {
114                  "rem": [
115                    [
116                      "@email:replace",
117                      "s",
118                      12,
119                      19
120                    ]
121                  ],
122                  "len": 28
123                }
124              }
125            },
126            "gen_ai.system.message": {
127              "value": {
128                "": {
129                  "rem": [
130                    [
131                      "@email:replace",
132                      "s",
133                      12,
134                      19
135                    ]
136                  ],
137                  "len": 28
138                }
139              }
140            },
141            "gen_ai.system_instructions": {
142              "value": {
143                "": {
144                  "rem": [
145                    [
146                      "@bearer:replace",
147                      "s",
148                      37,
149                      51
150                    ],
151                    [
152                      "@creditcard:replace",
153                      "s",
154                      63,
155                      75
156                    ],
157                    [
158                      "@iban:replace",
159                      "s",
160                      84,
161                      90
162                    ],
163                    [
164                      "@usssn:replace",
165                      "s",
166                      96,
167                      104
168                    ],
169                    [
170                      "@email:replace",
171                      "s",
172                      118,
173                      125
174                    ]
175                  ],
176                  "len": 175
177                }
178              }
179            },
180            "gen_ai.tool.call.result": {
181              "value": {
182                "": {
183                  "rem": [
184                    [
185                      "@email:replace",
186                      "s",
187                      43,
188                      50
189                    ]
190                  ],
191                  "len": 63
192                }
193              }
194            },
195            "mcp.tool.result.content": {
196              "value": {
197                "": {
198                  "rem": [
199                    [
200                      "@email:replace",
201                      "s",
202                      8,
203                      15
204                    ]
205                  ],
206                  "len": 24
207                }
208              }
209            }
210          }
211        }
212        "###);
213    }
214
215    #[test]
216    fn test_scrub_mcp_results_with_only_advanced_rules() {
217        let mut attributes = Attributes::new();
218        attributes.insert("gen_ai.tool.call.result", "user@example.com");
219        attributes.insert("mcp.tool.result.content", "user@example.com");
220        let mut span = Annotated::new(SpanV2 {
221            attributes: Annotated::new(attributes),
222            ..Default::default()
223        });
224        let settings = DataScrubbingConfig {
225            scrub_data: true,
226            scrub_defaults: false,
227            ..Default::default()
228        };
229        let config = settings.pii_config();
230        let original = span.clone();
231
232        eap::scrub(ValueType::Span, &mut span, None, config.as_ref()).unwrap();
233        assert_eq!(span, original);
234
235        let advanced = serde_json::from_value::<PiiConfig>(serde_json::json!({
236            "applications": {
237                "attributes.'gen_ai.tool.call.result'.value || attributes.'mcp.tool.result.content'.value": [
238                    "@email:replace"
239                ]
240            }
241        }))
242        .unwrap();
243        eap::scrub(ValueType::Span, &mut span, Some(&advanced), config.as_ref()).unwrap();
244
245        let attributes = span.value().unwrap().attributes.value().unwrap();
246        for key in ["gen_ai.tool.call.result", "mcp.tool.result.content"] {
247            assert_eq!(
248                attributes.get_value(key).and_then(|value| value.as_str()),
249                Some("[email]"),
250                "{key}"
251            );
252        }
253    }
254
255    #[test]
256    fn test_scrub_attributes_pii_default_rules() {
257        // `sentry.description`, `user.name`, `sentry.release`, `url.domain`, and `url.path` are marked as follows in `sentry-conventions`:
258        // * `sentry.description`: `true`
259        // * `user.name`: `true`
260        // * `sentry.release`: `false`
261        // * `url.domain`: `maybe`
262        // * `url.path`: `true`
263        // Therefore, default rules should scrub the `true` attributes and leave the `false` and `maybe` attributes intact.
264        let json = r#"{
265              "sentry.description": {
266                  "type": "string",
267                  "value": "secret123"
268              },
269              "user.name": {
270                  "type": "string",
271                  "value": "secret123"
272              },
273              "sentry.release": {
274                  "type": "string",
275                  "value": "secret123"
276              },
277              "url.domain": {
278                  "type": "string",
279                  "value": "secret123"
280              },
281              "url.path": {
282                  "type": "string",
283                  "value": "secret123"
284              },
285              "password": {
286                  "type": "string",
287                  "value": "secret123"
288              },
289              "secret": {
290                  "type": "string",
291                  "value": "topsecret"
292              },
293              "api_key": {
294                  "type": "string",
295                  "value": "sk-1234567890abcdef"
296              },
297              "auth_token": {
298                  "type": "string",
299                  "value": "bearer_token_123"
300              },
301              "credit_card": {
302                  "type": "string",
303                  "value": "4571234567890112"
304              },
305              "visa_card": {
306                  "type": "string",
307                  "value": "4571 2345 6789 0112"
308              },
309              "bank_account": {
310                  "type": "string",
311                  "value": "DE89370400440532013000"
312              },
313              "iban_code": {
314                  "type": "string",
315                  "value": "NO9386011117945"
316              },
317              "authorization": {
318                  "type": "string",
319                  "value": "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"
320              },
321              "private_key": {
322                  "type": "string",
323                  "value": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKB\n-----END PRIVATE KEY-----"
324              },
325              "database_url": {
326                  "type": "string",
327                  "value": "https://username:password@example.com/db"
328              },
329              "file_path": {
330                  "type": "string",
331                  "value": "C:\\Users\\john\\Documents\\secret.txt"
332              },
333              "unix_path": {
334                  "type": "string",
335                  "value": "/home/alice/private/data.json"
336              },
337              "social_security": {
338                  "type": "string",
339                  "value": "078-05-1120"
340              },
341              "user_email": {
342                  "type": "string",
343                  "value": "john.doe@example.com"
344              },
345              "client_ip": {
346                  "type": "string",
347                  "value": "192.168.1.100"
348              },
349              "ipv6_addr": {
350                  "type": "string",
351                  "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
352              },
353              "mac_address": {
354                  "type": "string",
355                  "value": "4a:00:04:10:9b:50"
356              },
357              "session_id": {
358                  "type": "string",
359                  "value": "ceee0822-ed8f-4622-b2a3-789e73e75cd1"
360              },
361              "device_imei": {
362                  "type": "string",
363                  "value": "356938035643809"
364              },
365              "public_data": {
366                  "type": "string",
367                  "value": "public_data"
368              },
369              "very_sensitive_data": {
370                  "type": "string",
371                  "value": "very_sensitive_data"
372              }
373        }"#;
374
375        let mut data = Annotated::<Attributes>::from_json(json).unwrap();
376
377        let data_scrubbing_config = DataScrubbingConfig {
378            scrub_data: true,
379            scrub_defaults: true,
380            scrub_ip_addresses: true,
381            exclude_fields: vec!["public_data".to_owned()],
382            sensitive_fields: vec![
383                "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed.
384                "very_sensitive_data".to_owned(),
385            ],
386            ..Default::default()
387        };
388
389        let scrubbing_config = data_scrubbing_config.pii_config();
390
391        eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap();
392
393        insta::assert_json_snapshot!(SerializableAnnotated(&data));
394    }
395
396    #[test]
397    fn test_scrub_attributes_pii_custom_object_rules() {
398        // `sentry.description`, `user.name`, `sentry.release`, `url.domain`, and `url.path` are marked as follows in `sentry-conventions`:
399        // * `sentry.description`: `true`
400        // * `user.name`: `true`
401        // * `sentry.release`: `false`
402        // * `url.domain`: `maybe`
403        // * `url.path`: `true`
404        // Therefore, custom rules should scrub the `true` and `maybe` attributes addressed explicitly, but not the `false` attribute.
405        let json = r#"
406        {
407          "sentry.description": {
408              "type": "string",
409              "value": "secret123"
410          },
411          "user.name": {
412              "type": "string",
413              "value": "secret123"
414          },
415          "sentry.release": {
416              "type": "string",
417              "value": "secret123"
418          },
419          "url.path": {
420              "type": "string",
421              "value": "secret123"
422          },
423          "url.domain": {
424              "type": "string",
425              "value": "secret123"
426          },
427          "password": {
428              "type": "string",
429              "value": "default_scrubbing_rules_are_off"
430          },
431          "test_field_mac": {
432              "type": "string",
433              "value": "4a:00:04:10:9b:50"
434          },
435          "test_field_imei": {
436              "type": "string",
437              "value": "356938035643809"
438          },
439          "test_field_uuid": {
440              "type": "string",
441              "value": "123e4567-e89b-12d3-a456-426614174000"
442          },
443          "test_field_regex_passes": {
444              "type": "string",
445              "value": "wxyz"
446          },
447          "test_field_regex_fails": {
448              "type": "string",
449              "value": "abc"
450          }
451        }"#;
452
453        let mut data = Annotated::<Attributes>::from_json(json).unwrap();
454
455        let scrubbing_config = DataScrubbingConfig {
456            scrub_data: true,
457            scrub_defaults: false,
458            scrub_ip_addresses: false,
459            ..Default::default()
460        };
461
462        let scrubbing_config = scrubbing_config.pii_config();
463
464        let config = serde_json::from_value::<PiiConfig>(serde_json::json!(
465        {
466            "rules": {
467                "project:0": {
468                    "type": "mac",
469                    "redaction": {
470                        "method": "replace",
471                        "text": "ITS_GONE"
472                    }
473                },
474                "project:1": {
475                    "type": "imei",
476                    "redaction": {
477                        "method": "replace",
478                        "text": "ITS_GONE"
479                    }
480                },
481                "project:2": {
482                    "type": "uuid",
483                    "redaction": {
484                        "method": "replace",
485                        "text": "BYE"
486                    }
487                },
488                "project:3": {
489                    "type": "pattern",
490                    "pattern": "[w-z]+",
491                    "redaction": {
492                        "method": "replace",
493                        "text": "REGEXED"
494                    }
495                },
496                "project:4": {
497                    "type": "anything",
498                    "redaction": {
499                        "method": "replace",
500                        "text": "[USER NAME]"
501                    }
502                },
503                "project:5": {
504                    "type": "anything",
505                    "redaction": {
506                        "method": "replace",
507                        "text": "[RELEASE]"
508                    }
509                },
510                "project:6": {
511                    "type": "anything",
512                    "redaction": {
513                        "method": "replace",
514                        "text": "[URL PATH]"
515                    }
516                },
517                "project:7": {
518                    "type": "anything",
519                    "redaction": {
520                        "method": "replace",
521                        "text": "[DESCRIPTION]"
522                    }
523                },
524                "project:8": {
525                    "type": "anything",
526                    "redaction": {
527                        "method": "replace",
528                        "text": "[URL DOMAIN]"
529                    }
530                }
531            },
532            "applications": {
533                "test_field_mac.value": [
534                    "project:0"
535                ],
536                "test_field_imei.value": [
537                    "project:1"
538                ],
539                "test_field_uuid.value": [
540                    "project:2"
541                ],
542                "test_field_regex_passes.value || test_field_regex_fails.value": [
543                    "project:3"
544                ],
545                "'user.name'.value": [
546                    "project:4"
547                ],
548                "'sentry.release'.value": [
549                    "project:5"
550                ],
551                "'url.path'.value": [
552                    "project:6"
553                ],
554                "'sentry.description'.value": [
555                    "project:7"
556                ],
557                "'url.domain'.value": [
558                    "project:8"
559                ]
560            }
561        }
562        ))
563        .unwrap();
564
565        eap::scrub(
566            ValueType::Span,
567            &mut data,
568            Some(&config),
569            scrubbing_config.as_ref(),
570        )
571        .unwrap();
572
573        insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###"
574        {
575          "password": {
576            "type": "string",
577            "value": "default_scrubbing_rules_are_off"
578          },
579          "sentry.description": {
580            "type": "string",
581            "value": "[DESCRIPTION]"
582          },
583          "sentry.release": {
584            "type": "string",
585            "value": "secret123"
586          },
587          "test_field_imei": {
588            "type": "string",
589            "value": "ITS_GONE"
590          },
591          "test_field_mac": {
592            "type": "string",
593            "value": "ITS_GONE"
594          },
595          "test_field_regex_fails": {
596            "type": "string",
597            "value": "abc"
598          },
599          "test_field_regex_passes": {
600            "type": "string",
601            "value": "REGEXED"
602          },
603          "test_field_uuid": {
604            "type": "string",
605            "value": "BYE"
606          },
607          "url.domain": {
608            "type": "string",
609            "value": "[URL DOMAIN]"
610          },
611          "url.path": {
612            "type": "string",
613            "value": "[URL PATH]"
614          },
615          "user.name": {
616            "type": "string",
617            "value": "[USER NAME]"
618          },
619          "_meta": {
620            "sentry.description": {
621              "value": {
622                "": {
623                  "rem": [
624                    [
625                      "project:7",
626                      "s",
627                      0,
628                      13
629                    ]
630                  ],
631                  "len": 9
632                }
633              }
634            },
635            "test_field_imei": {
636              "value": {
637                "": {
638                  "rem": [
639                    [
640                      "project:1",
641                      "s",
642                      0,
643                      8
644                    ]
645                  ],
646                  "len": 15
647                }
648              }
649            },
650            "test_field_mac": {
651              "value": {
652                "": {
653                  "rem": [
654                    [
655                      "project:0",
656                      "s",
657                      0,
658                      8
659                    ]
660                  ],
661                  "len": 17
662                }
663              }
664            },
665            "test_field_regex_passes": {
666              "value": {
667                "": {
668                  "rem": [
669                    [
670                      "project:3",
671                      "s",
672                      0,
673                      7
674                    ]
675                  ],
676                  "len": 4
677                }
678              }
679            },
680            "test_field_uuid": {
681              "value": {
682                "": {
683                  "rem": [
684                    [
685                      "project:2",
686                      "s",
687                      0,
688                      3
689                    ]
690                  ],
691                  "len": 36
692                }
693              }
694            },
695            "url.domain": {
696              "value": {
697                "": {
698                  "rem": [
699                    [
700                      "project:8",
701                      "s",
702                      0,
703                      12
704                    ]
705                  ],
706                  "len": 9
707                }
708              }
709            },
710            "url.path": {
711              "value": {
712                "": {
713                  "rem": [
714                    [
715                      "project:6",
716                      "s",
717                      0,
718                      10
719                    ]
720                  ],
721                  "len": 9
722                }
723              }
724            },
725            "user.name": {
726              "value": {
727                "": {
728                  "rem": [
729                    [
730                      "project:4",
731                      "s",
732                      0,
733                      11
734                    ]
735                  ],
736                  "len": 9
737                }
738              }
739            }
740          }
741        }
742        "###);
743    }
744
745    #[test]
746    fn test_scrub_attributes_sensitive_fields() {
747        let json = r#"
748        {
749            "normal_field": {
750                "type": "string",
751                "value": "normal_data"
752            },
753            "sensitive_custom": {
754                "type": "string",
755                "value": "should_be_removed"
756            },
757            "another_sensitive": {
758                "type": "integer",
759                "value": 42
760            },
761            "my_value": {
762                "type": "string",
763                "value": "this_should_be_removed_as_sensitive"
764            }
765        }
766        "#;
767
768        let mut data = Annotated::<Attributes>::from_json(json).unwrap();
769
770        let scrubbing_config = DataScrubbingConfig {
771            scrub_data: true,
772            scrub_defaults: false,
773            scrub_ip_addresses: false,
774            sensitive_fields: vec![
775                "value".to_owned(), // Make sure the inner 'value' of the attribute object isn't scrubbed.
776                "sensitive_custom".to_owned(),
777                "another_sensitive".to_owned(),
778            ],
779            ..Default::default()
780        };
781
782        let scrubbing_config = scrubbing_config.pii_config();
783
784        eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap();
785
786        insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###"
787        {
788          "another_sensitive": {
789            "type": "integer",
790            "value": null
791          },
792          "my_value": {
793            "type": "string",
794            "value": "[Filtered]"
795          },
796          "normal_field": {
797            "type": "string",
798            "value": "normal_data"
799          },
800          "sensitive_custom": {
801            "type": "string",
802            "value": "[Filtered]"
803          },
804          "_meta": {
805            "another_sensitive": {
806              "value": {
807                "": {
808                  "rem": [
809                    [
810                      "strip-fields",
811                      "x"
812                    ]
813                  ]
814                }
815              }
816            },
817            "my_value": {
818              "value": {
819                "": {
820                  "rem": [
821                    [
822                      "strip-fields",
823                      "s",
824                      0,
825                      10
826                    ]
827                  ],
828                  "len": 35
829                }
830              }
831            },
832            "sensitive_custom": {
833              "value": {
834                "": {
835                  "rem": [
836                    [
837                      "strip-fields",
838                      "s",
839                      0,
840                      10
841                    ]
842                  ],
843                  "len": 17
844                }
845              }
846            }
847          }
848        }
849        "###);
850    }
851
852    #[test]
853    fn test_scrub_attributes_safe_fields() {
854        let json = r#"
855        {
856            "password": {
857                "type": "string",
858                "value": "secret123"
859            },
860            "credit_card": {
861                "type": "string",
862                "value": "4242424242424242"
863            },
864            "secret": {
865                "type": "string",
866                "value": "this_should_stay"
867            }
868        }
869        "#;
870
871        let mut data = Annotated::<Attributes>::from_json(json).unwrap();
872
873        let scrubbing_config = DataScrubbingConfig {
874            scrub_data: true,
875            scrub_defaults: true,
876            scrub_ip_addresses: false,
877            exclude_fields: vec!["secret".to_owned()], // Only 'secret' is safe
878            ..Default::default()
879        };
880
881        let scrubbing_config = scrubbing_config.pii_config();
882
883        eap::scrub(ValueType::Span, &mut data, None, scrubbing_config.as_ref()).unwrap();
884
885        insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###"
886        {
887          "credit_card": {
888            "type": "string",
889            "value": "[Filtered]"
890          },
891          "password": {
892            "type": "string",
893            "value": "[Filtered]"
894          },
895          "secret": {
896            "type": "string",
897            "value": "this_should_stay"
898          },
899          "_meta": {
900            "credit_card": {
901              "value": {
902                "": {
903                  "rem": [
904                    [
905                      "@creditcard:filter",
906                      "s",
907                      0,
908                      10
909                    ]
910                  ],
911                  "len": 16
912                }
913              }
914            },
915            "password": {
916              "value": {
917                "": {
918                  "rem": [
919                    [
920                      "@password:filter",
921                      "s",
922                      0,
923                      10
924                    ]
925                  ],
926                  "len": 9
927                }
928              }
929            }
930          }
931        }
932        "###);
933    }
934
935    #[test]
936    fn test_scrub_attributes_implicit_attribute_value() {
937        let json = r#"
938        {
939            "remove_this_string_abc123": {
940                "type": "string",
941                "value": "abc123"
942            }
943        }
944        "#;
945
946        let mut data = Annotated::<Attributes>::from_json(json).unwrap();
947
948        let config = serde_json::from_value::<PiiConfig>(serde_json::json!({
949            "rules": {
950                "remove_abc123": {
951                    "type": "pattern",
952                    "pattern": "abc123",
953                    "redaction": {
954                        "method": "replace",
955                        "text": "abc---"
956                    }
957                },
958            },
959            "applications": {
960                "remove_this_string_abc123": ["remove_abc123"], // This selector should NOT match
961            }
962        }))
963        .unwrap();
964
965        let scrubbing_config = DataScrubbingConfig {
966            scrub_data: true,
967            scrub_defaults: true,
968            ..Default::default()
969        };
970
971        let scrubbing_config = scrubbing_config.pii_config();
972
973        eap::scrub(
974            ValueType::OurLog,
975            &mut data,
976            Some(&config),
977            scrubbing_config.as_ref(),
978        )
979        .unwrap();
980
981        insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###"
982        {
983          "remove_this_string_abc123": {
984            "type": "string",
985            "value": "abc123"
986          }
987        }
988        "###);
989
990        let config = serde_json::from_value::<PiiConfig>(serde_json::json!({
991            "rules": {
992                "remove_abc123": {
993                    "type": "pattern",
994                    "pattern": "abc123",
995                    "redaction": {
996                        "method": "replace",
997                        "text": "abc---"
998                    }
999                },
1000            },
1001            "applications": {
1002                "remove_this_string_abc123.value": ["remove_abc123"],
1003            }
1004        }))
1005        .unwrap();
1006
1007        eap::scrub(
1008            ValueType::OurLog,
1009            &mut data,
1010            Some(&config),
1011            scrubbing_config.as_ref(),
1012        )
1013        .unwrap();
1014
1015        insta::assert_json_snapshot!(SerializableAnnotated(&data), @r###"
1016        {
1017          "remove_this_string_abc123": {
1018            "type": "string",
1019            "value": "abc---"
1020          },
1021          "_meta": {
1022            "remove_this_string_abc123": {
1023              "value": {
1024                "": {
1025                  "rem": [
1026                    [
1027                      "remove_abc123",
1028                      "s",
1029                      0,
1030                      6
1031                    ]
1032                  ],
1033                  "len": 6
1034                }
1035              }
1036            }
1037          }
1038        }
1039        "###);
1040    }
1041
1042    macro_rules! attribute_rule_test {
1043        ($test_name:ident, $rule_type:expr, $test_value:expr, @$snapshot:literal) => {
1044            #[test]
1045            fn $test_name() {
1046                let config = serde_json::from_value::<PiiConfig>(serde_json::json!({
1047                    "applications": {
1048                        "$string": [$rule_type]
1049                    }
1050                }))
1051                .unwrap();
1052
1053                let mut scrubbing_config = crate::DataScrubbingConfig::default();
1054                scrubbing_config.scrub_data = true;
1055                scrubbing_config.scrub_defaults = false;
1056                scrubbing_config.scrub_ip_addresses = false;
1057
1058                let scrubbing_config = scrubbing_config.pii_config();
1059
1060                let span_json = format!(r#"{{
1061                    "start_timestamp": 1544719859.0,
1062                    "end_timestamp": 1544719860.0,
1063                    "trace_id": "5b8efff798038103d269b633813fc60c",
1064                    "span_id": "eee19b7ec3c1b174",
1065                    "name": "test",
1066                    "attributes": {{
1067                        "{rule_type}|{value}": {{
1068                            "type": "string",
1069                            "value": "{value}"
1070                        }}
1071                    }}
1072                }}"#,
1073                rule_type = $rule_type,
1074                value = $test_value
1075                    .replace('\\', "\\\\")
1076                    .replace('"', "\\\"")
1077                    .replace('\n', "\\n")
1078                );
1079
1080                let mut span = Annotated::<SpanV2>::from_json(&span_json).unwrap();
1081
1082                eap::scrub(ValueType::Span, &mut span, Some(&config), scrubbing_config.as_ref()).unwrap();
1083
1084                insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&span.value().unwrap().attributes), @$snapshot););
1085
1086                let log_json = format!(r#"{{
1087                    "timestamp": 1544719860.0,
1088                    "trace_id": "5b8efff798038103d269b633813fc60c",
1089                    "span_id": "eee19b7ec3c1b174",
1090                    "level": "info",
1091                    "body": "Test log",
1092                    "attributes": {{
1093                        "{rule_type}|{value}": {{
1094                            "type": "string",
1095                            "value": "{value}"
1096                        }}
1097                    }}
1098                }}"#,
1099                rule_type = $rule_type,
1100                value = $test_value
1101                    .replace('\\', "\\\\")
1102                    .replace('"', "\\\"")
1103                    .replace('\n', "\\n")
1104                );
1105
1106                let mut log = Annotated::<OurLog>::from_json(&log_json).unwrap();
1107
1108                eap::scrub(ValueType::OurLog, &mut log, Some(&config), scrubbing_config.as_ref()).unwrap();
1109
1110                insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&log.value().unwrap().attributes), @$snapshot););
1111
1112                let metric_json = format!(r#"{{
1113                    "timestamp": 1544719860.0,
1114                    "trace_id": "5b8efff798038103d269b633813fc60c",
1115                    "name": "test.metric",
1116                    "type": "counter",
1117                    "value": 1.0,
1118                    "attributes": {{
1119                        "{rule_type}|{value}": {{
1120                            "type": "string",
1121                            "value": "{value}"
1122                        }}
1123                    }}
1124                }}"#,
1125                rule_type = $rule_type,
1126                value = $test_value
1127                    .replace('\\', "\\\\")
1128                    .replace('"', "\\\"")
1129                    .replace('\n', "\\n")
1130                );
1131
1132                let mut metric = Annotated::<TraceMetric>::from_json(&metric_json).unwrap();
1133
1134                eap::scrub(ValueType::TraceMetric, &mut metric, Some(&config), scrubbing_config.as_ref()).unwrap();
1135
1136                insta::allow_duplicates!(insta::assert_json_snapshot!(SerializableAnnotated(&metric.value().unwrap().attributes), @$snapshot););
1137            }
1138        };
1139    }
1140
1141    // IP rules
1142    attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip, "@ip", "127.0.0.1", @r###"
1143    {
1144      "@ip|127.0.0.1": {
1145        "type": "string",
1146        "value": "[ip]"
1147      },
1148      "_meta": {
1149        "@ip|127.0.0.1": {
1150          "value": {
1151            "": {
1152              "rem": [
1153                [
1154                  "@ip",
1155                  "s",
1156                  0,
1157                  4
1158                ]
1159              ],
1160              "len": 9
1161            }
1162          }
1163        }
1164      }
1165    }
1166    "###);
1167
1168    attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_replace, "@ip:replace", "127.0.0.1", @r###"
1169    {
1170      "@ip:replace|127.0.0.1": {
1171        "type": "string",
1172        "value": "[ip]"
1173      },
1174      "_meta": {
1175        "@ip:replace|127.0.0.1": {
1176          "value": {
1177            "": {
1178              "rem": [
1179                [
1180                  "@ip:replace",
1181                  "s",
1182                  0,
1183                  4
1184                ]
1185              ],
1186              "len": 9
1187            }
1188          }
1189        }
1190      }
1191    }
1192    "###);
1193
1194    attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_remove, "@ip:remove", "127.0.0.1", @r###"
1195    {
1196      "@ip:remove|127.0.0.1": {
1197        "type": "string",
1198        "value": ""
1199      },
1200      "_meta": {
1201        "@ip:remove|127.0.0.1": {
1202          "value": {
1203            "": {
1204              "rem": [
1205                [
1206                  "@ip:remove",
1207                  "x",
1208                  0,
1209                  0
1210                ]
1211              ],
1212              "len": 9
1213            }
1214          }
1215        }
1216      }
1217    }
1218    "###);
1219
1220    attribute_rule_test!(test_scrub_attributes_pii_string_rules_ip_mask, "@ip:mask", "127.0.0.1", @r###"
1221    {
1222      "@ip:mask|127.0.0.1": {
1223        "type": "string",
1224        "value": "*********"
1225      },
1226      "_meta": {
1227        "@ip:mask|127.0.0.1": {
1228          "value": {
1229            "": {
1230              "rem": [
1231                [
1232                  "@ip:mask",
1233                  "m",
1234                  0,
1235                  9
1236                ]
1237              ],
1238              "len": 9
1239            }
1240          }
1241        }
1242      }
1243    }
1244    "###);
1245
1246    // Email rules
1247    attribute_rule_test!(test_scrub_attributes_pii_string_rules_email, "@email", "test@example.com", @r###"
1248    {
1249      "@email|test@example.com": {
1250        "type": "string",
1251        "value": "[email]"
1252      },
1253      "_meta": {
1254        "@email|test@example.com": {
1255          "value": {
1256            "": {
1257              "rem": [
1258                [
1259                  "@email",
1260                  "s",
1261                  0,
1262                  7
1263                ]
1264              ],
1265              "len": 16
1266            }
1267          }
1268        }
1269      }
1270    }
1271    "###);
1272
1273    attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_replace, "@email:replace", "test@example.com", @r###"
1274    {
1275      "@email:replace|test@example.com": {
1276        "type": "string",
1277        "value": "[email]"
1278      },
1279      "_meta": {
1280        "@email:replace|test@example.com": {
1281          "value": {
1282            "": {
1283              "rem": [
1284                [
1285                  "@email:replace",
1286                  "s",
1287                  0,
1288                  7
1289                ]
1290              ],
1291              "len": 16
1292            }
1293          }
1294        }
1295      }
1296    }
1297    "###);
1298
1299    attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_remove, "@email:remove", "test@example.com", @r###"
1300    {
1301      "@email:remove|test@example.com": {
1302        "type": "string",
1303        "value": ""
1304      },
1305      "_meta": {
1306        "@email:remove|test@example.com": {
1307          "value": {
1308            "": {
1309              "rem": [
1310                [
1311                  "@email:remove",
1312                  "x",
1313                  0,
1314                  0
1315                ]
1316              ],
1317              "len": 16
1318            }
1319          }
1320        }
1321      }
1322    }
1323    "###);
1324
1325    attribute_rule_test!(test_scrub_attributes_pii_string_rules_email_mask, "@email:mask", "test@example.com", @r###"
1326    {
1327      "@email:mask|test@example.com": {
1328        "type": "string",
1329        "value": "****************"
1330      },
1331      "_meta": {
1332        "@email:mask|test@example.com": {
1333          "value": {
1334            "": {
1335              "rem": [
1336                [
1337                  "@email:mask",
1338                  "m",
1339                  0,
1340                  16
1341                ]
1342              ],
1343              "len": 16
1344            }
1345          }
1346        }
1347      }
1348    }
1349    "###);
1350
1351    // Credit card rules
1352    attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard, "@creditcard", "4242424242424242", @r###"
1353    {
1354      "@creditcard|4242424242424242": {
1355        "type": "string",
1356        "value": "[creditcard]"
1357      },
1358      "_meta": {
1359        "@creditcard|4242424242424242": {
1360          "value": {
1361            "": {
1362              "rem": [
1363                [
1364                  "@creditcard",
1365                  "s",
1366                  0,
1367                  12
1368                ]
1369              ],
1370              "len": 16
1371            }
1372          }
1373        }
1374      }
1375    }
1376    "###);
1377
1378    attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_replace, "@creditcard:replace", "4242424242424242", @r###"
1379    {
1380      "@creditcard:replace|4242424242424242": {
1381        "type": "string",
1382        "value": "[creditcard]"
1383      },
1384      "_meta": {
1385        "@creditcard:replace|4242424242424242": {
1386          "value": {
1387            "": {
1388              "rem": [
1389                [
1390                  "@creditcard:replace",
1391                  "s",
1392                  0,
1393                  12
1394                ]
1395              ],
1396              "len": 16
1397            }
1398          }
1399        }
1400      }
1401    }
1402    "###);
1403
1404    attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_remove, "@creditcard:remove", "4242424242424242", @r###"
1405    {
1406      "@creditcard:remove|4242424242424242": {
1407        "type": "string",
1408        "value": ""
1409      },
1410      "_meta": {
1411        "@creditcard:remove|4242424242424242": {
1412          "value": {
1413            "": {
1414              "rem": [
1415                [
1416                  "@creditcard:remove",
1417                  "x",
1418                  0,
1419                  0
1420                ]
1421              ],
1422              "len": 16
1423            }
1424          }
1425        }
1426      }
1427    }
1428    "###);
1429
1430    attribute_rule_test!(test_scrub_attributes_pii_string_rules_creditcard_mask, "@creditcard:mask", "4242424242424242", @r###"
1431    {
1432      "@creditcard:mask|4242424242424242": {
1433        "type": "string",
1434        "value": "****************"
1435      },
1436      "_meta": {
1437        "@creditcard:mask|4242424242424242": {
1438          "value": {
1439            "": {
1440              "rem": [
1441                [
1442                  "@creditcard:mask",
1443                  "m",
1444                  0,
1445                  16
1446                ]
1447              ],
1448              "len": 16
1449            }
1450          }
1451        }
1452      }
1453    }
1454    "###);
1455
1456    // IBAN rules
1457    attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban, "@iban", "DE89370400440532013000", @r###"
1458    {
1459      "@iban|DE89370400440532013000": {
1460        "type": "string",
1461        "value": "[iban]"
1462      },
1463      "_meta": {
1464        "@iban|DE89370400440532013000": {
1465          "value": {
1466            "": {
1467              "rem": [
1468                [
1469                  "@iban",
1470                  "s",
1471                  0,
1472                  6
1473                ]
1474              ],
1475              "len": 22
1476            }
1477          }
1478        }
1479      }
1480    }
1481    "###);
1482
1483    attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_replace, "@iban:replace", "DE89370400440532013000", @r###"
1484    {
1485      "@iban:replace|DE89370400440532013000": {
1486        "type": "string",
1487        "value": "[iban]"
1488      },
1489      "_meta": {
1490        "@iban:replace|DE89370400440532013000": {
1491          "value": {
1492            "": {
1493              "rem": [
1494                [
1495                  "@iban:replace",
1496                  "s",
1497                  0,
1498                  6
1499                ]
1500              ],
1501              "len": 22
1502            }
1503          }
1504        }
1505      }
1506    }
1507    "###);
1508
1509    attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_remove, "@iban:remove", "DE89370400440532013000", @r###"
1510    {
1511      "@iban:remove|DE89370400440532013000": {
1512        "type": "string",
1513        "value": ""
1514      },
1515      "_meta": {
1516        "@iban:remove|DE89370400440532013000": {
1517          "value": {
1518            "": {
1519              "rem": [
1520                [
1521                  "@iban:remove",
1522                  "x",
1523                  0,
1524                  0
1525                ]
1526              ],
1527              "len": 22
1528            }
1529          }
1530        }
1531      }
1532    }
1533    "###);
1534
1535    attribute_rule_test!(test_scrub_attributes_pii_string_rules_iban_mask, "@iban:mask", "DE89370400440532013000", @r###"
1536    {
1537      "@iban:mask|DE89370400440532013000": {
1538        "type": "string",
1539        "value": "**********************"
1540      },
1541      "_meta": {
1542        "@iban:mask|DE89370400440532013000": {
1543          "value": {
1544            "": {
1545              "rem": [
1546                [
1547                  "@iban:mask",
1548                  "m",
1549                  0,
1550                  22
1551                ]
1552              ],
1553              "len": 22
1554            }
1555          }
1556        }
1557      }
1558    }
1559    "###);
1560
1561    // MAC address rules
1562    attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac, "@mac", "4a:00:04:10:9b:50", @r###"
1563    {
1564      "@mac|4a:00:04:10:9b:50": {
1565        "type": "string",
1566        "value": "*****************"
1567      },
1568      "_meta": {
1569        "@mac|4a:00:04:10:9b:50": {
1570          "value": {
1571            "": {
1572              "rem": [
1573                [
1574                  "@mac",
1575                  "m",
1576                  0,
1577                  17
1578                ]
1579              ],
1580              "len": 17
1581            }
1582          }
1583        }
1584      }
1585    }
1586    "###);
1587
1588    attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_replace, "@mac:replace", "4a:00:04:10:9b:50", @r###"
1589    {
1590      "@mac:replace|4a:00:04:10:9b:50": {
1591        "type": "string",
1592        "value": "[mac]"
1593      },
1594      "_meta": {
1595        "@mac:replace|4a:00:04:10:9b:50": {
1596          "value": {
1597            "": {
1598              "rem": [
1599                [
1600                  "@mac:replace",
1601                  "s",
1602                  0,
1603                  5
1604                ]
1605              ],
1606              "len": 17
1607            }
1608          }
1609        }
1610      }
1611    }
1612    "###);
1613
1614    attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_remove, "@mac:remove", "4a:00:04:10:9b:50", @r###"
1615    {
1616      "@mac:remove|4a:00:04:10:9b:50": {
1617        "type": "string",
1618        "value": ""
1619      },
1620      "_meta": {
1621        "@mac:remove|4a:00:04:10:9b:50": {
1622          "value": {
1623            "": {
1624              "rem": [
1625                [
1626                  "@mac:remove",
1627                  "x",
1628                  0,
1629                  0
1630                ]
1631              ],
1632              "len": 17
1633            }
1634          }
1635        }
1636      }
1637    }
1638    "###);
1639
1640    attribute_rule_test!(test_scrub_attributes_pii_string_rules_mac_mask, "@mac:mask", "4a:00:04:10:9b:50", @r###"
1641    {
1642      "@mac:mask|4a:00:04:10:9b:50": {
1643        "type": "string",
1644        "value": "*****************"
1645      },
1646      "_meta": {
1647        "@mac:mask|4a:00:04:10:9b:50": {
1648          "value": {
1649            "": {
1650              "rem": [
1651                [
1652                  "@mac:mask",
1653                  "m",
1654                  0,
1655                  17
1656                ]
1657              ],
1658              "len": 17
1659            }
1660          }
1661        }
1662      }
1663    }
1664    "###);
1665
1666    // UUID rules
1667    attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid, "@uuid", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###"
1668    {
1669      "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1670        "type": "string",
1671        "value": "************************************"
1672      },
1673      "_meta": {
1674        "@uuid|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1675          "value": {
1676            "": {
1677              "rem": [
1678                [
1679                  "@uuid",
1680                  "m",
1681                  0,
1682                  36
1683                ]
1684              ],
1685              "len": 36
1686            }
1687          }
1688        }
1689      }
1690    }
1691    "###);
1692
1693    attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_replace, "@uuid:replace", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###"
1694    {
1695      "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1696        "type": "string",
1697        "value": "[uuid]"
1698      },
1699      "_meta": {
1700        "@uuid:replace|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1701          "value": {
1702            "": {
1703              "rem": [
1704                [
1705                  "@uuid:replace",
1706                  "s",
1707                  0,
1708                  6
1709                ]
1710              ],
1711              "len": 36
1712            }
1713          }
1714        }
1715      }
1716    }
1717    "###);
1718
1719    attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_remove, "@uuid:remove", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###"
1720    {
1721      "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1722        "type": "string",
1723        "value": ""
1724      },
1725      "_meta": {
1726        "@uuid:remove|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1727          "value": {
1728            "": {
1729              "rem": [
1730                [
1731                  "@uuid:remove",
1732                  "x",
1733                  0,
1734                  0
1735                ]
1736              ],
1737              "len": 36
1738            }
1739          }
1740        }
1741      }
1742    }
1743    "###);
1744
1745    attribute_rule_test!(test_scrub_attributes_pii_string_rules_uuid_mask, "@uuid:mask", "ceee0822-ed8f-4622-b2a3-789e73e75cd1", @r###"
1746    {
1747      "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1748        "type": "string",
1749        "value": "************************************"
1750      },
1751      "_meta": {
1752        "@uuid:mask|ceee0822-ed8f-4622-b2a3-789e73e75cd1": {
1753          "value": {
1754            "": {
1755              "rem": [
1756                [
1757                  "@uuid:mask",
1758                  "m",
1759                  0,
1760                  36
1761                ]
1762              ],
1763              "len": 36
1764            }
1765          }
1766        }
1767      }
1768    }
1769    "###);
1770
1771    // IMEI rules
1772    attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei, "@imei", "356938035643809", @r###"
1773    {
1774      "@imei|356938035643809": {
1775        "type": "string",
1776        "value": "[imei]"
1777      },
1778      "_meta": {
1779        "@imei|356938035643809": {
1780          "value": {
1781            "": {
1782              "rem": [
1783                [
1784                  "@imei",
1785                  "s",
1786                  0,
1787                  6
1788                ]
1789              ],
1790              "len": 15
1791            }
1792          }
1793        }
1794      }
1795    }
1796    "###);
1797
1798    attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_replace, "@imei:replace", "356938035643809", @r###"
1799    {
1800      "@imei:replace|356938035643809": {
1801        "type": "string",
1802        "value": "[imei]"
1803      },
1804      "_meta": {
1805        "@imei:replace|356938035643809": {
1806          "value": {
1807            "": {
1808              "rem": [
1809                [
1810                  "@imei:replace",
1811                  "s",
1812                  0,
1813                  6
1814                ]
1815              ],
1816              "len": 15
1817            }
1818          }
1819        }
1820      }
1821    }
1822    "###);
1823
1824    attribute_rule_test!(test_scrub_attributes_pii_string_rules_imei_remove, "@imei:remove", "356938035643809", @r###"
1825    {
1826      "@imei:remove|356938035643809": {
1827        "type": "string",
1828        "value": ""
1829      },
1830      "_meta": {
1831        "@imei:remove|356938035643809": {
1832          "value": {
1833            "": {
1834              "rem": [
1835                [
1836                  "@imei:remove",
1837                  "x",
1838                  0,
1839                  0
1840                ]
1841              ],
1842              "len": 15
1843            }
1844          }
1845        }
1846      }
1847    }
1848    "###);
1849
1850    // PEM key rules
1851    attribute_rule_test!(
1852        test_scrub_attributes_pii_string_rules_pemkey,
1853        "@pemkey",
1854        "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----",
1855    @r###"
1856    {
1857      "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1858        "type": "string",
1859        "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----"
1860      },
1861      "_meta": {
1862        "@pemkey|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1863          "value": {
1864            "": {
1865              "rem": [
1866                [
1867                  "@pemkey",
1868                  "s",
1869                  31,
1870                  39
1871                ]
1872              ],
1873              "len": 124
1874            }
1875          }
1876        }
1877      }
1878    }
1879    "###);
1880
1881    attribute_rule_test!(
1882        test_scrub_attributes_pii_string_rules_pemkey_replace,
1883        "@pemkey:replace",
1884        "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----",
1885    @r###"
1886    {
1887      "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1888        "type": "string",
1889        "value": "-----BEGIN EC PRIVATE KEY-----\n[pemkey]\n-----END EC PRIVATE KEY-----"
1890      },
1891      "_meta": {
1892        "@pemkey:replace|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1893          "value": {
1894            "": {
1895              "rem": [
1896                [
1897                  "@pemkey:replace",
1898                  "s",
1899                  31,
1900                  39
1901                ]
1902              ],
1903              "len": 124
1904            }
1905          }
1906        }
1907      }
1908    }
1909    "###);
1910
1911    attribute_rule_test!(
1912        test_scrub_attributes_pii_string_rules_pemkey_remove,
1913        "@pemkey:remove",
1914        "-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----",
1915    @r###"
1916    {
1917      "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1918        "type": "string",
1919        "value": "-----BEGIN EC PRIVATE KEY-----\n\n-----END EC PRIVATE KEY-----"
1920      },
1921      "_meta": {
1922        "@pemkey:remove|-----BEGIN EC PRIVATE KEY-----\nMIHbAgEBBEFbLvIaAaez3q0u6BQYMHZ28B7iSdMPPaODUMGkdorl3ShgTbYmzqGL\n-----END EC PRIVATE KEY-----": {
1923          "value": {
1924            "": {
1925              "rem": [
1926                [
1927                  "@pemkey:remove",
1928                  "x",
1929                  31,
1930                  31
1931                ]
1932              ],
1933              "len": 124
1934            }
1935          }
1936        }
1937      }
1938    }
1939    "###);
1940
1941    // URL auth rules
1942    attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth, "@urlauth", "https://username:password@example.com/", @r###"
1943    {
1944      "@urlauth|https://username:password@example.com/": {
1945        "type": "string",
1946        "value": "https://[auth]@example.com/"
1947      },
1948      "_meta": {
1949        "@urlauth|https://username:password@example.com/": {
1950          "value": {
1951            "": {
1952              "rem": [
1953                [
1954                  "@urlauth",
1955                  "s",
1956                  8,
1957                  14
1958                ]
1959              ],
1960              "len": 38
1961            }
1962          }
1963        }
1964      }
1965    }
1966    "###);
1967
1968    attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_replace, "@urlauth:replace", "https://username:password@example.com/", @r###"
1969    {
1970      "@urlauth:replace|https://username:password@example.com/": {
1971        "type": "string",
1972        "value": "https://[auth]@example.com/"
1973      },
1974      "_meta": {
1975        "@urlauth:replace|https://username:password@example.com/": {
1976          "value": {
1977            "": {
1978              "rem": [
1979                [
1980                  "@urlauth:replace",
1981                  "s",
1982                  8,
1983                  14
1984                ]
1985              ],
1986              "len": 38
1987            }
1988          }
1989        }
1990      }
1991    }
1992    "###);
1993
1994    attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_remove, "@urlauth:remove", "https://username:password@example.com/", @r###"
1995    {
1996      "@urlauth:remove|https://username:password@example.com/": {
1997        "type": "string",
1998        "value": "https://@example.com/"
1999      },
2000      "_meta": {
2001        "@urlauth:remove|https://username:password@example.com/": {
2002          "value": {
2003            "": {
2004              "rem": [
2005                [
2006                  "@urlauth:remove",
2007                  "x",
2008                  8,
2009                  8
2010                ]
2011              ],
2012              "len": 38
2013            }
2014          }
2015        }
2016      }
2017    }
2018    "###);
2019
2020    attribute_rule_test!(test_scrub_attributes_pii_string_rules_urlauth_mask, "@urlauth:mask", "https://username:password@example.com/", @r###"
2021    {
2022      "@urlauth:mask|https://username:password@example.com/": {
2023        "type": "string",
2024        "value": "https://*****************@example.com/"
2025      },
2026      "_meta": {
2027        "@urlauth:mask|https://username:password@example.com/": {
2028          "value": {
2029            "": {
2030              "rem": [
2031                [
2032                  "@urlauth:mask",
2033                  "m",
2034                  8,
2035                  25
2036                ]
2037              ],
2038              "len": 38
2039            }
2040          }
2041        }
2042      }
2043    }
2044    "###);
2045
2046    // US SSN rules
2047    attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn, "@usssn", "078-05-1120", @r###"
2048    {
2049      "@usssn|078-05-1120": {
2050        "type": "string",
2051        "value": "***********"
2052      },
2053      "_meta": {
2054        "@usssn|078-05-1120": {
2055          "value": {
2056            "": {
2057              "rem": [
2058                [
2059                  "@usssn",
2060                  "m",
2061                  0,
2062                  11
2063                ]
2064              ],
2065              "len": 11
2066            }
2067          }
2068        }
2069      }
2070    }
2071    "###);
2072
2073    attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_replace, "@usssn:replace", "078-05-1120", @r###"
2074    {
2075      "@usssn:replace|078-05-1120": {
2076        "type": "string",
2077        "value": "[us-ssn]"
2078      },
2079      "_meta": {
2080        "@usssn:replace|078-05-1120": {
2081          "value": {
2082            "": {
2083              "rem": [
2084                [
2085                  "@usssn:replace",
2086                  "s",
2087                  0,
2088                  8
2089                ]
2090              ],
2091              "len": 11
2092            }
2093          }
2094        }
2095      }
2096    }
2097    "###);
2098
2099    attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_remove, "@usssn:remove", "078-05-1120", @r###"
2100    {
2101      "@usssn:remove|078-05-1120": {
2102        "type": "string",
2103        "value": ""
2104      },
2105      "_meta": {
2106        "@usssn:remove|078-05-1120": {
2107          "value": {
2108            "": {
2109              "rem": [
2110                [
2111                  "@usssn:remove",
2112                  "x",
2113                  0,
2114                  0
2115                ]
2116              ],
2117              "len": 11
2118            }
2119          }
2120        }
2121      }
2122    }
2123    "###);
2124
2125    attribute_rule_test!(test_scrub_attributes_pii_string_rules_usssn_mask, "@usssn:mask", "078-05-1120", @r###"
2126    {
2127      "@usssn:mask|078-05-1120": {
2128        "type": "string",
2129        "value": "***********"
2130      },
2131      "_meta": {
2132        "@usssn:mask|078-05-1120": {
2133          "value": {
2134            "": {
2135              "rem": [
2136                [
2137                  "@usssn:mask",
2138                  "m",
2139                  0,
2140                  11
2141                ]
2142              ],
2143              "len": 11
2144            }
2145          }
2146        }
2147      }
2148    }
2149    "###);
2150
2151    // User path rules
2152    attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath, "@userpath", "/Users/john/Documents", @r###"
2153    {
2154      "@userpath|/Users/john/Documents": {
2155        "type": "string",
2156        "value": "/Users/[user]/Documents"
2157      },
2158      "_meta": {
2159        "@userpath|/Users/john/Documents": {
2160          "value": {
2161            "": {
2162              "rem": [
2163                [
2164                  "@userpath",
2165                  "s",
2166                  7,
2167                  13
2168                ]
2169              ],
2170              "len": 21
2171            }
2172          }
2173        }
2174      }
2175    }
2176    "###);
2177
2178    attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_replace, "@userpath:replace", "/Users/john/Documents", @r###"
2179    {
2180      "@userpath:replace|/Users/john/Documents": {
2181        "type": "string",
2182        "value": "/Users/[user]/Documents"
2183      },
2184      "_meta": {
2185        "@userpath:replace|/Users/john/Documents": {
2186          "value": {
2187            "": {
2188              "rem": [
2189                [
2190                  "@userpath:replace",
2191                  "s",
2192                  7,
2193                  13
2194                ]
2195              ],
2196              "len": 21
2197            }
2198          }
2199        }
2200      }
2201    }
2202    "###);
2203
2204    attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_remove, "@userpath:remove", "/Users/john/Documents", @r###"
2205    {
2206      "@userpath:remove|/Users/john/Documents": {
2207        "type": "string",
2208        "value": "/Users//Documents"
2209      },
2210      "_meta": {
2211        "@userpath:remove|/Users/john/Documents": {
2212          "value": {
2213            "": {
2214              "rem": [
2215                [
2216                  "@userpath:remove",
2217                  "x",
2218                  7,
2219                  7
2220                ]
2221              ],
2222              "len": 21
2223            }
2224          }
2225        }
2226      }
2227    }
2228    "###);
2229
2230    attribute_rule_test!(test_scrub_attributes_pii_string_rules_userpath_mask, "@userpath:mask", "/Users/john/Documents", @r###"
2231    {
2232      "@userpath:mask|/Users/john/Documents": {
2233        "type": "string",
2234        "value": "/Users/****/Documents"
2235      },
2236      "_meta": {
2237        "@userpath:mask|/Users/john/Documents": {
2238          "value": {
2239            "": {
2240              "rem": [
2241                [
2242                  "@userpath:mask",
2243                  "m",
2244                  7,
2245                  11
2246                ]
2247              ],
2248              "len": 21
2249            }
2250          }
2251        }
2252      }
2253    }
2254    "###);
2255
2256    // Password rules
2257    // @password defaults to remove
2258    attribute_rule_test!(test_scrub_attributes_pii_string_rules_password, "@password", "my_password_123", @r###"
2259    {
2260      "@password|my_password_123": {
2261        "type": "string",
2262        "value": ""
2263      },
2264      "_meta": {
2265        "@password|my_password_123": {
2266          "value": {
2267            "": {
2268              "rem": [
2269                [
2270                  "@password",
2271                  "x",
2272                  0,
2273                  0
2274                ]
2275              ],
2276              "len": 15
2277            }
2278          }
2279        }
2280      }
2281    }
2282    "###);
2283
2284    attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_remove, "@password:remove", "my_password_123", @r###"
2285    {
2286      "@password:remove|my_password_123": {
2287        "type": "string",
2288        "value": ""
2289      },
2290      "_meta": {
2291        "@password:remove|my_password_123": {
2292          "value": {
2293            "": {
2294              "rem": [
2295                [
2296                  "@password:remove",
2297                  "x",
2298                  0,
2299                  0
2300                ]
2301              ],
2302              "len": 15
2303            }
2304          }
2305        }
2306      }
2307    }
2308    "###);
2309
2310    attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_replace, "@password:replace", "my_password_123", @r###"
2311    {
2312      "@password:replace|my_password_123": {
2313        "type": "string",
2314        "value": "[password]"
2315      },
2316      "_meta": {
2317        "@password:replace|my_password_123": {
2318          "value": {
2319            "": {
2320              "rem": [
2321                [
2322                  "@password:replace",
2323                  "s",
2324                  0,
2325                  10
2326                ]
2327              ],
2328              "len": 15
2329            }
2330          }
2331        }
2332      }
2333    }
2334    "###);
2335
2336    attribute_rule_test!(test_scrub_attributes_pii_string_rules_password_mask, "@password:mask", "my_password_123", @r###"
2337    {
2338      "@password:mask|my_password_123": {
2339        "type": "string",
2340        "value": "***************"
2341      },
2342      "_meta": {
2343        "@password:mask|my_password_123": {
2344          "value": {
2345            "": {
2346              "rem": [
2347                [
2348                  "@password:mask",
2349                  "m",
2350                  0,
2351                  15
2352                ]
2353              ],
2354              "len": 15
2355            }
2356          }
2357        }
2358      }
2359    }
2360    "###);
2361
2362    // Bearer token rules
2363    attribute_rule_test!(test_scrub_attributes_pii_string_rules_bearer, "@bearer", "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9", @r###"
2364    {
2365      "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2366        "type": "string",
2367        "value": "Bearer [token]"
2368      },
2369      "_meta": {
2370        "@bearer|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2371          "value": {
2372            "": {
2373              "rem": [
2374                [
2375                  "@bearer",
2376                  "s",
2377                  0,
2378                  14
2379                ]
2380              ],
2381              "len": 43
2382            }
2383          }
2384        }
2385      }
2386    }
2387    "###);
2388
2389    attribute_rule_test!(
2390        test_scrub_attributes_pii_string_rules_bearer_replace,
2391        "@bearer:replace",
2392        "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
2393    @r###"
2394    {
2395      "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2396        "type": "string",
2397        "value": "Bearer [token]"
2398      },
2399      "_meta": {
2400        "@bearer:replace|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2401          "value": {
2402            "": {
2403              "rem": [
2404                [
2405                  "@bearer:replace",
2406                  "s",
2407                  0,
2408                  14
2409                ]
2410              ],
2411              "len": 43
2412            }
2413          }
2414        }
2415      }
2416    }
2417    "###);
2418
2419    attribute_rule_test!(
2420        test_scrub_attributes_pii_string_rules_bearer_remove,
2421        "@bearer:remove",
2422        "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
2423    @r###"
2424    {
2425      "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2426        "type": "string",
2427        "value": ""
2428      },
2429      "_meta": {
2430        "@bearer:remove|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2431          "value": {
2432            "": {
2433              "rem": [
2434                [
2435                  "@bearer:remove",
2436                  "x",
2437                  0,
2438                  0
2439                ]
2440              ],
2441              "len": 43
2442            }
2443          }
2444        }
2445      }
2446    }
2447    "###);
2448
2449    attribute_rule_test!(
2450        test_scrub_attributes_pii_string_rules_bearer_mask,
2451        "@bearer:mask",
2452        "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9",
2453    @r###"
2454    {
2455      "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2456        "type": "string",
2457        "value": "*******************************************"
2458      },
2459      "_meta": {
2460        "@bearer:mask|Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9": {
2461          "value": {
2462            "": {
2463              "rem": [
2464                [
2465                  "@bearer:mask",
2466                  "m",
2467                  0,
2468                  43
2469                ]
2470              ],
2471              "len": 43
2472            }
2473          }
2474        }
2475      }
2476    }
2477    "###);
2478}