Skip to main content

relay_pii/
attachments.rs

1use regex::Match;
2use relay_event_schema::processor::{FieldAttrs, Pii, ProcessingState, ValueType};
3use smallvec::SmallVec;
4use std::borrow::Cow;
5use std::iter::FusedIterator;
6use utf16string::{LittleEndian, WStr};
7
8use crate::compiledconfig::RuleRef;
9use crate::regex::ScrubbingRegexRef;
10use crate::regexes::{ReplaceBehavior, get_regex_for_rule_type};
11use crate::{CompiledPiiConfig, JsonScrubError, JsonScrubVisitor, Redaction, transform, utils};
12
13/// The minimum length a string needs to be in a binary blob.
14///
15/// This module extracts encoded strings from within binary blobs, this specifies the
16/// minimum length we require those strings to be before we accept them to match scrubbing
17/// selectors on.
18const MIN_STRING_LEN: usize = 5;
19
20fn apply_regex_to_utf8_bytes(
21    data: &mut [u8],
22    rule: &RuleRef,
23    regex: ScrubbingRegexRef<'_>,
24    replace_behavior: &ReplaceBehavior,
25) -> SmallVec<[(usize, usize); 1]> {
26    let mut matches = SmallVec::<[(usize, usize); 1]>::new();
27
28    let regex = match regex.to_bytes() {
29        Ok(x) => x,
30        Err(e) => {
31            // XXX: This is not going to fly long-term
32            // Idea: Disable unicode support for regexes entirely, that drastically increases the
33            // likelihood this conversion will never fail.
34
35            // If we see this error in production, it means we need to add more regex validation
36            // to `validate_pii_config` (which is called sentry-side).
37            relay_log::error!(
38                error = &e as &dyn std::error::Error,
39                pattern = regex.as_str(),
40                "Regex failed to compile in non-unicode mode",
41            );
42            return matches;
43        }
44    };
45
46    for captures in regex.captures_iter(data) {
47        for (idx, group) in captures.iter().enumerate() {
48            if let Some(group) = group {
49                if group.start() == group.end() {
50                    continue;
51                }
52
53                match replace_behavior {
54                    ReplaceBehavior::Groups(replace_groups) => {
55                        if replace_groups.contains(&(idx as u8)) {
56                            matches.push((group.start(), group.end()));
57                        }
58                    }
59                    ReplaceBehavior::Value => {
60                        matches.push((0, data.len()));
61                        break;
62                    }
63                }
64            }
65        }
66    }
67
68    for (start, end) in matches.iter() {
69        data[*start..*end].apply_redaction(&rule.redaction);
70    }
71    matches
72}
73
74fn apply_regex_to_utf16le_bytes(
75    data: &mut [u8],
76    rule: &RuleRef,
77    regex: ScrubbingRegexRef<'_>,
78    replace_behavior: &ReplaceBehavior,
79) -> bool {
80    let mut changed = false;
81    for segment in WStrSegmentIter::new(data) {
82        match replace_behavior {
83            ReplaceBehavior::Value => {
84                let matches = regex
85                    .captures_iter(&segment.decoded)
86                    .filter_map(|captures| captures.get(0));
87                for re_match in matches {
88                    changed = true;
89                    let match_wstr = get_wstr_match(&segment.decoded, re_match, segment.encoded);
90                    match_wstr.apply_redaction(&rule.redaction);
91                }
92            }
93            ReplaceBehavior::Groups(replace_groups) => {
94                for captures in regex.captures_iter(&segment.decoded) {
95                    for group_idx in replace_groups.iter() {
96                        if let Some(re_match) = captures.get(*group_idx as usize) {
97                            changed = true;
98                            let match_wstr =
99                                get_wstr_match(&segment.decoded, re_match, segment.encoded);
100                            match_wstr.apply_redaction(&rule.redaction);
101                        }
102                    }
103                }
104            }
105        }
106    }
107    changed
108}
109
110/// Extract the matching encoded slice from the encoded string.
111fn get_wstr_match<'a>(
112    all_text: &str,
113    re_match: Match,
114    all_encoded: &'a mut WStr<LittleEndian>,
115) -> &'a mut WStr<LittleEndian> {
116    let mut encoded_start = 0;
117    let mut encoded_end = all_encoded.len();
118
119    let offsets_iter = all_text.char_indices().zip(all_encoded.char_indices());
120    for ((text_offset, _text_char), (encoded_offset, _encoded_char)) in offsets_iter {
121        if text_offset == re_match.start() {
122            encoded_start = encoded_offset;
123        }
124        if text_offset == re_match.end() {
125            encoded_end = encoded_offset;
126            break;
127        }
128    }
129    &mut all_encoded[encoded_start..encoded_end]
130}
131
132/// Traits to modify the strings in ways we need.
133trait StringMods: AsRef<[u8]> {
134    /// Replace this string's contents by repeating the given character into it.
135    ///
136    /// # Panics
137    ///
138    /// The `fill_char` has to encode to the smallest encoding unit, otherwise this will
139    /// panic.  Using an ASCII replacement character is usually safe in most encodings.
140    fn fill_content(&mut self, fill_char: char);
141
142    /// Replace this string's contents with the given replacement string.
143    ///
144    /// If the replacement string encodes to a shorter byte-slice than the current string
145    /// any remaining space will be filled with the padding character.
146    ///
147    /// If the replacement string encodes to a longer byte-slice than the current string the
148    /// replacement string is truncated.  If this does not align with a character boundary
149    /// in the replacement string it is further trucated to the previous character boundary
150    /// and the remainder is filled with the padding char.
151    ///
152    /// # Panics
153    ///
154    /// The `padding` character has to encode to the smallest encoding unit, otherwise this
155    /// will panic.  Using an ASCII padding character is usually safe in most encodings.
156    fn swap_content(&mut self, replacement: &str, padding: char);
157
158    /// Apply a PII scrubbing redaction to this string slice.
159    fn apply_redaction(&mut self, redaction: &Redaction) {
160        const PADDING: char = '*';
161        const MASK: char = '*';
162
163        match redaction {
164            Redaction::Default | Redaction::Remove => {
165                self.fill_content(PADDING);
166            }
167            Redaction::Mask => {
168                self.fill_content(MASK);
169            }
170            Redaction::Hash => {
171                let hashed = utils::hash_value(self.as_ref());
172                self.swap_content(&hashed, PADDING);
173            }
174            Redaction::Replace(replace) => {
175                self.swap_content(replace.text.as_str(), PADDING);
176            }
177            Redaction::Other => relay_log::debug!("Incoming redaction is not supported"),
178        }
179    }
180}
181
182impl StringMods for WStr<LittleEndian> {
183    fn fill_content(&mut self, fill_char: char) {
184        // If fill_char is too wide, fill_char.encode_utf16() will panic, fulfilling the
185        // trait's contract that we must panic if fill_char is too wide.
186        let mut buf = [0u16; 1];
187        let fill_u16 = fill_char.encode_utf16(&mut buf[..]);
188        let fill_buf = fill_u16[0].to_le_bytes();
189
190        unsafe {
191            let chunks = self
192                .as_bytes_mut()
193                .as_chunks_mut::<{ std::mem::size_of::<u16>() }>();
194            for chunk in chunks.0 {
195                chunk.copy_from_slice(&fill_buf);
196            }
197        }
198    }
199
200    fn swap_content(&mut self, replacement: &str, padding: char) {
201        // If the padding char is too wide, padding.encode_utf16() will panic, fulfilling
202        // the trait's contract that we must panic in this case.
203        let len = self.len();
204
205        let mut buf = [0u16; 1];
206        padding.encode_utf16(&mut buf[..]);
207        let fill_buf = buf[0].to_le_bytes();
208
209        let mut offset = 0;
210        for code in replacement.encode_utf16() {
211            let char_len = if 0xD800 & code == 0xD800 {
212                std::mem::size_of::<u16>() * 2 // leading surrogate
213            } else {
214                std::mem::size_of::<u16>()
215            };
216            if (len - offset) < char_len {
217                break; // Not enough space for this char
218            }
219            unsafe {
220                let target = &mut self.as_bytes_mut()[offset..offset + std::mem::size_of::<u16>()];
221                target.copy_from_slice(&code.to_le_bytes());
222            }
223            offset += std::mem::size_of::<u16>();
224        }
225
226        unsafe {
227            let remainder_bytes = &mut self.as_bytes_mut()[offset..];
228            let chunks = remainder_bytes.as_chunks_mut::<{ std::mem::size_of::<u16>() }>();
229            for chunk in chunks.0 {
230                chunk.copy_from_slice(&fill_buf);
231            }
232        }
233    }
234}
235
236impl StringMods for [u8] {
237    fn fill_content(&mut self, fill_char: char) {
238        // If fill_char is too wide, fill_char.encode_utf16() will panic, fulfilling the
239        // trait's contract that we must panic if fill_char is too wide.
240        let mut buf = [0u8; 1];
241        fill_char.encode_utf8(&mut buf[..]);
242        for byte in self {
243            *byte = buf[0];
244        }
245    }
246
247    fn swap_content(&mut self, replacement: &str, padding: char) {
248        // If the padding char is too wide, padding.encode_utf16() will panic, fulfilling
249        // the trait's contract that we must panic in this case.
250        let mut buf = [0u8; 1];
251        padding.encode_utf8(&mut buf[..]);
252
253        let cutoff = replacement.len().min(self.len());
254        let (left, right) = self.split_at_mut(cutoff);
255        left.copy_from_slice(&replacement.as_bytes()[..cutoff]);
256
257        for byte in right {
258            *byte = buf[0];
259        }
260    }
261}
262
263/// An iterator over segments of text in binary data.
264///
265/// This iterator will look for blocks of UTF-16 encoded text with little-endian byte order
266/// in a block of binary data and yield those slices as segments with both the decoded and
267/// encoded text.
268struct WStrSegmentIter<'a> {
269    data: &'a mut [u8],
270    offset: usize,
271}
272
273impl<'a> WStrSegmentIter<'a> {
274    fn new(data: &'a mut [u8]) -> Self {
275        Self { data, offset: 0 }
276    }
277}
278
279impl<'a> Iterator for WStrSegmentIter<'a> {
280    type Item = WStrSegment<'a>;
281
282    fn next(&mut self) -> Option<Self::Item> {
283        loop {
284            if self.offset >= self.data.len() {
285                return None;
286            }
287
288            let slice = match WStr::from_utf16le_mut(&mut self.data[self.offset..]) {
289                Ok(wstr) => {
290                    self.offset += wstr.len();
291                    unsafe { wstr.as_bytes_mut() }
292                }
293                Err(err) => {
294                    let start = self.offset;
295                    let end = start + err.valid_up_to();
296                    match err.error_len() {
297                        Some(len) => self.offset += err.valid_up_to() + len,
298                        None => self.offset = self.data.len(),
299                    }
300                    &mut self.data[start..end]
301                }
302            };
303
304            // We are handing out multiple mutable slices from the same mutable slice.  This
305            // is safe because we know they are not overlapping.  However the compiler
306            // doesn't know this so we need to transmute the lifetimes of the slices we
307            // return with std::slice::from_raw_parts_mut().
308            let ptr = slice.as_mut_ptr();
309            let len = slice.len();
310            let encoded = unsafe {
311                WStr::from_utf16le_unchecked_mut(std::slice::from_raw_parts_mut(ptr, len))
312            };
313
314            if encoded.chars().take(MIN_STRING_LEN).count() < MIN_STRING_LEN {
315                continue;
316            }
317            let decoded = encoded.to_utf8();
318            return Some(WStrSegment { encoded, decoded });
319        }
320    }
321}
322
323impl FusedIterator for WStrSegmentIter<'_> {}
324
325/// An encoded string segment in a larger data block.
326///
327/// The slice of data will contain the entire block which will be valid according to the
328/// encoding.  This will be a unique sub-slice of the data in [`WStrSegmentIter`] as the
329/// iterator will not yield overlapping slices.
330///
331/// While the `data` field is mutable, after mutating this the string in `decoded` will no
332/// longer match.
333struct WStrSegment<'a> {
334    /// The raw bytes of this segment.
335    encoded: &'a mut WStr<LittleEndian>,
336    /// The decoded string of this segment.
337    decoded: String,
338}
339
340/// A PII processor for attachment files.
341pub struct PiiAttachmentsProcessor<'a> {
342    compiled_config: &'a CompiledPiiConfig,
343    root_state: ProcessingState<'static>,
344}
345
346/// Which encodings to scrub for `scrub_bytes`.
347pub enum ScrubEncodings {
348    /// Scrub UTF-8.
349    Utf8,
350
351    /// Scrub UTF-16LE (little endian).
352    Utf16Le,
353
354    /// Attempt to scrub in all available encodings.
355    All,
356}
357
358impl<'a> PiiAttachmentsProcessor<'a> {
359    /// Creates a new `PiiAttachmentsProcessor` from the given PII config.
360    pub fn new(compiled_config: &'a CompiledPiiConfig) -> Self {
361        // this constructor needs to be cheap... a new PiiProcessor is created for each event. Move
362        // any init logic into CompiledPiiConfig::new.
363
364        let root_state =
365            ProcessingState::root().enter_borrowed("", None, Some(ValueType::Attachments));
366
367        PiiAttachmentsProcessor {
368            compiled_config,
369            root_state,
370        }
371    }
372
373    /// Returns the processing state for the file with the given name.
374    pub(crate) fn state<'s>(
375        &'s self,
376        filename: &'s str,
377        value_type: ValueType,
378    ) -> ProcessingState<'s> {
379        self.root_state.enter_borrowed(
380            filename,
381            Some(Cow::Owned(FieldAttrs::new().pii(Pii::True))),
382            Some(value_type),
383        )
384    }
385
386    /// Applies PII rules to a plain buffer.
387    ///
388    /// Returns `true`, if the buffer was modified.
389    pub(crate) fn scrub_bytes(
390        &self,
391        data: &mut [u8],
392        state: &ProcessingState<'_>,
393        encodings: ScrubEncodings,
394    ) -> bool {
395        let pii = state.pii();
396        if pii == Pii::False {
397            return false;
398        }
399
400        let mut changed = false;
401
402        for (selector, rules) in &self.compiled_config.applications {
403            if selector.matches_path(&state.path()) {
404                for rule in rules {
405                    // Note:
406                    //
407                    // - We ignore pattern_type and just treat every regex like a value regex (i.e.
408                    //   redactPair becomes pattern rule). Very unlikely anybody would want that
409                    //   behavior (e.g.  "Remove passwords on **" would remove a file called
410                    //   "passwords.txt", but also "author.txt").  Just use selectors!
411                    //
412                    // - We impose severe restrictions on how redaction methods work, as we must
413                    //   not change the lengths of attachments.
414                    for (_pattern_type, regex, replace_behavior) in
415                        get_regex_for_rule_type(&rule.ty)
416                    {
417                        match encodings {
418                            ScrubEncodings::Utf8 => {
419                                let matches =
420                                    apply_regex_to_utf8_bytes(data, rule, regex, &replace_behavior);
421                                changed |= !(matches.is_empty());
422                            }
423                            ScrubEncodings::Utf16Le => {
424                                changed |= apply_regex_to_utf16le_bytes(
425                                    data,
426                                    rule,
427                                    regex,
428                                    &replace_behavior,
429                                );
430                            }
431                            ScrubEncodings::All => {
432                                let matches =
433                                    apply_regex_to_utf8_bytes(data, rule, regex, &replace_behavior);
434                                changed |= !(matches.is_empty());
435
436                                // Only scrub regions with the UTF-16 scrubber if they haven't been
437                                // scrubbed yet.
438                                let unscrubbed_ranges = matches
439                                    .into_iter()
440                                    .chain(std::iter::once((data.len(), 0)))
441                                    .scan((0usize, 0usize), |previous, current| {
442                                        let start = if previous.1 % 2 == 0 {
443                                            previous.1
444                                        } else {
445                                            previous.1 + 1
446                                        };
447                                        let item = (start, current.0);
448                                        *previous = current;
449                                        Some(item)
450                                    })
451                                    .filter(|(start, end)| end > start);
452                                for (start, end) in unscrubbed_ranges {
453                                    changed |= apply_regex_to_utf16le_bytes(
454                                        &mut data[start..end],
455                                        rule,
456                                        regex,
457                                        &replace_behavior,
458                                    );
459                                }
460                            }
461                        }
462                    }
463                }
464            }
465        }
466
467        changed
468    }
469
470    /// Applies PII scrubbing rules to a plain attachment.
471    ///
472    /// Returns `true`, if the attachment was modified.
473    pub fn scrub_attachment(&self, filename: &str, data: &mut [u8]) -> bool {
474        let state = self.state(filename, ValueType::Binary);
475        self.scrub_bytes(data, &state, ScrubEncodings::All)
476    }
477
478    /// Scrub a filepath, preserving the basename.
479    pub fn scrub_utf8_filepath(&self, path: &mut str, state: &ProcessingState<'_>) -> bool {
480        if let Some(index) = path.rfind(['/', '\\']) {
481            let data = unsafe { &mut path.as_bytes_mut()[..index] };
482            self.scrub_bytes(data, state, ScrubEncodings::Utf8)
483        } else {
484            false
485        }
486    }
487
488    /// Scrub a filepath, preserving the basename.
489    pub fn scrub_utf16_filepath(
490        &self,
491        path: &mut WStr<LittleEndian>,
492        state: &ProcessingState<'_>,
493    ) -> bool {
494        let index = path
495            .char_indices()
496            .rev()
497            .find_map(|(i, c)| if c == '/' || c == '\\' { Some(i) } else { None });
498
499        if let Some(index) = index {
500            let data = unsafe { &mut path.as_bytes_mut()[..index] };
501            self.scrub_bytes(data, state, ScrubEncodings::Utf16Le)
502        } else {
503            false
504        }
505    }
506
507    /// Applies PII rules to the given JSON.
508    ///
509    /// This function will perform PII scrubbing using `serde_transcode`, which means that it
510    /// does not have to read the entire document in memory but will rather perform in on a
511    /// per-item basis using a streaming approach.
512    ///
513    /// Returns a scrubbed copy of the JSON document.
514    pub fn scrub_json(&self, payload: &[u8]) -> Result<Vec<u8>, JsonScrubError> {
515        let output = Vec::new();
516
517        let visitor = JsonScrubVisitor::new(self.compiled_config);
518
519        let mut deserializer_inner = serde_json::Deserializer::from_slice(payload);
520        let deserializer = transform::Deserializer::new(&mut deserializer_inner, visitor);
521
522        let mut serializer = serde_json::Serializer::new(output);
523        serde_transcode::transcode(deserializer, &mut serializer)
524            .map_err(|_| JsonScrubError::TranscodeFailed)?;
525        Ok(serializer.into_inner())
526    }
527}
528
529#[cfg(test)]
530mod tests {
531    use itertools::Itertools;
532    use regex::Regex;
533
534    use super::*;
535    use crate::PiiConfig;
536
537    enum AttachmentBytesTestCase<'a> {
538        Builtin {
539            selector: &'a str,
540            rule: &'a str,
541            filename: &'a str,
542            value_type: ValueType,
543            input: &'a [u8],
544            output: &'a [u8],
545            changed: bool,
546        },
547        Regex {
548            selector: &'a str,
549            regex: &'a str,
550            filename: &'a str,
551            value_type: ValueType,
552            input: &'a [u8],
553            output: &'a [u8],
554            changed: bool,
555        },
556    }
557
558    impl AttachmentBytesTestCase<'_> {
559        fn run(self) {
560            let (config, filename, value_type, input, expected, changed) = match self {
561                AttachmentBytesTestCase::Builtin {
562                    selector,
563                    rule,
564                    filename,
565                    value_type,
566                    input,
567                    output,
568                    changed,
569                } => {
570                    let config = serde_json::from_value::<PiiConfig>(serde_json::json!(
571                        {
572                            "applications": {
573                                selector: [rule]
574                            }
575                        }
576                    ))
577                    .unwrap();
578                    (config, filename, value_type, input, output, changed)
579                }
580                AttachmentBytesTestCase::Regex {
581                    selector,
582                    regex,
583                    filename,
584                    value_type,
585                    input,
586                    output,
587                    changed,
588                } => {
589                    let config = serde_json::from_value::<PiiConfig>(serde_json::json!(
590                        {
591                            "rules": {
592                                "custom": {
593                                    "type": "pattern",
594                                    "pattern": regex,
595                                    "redaction": {
596                                      "method": "remove"
597                                    }
598                                }
599                            },
600                            "applications": {
601                                selector: ["custom"]
602                            }
603                        }
604                    ))
605                    .unwrap();
606                    (config, filename, value_type, input, output, changed)
607                }
608            };
609
610            let mut actual = input.to_owned();
611            let processor = PiiAttachmentsProcessor::new(config.compiled());
612            let state = processor.state(filename, value_type);
613            let has_changed = processor.scrub_bytes(&mut actual, &state, ScrubEncodings::All);
614
615            assert!(
616                actual == expected,
617                "`actual == expected` in line {}:\n{}\n{}",
618                line!(),
619                pretty_hex::pretty_hex(&actual),
620                pretty_hex::pretty_hex(&expected),
621            );
622
623            assert_eq!(changed, has_changed);
624        }
625    }
626
627    fn utf16le(s: &str) -> Vec<u8> {
628        s.encode_utf16()
629            .map(|u| u.to_le_bytes())
630            .collect::<Vec<[u8; 2]>>()
631            .iter()
632            .flatten()
633            .copied()
634            .collect()
635    }
636
637    #[test]
638    fn test_ip_replace_padding() {
639        AttachmentBytesTestCase::Builtin {
640            selector: "$binary",
641            rule: "@ip",
642            filename: "foo.txt",
643            value_type: ValueType::Binary,
644            input: b"before 127.0.0.1 after",
645            output: b"before [ip]***** after",
646            changed: true,
647        }
648        .run();
649    }
650
651    #[test]
652    fn test_ip_replace_padding_utf16() {
653        AttachmentBytesTestCase::Builtin {
654            selector: "$binary",
655            rule: "@ip",
656            filename: "foo.txt",
657            value_type: ValueType::Binary,
658            input: utf16le("before 127.0.0.1 after").as_slice(),
659            output: utf16le("before [ip]***** after").as_slice(),
660            changed: true,
661        }
662        .run();
663    }
664
665    #[test]
666    fn test_ip_hash_trunchating() {
667        AttachmentBytesTestCase::Builtin {
668            selector: "$binary",
669            rule: "@ip:hash",
670            filename: "foo.txt",
671            value_type: ValueType::Binary,
672            input: b"before 127.0.0.1 after",
673            output: b"before AE12FE3B5 after",
674            changed: true,
675        }
676        .run();
677    }
678
679    #[test]
680    fn test_ip_hash_trunchating_utf16() {
681        AttachmentBytesTestCase::Builtin {
682            selector: "$binary",
683            rule: "@ip:hash",
684            filename: "foo.txt",
685            value_type: ValueType::Binary,
686            input: utf16le("before 127.0.0.1 after").as_slice(),
687            output: utf16le("before 3FA8F5A46 after").as_slice(),
688            changed: true,
689        }
690        .run();
691    }
692
693    #[test]
694    fn test_ip_masking() {
695        AttachmentBytesTestCase::Builtin {
696            selector: "$binary",
697            rule: "@ip:mask",
698            filename: "foo.txt",
699            value_type: ValueType::Binary,
700            input: b"before 127.0.0.1 after",
701            output: b"before ********* after",
702            changed: true,
703        }
704        .run();
705    }
706
707    #[test]
708    fn test_ip_masking_utf16() {
709        AttachmentBytesTestCase::Builtin {
710            selector: "$binary",
711            rule: "@ip:mask",
712            filename: "foo.txt",
713            value_type: ValueType::Binary,
714            input: utf16le("before 127.0.0.1 after").as_slice(),
715            output: utf16le("before ********* after").as_slice(),
716            changed: true,
717        }
718        .run();
719    }
720
721    #[test]
722    fn test_ip_removing() {
723        AttachmentBytesTestCase::Builtin {
724            selector: "$binary",
725            rule: "@ip:remove",
726            filename: "foo.txt",
727            value_type: ValueType::Binary,
728            input: b"before 127.0.0.1 after",
729            output: b"before ********* after",
730            changed: true,
731        }
732        .run();
733    }
734
735    #[test]
736    fn test_ip_removing_utf16() {
737        AttachmentBytesTestCase::Builtin {
738            selector: "$binary",
739            rule: "@ip:remove",
740            filename: "foo.txt",
741            value_type: ValueType::Binary,
742            input: utf16le("before 127.0.0.1 after").as_slice(),
743            output: utf16le("before ********* after").as_slice(),
744            changed: true,
745        }
746        .run();
747    }
748
749    #[test]
750    fn test_creditcard_checksum() {
751        AttachmentBytesTestCase::Builtin {
752            selector: "$binary",
753            rule: "@creditcard:mask",
754            filename: "foo.txt",
755            value_type: ValueType::Binary,
756            input: b"a 4510944073809921 b 5100 4111 1111 1111 1111 c 4111 1111 1111 1111 d",
757            output: b"a 4510944073809921 b 5100 4111 1111 1111 1111 c ******************* d",
758            changed: true,
759        }
760        .run();
761    }
762
763    #[test]
764    fn test_creditcard_checksum_utf16() {
765        AttachmentBytesTestCase::Builtin {
766            selector: "$binary",
767            rule: "@creditcard:mask",
768            filename: "foo.txt",
769            value_type: ValueType::Binary,
770            input: utf16le("a 4510944073809921 b 5100 4111 1111 1111 1111 c 4111 1111 1111 1111 d")
771                .as_slice(),
772            output: utf16le(
773                "a 4510944073809921 b 5100 4111 1111 1111 1111 c ******************* d",
774            )
775            .as_slice(),
776            changed: true,
777        }
778        .run();
779    }
780
781    #[test]
782    fn test_selectors() {
783        for wrong_selector in &[
784            "$string",
785            "$number",
786            "$attachments.* && $string",
787            "$attachments",
788            "** && !$binary",
789        ] {
790            AttachmentBytesTestCase::Builtin {
791                selector: wrong_selector,
792                rule: "@ip:mask",
793                filename: "foo.txt",
794                value_type: ValueType::Binary,
795                input: b"before 127.0.0.1 after",
796                output: b"before 127.0.0.1 after",
797                changed: false,
798            }
799            .run();
800        }
801    }
802
803    #[test]
804    fn test_all_the_bytes() {
805        AttachmentBytesTestCase::Builtin {
806            selector: "$binary",
807            rule: "@anything:remove",
808            filename: "foo.txt",
809            value_type: ValueType::Binary,
810            input: (0..255u8).collect::<Vec<_>>().as_slice(),
811            output: &[b'*'; 255],
812            changed: true,
813        }
814        .run();
815    }
816
817    #[test]
818    fn test_bytes_regexes() {
819        // Test that specifically bytes patterns that are not valid UTF-8 can be matched against.
820        //
821        // From https://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805
822        let samples: &[&[u8]] = &[
823            b"\xc3\x28",                 // Invalid 2 Octet Sequence
824            b"\xa0\xa1",                 // Invalid Sequence Identifier
825            b"\xe2\x28\xa1",             // Invalid 3 Octet Sequence (in 2nd Octet)
826            b"\xe2\x82\x28",             // Invalid 3 Octet Sequence (in 3rd Octet)
827            b"\xf0\x28\x8c\xbc",         // Invalid 4 Octet Sequence (in 2nd Octet)
828            b"\xf0\x90\x28\xbc",         // Invalid 4 Octet Sequence (in 3rd Octet)
829            b"\xf0\x28\x8c\x28",         // Invalid 4 Octet Sequence (in 4th Octet)
830            b"\xf8\xa1\xa1\xa1\xa1",     // Valid 5 Octet Sequence (but not Unicode!)
831            b"\xfc\xa1\xa1\xa1\xa1\xa1", // Valid 6 Octet Sequence (but not Unicode!)
832        ];
833
834        for bytes in samples {
835            assert!(String::from_utf8(bytes.to_vec()).is_err());
836
837            AttachmentBytesTestCase::Regex {
838                selector: "$binary",
839                regex: &bytes.iter().map(|x| format!("\\x{x:02x}")).join(""),
840                filename: "foo.txt",
841                value_type: ValueType::Binary,
842                input: bytes,
843                output: &vec![b'*'; bytes.len()],
844                changed: true,
845            }
846            .run()
847        }
848    }
849
850    #[test]
851    fn test_segments_all_data() {
852        let mut data = Vec::from(&b"h\x00e\x00l\x00l\x00o\x00"[..]);
853        let mut iter = WStrSegmentIter::new(&mut data[..]);
854
855        let segment = iter.next().unwrap();
856        assert_eq!(segment.decoded, "hello");
857        assert_eq!(segment.encoded.as_bytes(), b"h\x00e\x00l\x00l\x00o\x00");
858
859        assert!(iter.next().is_none());
860    }
861
862    #[test]
863    fn test_segments_middle_2_byte_aligned() {
864        let mut data = Vec::from(&b"\xd8\xd8\xd8\xd8h\x00e\x00l\x00l\x00o\x00\xd8\xd8"[..]);
865        let mut iter = WStrSegmentIter::new(&mut data[..]);
866
867        let segment = iter.next().unwrap();
868        assert_eq!(segment.decoded, "hello");
869        assert_eq!(segment.encoded.as_bytes(), b"h\x00e\x00l\x00l\x00o\x00");
870
871        assert!(iter.next().is_none());
872    }
873
874    #[test]
875    fn test_segments_middle_2_byte_aligned_mutation() {
876        let mut data = Vec::from(&b"\xd8\xd8\xd8\xd8h\x00e\x00l\x00l\x00o\x00\xd8\xd8"[..]);
877        let mut iter = WStrSegmentIter::new(&mut data[..]);
878
879        let segment = iter.next().unwrap();
880        unsafe {
881            segment
882                .encoded
883                .as_bytes_mut()
884                .copy_from_slice(&b"w\x00o\x00r\x00l\x00d\x00"[..]);
885        }
886
887        assert!(iter.next().is_none());
888
889        assert_eq!(data, b"\xd8\xd8\xd8\xd8w\x00o\x00r\x00l\x00d\x00\xd8\xd8");
890    }
891
892    #[test]
893    fn test_segments_middle_unaligned() {
894        let mut data = Vec::from(&b"\xd8\xd8\xd8h\x00e\x00l\x00l\x00o\x00\xd8\xd8"[..]);
895        let mut iter = WStrSegmentIter::new(&mut data);
896
897        // Off-by-one is devastating, nearly everything is valid unicode.
898        let segment = iter.next().unwrap();
899        assert_eq!(segment.decoded, "棘攀氀氀漀");
900
901        assert!(iter.next().is_none());
902    }
903
904    #[test]
905    fn test_segments_end_aligned() {
906        let mut data = Vec::from(&b"\xd8\xd8h\x00e\x00l\x00l\x00o\x00"[..]);
907        let mut iter = WStrSegmentIter::new(&mut data);
908
909        let segment = iter.next().unwrap();
910        assert_eq!(segment.decoded, "hello");
911
912        assert!(iter.next().is_none());
913    }
914
915    #[test]
916    fn test_segments_garbage() {
917        let mut data = Vec::from(&b"\xd8\xd8"[..]);
918        let mut iter = WStrSegmentIter::new(&mut data);
919
920        assert!(iter.next().is_none());
921    }
922
923    #[test]
924    fn test_segments_too_short() {
925        let mut data = Vec::from(&b"\xd8\xd8y\x00o\x00\xd8\xd8h\x00e\x00l\x00l\x00o\x00"[..]);
926        let mut iter = WStrSegmentIter::new(&mut data);
927
928        let segment = iter.next().unwrap();
929        assert_eq!(segment.decoded, "hello");
930
931        assert!(iter.next().is_none());
932    }
933
934    #[test]
935    fn test_segments_multiple() {
936        let mut data =
937            Vec::from(&b"\xd8\xd8h\x00e\x00l\x00l\x00o\x00\xd8\xd8w\x00o\x00r\x00l\x00d\x00"[..]);
938
939        let mut iter = WStrSegmentIter::new(&mut data);
940
941        let segment = iter.next().unwrap();
942        assert_eq!(segment.decoded, "hello");
943
944        let segment = iter.next().unwrap();
945        assert_eq!(segment.decoded, "world");
946
947        assert!(iter.next().is_none());
948    }
949
950    #[test]
951    fn test_fill_content_wstr() {
952        let mut b = Vec::from(&b"h\x00e\x00l\x00l\x00o\x00"[..]);
953        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
954        s.fill_content('x');
955        assert_eq!(b.as_slice(), b"x\x00x\x00x\x00x\x00x\x00");
956    }
957
958    #[test]
959    #[should_panic]
960    fn test_fill_content_wstr_panic() {
961        let mut b = Vec::from(&b"h\x00e\x00y\x00"[..]);
962        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
963        s.fill_content('\u{10000}');
964    }
965
966    #[test]
967    fn test_swap_content_wstr() {
968        // Exact same size
969        let mut b = Vec::from(&b"h\x00e\x00l\x00l\x00o\x00"[..]);
970        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
971        s.swap_content("world", 'x');
972        assert_eq!(b.as_slice(), b"w\x00o\x00r\x00l\x00d\x00");
973
974        // Shorter, padding fits
975        let mut b = Vec::from(&b"h\x00e\x00l\x00l\x00o\x00"[..]);
976        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
977        s.swap_content("hey", 'x');
978        assert_eq!(b.as_slice(), b"h\x00e\x00y\x00x\x00x\x00");
979
980        // Longer, truncated fits
981        let mut b = Vec::from(&b"h\x00e\x00y\x00"[..]);
982        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
983        s.swap_content("world", 'x');
984        assert_eq!(b.as_slice(), b"w\x00o\x00r\x00");
985
986        // Longer, truncated + padding
987        let mut b = Vec::from(&b"h\x00e\x00y\x00"[..]);
988        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
989        s.swap_content("yo\u{10000}", 'x');
990        assert_eq!(b.as_slice(), b"y\x00o\x00x\x00");
991    }
992
993    #[test]
994    #[should_panic]
995    fn test_swap_content_wstr_panic() {
996        let mut b = Vec::from(&b"h\x00e\x00y\x00"[..]);
997        let s = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
998        s.swap_content("yo", '\u{10000}');
999    }
1000
1001    #[test]
1002    #[allow(clippy::trivial_regex)]
1003    fn test_get_wstr_match() {
1004        let s = "hello there";
1005        let mut b = Vec::from(&b"h\x00e\x00l\x00l\x00o\x00 \x00t\x00h\x00e\x00r\x00e\x00"[..]);
1006        let w = WStr::from_utf16le_mut(b.as_mut_slice()).unwrap();
1007
1008        // Partial match
1009        let re = Regex::new("hello").unwrap();
1010        let re_match = re.find(s).unwrap();
1011        let m = get_wstr_match(s, re_match, w);
1012        assert_eq!(m.as_bytes(), b"h\x00e\x00l\x00l\x00o\x00");
1013
1014        // Full match
1015        let re = Regex::new(".*").unwrap();
1016        let re_match = re.find(s).unwrap();
1017        let m = get_wstr_match(s, re_match, w);
1018        assert_eq!(
1019            m.as_bytes(),
1020            b"h\x00e\x00l\x00l\x00o\x00 \x00t\x00h\x00e\x00r\x00e\x00"
1021        );
1022    }
1023}