relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.max_bytes(),
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        if let Some(size_state) = self.size_state.last() {
97            // If our current depth is the one where we found a bag_size attribute, this means we
98            // are done processing a databag. Pop the bag size state.
99            if state.depth() == size_state.encountered_at_depth {
100                self.size_state.pop().unwrap();
101            }
102        }
103
104        // After processing a value, update the remaining bag sizes. We have a separate if-let
105        // here in case somebody defines nested databags (a struct with bag_size that contains
106        // another struct with a different bag_size), in case we just exited a databag we want
107        // to update the bag_size_state of the outer databag with the remaining size.
108        //
109        // This also has to happen after string trimming, which is why it's running in
110        // after_process.
111        if state.entered_anything() && !self.size_state.is_empty() {
112            // Do not subtract if state is from newtype struct.
113            let item_length = state
114                .bytes_size()
115                .unwrap_or_else(|| relay_protocol::estimate_size_flat(value) + 1);
116            for size_state in self.size_state.iter_mut() {
117                size_state.size_remaining = size_state
118                    .size_remaining
119                    .map(|size| size.saturating_sub(item_length));
120            }
121        }
122
123        Ok(())
124    }
125
126    fn process_string(
127        &mut self,
128        value: &mut String,
129        meta: &mut Meta,
130        state: &ProcessingState<'_>,
131    ) -> ProcessingResult {
132        if let Some(max_chars) = state.max_chars() {
133            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
134        }
135
136        if !state.attrs().trim {
137            return Ok(());
138        }
139
140        if let Some(size_remaining) = self.remaining_size() {
141            trim_string(value, meta, size_remaining, 0);
142        }
143
144        Ok(())
145    }
146
147    fn process_array<T>(
148        &mut self,
149        value: &mut Array<T>,
150        meta: &mut Meta,
151        state: &ProcessingState<'_>,
152    ) -> ProcessingResult
153    where
154        T: ProcessValue,
155    {
156        if !state.attrs().trim {
157            return Ok(());
158        }
159
160        // If we need to check the bag size, then we go down a different path
161        if !self.size_state.is_empty() {
162            let original_length = value.len();
163
164            if self.should_remove_container(value, state) {
165                return Err(ProcessingAction::DeleteValueHard);
166            }
167
168            let mut split_index = None;
169            for (index, item) in value.iter_mut().enumerate() {
170                if self.remaining_size() == Some(0) {
171                    split_index = Some(index);
172                    break;
173                }
174
175                let item_state = state.enter_index(index, None, ValueType::for_field(item));
176                processor::process_value(item, self, &item_state)?;
177            }
178
179            if let Some(split_index) = split_index {
180                let _ = value.split_off(split_index);
181            }
182
183            if value.len() != original_length {
184                meta.set_original_length(Some(original_length));
185            }
186        } else {
187            value.process_child_values(self, state)?;
188        }
189
190        Ok(())
191    }
192
193    fn process_object<T>(
194        &mut self,
195        value: &mut Object<T>,
196        meta: &mut Meta,
197        state: &ProcessingState<'_>,
198    ) -> ProcessingResult
199    where
200        T: ProcessValue,
201    {
202        if !state.attrs().trim {
203            return Ok(());
204        }
205
206        // If we need to check the bag size, then we go down a different path
207        if !self.size_state.is_empty() {
208            let original_length = value.len();
209
210            if self.should_remove_container(value, state) {
211                return Err(ProcessingAction::DeleteValueHard);
212            }
213
214            let mut split_key = None;
215            for (key, item) in value.iter_mut() {
216                if self.remaining_size() == Some(0) {
217                    split_key = Some(key.to_owned());
218                    break;
219                }
220
221                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
222                processor::process_value(item, self, &item_state)?;
223            }
224
225            if let Some(split_key) = split_key {
226                let _ = value.split_off(&split_key);
227            }
228
229            if value.len() != original_length {
230                meta.set_original_length(Some(original_length));
231            }
232        } else {
233            value.process_child_values(self, state)?;
234        }
235
236        Ok(())
237    }
238
239    fn process_value(
240        &mut self,
241        value: &mut Value,
242        _meta: &mut Meta,
243        state: &ProcessingState<'_>,
244    ) -> ProcessingResult {
245        if !state.attrs().trim {
246            return Ok(());
247        }
248
249        match value {
250            Value::Array(_) | Value::Object(_) => {
251                if self.remaining_depth(state) == Some(1)
252                    && let Ok(x) = serde_json::to_string(&value)
253                {
254                    // Error case should not be possible
255                    *value = Value::String(x);
256                }
257            }
258            _ => (),
259        }
260
261        value.process_child_values(self, state)?;
262        Ok(())
263    }
264
265    fn process_replay(
266        &mut self,
267        replay: &mut Replay,
268        _: &mut Meta,
269        state: &ProcessingState<'_>,
270    ) -> ProcessingResult {
271        replay.process_child_values(self, state)
272    }
273
274    fn process_raw_stacktrace(
275        &mut self,
276        stacktrace: &mut RawStacktrace,
277        _meta: &mut Meta,
278        state: &ProcessingState<'_>,
279    ) -> ProcessingResult {
280        if !state.attrs().trim {
281            return Ok(());
282        }
283
284        processor::apply(&mut stacktrace.frames, |frames, meta| {
285            enforce_frame_hard_limit(frames, meta, 200, 50);
286            Ok(())
287        })?;
288
289        stacktrace.process_child_values(self, state)?;
290
291        processor::apply(&mut stacktrace.frames, |frames, _meta| {
292            slim_frame_data(frames, 50);
293            Ok(())
294        })?;
295
296        Ok(())
297    }
298}
299
300/// Trims the string to the given maximum length and updates meta data.
301pub(crate) fn trim_string(
302    value: &mut String,
303    meta: &mut Meta,
304    max_chars: usize,
305    max_chars_allowance: usize,
306) {
307    let hard_limit = max_chars + max_chars_allowance;
308
309    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
310        return;
311    }
312
313    processor::process_chunked_value(value, meta, |chunks| {
314        let mut length = 0;
315        let mut new_chunks = vec![];
316
317        for chunk in chunks {
318            let chunk_chars = chunk.count();
319
320            // if the entire chunk fits, just put it in
321            if length + chunk_chars < max_chars {
322                new_chunks.push(chunk);
323                length += chunk_chars;
324                continue;
325            }
326
327            match chunk {
328                // if there is enough space for this chunk and the 3 character
329                // ellipsis marker we can push the remaining chunk
330                Chunk::Redaction { .. } => {
331                    if length + chunk_chars + 3 < hard_limit {
332                        new_chunks.push(chunk);
333                    }
334                }
335
336                // if this is a text chunk, we can put the remaining characters in.
337                Chunk::Text { text } => {
338                    let mut remaining = String::new();
339                    for c in text.chars() {
340                        if length + 3 < max_chars {
341                            remaining.push(c);
342                        } else {
343                            break;
344                        }
345                        length += 1;
346                    }
347
348                    new_chunks.push(Chunk::Text {
349                        text: Cow::Owned(remaining),
350                    });
351                }
352            }
353
354            new_chunks.push(Chunk::Redaction {
355                text: Cow::Borrowed("..."),
356                rule_id: Cow::Borrowed("!limit"),
357                ty: RemarkType::Substituted,
358            });
359            break;
360        }
361
362        new_chunks
363    });
364}
365
366/// Trim down the frame list to a hard limit.
367///
368/// The total limit is `recent_frames` + `old_frames`.
369/// `recent_frames` is the number of frames to keep from the beginning of the list,
370/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
371/// the oldest frames up the stack.
372///
373/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
374/// the recursion.
375fn enforce_frame_hard_limit(
376    frames: &mut Array<Frame>,
377    meta: &mut Meta,
378    recent_frames: usize,
379    old_frames: usize,
380) {
381    let original_length = frames.len();
382    let limit = recent_frames + old_frames;
383    if original_length > limit {
384        meta.set_original_length(Some(original_length));
385        let _ = frames.drain(old_frames..original_length - recent_frames);
386    }
387}
388
389/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
390///
391/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
392fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
393    let frames_len = frames.len();
394
395    if frames_len <= frame_allowance {
396        return;
397    }
398
399    // Avoid ownership issues by only storing indices
400    let mut app_frame_indices = Vec::with_capacity(frames_len);
401    let mut system_frame_indices = Vec::with_capacity(frames_len);
402
403    for (i, frame) in frames.iter().enumerate() {
404        if let Some(frame) = frame.value() {
405            match frame.in_app.value() {
406                Some(true) => app_frame_indices.push(i),
407                _ => system_frame_indices.push(i),
408            }
409        }
410    }
411
412    let app_count = app_frame_indices.len();
413    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
414    let system_frames_to_remove = system_frame_indices
415        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
416        .unwrap_or(&[]);
417
418    let remaining = frames_len
419        .saturating_sub(frame_allowance)
420        .saturating_sub(system_frames_to_remove.len());
421    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
422    let app_frames_to_remove = app_frame_indices
423        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
424        .unwrap_or(&[]);
425
426    // TODO: Which annotation to set?
427
428    for i in system_frames_to_remove.iter().chain(app_frames_to_remove) {
429        if let Some(frame) = frames.get_mut(*i)
430            && let Some(ref mut frame) = frame.value_mut().as_mut()
431        {
432            frame.vars = Annotated::empty();
433            frame.pre_context = Annotated::empty();
434            frame.post_context = Annotated::empty();
435        }
436    }
437}
438
439#[cfg(test)]
440mod tests {
441    use std::iter::repeat_n;
442
443    use crate::MaxChars;
444    use chrono::DateTime;
445    use relay_event_schema::protocol::{
446        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, PairList, SentryTags, Span,
447        SpanId, TagEntry, Tags, Timestamp, TraceId, Values,
448    };
449    use relay_protocol::{FromValue, IntoValue, Map, Remark, SerializableAnnotated, get_value};
450    use similar_asserts::assert_eq;
451
452    use super::*;
453
454    #[test]
455    fn test_string_trimming() {
456        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
457        processor::apply(&mut value, |v, m| {
458            trim_string(v, m, 20, 0);
459            Ok(())
460        })
461        .unwrap();
462
463        assert_eq!(
464            value,
465            Annotated(Some("This is my long s...".into()), {
466                let mut meta = Meta::default();
467                meta.add_remark(Remark {
468                    ty: RemarkType::Substituted,
469                    rule_id: "!limit".to_owned(),
470                    range: Some((17, 20)),
471                });
472                meta.set_original_length(Some(46));
473                meta
474            })
475        );
476    }
477
478    #[test]
479    fn test_basic_trimming() {
480        let mut processor = TrimmingProcessor::new();
481
482        let mut event = Annotated::new(Event {
483            logger: Annotated::new("x".repeat(300)),
484            ..Default::default()
485        });
486
487        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
488
489        let mut expected = Annotated::new("x".repeat(300));
490        processor::apply(&mut expected, |v, m| {
491            trim_string(v, m, MaxChars::Logger.limit(), 0);
492            Ok(())
493        })
494        .unwrap();
495
496        assert_eq!(event.value().unwrap().logger, expected);
497    }
498
499    #[test]
500    fn test_max_char_allowance() {
501        let string = "This string requires some allowance to fit!";
502        let mut value = Annotated::new(string.to_owned()); // len == 43
503        processor::apply(&mut value, |v, m| {
504            trim_string(v, m, 40, 5);
505            Ok(())
506        })
507        .unwrap();
508
509        assert_eq!(value, Annotated::new(string.to_owned()));
510    }
511
512    #[test]
513    fn test_databag_stripping() {
514        let mut processor = TrimmingProcessor::new();
515
516        fn make_nested_object(depth: usize) -> Annotated<Value> {
517            if depth == 0 {
518                return Annotated::new(Value::String("max depth".to_owned()));
519            }
520            let mut rv = Object::new();
521            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
522            Annotated::new(Value::Object(rv))
523        }
524
525        let databag = Annotated::new({
526            let mut map = Object::new();
527            map.insert(
528                "key_1".to_owned(),
529                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
530            );
531            map.insert(
532                "key_2".to_owned(),
533                make_nested_object(8).map_value(ExtraValue),
534            );
535            map.insert(
536                "key_3".to_owned(),
537                // innermost key (string) is entering json stringify codepath
538                make_nested_object(5).map_value(ExtraValue),
539            );
540            map
541        });
542        let mut event = Annotated::new(Event {
543            extra: databag,
544            ..Default::default()
545        });
546
547        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
548        let stripped_extra = &event.value().unwrap().extra;
549        let json = stripped_extra.to_json_pretty().unwrap();
550
551        assert_eq!(
552            json,
553            r#"{
554  "key_1": "value 1",
555  "key_2": {
556    "key8": {
557      "key7": {
558        "key6": {
559          "key5": {
560            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
561          }
562        }
563      }
564    }
565  },
566  "key_3": {
567    "key5": {
568      "key4": {
569        "key3": {
570          "key2": {
571            "key1": "max depth"
572          }
573        }
574      }
575    }
576  }
577}"#
578        );
579    }
580
581    #[test]
582    fn test_databag_array_stripping() {
583        let mut processor = TrimmingProcessor::new();
584
585        let databag = Annotated::new({
586            let mut map = Object::new();
587            for idx in 0..100 {
588                map.insert(
589                    format!("key_{idx}"),
590                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
591                );
592            }
593            map
594        });
595        let mut event = Annotated::new(Event {
596            extra: databag,
597            ..Default::default()
598        });
599
600        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
601        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
602
603        insta::assert_ron_snapshot!(stripped_extra);
604    }
605
606    /// Tests that a trimming a string takes a lower outer limit into account.
607    #[test]
608    fn test_string_trimming_limits() {
609        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
610        struct Outer {
611            #[metastructure(max_bytes = 10)]
612            inner: Annotated<Inner>,
613        }
614
615        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
616        struct Inner {
617            #[metastructure(max_bytes = 20)]
618            innerer: Annotated<String>,
619        }
620
621        let mut processor = TrimmingProcessor::new();
622
623        let mut outer = Annotated::new({
624            Outer {
625                inner: Annotated::new(Inner {
626                    innerer: Annotated::new("This string is 28 bytes long".into()),
627                }),
628            }
629        });
630
631        processor::process_value(&mut outer, &mut processor, ProcessingState::root()).unwrap();
632        let stripped = SerializableAnnotated(&outer);
633
634        insta::assert_ron_snapshot!(stripped, @r###"
635        {
636          "inner": {
637            "innerer": "This st...",
638          },
639          "_meta": {
640            "inner": {
641              "innerer": {
642                "": Meta(Some(MetaInner(
643                  rem: [
644                    [
645                      "!limit",
646                      s,
647                      7,
648                      10,
649                    ],
650                  ],
651                  len: Some(28),
652                ))),
653              },
654            },
655          },
656        }
657        "###);
658    }
659
660    #[test]
661    fn test_tags_stripping() {
662        let mut processor = TrimmingProcessor::new();
663
664        let mut event = Annotated::new(Event {
665            tags: Annotated::new(Tags(
666                vec![Annotated::new(TagEntry(
667                    Annotated::new("x".repeat(300)),
668                    Annotated::new("x".repeat(300)),
669                ))]
670                .into(),
671            )),
672            ..Default::default()
673        });
674
675        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
676        let json = event
677            .value()
678            .unwrap()
679            .tags
680            .payload_to_json_pretty()
681            .unwrap();
682
683        assert_eq!(
684            json,
685            r#"[
686  [
687    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
688    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
689  ]
690]"#
691        );
692    }
693
694    #[test]
695    fn test_databag_state_leak() {
696        let event = Annotated::new(Event {
697            breadcrumbs: Annotated::new(Values::new(
698                repeat_n(
699                    Annotated::new(Breadcrumb {
700                        data: {
701                            let mut map = Map::new();
702                            map.insert(
703                                "spamspamspam".to_owned(),
704                                Annotated::new(Value::String("blablabla".to_owned())),
705                            );
706                            Annotated::new(map)
707                        },
708                        ..Default::default()
709                    }),
710                    200,
711                )
712                .collect(),
713            )),
714            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
715                ty: Annotated::new("TypeError".to_owned()),
716                value: Annotated::new("important error message".to_owned().into()),
717                stacktrace: Annotated::new(
718                    RawStacktrace {
719                        frames: Annotated::new(
720                            repeat_n(
721                                Annotated::new(Frame {
722                                    function: Annotated::new("importantFunctionName".to_owned()),
723                                    symbol: Annotated::new("important_symbol".to_owned()),
724                                    ..Default::default()
725                                }),
726                                200,
727                            )
728                            .collect(),
729                        ),
730                        ..Default::default()
731                    }
732                    .into(),
733                ),
734                ..Default::default()
735            })])),
736            ..Default::default()
737        });
738
739        let mut processor = TrimmingProcessor::new();
740        let mut stripped_event = event.clone();
741        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
742            .unwrap();
743
744        assert_eq!(
745            event.to_json_pretty().unwrap(),
746            stripped_event.to_json_pretty().unwrap()
747        );
748    }
749
750    #[test]
751    fn test_custom_context_trimming() {
752        let mut contexts = Contexts::new();
753        for i in 1..2 {
754            contexts.insert(format!("despacito{i}"), {
755                let mut context = Object::new();
756                context.insert(
757                    "foo".to_owned(),
758                    Annotated::new(Value::String("a".repeat(4000))),
759                );
760                context.insert(
761                    "bar".to_owned(),
762                    Annotated::new(Value::String("a".repeat(5000))),
763                );
764                Context::Other(context)
765            });
766        }
767
768        let mut contexts = Annotated::new(contexts);
769        let mut processor = TrimmingProcessor::new();
770        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
771
772        let contexts = contexts.value().unwrap();
773        for i in 1..2 {
774            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
775                Context::Other(x) => x,
776                _ => panic!("Context has changed type!"),
777            };
778
779            assert_eq!(
780                other
781                    .get("bar")
782                    .unwrap()
783                    .value()
784                    .unwrap()
785                    .as_str()
786                    .unwrap()
787                    .len(),
788                5000
789            );
790            assert_eq!(
791                other
792                    .get("foo")
793                    .unwrap()
794                    .value()
795                    .unwrap()
796                    .as_str()
797                    .unwrap()
798                    .len(),
799                3189
800            );
801        }
802    }
803
804    #[test]
805    fn test_extra_trimming_long_arrays() {
806        let mut extra = Object::new();
807        extra.insert("foo".to_owned(), {
808            Annotated::new(ExtraValue(Value::Array(
809                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
810            )))
811        });
812
813        let mut event = Annotated::new(Event {
814            extra: Annotated::new(extra),
815            ..Default::default()
816        });
817
818        let mut processor = TrimmingProcessor::new();
819        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
820
821        let arr = match event
822            .value()
823            .unwrap()
824            .extra
825            .value()
826            .unwrap()
827            .get("foo")
828            .unwrap()
829            .value()
830            .unwrap()
831        {
832            ExtraValue(Value::Array(x)) => x,
833            x => panic!("Wrong type: {x:?}"),
834        };
835
836        // this is larger / 2 for the extra value
837        assert_eq!(arr.len(), 8192);
838    }
839
840    // TODO(ja): Enable this test
841    // #[test]
842    // fn test_newtypes_do_not_add_to_depth() {
843    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
844    //     struct WrappedString(String);
845
846    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
847    //     struct StructChild2 {
848    //         inner: Annotated<WrappedString>,
849    //     }
850
851    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
852    //     struct StructChild {
853    //         inner: Annotated<StructChild2>,
854    //     }
855
856    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
857    //     struct Struct {
858    //         #[metastructure(bag_size = "small")]
859    //         inner: Annotated<StructChild>,
860    //     }
861
862    //     let mut value = Annotated::new(Struct {
863    //         inner: Annotated::new(StructChild {
864    //             inner: Annotated::new(StructChild2 {
865    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
866    //             }),
867    //         }),
868    //     });
869
870    //     let mut processor = TrimmingProcessor::new();
871    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
872
873    //     // Ensure stack does not leak with newtypes
874    //     assert!(processor.bag_size_state.is_empty());
875
876    //     assert_eq!(
877    //         value.to_json().unwrap(),
878    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
879    //     );
880    // }
881
882    #[test]
883    fn test_frameqty_equals_limit() {
884        fn create_frame(filename: &str) -> Annotated<Frame> {
885            Annotated::new(Frame {
886                filename: Annotated::new(filename.into()),
887                ..Default::default()
888            })
889        }
890
891        let mut frames = Annotated::new(vec![
892            create_frame("foo3.py"),
893            create_frame("foo4.py"),
894            create_frame("foo5.py"),
895        ]);
896
897        processor::apply(&mut frames, |f, m| {
898            enforce_frame_hard_limit(f, m, 3, 0);
899            Ok(())
900        })
901        .unwrap();
902
903        processor::apply(&mut frames, |f, m| {
904            enforce_frame_hard_limit(f, m, 1, 2);
905            Ok(())
906        })
907        .unwrap();
908
909        // original_length isn't set, when limit is equal to length, as no trimming took place.
910        assert!(frames.meta().original_length().is_none());
911    }
912
913    #[test]
914    fn test_frame_hard_limit() {
915        fn create_frame(filename: &str) -> Annotated<Frame> {
916            Annotated::new(Frame {
917                filename: Annotated::new(filename.into()),
918                ..Default::default()
919            })
920        }
921
922        let mut frames = Annotated::new(vec![
923            create_frame("foo1.py"),
924            create_frame("foo2.py"),
925            create_frame("foo3.py"),
926            create_frame("foo4.py"),
927            create_frame("foo5.py"),
928        ]);
929
930        processor::apply(&mut frames, |f, m| {
931            enforce_frame_hard_limit(f, m, 3, 0);
932            Ok(())
933        })
934        .unwrap();
935
936        let mut expected_meta = Meta::default();
937        expected_meta.set_original_length(Some(5));
938
939        assert_eq!(
940            frames,
941            Annotated(
942                Some(vec![
943                    create_frame("foo3.py"),
944                    create_frame("foo4.py"),
945                    create_frame("foo5.py"),
946                ]),
947                expected_meta
948            )
949        );
950    }
951
952    #[test]
953    fn test_frame_hard_limit_recent_old() {
954        fn create_frame(filename: &str) -> Annotated<Frame> {
955            Annotated::new(Frame {
956                filename: Annotated::new(filename.into()),
957                ..Default::default()
958            })
959        }
960
961        let mut frames = Annotated::new(vec![
962            create_frame("foo1.py"),
963            create_frame("foo2.py"),
964            create_frame("foo3.py"),
965            create_frame("foo4.py"),
966            create_frame("foo5.py"),
967        ]);
968
969        processor::apply(&mut frames, |f, m| {
970            enforce_frame_hard_limit(f, m, 2, 1);
971            Ok(())
972        })
973        .unwrap();
974
975        let mut expected_meta = Meta::default();
976        expected_meta.set_original_length(Some(5));
977
978        assert_eq!(
979            frames,
980            Annotated(
981                Some(vec![
982                    create_frame("foo1.py"),
983                    create_frame("foo4.py"),
984                    create_frame("foo5.py"),
985                ]),
986                expected_meta
987            )
988        );
989    }
990
991    #[test]
992    fn test_slim_frame_data_under_max() {
993        let mut frames = vec![Annotated::new(Frame {
994            filename: Annotated::new("foo".into()),
995            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
996            context_line: Annotated::new("b".to_owned()),
997            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
998            ..Default::default()
999        })];
1000
1001        let old_frames = frames.clone();
1002        slim_frame_data(&mut frames, 4);
1003
1004        assert_eq!(frames, old_frames);
1005    }
1006
1007    #[test]
1008    fn test_slim_frame_data_over_max() {
1009        let mut frames = vec![];
1010
1011        for n in 0..5 {
1012            frames.push(Annotated::new(Frame {
1013                filename: Annotated::new(format!("foo {n}").into()),
1014                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1015                context_line: Annotated::new("b".to_owned()),
1016                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1017                ..Default::default()
1018            }));
1019        }
1020
1021        slim_frame_data(&mut frames, 4);
1022
1023        let expected = vec![
1024            Annotated::new(Frame {
1025                filename: Annotated::new("foo 0".into()),
1026                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1027                context_line: Annotated::new("b".to_owned()),
1028                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1029                ..Default::default()
1030            }),
1031            Annotated::new(Frame {
1032                filename: Annotated::new("foo 1".into()),
1033                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1034                context_line: Annotated::new("b".to_owned()),
1035                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1036                ..Default::default()
1037            }),
1038            Annotated::new(Frame {
1039                filename: Annotated::new("foo 2".into()),
1040                context_line: Annotated::new("b".to_owned()),
1041                ..Default::default()
1042            }),
1043            Annotated::new(Frame {
1044                filename: Annotated::new("foo 3".into()),
1045                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1046                context_line: Annotated::new("b".to_owned()),
1047                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1048                ..Default::default()
1049            }),
1050            Annotated::new(Frame {
1051                filename: Annotated::new("foo 4".into()),
1052                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1053                context_line: Annotated::new("b".to_owned()),
1054                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1055                ..Default::default()
1056            }),
1057        ];
1058
1059        assert_eq!(frames, expected);
1060    }
1061
1062    #[test]
1063    fn test_too_many_spans_trimmed() {
1064        let span = Span {
1065            platform: Annotated::new("a".repeat(1024 * 90)),
1066            sentry_tags: Annotated::new(SentryTags {
1067                release: Annotated::new("b".repeat(1024 * 100)),
1068                ..Default::default()
1069            }),
1070            ..Default::default()
1071        };
1072        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1073            .take(10)
1074            .collect();
1075
1076        let mut event = Annotated::new(Event {
1077            spans: Annotated::new(spans.clone()),
1078            ..Default::default()
1079        });
1080
1081        let mut processor = TrimmingProcessor::new();
1082        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1083
1084        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1085        assert_eq!(trimmed_spans.len(), 5);
1086
1087        // The actual spans were not touched:
1088        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1089    }
1090
1091    #[test]
1092    fn test_untrimmable_fields() {
1093        let original_description = "a".repeat(819163);
1094        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1095        let mut event = Annotated::new(Event {
1096            spans: Annotated::new(vec![
1097                Span {
1098                    description: original_description.clone().into(),
1099                    ..Default::default()
1100                }
1101                .into(),
1102                Span {
1103                    trace_id: original_trace_id.into(),
1104                    ..Default::default()
1105                }
1106                .into(),
1107            ]),
1108            ..Default::default()
1109        });
1110
1111        let mut processor = TrimmingProcessor::new();
1112        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1113
1114        assert_eq!(
1115            get_value!(event.spans[0].description!),
1116            &original_description
1117        );
1118        // Trace ID would be trimmed without `trim = "false"`
1119        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1120    }
1121
1122    #[test]
1123    fn test_untrimmable_fields_drop() {
1124        let original_description = "a".repeat(819164);
1125        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1126        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1127        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1128        let original_op = "e".repeat(129);
1129
1130        let mut event = Annotated::new(Event {
1131            spans: Annotated::new(vec![
1132                Span {
1133                    description: original_description.clone().into(),
1134                    ..Default::default()
1135                }
1136                .into(),
1137                Span {
1138                    span_id: original_span_id.into(),
1139                    trace_id: original_trace_id.into(),
1140                    segment_id: original_segment_id.into(),
1141                    is_segment: false.into(),
1142                    op: original_op.clone().into(),
1143                    start_timestamp: Timestamp(
1144                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1145                            .unwrap()
1146                            .into(),
1147                    )
1148                    .into(),
1149                    timestamp: Timestamp(
1150                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1151                            .unwrap()
1152                            .into(),
1153                    )
1154                    .into(),
1155                    ..Default::default()
1156                }
1157                .into(),
1158            ]),
1159            ..Default::default()
1160        });
1161
1162        let mut processor = TrimmingProcessor::new();
1163        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1164
1165        assert_eq!(
1166            get_value!(event.spans[0].description!),
1167            &original_description
1168        );
1169        // These fields would be dropped without `trim = "false"`
1170        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1171        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1172        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1173        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1174        // span.op is trimmed to its max_chars, but not dropped:
1175        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1176        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1177        assert!(get_value!(event.spans[1].timestamp).is_some());
1178    }
1179
1180    #[test]
1181    fn test_too_long_tags() {
1182        let mut event = Annotated::new(Event {
1183        tags: Annotated::new(Tags(PairList(
1184            vec![Annotated::new(TagEntry(
1185                Annotated::new("foobar".to_owned()),
1186                Annotated::new("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx".to_owned()),
1187            )), Annotated::new(TagEntry(
1188                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1189                Annotated::new("bar".to_owned()),
1190            ))]),
1191        )),
1192        ..Event::default()
1193    });
1194
1195        let mut processor = TrimmingProcessor::new();
1196        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1197
1198        insta::assert_debug_snapshot!(get_value!(event.tags!), @r###"
1199        Tags(
1200            PairList(
1201                [
1202                    TagEntry(
1203                        "foobar",
1204                        Annotated(
1205                            "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
1206                            Meta {
1207                                remarks: [
1208                                    Remark {
1209                                        ty: Substituted,
1210                                        rule_id: "!limit",
1211                                        range: Some(
1212                                            (
1213                                                197,
1214                                                200,
1215                                            ),
1216                                        ),
1217                                    },
1218                                ],
1219                                errors: [],
1220                                original_length: Some(
1221                                    203,
1222                                ),
1223                                original_value: None,
1224                            },
1225                        ),
1226                    ),
1227                    TagEntry(
1228                        Annotated(
1229                            "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo...",
1230                            Meta {
1231                                remarks: [
1232                                    Remark {
1233                                        ty: Substituted,
1234                                        rule_id: "!limit",
1235                                        range: Some(
1236                                            (
1237                                                197,
1238                                                200,
1239                                            ),
1240                                        ),
1241                                    },
1242                                ],
1243                                errors: [],
1244                                original_length: Some(
1245                                    203,
1246                                ),
1247                                original_value: None,
1248                            },
1249                        ),
1250                        "bar",
1251                    ),
1252                ],
1253            ),
1254        )
1255        "###);
1256    }
1257
1258    #[test]
1259    fn test_fixed_item_size() {
1260        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1261        struct TestObject {
1262            #[metastructure(max_bytes = 28)]
1263            inner: Annotated<TestObjectInner>,
1264        }
1265        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1266        struct TestObjectInner {
1267            #[metastructure(max_chars = 10, trim = true)]
1268            body: Annotated<String>,
1269            // This should neither be trimmed nor factor into size calculations.
1270            #[metastructure(trim = false, bytes_size = "always_zero")]
1271            number: Annotated<u64>,
1272            // This should count as 10B.
1273            #[metastructure(trim = false, bytes_size = 10)]
1274            other_number: Annotated<u64>,
1275            #[metastructure(trim = true)]
1276            footer: Annotated<String>,
1277        }
1278
1279        fn always_zero(_state: &ProcessingState) -> Option<usize> {
1280            Some(0)
1281        }
1282
1283        let mut object = Annotated::new(TestObject {
1284            inner: Annotated::new(TestObjectInner {
1285                body: Annotated::new("Longer than 10 chars".to_owned()),
1286                number: Annotated::new(13),
1287                other_number: Annotated::new(12),
1288                footer: Annotated::new("There should only be 'Th...' left".to_owned()),
1289            }),
1290        });
1291
1292        let mut processor = TrimmingProcessor::new();
1293        processor::process_value(&mut object, &mut processor, ProcessingState::root()).unwrap();
1294
1295        // * `body` gets trimmed to 13B (10 chars + `...`)
1296        // * `number` counts as 0B
1297        // * `other_number` counts as 10B
1298        // That leaves 5B for the `footer`.
1299        insta::assert_ron_snapshot!(SerializableAnnotated(&object), @r###"
1300        {
1301          "inner": {
1302            "body": "Longer ...",
1303            "number": 13,
1304            "other_number": 12,
1305            "footer": "Th...",
1306          },
1307          "_meta": {
1308            "inner": {
1309              "body": {
1310                "": Meta(Some(MetaInner(
1311                  rem: [
1312                    [
1313                      "!limit",
1314                      s,
1315                      7,
1316                      10,
1317                    ],
1318                  ],
1319                  len: Some(20),
1320                ))),
1321              },
1322              "footer": {
1323                "": Meta(Some(MetaInner(
1324                  rem: [
1325                    [
1326                      "!limit",
1327                      s,
1328                      2,
1329                      5,
1330                    ],
1331                  ],
1332                  len: Some(33),
1333                ))),
1334              },
1335            },
1336          },
1337        }
1338        "###);
1339    }
1340}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
