relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.attrs().max_bytes.is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.attrs().max_bytes,
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        if let Some(size_state) = self.size_state.last() {
97            // If our current depth is the one where we found a bag_size attribute, this means we
98            // are done processing a databag. Pop the bag size state.
99            if state.depth() == size_state.encountered_at_depth {
100                self.size_state.pop().unwrap();
101            }
102        }
103
104        for size_state in self.size_state.iter_mut() {
105            // After processing a value, update the remaining bag sizes. We have a separate if-let
106            // here in case somebody defines nested databags (a struct with bag_size that contains
107            // another struct with a different bag_size), in case we just exited a databag we want
108            // to update the bag_size_state of the outer databag with the remaining size.
109            //
110            // This also has to happen after string trimming, which is why it's running in
111            // after_process.
112
113            if state.entered_anything() {
114                // Do not subtract if state is from newtype struct.
115                let item_length = relay_protocol::estimate_size_flat(value) + 1;
116                size_state.size_remaining = size_state
117                    .size_remaining
118                    .map(|size| size.saturating_sub(item_length));
119            }
120        }
121
122        Ok(())
123    }
124
125    fn process_string(
126        &mut self,
127        value: &mut String,
128        meta: &mut Meta,
129        state: &ProcessingState<'_>,
130    ) -> ProcessingResult {
131        if let Some(max_chars) = state.attrs().max_chars {
132            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
133        }
134
135        if !state.attrs().trim {
136            return Ok(());
137        }
138
139        if let Some(size_state) = self.size_state.last() {
140            if let Some(size_remaining) = size_state.size_remaining {
141                trim_string(value, meta, size_remaining, 0);
142            }
143        }
144
145        Ok(())
146    }
147
148    fn process_array<T>(
149        &mut self,
150        value: &mut Array<T>,
151        meta: &mut Meta,
152        state: &ProcessingState<'_>,
153    ) -> ProcessingResult
154    where
155        T: ProcessValue,
156    {
157        if !state.attrs().trim {
158            return Ok(());
159        }
160
161        // If we need to check the bag size, then we go down a different path
162        if !self.size_state.is_empty() {
163            let original_length = value.len();
164
165            if self.should_remove_container(value, state) {
166                return Err(ProcessingAction::DeleteValueHard);
167            }
168
169            let mut split_index = None;
170            for (index, item) in value.iter_mut().enumerate() {
171                if self.remaining_size() == Some(0) {
172                    split_index = Some(index);
173                    break;
174                }
175
176                let item_state = state.enter_index(index, None, ValueType::for_field(item));
177                processor::process_value(item, self, &item_state)?;
178            }
179
180            if let Some(split_index) = split_index {
181                let _ = value.split_off(split_index);
182            }
183
184            if value.len() != original_length {
185                meta.set_original_length(Some(original_length));
186            }
187        } else {
188            value.process_child_values(self, state)?;
189        }
190
191        Ok(())
192    }
193
194    fn process_object<T>(
195        &mut self,
196        value: &mut Object<T>,
197        meta: &mut Meta,
198        state: &ProcessingState<'_>,
199    ) -> ProcessingResult
200    where
201        T: ProcessValue,
202    {
203        if !state.attrs().trim {
204            return Ok(());
205        }
206
207        // If we need to check the bag size, then we go down a different path
208        if !self.size_state.is_empty() {
209            let original_length = value.len();
210
211            if self.should_remove_container(value, state) {
212                return Err(ProcessingAction::DeleteValueHard);
213            }
214
215            let mut split_key = None;
216            for (key, item) in value.iter_mut() {
217                if self.remaining_size() == Some(0) {
218                    split_key = Some(key.to_owned());
219                    break;
220                }
221
222                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
223                processor::process_value(item, self, &item_state)?;
224            }
225
226            if let Some(split_key) = split_key {
227                let _ = value.split_off(&split_key);
228            }
229
230            if value.len() != original_length {
231                meta.set_original_length(Some(original_length));
232            }
233        } else {
234            value.process_child_values(self, state)?;
235        }
236
237        Ok(())
238    }
239
240    fn process_value(
241        &mut self,
242        value: &mut Value,
243        _meta: &mut Meta,
244        state: &ProcessingState<'_>,
245    ) -> ProcessingResult {
246        if !state.attrs().trim {
247            return Ok(());
248        }
249
250        match value {
251            Value::Array(_) | Value::Object(_) => {
252                if self.remaining_depth(state) == Some(1) {
253                    if let Ok(x) = serde_json::to_string(&value) {
254                        // Error case should not be possible
255                        *value = Value::String(x);
256                    }
257                }
258            }
259            _ => (),
260        }
261
262        value.process_child_values(self, state)?;
263        Ok(())
264    }
265
266    fn process_replay(
267        &mut self,
268        replay: &mut Replay,
269        _: &mut Meta,
270        state: &ProcessingState<'_>,
271    ) -> ProcessingResult {
272        replay.process_child_values(self, state)
273    }
274
275    fn process_raw_stacktrace(
276        &mut self,
277        stacktrace: &mut RawStacktrace,
278        _meta: &mut Meta,
279        state: &ProcessingState<'_>,
280    ) -> ProcessingResult {
281        if !state.attrs().trim {
282            return Ok(());
283        }
284
285        processor::apply(&mut stacktrace.frames, |frames, meta| {
286            enforce_frame_hard_limit(frames, meta, 200, 50);
287            Ok(())
288        })?;
289
290        stacktrace.process_child_values(self, state)?;
291
292        processor::apply(&mut stacktrace.frames, |frames, _meta| {
293            slim_frame_data(frames, 50);
294            Ok(())
295        })?;
296
297        Ok(())
298    }
299}
300
301/// Trims the string to the given maximum length and updates meta data.
302fn trim_string(value: &mut String, meta: &mut Meta, max_chars: usize, max_chars_allowance: usize) {
303    let hard_limit = max_chars + max_chars_allowance;
304
305    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
306        return;
307    }
308
309    processor::process_chunked_value(value, meta, |chunks| {
310        let mut length = 0;
311        let mut new_chunks = vec![];
312
313        for chunk in chunks {
314            let chunk_chars = chunk.count();
315
316            // if the entire chunk fits, just put it in
317            if length + chunk_chars < max_chars {
318                new_chunks.push(chunk);
319                length += chunk_chars;
320                continue;
321            }
322
323            match chunk {
324                // if there is enough space for this chunk and the 3 character
325                // ellipsis marker we can push the remaining chunk
326                Chunk::Redaction { .. } => {
327                    if length + chunk_chars + 3 < hard_limit {
328                        new_chunks.push(chunk);
329                    }
330                }
331
332                // if this is a text chunk, we can put the remaining characters in.
333                Chunk::Text { text } => {
334                    let mut remaining = String::new();
335                    for c in text.chars() {
336                        if length + 3 < max_chars {
337                            remaining.push(c);
338                        } else {
339                            break;
340                        }
341                        length += 1;
342                    }
343
344                    new_chunks.push(Chunk::Text {
345                        text: Cow::Owned(remaining),
346                    });
347                }
348            }
349
350            new_chunks.push(Chunk::Redaction {
351                text: Cow::Borrowed("..."),
352                rule_id: Cow::Borrowed("!limit"),
353                ty: RemarkType::Substituted,
354            });
355            break;
356        }
357
358        new_chunks
359    });
360}
361
362/// Trim down the frame list to a hard limit.
363///
364/// The total limit is `recent_frames` + `old_frames`.
365/// `recent_frames` is the number of frames to keep from the beginning of the list,
366/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
367/// the oldest frames up the stack.
368///
369/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
370/// the recursion.
371fn enforce_frame_hard_limit(
372    frames: &mut Array<Frame>,
373    meta: &mut Meta,
374    recent_frames: usize,
375    old_frames: usize,
376) {
377    let original_length = frames.len();
378    let limit = recent_frames + old_frames;
379    if original_length > limit {
380        meta.set_original_length(Some(original_length));
381        let _ = frames.drain(old_frames..original_length - recent_frames);
382    }
383}
384
385/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
386///
387/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
388fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
389    let frames_len = frames.len();
390
391    if frames_len <= frame_allowance {
392        return;
393    }
394
395    // Avoid ownership issues by only storing indices
396    let mut app_frame_indices = Vec::with_capacity(frames_len);
397    let mut system_frame_indices = Vec::with_capacity(frames_len);
398
399    for (i, frame) in frames.iter().enumerate() {
400        if let Some(frame) = frame.value() {
401            match frame.in_app.value() {
402                Some(true) => app_frame_indices.push(i),
403                _ => system_frame_indices.push(i),
404            }
405        }
406    }
407
408    let app_count = app_frame_indices.len();
409    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
410    let system_frames_to_remove = system_frame_indices
411        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
412        .unwrap_or(&[]);
413
414    let remaining = frames_len
415        .saturating_sub(frame_allowance)
416        .saturating_sub(system_frames_to_remove.len());
417    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
418    let app_frames_to_remove = app_frame_indices
419        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
420        .unwrap_or(&[]);
421
422    // TODO: Which annotation to set?
423
424    for i in system_frames_to_remove.iter().chain(app_frames_to_remove) {
425        if let Some(frame) = frames.get_mut(*i) {
426            if let Some(ref mut frame) = frame.value_mut().as_mut() {
427                frame.vars = Annotated::empty();
428                frame.pre_context = Annotated::empty();
429                frame.post_context = Annotated::empty();
430            }
431        }
432    }
433}
434
435#[cfg(test)]
436mod tests {
437    use std::iter::repeat_n;
438
439    use crate::MaxChars;
440    use chrono::DateTime;
441    use relay_event_schema::protocol::{
442        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, SentryTags, Span, SpanId,
443        TagEntry, Tags, Timestamp, TraceId, Values,
444    };
445    use relay_protocol::{Map, Remark, SerializableAnnotated, get_value};
446    use similar_asserts::assert_eq;
447
448    use super::*;
449
450    #[test]
451    fn test_string_trimming() {
452        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
453        processor::apply(&mut value, |v, m| {
454            trim_string(v, m, 20, 0);
455            Ok(())
456        })
457        .unwrap();
458
459        assert_eq!(
460            value,
461            Annotated(Some("This is my long s...".into()), {
462                let mut meta = Meta::default();
463                meta.add_remark(Remark {
464                    ty: RemarkType::Substituted,
465                    rule_id: "!limit".to_owned(),
466                    range: Some((17, 20)),
467                });
468                meta.set_original_length(Some(46));
469                meta
470            })
471        );
472    }
473
474    #[test]
475    fn test_basic_trimming() {
476        let mut processor = TrimmingProcessor::new();
477
478        let mut event = Annotated::new(Event {
479            logger: Annotated::new("x".repeat(300)),
480            ..Default::default()
481        });
482
483        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
484
485        let mut expected = Annotated::new("x".repeat(300));
486        processor::apply(&mut expected, |v, m| {
487            trim_string(v, m, MaxChars::Logger.limit(), 0);
488            Ok(())
489        })
490        .unwrap();
491
492        assert_eq!(event.value().unwrap().logger, expected);
493    }
494
495    #[test]
496    fn test_max_char_allowance() {
497        let string = "This string requires some allowance to fit!";
498        let mut value = Annotated::new(string.to_owned()); // len == 43
499        processor::apply(&mut value, |v, m| {
500            trim_string(v, m, 40, 5);
501            Ok(())
502        })
503        .unwrap();
504
505        assert_eq!(value, Annotated::new(string.to_owned()));
506    }
507
508    #[test]
509    fn test_databag_stripping() {
510        let mut processor = TrimmingProcessor::new();
511
512        fn make_nested_object(depth: usize) -> Annotated<Value> {
513            if depth == 0 {
514                return Annotated::new(Value::String("max depth".to_owned()));
515            }
516            let mut rv = Object::new();
517            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
518            Annotated::new(Value::Object(rv))
519        }
520
521        let databag = Annotated::new({
522            let mut map = Object::new();
523            map.insert(
524                "key_1".to_owned(),
525                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
526            );
527            map.insert(
528                "key_2".to_owned(),
529                make_nested_object(8).map_value(ExtraValue),
530            );
531            map.insert(
532                "key_3".to_owned(),
533                // innermost key (string) is entering json stringify codepath
534                make_nested_object(5).map_value(ExtraValue),
535            );
536            map
537        });
538        let mut event = Annotated::new(Event {
539            extra: databag,
540            ..Default::default()
541        });
542
543        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
544        let stripped_extra = &event.value().unwrap().extra;
545        let json = stripped_extra.to_json_pretty().unwrap();
546
547        assert_eq!(
548            json,
549            r#"{
550  "key_1": "value 1",
551  "key_2": {
552    "key8": {
553      "key7": {
554        "key6": {
555          "key5": {
556            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
557          }
558        }
559      }
560    }
561  },
562  "key_3": {
563    "key5": {
564      "key4": {
565        "key3": {
566          "key2": {
567            "key1": "max depth"
568          }
569        }
570      }
571    }
572  }
573}"#
574        );
575    }
576
577    #[test]
578    fn test_databag_array_stripping() {
579        let mut processor = TrimmingProcessor::new();
580
581        let databag = Annotated::new({
582            let mut map = Object::new();
583            for idx in 0..100 {
584                map.insert(
585                    format!("key_{idx}"),
586                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
587                );
588            }
589            map
590        });
591        let mut event = Annotated::new(Event {
592            extra: databag,
593            ..Default::default()
594        });
595
596        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
597        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
598
599        insta::assert_ron_snapshot!(stripped_extra);
600    }
601
602    #[test]
603    fn test_tags_stripping() {
604        let mut processor = TrimmingProcessor::new();
605
606        let mut event = Annotated::new(Event {
607            tags: Annotated::new(Tags(
608                vec![Annotated::new(TagEntry(
609                    Annotated::new("x".repeat(300)),
610                    Annotated::new("x".repeat(300)),
611                ))]
612                .into(),
613            )),
614            ..Default::default()
615        });
616
617        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
618        let json = event
619            .value()
620            .unwrap()
621            .tags
622            .payload_to_json_pretty()
623            .unwrap();
624
625        assert_eq!(
626            json,
627            r#"[
628  [
629    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
630    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
631  ]
632]"#
633        );
634    }
635
636    #[test]
637    fn test_databag_state_leak() {
638        let event = Annotated::new(Event {
639            breadcrumbs: Annotated::new(Values::new(
640                repeat_n(
641                    Annotated::new(Breadcrumb {
642                        data: {
643                            let mut map = Map::new();
644                            map.insert(
645                                "spamspamspam".to_owned(),
646                                Annotated::new(Value::String("blablabla".to_owned())),
647                            );
648                            Annotated::new(map)
649                        },
650                        ..Default::default()
651                    }),
652                    200,
653                )
654                .collect(),
655            )),
656            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
657                ty: Annotated::new("TypeError".to_owned()),
658                value: Annotated::new("important error message".to_owned().into()),
659                stacktrace: Annotated::new(
660                    RawStacktrace {
661                        frames: Annotated::new(
662                            repeat_n(
663                                Annotated::new(Frame {
664                                    function: Annotated::new("importantFunctionName".to_owned()),
665                                    symbol: Annotated::new("important_symbol".to_owned()),
666                                    ..Default::default()
667                                }),
668                                200,
669                            )
670                            .collect(),
671                        ),
672                        ..Default::default()
673                    }
674                    .into(),
675                ),
676                ..Default::default()
677            })])),
678            ..Default::default()
679        });
680
681        let mut processor = TrimmingProcessor::new();
682        let mut stripped_event = event.clone();
683        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
684            .unwrap();
685
686        assert_eq!(
687            event.to_json_pretty().unwrap(),
688            stripped_event.to_json_pretty().unwrap()
689        );
690    }
691
692    #[test]
693    fn test_custom_context_trimming() {
694        let mut contexts = Contexts::new();
695        for i in 1..2 {
696            contexts.insert(format!("despacito{i}"), {
697                let mut context = Object::new();
698                context.insert(
699                    "foo".to_owned(),
700                    Annotated::new(Value::String("a".repeat(4000))),
701                );
702                context.insert(
703                    "bar".to_owned(),
704                    Annotated::new(Value::String("a".repeat(5000))),
705                );
706                Context::Other(context)
707            });
708        }
709
710        let mut contexts = Annotated::new(contexts);
711        let mut processor = TrimmingProcessor::new();
712        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
713
714        let contexts = contexts.value().unwrap();
715        for i in 1..2 {
716            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
717                Context::Other(x) => x,
718                _ => panic!("Context has changed type!"),
719            };
720
721            assert_eq!(
722                other
723                    .get("bar")
724                    .unwrap()
725                    .value()
726                    .unwrap()
727                    .as_str()
728                    .unwrap()
729                    .len(),
730                5000
731            );
732            assert_eq!(
733                other
734                    .get("foo")
735                    .unwrap()
736                    .value()
737                    .unwrap()
738                    .as_str()
739                    .unwrap()
740                    .len(),
741                3189
742            );
743        }
744    }
745
746    #[test]
747    fn test_extra_trimming_long_arrays() {
748        let mut extra = Object::new();
749        extra.insert("foo".to_owned(), {
750            Annotated::new(ExtraValue(Value::Array(
751                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
752            )))
753        });
754
755        let mut event = Annotated::new(Event {
756            extra: Annotated::new(extra),
757            ..Default::default()
758        });
759
760        let mut processor = TrimmingProcessor::new();
761        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
762
763        let arr = match event
764            .value()
765            .unwrap()
766            .extra
767            .value()
768            .unwrap()
769            .get("foo")
770            .unwrap()
771            .value()
772            .unwrap()
773        {
774            ExtraValue(Value::Array(x)) => x,
775            x => panic!("Wrong type: {x:?}"),
776        };
777
778        // this is larger / 2 for the extra value
779        assert_eq!(arr.len(), 8192);
780    }
781
782    // TODO(ja): Enable this test
783    // #[test]
784    // fn test_newtypes_do_not_add_to_depth() {
785    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
786    //     struct WrappedString(String);
787
788    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
789    //     struct StructChild2 {
790    //         inner: Annotated<WrappedString>,
791    //     }
792
793    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
794    //     struct StructChild {
795    //         inner: Annotated<StructChild2>,
796    //     }
797
798    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
799    //     struct Struct {
800    //         #[metastructure(bag_size = "small")]
801    //         inner: Annotated<StructChild>,
802    //     }
803
804    //     let mut value = Annotated::new(Struct {
805    //         inner: Annotated::new(StructChild {
806    //             inner: Annotated::new(StructChild2 {
807    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
808    //             }),
809    //         }),
810    //     });
811
812    //     let mut processor = TrimmingProcessor::new();
813    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
814
815    //     // Ensure stack does not leak with newtypes
816    //     assert!(processor.bag_size_state.is_empty());
817
818    //     assert_eq!(
819    //         value.to_json().unwrap(),
820    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
821    //     );
822    // }
823
824    #[test]
825    fn test_frameqty_equals_limit() {
826        fn create_frame(filename: &str) -> Annotated<Frame> {
827            Annotated::new(Frame {
828                filename: Annotated::new(filename.into()),
829                ..Default::default()
830            })
831        }
832
833        let mut frames = Annotated::new(vec![
834            create_frame("foo3.py"),
835            create_frame("foo4.py"),
836            create_frame("foo5.py"),
837        ]);
838
839        processor::apply(&mut frames, |f, m| {
840            enforce_frame_hard_limit(f, m, 3, 0);
841            Ok(())
842        })
843        .unwrap();
844
845        processor::apply(&mut frames, |f, m| {
846            enforce_frame_hard_limit(f, m, 1, 2);
847            Ok(())
848        })
849        .unwrap();
850
851        // original_length isn't set, when limit is equal to length, as no trimming took place.
852        assert!(frames.meta().original_length().is_none());
853    }
854
855    #[test]
856    fn test_frame_hard_limit() {
857        fn create_frame(filename: &str) -> Annotated<Frame> {
858            Annotated::new(Frame {
859                filename: Annotated::new(filename.into()),
860                ..Default::default()
861            })
862        }
863
864        let mut frames = Annotated::new(vec![
865            create_frame("foo1.py"),
866            create_frame("foo2.py"),
867            create_frame("foo3.py"),
868            create_frame("foo4.py"),
869            create_frame("foo5.py"),
870        ]);
871
872        processor::apply(&mut frames, |f, m| {
873            enforce_frame_hard_limit(f, m, 3, 0);
874            Ok(())
875        })
876        .unwrap();
877
878        let mut expected_meta = Meta::default();
879        expected_meta.set_original_length(Some(5));
880
881        assert_eq!(
882            frames,
883            Annotated(
884                Some(vec![
885                    create_frame("foo3.py"),
886                    create_frame("foo4.py"),
887                    create_frame("foo5.py"),
888                ]),
889                expected_meta
890            )
891        );
892    }
893
894    #[test]
895    fn test_frame_hard_limit_recent_old() {
896        fn create_frame(filename: &str) -> Annotated<Frame> {
897            Annotated::new(Frame {
898                filename: Annotated::new(filename.into()),
899                ..Default::default()
900            })
901        }
902
903        let mut frames = Annotated::new(vec![
904            create_frame("foo1.py"),
905            create_frame("foo2.py"),
906            create_frame("foo3.py"),
907            create_frame("foo4.py"),
908            create_frame("foo5.py"),
909        ]);
910
911        processor::apply(&mut frames, |f, m| {
912            enforce_frame_hard_limit(f, m, 2, 1);
913            Ok(())
914        })
915        .unwrap();
916
917        let mut expected_meta = Meta::default();
918        expected_meta.set_original_length(Some(5));
919
920        assert_eq!(
921            frames,
922            Annotated(
923                Some(vec![
924                    create_frame("foo1.py"),
925                    create_frame("foo4.py"),
926                    create_frame("foo5.py"),
927                ]),
928                expected_meta
929            )
930        );
931    }
932
933    #[test]
934    fn test_slim_frame_data_under_max() {
935        let mut frames = vec![Annotated::new(Frame {
936            filename: Annotated::new("foo".into()),
937            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
938            context_line: Annotated::new("b".to_owned()),
939            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
940            ..Default::default()
941        })];
942
943        let old_frames = frames.clone();
944        slim_frame_data(&mut frames, 4);
945
946        assert_eq!(frames, old_frames);
947    }
948
949    #[test]
950    fn test_slim_frame_data_over_max() {
951        let mut frames = vec![];
952
953        for n in 0..5 {
954            frames.push(Annotated::new(Frame {
955                filename: Annotated::new(format!("foo {n}").into()),
956                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
957                context_line: Annotated::new("b".to_owned()),
958                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
959                ..Default::default()
960            }));
961        }
962
963        slim_frame_data(&mut frames, 4);
964
965        let expected = vec![
966            Annotated::new(Frame {
967                filename: Annotated::new("foo 0".into()),
968                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
969                context_line: Annotated::new("b".to_owned()),
970                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
971                ..Default::default()
972            }),
973            Annotated::new(Frame {
974                filename: Annotated::new("foo 1".into()),
975                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
976                context_line: Annotated::new("b".to_owned()),
977                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
978                ..Default::default()
979            }),
980            Annotated::new(Frame {
981                filename: Annotated::new("foo 2".into()),
982                context_line: Annotated::new("b".to_owned()),
983                ..Default::default()
984            }),
985            Annotated::new(Frame {
986                filename: Annotated::new("foo 3".into()),
987                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
988                context_line: Annotated::new("b".to_owned()),
989                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
990                ..Default::default()
991            }),
992            Annotated::new(Frame {
993                filename: Annotated::new("foo 4".into()),
994                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
995                context_line: Annotated::new("b".to_owned()),
996                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
997                ..Default::default()
998            }),
999        ];
1000
1001        assert_eq!(frames, expected);
1002    }
1003
1004    #[test]
1005    fn test_too_many_spans_trimmed() {
1006        let span = Span {
1007            platform: Annotated::new("a".repeat(1024 * 90)),
1008            sentry_tags: Annotated::new(SentryTags {
1009                release: Annotated::new("b".repeat(1024 * 100)),
1010                ..Default::default()
1011            }),
1012            ..Default::default()
1013        };
1014        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1015            .take(10)
1016            .collect();
1017
1018        let mut event = Annotated::new(Event {
1019            spans: Annotated::new(spans.clone()),
1020            ..Default::default()
1021        });
1022
1023        let mut processor = TrimmingProcessor::new();
1024        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1025
1026        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1027        assert_eq!(trimmed_spans.len(), 5);
1028
1029        // The actual spans were not touched:
1030        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1031    }
1032
1033    #[test]
1034    fn test_untrimmable_fields() {
1035        let original_description = "a".repeat(819163);
1036        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1037        let mut event = Annotated::new(Event {
1038            spans: Annotated::new(vec![
1039                Span {
1040                    description: original_description.clone().into(),
1041                    ..Default::default()
1042                }
1043                .into(),
1044                Span {
1045                    trace_id: original_trace_id.into(),
1046                    ..Default::default()
1047                }
1048                .into(),
1049            ]),
1050            ..Default::default()
1051        });
1052
1053        let mut processor = TrimmingProcessor::new();
1054        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1055
1056        assert_eq!(
1057            get_value!(event.spans[0].description!),
1058            &original_description
1059        );
1060        // Trace ID would be trimmed without `trim = "false"`
1061        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1062    }
1063
1064    #[test]
1065    fn test_untrimmable_fields_drop() {
1066        let original_description = "a".repeat(819164);
1067        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1068        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1069        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1070        let original_op = "e".repeat(129);
1071
1072        let mut event = Annotated::new(Event {
1073            spans: Annotated::new(vec![
1074                Span {
1075                    description: original_description.clone().into(),
1076                    ..Default::default()
1077                }
1078                .into(),
1079                Span {
1080                    span_id: original_span_id.into(),
1081                    trace_id: original_trace_id.into(),
1082                    segment_id: original_segment_id.into(),
1083                    is_segment: false.into(),
1084                    op: original_op.clone().into(),
1085                    start_timestamp: Timestamp(
1086                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1087                            .unwrap()
1088                            .into(),
1089                    )
1090                    .into(),
1091                    timestamp: Timestamp(
1092                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1093                            .unwrap()
1094                            .into(),
1095                    )
1096                    .into(),
1097                    ..Default::default()
1098                }
1099                .into(),
1100            ]),
1101            ..Default::default()
1102        });
1103
1104        let mut processor = TrimmingProcessor::new();
1105        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1106
1107        assert_eq!(
1108            get_value!(event.spans[0].description!),
1109            &original_description
1110        );
1111        // These fields would be dropped without `trim = "false"`
1112        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1113        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1114        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1115        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1116        // span.op is trimmed to its max_chars, but not dropped:
1117        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1118        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1119        assert!(get_value!(event.spans[1].timestamp).is_some());
1120    }
1121}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
