relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.attrs().max_bytes.is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.attrs().max_bytes,
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        if let Some(size_state) = self.size_state.last() {
97            // If our current depth is the one where we found a bag_size attribute, this means we
98            // are done processing a databag. Pop the bag size state.
99            if state.depth() == size_state.encountered_at_depth {
100                self.size_state.pop().unwrap();
101            }
102        }
103
104        for size_state in self.size_state.iter_mut() {
105            // After processing a value, update the remaining bag sizes. We have a separate if-let
106            // here in case somebody defines nested databags (a struct with bag_size that contains
107            // another struct with a different bag_size), in case we just exited a databag we want
108            // to update the bag_size_state of the outer databag with the remaining size.
109            //
110            // This also has to happen after string trimming, which is why it's running in
111            // after_process.
112
113            if state.entered_anything() {
114                // Do not subtract if state is from newtype struct.
115                let item_length = relay_protocol::estimate_size_flat(value) + 1;
116                size_state.size_remaining = size_state
117                    .size_remaining
118                    .map(|size| size.saturating_sub(item_length));
119            }
120        }
121
122        Ok(())
123    }
124
125    fn process_string(
126        &mut self,
127        value: &mut String,
128        meta: &mut Meta,
129        state: &ProcessingState<'_>,
130    ) -> ProcessingResult {
131        if let Some(max_chars) = state.attrs().max_chars {
132            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
133        }
134
135        if !state.attrs().trim {
136            return Ok(());
137        }
138
139        if let Some(size_state) = self.size_state.last() {
140            if let Some(size_remaining) = size_state.size_remaining {
141                trim_string(value, meta, size_remaining, 0);
142            }
143        }
144
145        Ok(())
146    }
147
148    fn process_array<T>(
149        &mut self,
150        value: &mut Array<T>,
151        meta: &mut Meta,
152        state: &ProcessingState<'_>,
153    ) -> ProcessingResult
154    where
155        T: ProcessValue,
156    {
157        if !state.attrs().trim {
158            return Ok(());
159        }
160
161        // If we need to check the bag size, then we go down a different path
162        if !self.size_state.is_empty() {
163            let original_length = value.len();
164
165            if self.should_remove_container(value, state) {
166                return Err(ProcessingAction::DeleteValueHard);
167            }
168
169            let mut split_index = None;
170            for (index, item) in value.iter_mut().enumerate() {
171                if self.remaining_size() == Some(0) {
172                    split_index = Some(index);
173                    break;
174                }
175
176                let item_state = state.enter_index(index, None, ValueType::for_field(item));
177                processor::process_value(item, self, &item_state)?;
178            }
179
180            if let Some(split_index) = split_index {
181                let _ = value.split_off(split_index);
182            }
183
184            if value.len() != original_length {
185                meta.set_original_length(Some(original_length));
186            }
187        } else {
188            value.process_child_values(self, state)?;
189        }
190
191        Ok(())
192    }
193
194    fn process_object<T>(
195        &mut self,
196        value: &mut Object<T>,
197        meta: &mut Meta,
198        state: &ProcessingState<'_>,
199    ) -> ProcessingResult
200    where
201        T: ProcessValue,
202    {
203        if !state.attrs().trim {
204            return Ok(());
205        }
206
207        // If we need to check the bag size, then we go down a different path
208        if !self.size_state.is_empty() {
209            let original_length = value.len();
210
211            if self.should_remove_container(value, state) {
212                return Err(ProcessingAction::DeleteValueHard);
213            }
214
215            let mut split_key = None;
216            for (key, item) in value.iter_mut() {
217                if self.remaining_size() == Some(0) {
218                    split_key = Some(key.to_owned());
219                    break;
220                }
221
222                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
223                processor::process_value(item, self, &item_state)?;
224            }
225
226            if let Some(split_key) = split_key {
227                let _ = value.split_off(&split_key);
228            }
229
230            if value.len() != original_length {
231                meta.set_original_length(Some(original_length));
232            }
233        } else {
234            value.process_child_values(self, state)?;
235        }
236
237        Ok(())
238    }
239
240    fn process_value(
241        &mut self,
242        value: &mut Value,
243        _meta: &mut Meta,
244        state: &ProcessingState<'_>,
245    ) -> ProcessingResult {
246        if !state.attrs().trim {
247            return Ok(());
248        }
249
250        match value {
251            Value::Array(_) | Value::Object(_) => {
252                if self.remaining_depth(state) == Some(1) {
253                    if let Ok(x) = serde_json::to_string(&value) {
254                        // Error case should not be possible
255                        *value = Value::String(x);
256                    }
257                }
258            }
259            _ => (),
260        }
261
262        value.process_child_values(self, state)?;
263        Ok(())
264    }
265
266    fn process_replay(
267        &mut self,
268        replay: &mut Replay,
269        _: &mut Meta,
270        state: &ProcessingState<'_>,
271    ) -> ProcessingResult {
272        replay.process_child_values(self, state)
273    }
274
275    fn process_raw_stacktrace(
276        &mut self,
277        stacktrace: &mut RawStacktrace,
278        _meta: &mut Meta,
279        state: &ProcessingState<'_>,
280    ) -> ProcessingResult {
281        if !state.attrs().trim {
282            return Ok(());
283        }
284
285        processor::apply(&mut stacktrace.frames, |frames, meta| {
286            enforce_frame_hard_limit(frames, meta, 200, 50);
287            Ok(())
288        })?;
289
290        stacktrace.process_child_values(self, state)?;
291
292        processor::apply(&mut stacktrace.frames, |frames, _meta| {
293            slim_frame_data(frames, 50);
294            Ok(())
295        })?;
296
297        Ok(())
298    }
299}
300
301/// Trims the string to the given maximum length and updates meta data.
302fn trim_string(value: &mut String, meta: &mut Meta, max_chars: usize, max_chars_allowance: usize) {
303    let hard_limit = max_chars + max_chars_allowance;
304
305    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
306        return;
307    }
308
309    processor::process_chunked_value(value, meta, |chunks| {
310        let mut length = 0;
311        let mut new_chunks = vec![];
312
313        for chunk in chunks {
314            let chunk_chars = chunk.count();
315
316            // if the entire chunk fits, just put it in
317            if length + chunk_chars < max_chars {
318                new_chunks.push(chunk);
319                length += chunk_chars;
320                continue;
321            }
322
323            match chunk {
324                // if there is enough space for this chunk and the 3 character
325                // ellipsis marker we can push the remaining chunk
326                Chunk::Redaction { .. } => {
327                    if length + chunk_chars + 3 < hard_limit {
328                        new_chunks.push(chunk);
329                    }
330                }
331
332                // if this is a text chunk, we can put the remaining characters in.
333                Chunk::Text { text } => {
334                    let mut remaining = String::new();
335                    for c in text.chars() {
336                        if length + 3 < max_chars {
337                            remaining.push(c);
338                        } else {
339                            break;
340                        }
341                        length += 1;
342                    }
343
344                    new_chunks.push(Chunk::Text {
345                        text: Cow::Owned(remaining),
346                    });
347                }
348            }
349
350            new_chunks.push(Chunk::Redaction {
351                text: Cow::Borrowed("..."),
352                rule_id: Cow::Borrowed("!limit"),
353                ty: RemarkType::Substituted,
354            });
355            break;
356        }
357
358        new_chunks
359    });
360}
361
362/// Trim down the frame list to a hard limit.
363///
364/// The total limit is `recent_frames` + `old_frames`.
365/// `recent_frames` is the number of frames to keep from the beginning of the list,
366/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
367/// the oldest frames up the stack.
368///
369/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
370/// the recursion.
371fn enforce_frame_hard_limit(
372    frames: &mut Array<Frame>,
373    meta: &mut Meta,
374    recent_frames: usize,
375    old_frames: usize,
376) {
377    let original_length = frames.len();
378    let limit = recent_frames + old_frames;
379    if original_length > limit {
380        meta.set_original_length(Some(original_length));
381        let _ = frames.drain(old_frames..original_length - recent_frames);
382    }
383}
384
385/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
386///
387/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
388fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
389    let frames_len = frames.len();
390
391    if frames_len <= frame_allowance {
392        return;
393    }
394
395    // Avoid ownership issues by only storing indices
396    let mut app_frame_indices = Vec::with_capacity(frames_len);
397    let mut system_frame_indices = Vec::with_capacity(frames_len);
398
399    for (i, frame) in frames.iter().enumerate() {
400        if let Some(frame) = frame.value() {
401            match frame.in_app.value() {
402                Some(true) => app_frame_indices.push(i),
403                _ => system_frame_indices.push(i),
404            }
405        }
406    }
407
408    let app_count = app_frame_indices.len();
409    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
410    let system_frames_to_remove = system_frame_indices
411        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
412        .unwrap_or(&[]);
413
414    let remaining = frames_len
415        .saturating_sub(frame_allowance)
416        .saturating_sub(system_frames_to_remove.len());
417    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
418    let app_frames_to_remove = app_frame_indices
419        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
420        .unwrap_or(&[]);
421
422    // TODO: Which annotation to set?
423
424    for i in system_frames_to_remove.iter().chain(app_frames_to_remove) {
425        if let Some(frame) = frames.get_mut(*i) {
426            if let Some(ref mut frame) = frame.value_mut().as_mut() {
427                frame.vars = Annotated::empty();
428                frame.pre_context = Annotated::empty();
429                frame.post_context = Annotated::empty();
430            }
431        }
432    }
433}
434
435#[cfg(test)]
436mod tests {
437    use std::iter::repeat_n;
438
439    use crate::MaxChars;
440    use chrono::DateTime;
441    use relay_event_schema::protocol::{
442        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, SentryTags, Span, SpanId,
443        TagEntry, Tags, Timestamp, TraceId, Values,
444    };
445    use relay_protocol::{Map, Remark, SerializableAnnotated, get_value};
446    use similar_asserts::assert_eq;
447
448    use super::*;
449
450    #[test]
451    fn test_string_trimming() {
452        let mut value =
453            Annotated::new("This is my long string I want to have trimmed!".to_string());
454        processor::apply(&mut value, |v, m| {
455            trim_string(v, m, 20, 0);
456            Ok(())
457        })
458        .unwrap();
459
460        assert_eq!(
461            value,
462            Annotated(Some("This is my long s...".into()), {
463                let mut meta = Meta::default();
464                meta.add_remark(Remark {
465                    ty: RemarkType::Substituted,
466                    rule_id: "!limit".to_string(),
467                    range: Some((17, 20)),
468                });
469                meta.set_original_length(Some(46));
470                meta
471            })
472        );
473    }
474
475    #[test]
476    fn test_basic_trimming() {
477        let mut processor = TrimmingProcessor::new();
478
479        let mut event = Annotated::new(Event {
480            logger: Annotated::new("x".repeat(300)),
481            ..Default::default()
482        });
483
484        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
485
486        let mut expected = Annotated::new("x".repeat(300));
487        processor::apply(&mut expected, |v, m| {
488            trim_string(v, m, MaxChars::Logger.limit(), 0);
489            Ok(())
490        })
491        .unwrap();
492
493        assert_eq!(event.value().unwrap().logger, expected);
494    }
495
496    #[test]
497    fn test_max_char_allowance() {
498        let string = "This string requires some allowance to fit!";
499        let mut value = Annotated::new(string.to_owned()); // len == 43
500        processor::apply(&mut value, |v, m| {
501            trim_string(v, m, 40, 5);
502            Ok(())
503        })
504        .unwrap();
505
506        assert_eq!(value, Annotated::new(string.to_owned()));
507    }
508
509    #[test]
510    fn test_databag_stripping() {
511        let mut processor = TrimmingProcessor::new();
512
513        fn make_nested_object(depth: usize) -> Annotated<Value> {
514            if depth == 0 {
515                return Annotated::new(Value::String("max depth".to_string()));
516            }
517            let mut rv = Object::new();
518            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
519            Annotated::new(Value::Object(rv))
520        }
521
522        let databag = Annotated::new({
523            let mut map = Object::new();
524            map.insert(
525                "key_1".to_string(),
526                Annotated::new(ExtraValue(Value::String("value 1".to_string()))),
527            );
528            map.insert(
529                "key_2".to_string(),
530                make_nested_object(8).map_value(ExtraValue),
531            );
532            map.insert(
533                "key_3".to_string(),
534                // innermost key (string) is entering json stringify codepath
535                make_nested_object(5).map_value(ExtraValue),
536            );
537            map
538        });
539        let mut event = Annotated::new(Event {
540            extra: databag,
541            ..Default::default()
542        });
543
544        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
545        let stripped_extra = &event.value().unwrap().extra;
546        let json = stripped_extra.to_json_pretty().unwrap();
547
548        assert_eq!(
549            json,
550            r#"{
551  "key_1": "value 1",
552  "key_2": {
553    "key8": {
554      "key7": {
555        "key6": {
556          "key5": {
557            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
558          }
559        }
560      }
561    }
562  },
563  "key_3": {
564    "key5": {
565      "key4": {
566        "key3": {
567          "key2": {
568            "key1": "max depth"
569          }
570        }
571      }
572    }
573  }
574}"#
575        );
576    }
577
578    #[test]
579    fn test_databag_array_stripping() {
580        let mut processor = TrimmingProcessor::new();
581
582        let databag = Annotated::new({
583            let mut map = Object::new();
584            for idx in 0..100 {
585                map.insert(
586                    format!("key_{idx}"),
587                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
588                );
589            }
590            map
591        });
592        let mut event = Annotated::new(Event {
593            extra: databag,
594            ..Default::default()
595        });
596
597        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
598        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
599
600        insta::assert_ron_snapshot!(stripped_extra);
601    }
602
603    #[test]
604    fn test_tags_stripping() {
605        let mut processor = TrimmingProcessor::new();
606
607        let mut event = Annotated::new(Event {
608            tags: Annotated::new(Tags(
609                vec![Annotated::new(TagEntry(
610                    Annotated::new("x".repeat(300)),
611                    Annotated::new("x".repeat(300)),
612                ))]
613                .into(),
614            )),
615            ..Default::default()
616        });
617
618        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
619        let json = event
620            .value()
621            .unwrap()
622            .tags
623            .payload_to_json_pretty()
624            .unwrap();
625
626        assert_eq!(
627            json,
628            r#"[
629  [
630    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
631    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
632  ]
633]"#
634        );
635    }
636
637    #[test]
638    fn test_databag_state_leak() {
639        let event = Annotated::new(Event {
640            breadcrumbs: Annotated::new(Values::new(
641                repeat_n(
642                    Annotated::new(Breadcrumb {
643                        data: {
644                            let mut map = Map::new();
645                            map.insert(
646                                "spamspamspam".to_string(),
647                                Annotated::new(Value::String("blablabla".to_string())),
648                            );
649                            Annotated::new(map)
650                        },
651                        ..Default::default()
652                    }),
653                    200,
654                )
655                .collect(),
656            )),
657            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
658                ty: Annotated::new("TypeError".to_string()),
659                value: Annotated::new("important error message".to_string().into()),
660                stacktrace: Annotated::new(
661                    RawStacktrace {
662                        frames: Annotated::new(
663                            repeat_n(
664                                Annotated::new(Frame {
665                                    function: Annotated::new("importantFunctionName".to_string()),
666                                    symbol: Annotated::new("important_symbol".to_string()),
667                                    ..Default::default()
668                                }),
669                                200,
670                            )
671                            .collect(),
672                        ),
673                        ..Default::default()
674                    }
675                    .into(),
676                ),
677                ..Default::default()
678            })])),
679            ..Default::default()
680        });
681
682        let mut processor = TrimmingProcessor::new();
683        let mut stripped_event = event.clone();
684        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
685            .unwrap();
686
687        assert_eq!(
688            event.to_json_pretty().unwrap(),
689            stripped_event.to_json_pretty().unwrap()
690        );
691    }
692
693    #[test]
694    fn test_custom_context_trimming() {
695        let mut contexts = Contexts::new();
696        for i in 1..2 {
697            contexts.insert(format!("despacito{i}"), {
698                let mut context = Object::new();
699                context.insert(
700                    "foo".to_string(),
701                    Annotated::new(Value::String("a".repeat(4000))),
702                );
703                context.insert(
704                    "bar".to_string(),
705                    Annotated::new(Value::String("a".repeat(5000))),
706                );
707                Context::Other(context)
708            });
709        }
710
711        let mut contexts = Annotated::new(contexts);
712        let mut processor = TrimmingProcessor::new();
713        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
714
715        let contexts = contexts.value().unwrap();
716        for i in 1..2 {
717            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
718                Context::Other(x) => x,
719                _ => panic!("Context has changed type!"),
720            };
721
722            assert_eq!(
723                other
724                    .get("bar")
725                    .unwrap()
726                    .value()
727                    .unwrap()
728                    .as_str()
729                    .unwrap()
730                    .len(),
731                5000
732            );
733            assert_eq!(
734                other
735                    .get("foo")
736                    .unwrap()
737                    .value()
738                    .unwrap()
739                    .as_str()
740                    .unwrap()
741                    .len(),
742                3189
743            );
744        }
745    }
746
747    #[test]
748    fn test_extra_trimming_long_arrays() {
749        let mut extra = Object::new();
750        extra.insert("foo".to_string(), {
751            Annotated::new(ExtraValue(Value::Array(
752                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
753            )))
754        });
755
756        let mut event = Annotated::new(Event {
757            extra: Annotated::new(extra),
758            ..Default::default()
759        });
760
761        let mut processor = TrimmingProcessor::new();
762        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
763
764        let arr = match event
765            .value()
766            .unwrap()
767            .extra
768            .value()
769            .unwrap()
770            .get("foo")
771            .unwrap()
772            .value()
773            .unwrap()
774        {
775            ExtraValue(Value::Array(x)) => x,
776            x => panic!("Wrong type: {x:?}"),
777        };
778
779        // this is larger / 2 for the extra value
780        assert_eq!(arr.len(), 8192);
781    }
782
783    // TODO(ja): Enable this test
784    // #[test]
785    // fn test_newtypes_do_not_add_to_depth() {
786    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
787    //     struct WrappedString(String);
788
789    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
790    //     struct StructChild2 {
791    //         inner: Annotated<WrappedString>,
792    //     }
793
794    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
795    //     struct StructChild {
796    //         inner: Annotated<StructChild2>,
797    //     }
798
799    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
800    //     struct Struct {
801    //         #[metastructure(bag_size = "small")]
802    //         inner: Annotated<StructChild>,
803    //     }
804
805    //     let mut value = Annotated::new(Struct {
806    //         inner: Annotated::new(StructChild {
807    //             inner: Annotated::new(StructChild2 {
808    //                 inner: Annotated::new(WrappedString("hi".to_string())),
809    //             }),
810    //         }),
811    //     });
812
813    //     let mut processor = TrimmingProcessor::new();
814    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
815
816    //     // Ensure stack does not leak with newtypes
817    //     assert!(processor.bag_size_state.is_empty());
818
819    //     assert_eq!(
820    //         value.to_json().unwrap(),
821    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
822    //     );
823    // }
824
825    #[test]
826    fn test_frameqty_equals_limit() {
827        fn create_frame(filename: &str) -> Annotated<Frame> {
828            Annotated::new(Frame {
829                filename: Annotated::new(filename.into()),
830                ..Default::default()
831            })
832        }
833
834        let mut frames = Annotated::new(vec![
835            create_frame("foo3.py"),
836            create_frame("foo4.py"),
837            create_frame("foo5.py"),
838        ]);
839
840        processor::apply(&mut frames, |f, m| {
841            enforce_frame_hard_limit(f, m, 3, 0);
842            Ok(())
843        })
844        .unwrap();
845
846        processor::apply(&mut frames, |f, m| {
847            enforce_frame_hard_limit(f, m, 1, 2);
848            Ok(())
849        })
850        .unwrap();
851
852        // original_length isn't set, when limit is equal to length, as no trimming took place.
853        assert!(frames.meta().original_length().is_none());
854    }
855
856    #[test]
857    fn test_frame_hard_limit() {
858        fn create_frame(filename: &str) -> Annotated<Frame> {
859            Annotated::new(Frame {
860                filename: Annotated::new(filename.into()),
861                ..Default::default()
862            })
863        }
864
865        let mut frames = Annotated::new(vec![
866            create_frame("foo1.py"),
867            create_frame("foo2.py"),
868            create_frame("foo3.py"),
869            create_frame("foo4.py"),
870            create_frame("foo5.py"),
871        ]);
872
873        processor::apply(&mut frames, |f, m| {
874            enforce_frame_hard_limit(f, m, 3, 0);
875            Ok(())
876        })
877        .unwrap();
878
879        let mut expected_meta = Meta::default();
880        expected_meta.set_original_length(Some(5));
881
882        assert_eq!(
883            frames,
884            Annotated(
885                Some(vec![
886                    create_frame("foo3.py"),
887                    create_frame("foo4.py"),
888                    create_frame("foo5.py"),
889                ]),
890                expected_meta
891            )
892        );
893    }
894
895    #[test]
896    fn test_frame_hard_limit_recent_old() {
897        fn create_frame(filename: &str) -> Annotated<Frame> {
898            Annotated::new(Frame {
899                filename: Annotated::new(filename.into()),
900                ..Default::default()
901            })
902        }
903
904        let mut frames = Annotated::new(vec![
905            create_frame("foo1.py"),
906            create_frame("foo2.py"),
907            create_frame("foo3.py"),
908            create_frame("foo4.py"),
909            create_frame("foo5.py"),
910        ]);
911
912        processor::apply(&mut frames, |f, m| {
913            enforce_frame_hard_limit(f, m, 2, 1);
914            Ok(())
915        })
916        .unwrap();
917
918        let mut expected_meta = Meta::default();
919        expected_meta.set_original_length(Some(5));
920
921        assert_eq!(
922            frames,
923            Annotated(
924                Some(vec![
925                    create_frame("foo1.py"),
926                    create_frame("foo4.py"),
927                    create_frame("foo5.py"),
928                ]),
929                expected_meta
930            )
931        );
932    }
933
934    #[test]
935    fn test_slim_frame_data_under_max() {
936        let mut frames = vec![Annotated::new(Frame {
937            filename: Annotated::new("foo".into()),
938            pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
939            context_line: Annotated::new("b".to_string()),
940            post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
941            ..Default::default()
942        })];
943
944        let old_frames = frames.clone();
945        slim_frame_data(&mut frames, 4);
946
947        assert_eq!(frames, old_frames);
948    }
949
950    #[test]
951    fn test_slim_frame_data_over_max() {
952        let mut frames = vec![];
953
954        for n in 0..5 {
955            frames.push(Annotated::new(Frame {
956                filename: Annotated::new(format!("foo {n}").into()),
957                pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
958                context_line: Annotated::new("b".to_string()),
959                post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
960                ..Default::default()
961            }));
962        }
963
964        slim_frame_data(&mut frames, 4);
965
966        let expected = vec![
967            Annotated::new(Frame {
968                filename: Annotated::new("foo 0".into()),
969                pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
970                context_line: Annotated::new("b".to_string()),
971                post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
972                ..Default::default()
973            }),
974            Annotated::new(Frame {
975                filename: Annotated::new("foo 1".into()),
976                pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
977                context_line: Annotated::new("b".to_string()),
978                post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
979                ..Default::default()
980            }),
981            Annotated::new(Frame {
982                filename: Annotated::new("foo 2".into()),
983                context_line: Annotated::new("b".to_string()),
984                ..Default::default()
985            }),
986            Annotated::new(Frame {
987                filename: Annotated::new("foo 3".into()),
988                pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
989                context_line: Annotated::new("b".to_string()),
990                post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
991                ..Default::default()
992            }),
993            Annotated::new(Frame {
994                filename: Annotated::new("foo 4".into()),
995                pre_context: Annotated::new(vec![Annotated::new("a".to_string())]),
996                context_line: Annotated::new("b".to_string()),
997                post_context: Annotated::new(vec![Annotated::new("c".to_string())]),
998                ..Default::default()
999            }),
1000        ];
1001
1002        assert_eq!(frames, expected);
1003    }
1004
1005    #[test]
1006    fn test_too_many_spans_trimmed() {
1007        let span = Span {
1008            platform: Annotated::new("a".repeat(1024 * 90)),
1009            sentry_tags: Annotated::new(SentryTags {
1010                release: Annotated::new("b".repeat(1024 * 100)),
1011                ..Default::default()
1012            }),
1013            ..Default::default()
1014        };
1015        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1016            .take(10)
1017            .collect();
1018
1019        let mut event = Annotated::new(Event {
1020            spans: Annotated::new(spans.clone()),
1021            ..Default::default()
1022        });
1023
1024        let mut processor = TrimmingProcessor::new();
1025        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1026
1027        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1028        assert_eq!(trimmed_spans.len(), 5);
1029
1030        // The actual spans were not touched:
1031        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1032    }
1033
1034    #[test]
1035    fn test_untrimmable_fields() {
1036        let original_description = "a".repeat(819163);
1037        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1038        let mut event = Annotated::new(Event {
1039            spans: Annotated::new(vec![
1040                Span {
1041                    description: original_description.clone().into(),
1042                    ..Default::default()
1043                }
1044                .into(),
1045                Span {
1046                    trace_id: original_trace_id.into(),
1047                    ..Default::default()
1048                }
1049                .into(),
1050            ]),
1051            ..Default::default()
1052        });
1053
1054        let mut processor = TrimmingProcessor::new();
1055        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1056
1057        assert_eq!(
1058            get_value!(event.spans[0].description!),
1059            &original_description
1060        );
1061        // Trace ID would be trimmed without `trim = "false"`
1062        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1063    }
1064
1065    #[test]
1066    fn test_untrimmable_fields_drop() {
1067        let original_description = "a".repeat(819164);
1068        let original_span_id = SpanId("b".repeat(48));
1069        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1070        let original_segment_id = SpanId("d".repeat(48));
1071        let original_op = "e".repeat(129);
1072
1073        let mut event = Annotated::new(Event {
1074            spans: Annotated::new(vec![
1075                Span {
1076                    description: original_description.clone().into(),
1077                    ..Default::default()
1078                }
1079                .into(),
1080                Span {
1081                    span_id: original_span_id.clone().into(),
1082                    trace_id: original_trace_id.into(),
1083                    segment_id: original_segment_id.clone().into(),
1084                    is_segment: false.into(),
1085                    op: original_op.clone().into(),
1086                    start_timestamp: Timestamp(
1087                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1088                            .unwrap()
1089                            .into(),
1090                    )
1091                    .into(),
1092                    timestamp: Timestamp(
1093                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1094                            .unwrap()
1095                            .into(),
1096                    )
1097                    .into(),
1098                    ..Default::default()
1099                }
1100                .into(),
1101            ]),
1102            ..Default::default()
1103        });
1104
1105        let mut processor = TrimmingProcessor::new();
1106        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1107
1108        assert_eq!(
1109            get_value!(event.spans[0].description!),
1110            &original_description
1111        );
1112        // These fields would be dropped without `trim = "false"`
1113        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1114        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1115        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1116        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1117        // span.op is trimmed to its max_chars, but not dropped:
1118        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1119        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1120        assert!(get_value!(event.spans[1].timestamp).is_some());
1121    }
1122}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
