relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.max_bytes(),
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        // If our current depth is the one where we found a bag_size attribute, this means we
97        // are done processing a databag. Pop the bag size state.
98        self.size_state
99            .pop_if(|size_state| state.depth() == size_state.encountered_at_depth);
100
101        // After processing a value, update the remaining bag sizes. We have a separate if-let
102        // here in case somebody defines nested databags (a struct with bag_size that contains
103        // another struct with a different bag_size), in case we just exited a databag we want
104        // to update the bag_size_state of the outer databag with the remaining size.
105        //
106        // This also has to happen after string trimming, which is why it's running in
107        // after_process.
108        if state.entered_anything() && !self.size_state.is_empty() {
109            // Do not subtract if state is from newtype struct.
110            let item_length = state
111                .bytes_size()
112                .unwrap_or_else(|| relay_protocol::estimate_size_flat(value) + 1);
113            for size_state in self.size_state.iter_mut() {
114                size_state.size_remaining = size_state
115                    .size_remaining
116                    .map(|size| size.saturating_sub(item_length));
117            }
118        }
119
120        Ok(())
121    }
122
123    fn process_string(
124        &mut self,
125        value: &mut String,
126        meta: &mut Meta,
127        state: &ProcessingState<'_>,
128    ) -> ProcessingResult {
129        if let Some(max_chars) = state.max_chars() {
130            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
131        }
132
133        if !state.attrs().trim {
134            return Ok(());
135        }
136
137        if let Some(size_remaining) = self.remaining_size() {
138            trim_string(value, meta, size_remaining, 0);
139        }
140
141        Ok(())
142    }
143
144    fn process_array<T>(
145        &mut self,
146        value: &mut Array<T>,
147        meta: &mut Meta,
148        state: &ProcessingState<'_>,
149    ) -> ProcessingResult
150    where
151        T: ProcessValue,
152    {
153        if !state.attrs().trim {
154            return Ok(());
155        }
156
157        // If we need to check the bag size, then we go down a different path
158        if !self.size_state.is_empty() {
159            let original_length = value.len();
160
161            if self.should_remove_container(value, state) {
162                return Err(ProcessingAction::DeleteValueHard);
163            }
164
165            let mut split_index = None;
166            for (index, item) in value.iter_mut().enumerate() {
167                if self.remaining_size() == Some(0) {
168                    split_index = Some(index);
169                    break;
170                }
171
172                let item_state = state.enter_index(index, None, ValueType::for_field(item));
173                processor::process_value(item, self, &item_state)?;
174            }
175
176            if let Some(split_index) = split_index {
177                let _ = value.split_off(split_index);
178            }
179
180            if value.len() != original_length {
181                meta.set_original_length(Some(original_length));
182            }
183        } else {
184            value.process_child_values(self, state)?;
185        }
186
187        Ok(())
188    }
189
190    fn process_object<T>(
191        &mut self,
192        value: &mut Object<T>,
193        meta: &mut Meta,
194        state: &ProcessingState<'_>,
195    ) -> ProcessingResult
196    where
197        T: ProcessValue,
198    {
199        if !state.attrs().trim {
200            return Ok(());
201        }
202
203        // If we need to check the bag size, then we go down a different path
204        if !self.size_state.is_empty() {
205            let original_length = value.len();
206
207            if self.should_remove_container(value, state) {
208                return Err(ProcessingAction::DeleteValueHard);
209            }
210
211            let mut split_key = None;
212            for (key, item) in value.iter_mut() {
213                if self.remaining_size() == Some(0) {
214                    split_key = Some(key.to_owned());
215                    break;
216                }
217
218                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
219                processor::process_value(item, self, &item_state)?;
220            }
221
222            if let Some(split_key) = split_key {
223                let _ = value.split_off(&split_key);
224            }
225
226            if value.len() != original_length {
227                meta.set_original_length(Some(original_length));
228            }
229        } else {
230            value.process_child_values(self, state)?;
231        }
232
233        Ok(())
234    }
235
236    fn process_value(
237        &mut self,
238        value: &mut Value,
239        _meta: &mut Meta,
240        state: &ProcessingState<'_>,
241    ) -> ProcessingResult {
242        if !state.attrs().trim {
243            return Ok(());
244        }
245
246        match value {
247            Value::Array(_) | Value::Object(_) => {
248                if self.remaining_depth(state) == Some(1)
249                    && let Ok(x) = serde_json::to_string(&value)
250                {
251                    // Error case should not be possible
252                    *value = Value::String(x);
253                }
254            }
255            _ => (),
256        }
257
258        value.process_child_values(self, state)?;
259        Ok(())
260    }
261
262    fn process_replay(
263        &mut self,
264        replay: &mut Replay,
265        _: &mut Meta,
266        state: &ProcessingState<'_>,
267    ) -> ProcessingResult {
268        replay.process_child_values(self, state)
269    }
270
271    fn process_raw_stacktrace(
272        &mut self,
273        stacktrace: &mut RawStacktrace,
274        _meta: &mut Meta,
275        state: &ProcessingState<'_>,
276    ) -> ProcessingResult {
277        if !state.attrs().trim {
278            return Ok(());
279        }
280
281        processor::apply(&mut stacktrace.frames, |frames, meta| {
282            enforce_frame_hard_limit(frames, meta, 200, 50);
283            Ok(())
284        })?;
285
286        stacktrace.process_child_values(self, state)?;
287
288        processor::apply(&mut stacktrace.frames, |frames, _meta| {
289            slim_frame_data(frames, 50);
290            Ok(())
291        })?;
292
293        Ok(())
294    }
295}
296
297/// Trims the string to the given maximum length and updates meta data.
298pub(crate) fn trim_string(
299    value: &mut String,
300    meta: &mut Meta,
301    max_chars: usize,
302    max_chars_allowance: usize,
303) {
304    let hard_limit = max_chars + max_chars_allowance;
305
306    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
307        return;
308    }
309
310    processor::process_chunked_value(value, meta, |chunks| {
311        let mut length = 0;
312        let mut new_chunks = vec![];
313
314        for chunk in chunks {
315            let chunk_chars = chunk.count();
316
317            // if the entire chunk fits, just put it in
318            if length + chunk_chars < max_chars {
319                new_chunks.push(chunk);
320                length += chunk_chars;
321                continue;
322            }
323
324            match chunk {
325                // if there is enough space for this chunk and the 3 character
326                // ellipsis marker we can push the remaining chunk
327                Chunk::Redaction { .. } => {
328                    if length + chunk_chars + 3 < hard_limit {
329                        new_chunks.push(chunk);
330                    }
331                }
332
333                // if this is a text chunk, we can put the remaining characters in.
334                Chunk::Text { text } => {
335                    let mut remaining = String::new();
336                    for c in text.chars() {
337                        if length + 3 < max_chars {
338                            remaining.push(c);
339                        } else {
340                            break;
341                        }
342                        length += 1;
343                    }
344
345                    new_chunks.push(Chunk::Text {
346                        text: Cow::Owned(remaining),
347                    });
348                }
349            }
350
351            new_chunks.push(Chunk::Redaction {
352                text: Cow::Borrowed("..."),
353                rule_id: Cow::Borrowed("!limit"),
354                ty: RemarkType::Substituted,
355            });
356            break;
357        }
358
359        new_chunks
360    });
361}
362
363/// Trim down the frame list to a hard limit.
364///
365/// The total limit is `recent_frames` + `old_frames`.
366/// `recent_frames` is the number of frames to keep from the beginning of the list,
367/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
368/// the oldest frames up the stack.
369///
370/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
371/// the recursion.
372fn enforce_frame_hard_limit(
373    frames: &mut Array<Frame>,
374    meta: &mut Meta,
375    recent_frames: usize,
376    old_frames: usize,
377) {
378    let original_length = frames.len();
379    let limit = recent_frames + old_frames;
380    if original_length > limit {
381        meta.set_original_length(Some(original_length));
382        let _ = frames.drain(old_frames..original_length - recent_frames);
383    }
384}
385
386/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
387///
388/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
389fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
390    let frames_len = frames.len();
391
392    if frames_len <= frame_allowance {
393        return;
394    }
395
396    // Avoid ownership issues by only storing indices
397    let mut app_frame_indices = Vec::with_capacity(frames_len);
398    let mut system_frame_indices = Vec::with_capacity(frames_len);
399
400    for (i, frame) in frames.iter().enumerate() {
401        if let Some(frame) = frame.value() {
402            match frame.in_app.value() {
403                Some(true) => app_frame_indices.push(i),
404                _ => system_frame_indices.push(i),
405            }
406        }
407    }
408
409    let app_count = app_frame_indices.len();
410    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
411    let system_frames_to_remove = system_frame_indices
412        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
413        .unwrap_or(&[]);
414
415    let remaining = frames_len
416        .saturating_sub(frame_allowance)
417        .saturating_sub(system_frames_to_remove.len());
418    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
419    let app_frames_to_remove = app_frame_indices
420        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
421        .unwrap_or(&[]);
422
423    // TODO: Which annotation to set?
424
425    for i in system_frames_to_remove.iter().chain(app_frames_to_remove) {
426        if let Some(frame) = frames.get_mut(*i)
427            && let Some(ref mut frame) = frame.value_mut().as_mut()
428        {
429            frame.vars = Annotated::empty();
430            frame.pre_context = Annotated::empty();
431            frame.post_context = Annotated::empty();
432        }
433    }
434}
435
436#[cfg(test)]
437mod tests {
438    use std::iter::repeat_n;
439
440    use crate::MaxChars;
441    use chrono::DateTime;
442    use relay_event_schema::protocol::{
443        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, PairList, SentryTags, Span,
444        SpanId, TagEntry, Tags, Timestamp, TraceId, Values,
445    };
446    use relay_protocol::{FromValue, IntoValue, Map, Remark, SerializableAnnotated, get_value};
447    use similar_asserts::assert_eq;
448
449    use super::*;
450
451    #[test]
452    fn test_string_trimming() {
453        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
454        processor::apply(&mut value, |v, m| {
455            trim_string(v, m, 20, 0);
456            Ok(())
457        })
458        .unwrap();
459
460        assert_eq!(
461            value,
462            Annotated(Some("This is my long s...".into()), {
463                let mut meta = Meta::default();
464                meta.add_remark(Remark {
465                    ty: RemarkType::Substituted,
466                    rule_id: "!limit".to_owned(),
467                    range: Some((17, 20)),
468                });
469                meta.set_original_length(Some(46));
470                meta
471            })
472        );
473    }
474
475    #[test]
476    fn test_basic_trimming() {
477        let mut processor = TrimmingProcessor::new();
478
479        let mut event = Annotated::new(Event {
480            logger: Annotated::new("x".repeat(300)),
481            ..Default::default()
482        });
483
484        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
485
486        let mut expected = Annotated::new("x".repeat(300));
487        processor::apply(&mut expected, |v, m| {
488            trim_string(v, m, MaxChars::Logger.limit(), 0);
489            Ok(())
490        })
491        .unwrap();
492
493        assert_eq!(event.value().unwrap().logger, expected);
494    }
495
496    #[test]
497    fn test_max_char_allowance() {
498        let string = "This string requires some allowance to fit!";
499        let mut value = Annotated::new(string.to_owned()); // len == 43
500        processor::apply(&mut value, |v, m| {
501            trim_string(v, m, 40, 5);
502            Ok(())
503        })
504        .unwrap();
505
506        assert_eq!(value, Annotated::new(string.to_owned()));
507    }
508
509    #[test]
510    fn test_databag_stripping() {
511        let mut processor = TrimmingProcessor::new();
512
513        fn make_nested_object(depth: usize) -> Annotated<Value> {
514            if depth == 0 {
515                return Annotated::new(Value::String("max depth".to_owned()));
516            }
517            let mut rv = Object::new();
518            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
519            Annotated::new(Value::Object(rv))
520        }
521
522        let databag = Annotated::new({
523            let mut map = Object::new();
524            map.insert(
525                "key_1".to_owned(),
526                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
527            );
528            map.insert(
529                "key_2".to_owned(),
530                make_nested_object(8).map_value(ExtraValue),
531            );
532            map.insert(
533                "key_3".to_owned(),
534                // innermost key (string) is entering json stringify codepath
535                make_nested_object(5).map_value(ExtraValue),
536            );
537            map
538        });
539        let mut event = Annotated::new(Event {
540            extra: databag,
541            ..Default::default()
542        });
543
544        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
545        let stripped_extra = &event.value().unwrap().extra;
546        let json = stripped_extra.to_json_pretty().unwrap();
547
548        assert_eq!(
549            json,
550            r#"{
551  "key_1": "value 1",
552  "key_2": {
553    "key8": {
554      "key7": {
555        "key6": {
556          "key5": {
557            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
558          }
559        }
560      }
561    }
562  },
563  "key_3": {
564    "key5": {
565      "key4": {
566        "key3": {
567          "key2": {
568            "key1": "max depth"
569          }
570        }
571      }
572    }
573  }
574}"#
575        );
576    }
577
578    #[test]
579    fn test_databag_array_stripping() {
580        let mut processor = TrimmingProcessor::new();
581
582        let databag = Annotated::new({
583            let mut map = Object::new();
584            for idx in 0..100 {
585                map.insert(
586                    format!("key_{idx}"),
587                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
588                );
589            }
590            map
591        });
592        let mut event = Annotated::new(Event {
593            extra: databag,
594            ..Default::default()
595        });
596
597        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
598        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
599
600        insta::assert_ron_snapshot!(stripped_extra);
601    }
602
603    /// Tests that a trimming a string takes a lower outer limit into account.
604    #[test]
605    fn test_string_trimming_limits() {
606        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
607        struct Outer {
608            #[metastructure(max_bytes = 10)]
609            inner: Annotated<Inner>,
610        }
611
612        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
613        struct Inner {
614            #[metastructure(max_bytes = 20)]
615            innerer: Annotated<String>,
616        }
617
618        let mut processor = TrimmingProcessor::new();
619
620        let mut outer = Annotated::new({
621            Outer {
622                inner: Annotated::new(Inner {
623                    innerer: Annotated::new("This string is 28 bytes long".into()),
624                }),
625            }
626        });
627
628        processor::process_value(&mut outer, &mut processor, ProcessingState::root()).unwrap();
629        let stripped = SerializableAnnotated(&outer);
630
631        insta::assert_ron_snapshot!(stripped, @r###"
632        {
633          "inner": {
634            "innerer": "This st...",
635          },
636          "_meta": {
637            "inner": {
638              "innerer": {
639                "": Meta(Some(MetaInner(
640                  rem: [
641                    [
642                      "!limit",
643                      s,
644                      7,
645                      10,
646                    ],
647                  ],
648                  len: Some(28),
649                ))),
650              },
651            },
652          },
653        }
654        "###);
655    }
656
657    #[test]
658    fn test_tags_stripping() {
659        let mut processor = TrimmingProcessor::new();
660
661        let mut event = Annotated::new(Event {
662            tags: Annotated::new(Tags(
663                vec![Annotated::new(TagEntry(
664                    Annotated::new("x".repeat(300)),
665                    Annotated::new("x".repeat(300)),
666                ))]
667                .into(),
668            )),
669            ..Default::default()
670        });
671
672        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
673        let json = event
674            .value()
675            .unwrap()
676            .tags
677            .payload_to_json_pretty()
678            .unwrap();
679
680        assert_eq!(
681            json,
682            r#"[
683  [
684    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
685    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
686  ]
687]"#
688        );
689    }
690
691    #[test]
692    fn test_databag_state_leak() {
693        let event = Annotated::new(Event {
694            breadcrumbs: Annotated::new(Values::new(
695                repeat_n(
696                    Annotated::new(Breadcrumb {
697                        data: {
698                            let mut map = Map::new();
699                            map.insert(
700                                "spamspamspam".to_owned(),
701                                Annotated::new(Value::String("blablabla".to_owned())),
702                            );
703                            Annotated::new(map)
704                        },
705                        ..Default::default()
706                    }),
707                    200,
708                )
709                .collect(),
710            )),
711            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
712                ty: Annotated::new("TypeError".to_owned()),
713                value: Annotated::new("important error message".to_owned().into()),
714                stacktrace: Annotated::new(
715                    RawStacktrace {
716                        frames: Annotated::new(
717                            repeat_n(
718                                Annotated::new(Frame {
719                                    function: Annotated::new("importantFunctionName".to_owned()),
720                                    symbol: Annotated::new("important_symbol".to_owned()),
721                                    ..Default::default()
722                                }),
723                                200,
724                            )
725                            .collect(),
726                        ),
727                        ..Default::default()
728                    }
729                    .into(),
730                ),
731                ..Default::default()
732            })])),
733            ..Default::default()
734        });
735
736        let mut processor = TrimmingProcessor::new();
737        let mut stripped_event = event.clone();
738        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
739            .unwrap();
740
741        assert_eq!(
742            event.to_json_pretty().unwrap(),
743            stripped_event.to_json_pretty().unwrap()
744        );
745    }
746
747    #[test]
748    fn test_custom_context_trimming() {
749        let mut contexts = Contexts::new();
750        for i in 1..2 {
751            contexts.insert(format!("despacito{i}"), {
752                let mut context = Object::new();
753                context.insert(
754                    "foo".to_owned(),
755                    Annotated::new(Value::String("a".repeat(4000))),
756                );
757                context.insert(
758                    "bar".to_owned(),
759                    Annotated::new(Value::String("a".repeat(5000))),
760                );
761                Context::Other(context)
762            });
763        }
764
765        let mut contexts = Annotated::new(contexts);
766        let mut processor = TrimmingProcessor::new();
767        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
768
769        let contexts = contexts.value().unwrap();
770        for i in 1..2 {
771            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
772                Context::Other(x) => x,
773                _ => panic!("Context has changed type!"),
774            };
775
776            assert_eq!(
777                other
778                    .get("bar")
779                    .unwrap()
780                    .value()
781                    .unwrap()
782                    .as_str()
783                    .unwrap()
784                    .len(),
785                5000
786            );
787            assert_eq!(
788                other
789                    .get("foo")
790                    .unwrap()
791                    .value()
792                    .unwrap()
793                    .as_str()
794                    .unwrap()
795                    .len(),
796                3189
797            );
798        }
799    }
800
801    #[test]
802    fn test_extra_trimming_long_arrays() {
803        let mut extra = Object::new();
804        extra.insert("foo".to_owned(), {
805            Annotated::new(ExtraValue(Value::Array(
806                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
807            )))
808        });
809
810        let mut event = Annotated::new(Event {
811            extra: Annotated::new(extra),
812            ..Default::default()
813        });
814
815        let mut processor = TrimmingProcessor::new();
816        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
817
818        let arr = match event
819            .value()
820            .unwrap()
821            .extra
822            .value()
823            .unwrap()
824            .get("foo")
825            .unwrap()
826            .value()
827            .unwrap()
828        {
829            ExtraValue(Value::Array(x)) => x,
830            x => panic!("Wrong type: {x:?}"),
831        };
832
833        // this is larger / 2 for the extra value
834        assert_eq!(arr.len(), 8192);
835    }
836
837    // TODO(ja): Enable this test
838    // #[test]
839    // fn test_newtypes_do_not_add_to_depth() {
840    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
841    //     struct WrappedString(String);
842
843    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
844    //     struct StructChild2 {
845    //         inner: Annotated<WrappedString>,
846    //     }
847
848    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
849    //     struct StructChild {
850    //         inner: Annotated<StructChild2>,
851    //     }
852
853    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
854    //     struct Struct {
855    //         #[metastructure(bag_size = "small")]
856    //         inner: Annotated<StructChild>,
857    //     }
858
859    //     let mut value = Annotated::new(Struct {
860    //         inner: Annotated::new(StructChild {
861    //             inner: Annotated::new(StructChild2 {
862    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
863    //             }),
864    //         }),
865    //     });
866
867    //     let mut processor = TrimmingProcessor::new();
868    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
869
870    //     // Ensure stack does not leak with newtypes
871    //     assert!(processor.bag_size_state.is_empty());
872
873    //     assert_eq!(
874    //         value.to_json().unwrap(),
875    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
876    //     );
877    // }
878
879    #[test]
880    fn test_frameqty_equals_limit() {
881        fn create_frame(filename: &str) -> Annotated<Frame> {
882            Annotated::new(Frame {
883                filename: Annotated::new(filename.into()),
884                ..Default::default()
885            })
886        }
887
888        let mut frames = Annotated::new(vec![
889            create_frame("foo3.py"),
890            create_frame("foo4.py"),
891            create_frame("foo5.py"),
892        ]);
893
894        processor::apply(&mut frames, |f, m| {
895            enforce_frame_hard_limit(f, m, 3, 0);
896            Ok(())
897        })
898        .unwrap();
899
900        processor::apply(&mut frames, |f, m| {
901            enforce_frame_hard_limit(f, m, 1, 2);
902            Ok(())
903        })
904        .unwrap();
905
906        // original_length isn't set, when limit is equal to length, as no trimming took place.
907        assert!(frames.meta().original_length().is_none());
908    }
909
910    #[test]
911    fn test_frame_hard_limit() {
912        fn create_frame(filename: &str) -> Annotated<Frame> {
913            Annotated::new(Frame {
914                filename: Annotated::new(filename.into()),
915                ..Default::default()
916            })
917        }
918
919        let mut frames = Annotated::new(vec![
920            create_frame("foo1.py"),
921            create_frame("foo2.py"),
922            create_frame("foo3.py"),
923            create_frame("foo4.py"),
924            create_frame("foo5.py"),
925        ]);
926
927        processor::apply(&mut frames, |f, m| {
928            enforce_frame_hard_limit(f, m, 3, 0);
929            Ok(())
930        })
931        .unwrap();
932
933        let mut expected_meta = Meta::default();
934        expected_meta.set_original_length(Some(5));
935
936        assert_eq!(
937            frames,
938            Annotated(
939                Some(vec![
940                    create_frame("foo3.py"),
941                    create_frame("foo4.py"),
942                    create_frame("foo5.py"),
943                ]),
944                expected_meta
945            )
946        );
947    }
948
949    #[test]
950    fn test_frame_hard_limit_recent_old() {
951        fn create_frame(filename: &str) -> Annotated<Frame> {
952            Annotated::new(Frame {
953                filename: Annotated::new(filename.into()),
954                ..Default::default()
955            })
956        }
957
958        let mut frames = Annotated::new(vec![
959            create_frame("foo1.py"),
960            create_frame("foo2.py"),
961            create_frame("foo3.py"),
962            create_frame("foo4.py"),
963            create_frame("foo5.py"),
964        ]);
965
966        processor::apply(&mut frames, |f, m| {
967            enforce_frame_hard_limit(f, m, 2, 1);
968            Ok(())
969        })
970        .unwrap();
971
972        let mut expected_meta = Meta::default();
973        expected_meta.set_original_length(Some(5));
974
975        assert_eq!(
976            frames,
977            Annotated(
978                Some(vec![
979                    create_frame("foo1.py"),
980                    create_frame("foo4.py"),
981                    create_frame("foo5.py"),
982                ]),
983                expected_meta
984            )
985        );
986    }
987
988    #[test]
989    fn test_slim_frame_data_under_max() {
990        let mut frames = vec![Annotated::new(Frame {
991            filename: Annotated::new("foo".into()),
992            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
993            context_line: Annotated::new("b".to_owned()),
994            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
995            ..Default::default()
996        })];
997
998        let old_frames = frames.clone();
999        slim_frame_data(&mut frames, 4);
1000
1001        assert_eq!(frames, old_frames);
1002    }
1003
1004    #[test]
1005    fn test_slim_frame_data_over_max() {
1006        let mut frames = vec![];
1007
1008        for n in 0..5 {
1009            frames.push(Annotated::new(Frame {
1010                filename: Annotated::new(format!("foo {n}").into()),
1011                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1012                context_line: Annotated::new("b".to_owned()),
1013                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1014                ..Default::default()
1015            }));
1016        }
1017
1018        slim_frame_data(&mut frames, 4);
1019
1020        let expected = vec![
1021            Annotated::new(Frame {
1022                filename: Annotated::new("foo 0".into()),
1023                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1024                context_line: Annotated::new("b".to_owned()),
1025                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1026                ..Default::default()
1027            }),
1028            Annotated::new(Frame {
1029                filename: Annotated::new("foo 1".into()),
1030                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1031                context_line: Annotated::new("b".to_owned()),
1032                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1033                ..Default::default()
1034            }),
1035            Annotated::new(Frame {
1036                filename: Annotated::new("foo 2".into()),
1037                context_line: Annotated::new("b".to_owned()),
1038                ..Default::default()
1039            }),
1040            Annotated::new(Frame {
1041                filename: Annotated::new("foo 3".into()),
1042                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1043                context_line: Annotated::new("b".to_owned()),
1044                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1045                ..Default::default()
1046            }),
1047            Annotated::new(Frame {
1048                filename: Annotated::new("foo 4".into()),
1049                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1050                context_line: Annotated::new("b".to_owned()),
1051                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1052                ..Default::default()
1053            }),
1054        ];
1055
1056        assert_eq!(frames, expected);
1057    }
1058
1059    #[test]
1060    fn test_too_many_spans_trimmed() {
1061        let span = Span {
1062            platform: Annotated::new("a".repeat(1024 * 90)),
1063            sentry_tags: Annotated::new(SentryTags {
1064                release: Annotated::new("b".repeat(1024 * 100)),
1065                ..Default::default()
1066            }),
1067            ..Default::default()
1068        };
1069        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1070            .take(10)
1071            .collect();
1072
1073        let mut event = Annotated::new(Event {
1074            spans: Annotated::new(spans.clone()),
1075            ..Default::default()
1076        });
1077
1078        let mut processor = TrimmingProcessor::new();
1079        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1080
1081        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1082        assert_eq!(trimmed_spans.len(), 5);
1083
1084        // The actual spans were not touched:
1085        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1086    }
1087
1088    #[test]
1089    fn test_untrimmable_fields() {
1090        let original_description = "a".repeat(819163);
1091        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1092        let mut event = Annotated::new(Event {
1093            spans: Annotated::new(vec![
1094                Span {
1095                    description: original_description.clone().into(),
1096                    ..Default::default()
1097                }
1098                .into(),
1099                Span {
1100                    trace_id: original_trace_id.into(),
1101                    ..Default::default()
1102                }
1103                .into(),
1104            ]),
1105            ..Default::default()
1106        });
1107
1108        let mut processor = TrimmingProcessor::new();
1109        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1110
1111        assert_eq!(
1112            get_value!(event.spans[0].description!),
1113            &original_description
1114        );
1115        // Trace ID would be trimmed without `trim = "false"`
1116        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1117    }
1118
1119    #[test]
1120    fn test_untrimmable_fields_drop() {
1121        let original_description = "a".repeat(819164);
1122        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1123        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1124        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1125        let original_op = "e".repeat(129);
1126
1127        let mut event = Annotated::new(Event {
1128            spans: Annotated::new(vec![
1129                Span {
1130                    description: original_description.clone().into(),
1131                    ..Default::default()
1132                }
1133                .into(),
1134                Span {
1135                    span_id: original_span_id.into(),
1136                    trace_id: original_trace_id.into(),
1137                    segment_id: original_segment_id.into(),
1138                    is_segment: false.into(),
1139                    op: original_op.clone().into(),
1140                    start_timestamp: Timestamp(
1141                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1142                            .unwrap()
1143                            .into(),
1144                    )
1145                    .into(),
1146                    timestamp: Timestamp(
1147                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1148                            .unwrap()
1149                            .into(),
1150                    )
1151                    .into(),
1152                    ..Default::default()
1153                }
1154                .into(),
1155            ]),
1156            ..Default::default()
1157        });
1158
1159        let mut processor = TrimmingProcessor::new();
1160        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1161
1162        assert_eq!(
1163            get_value!(event.spans[0].description!),
1164            &original_description
1165        );
1166        // These fields would be dropped without `trim = "false"`
1167        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1168        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1169        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1170        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1171        // span.op is trimmed to its max_chars, but not dropped:
1172        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1173        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1174        assert!(get_value!(event.spans[1].timestamp).is_some());
1175    }
1176
1177    #[test]
1178    fn test_too_long_tags() {
1179        let mut event = Annotated::new(Event {
1180        tags: Annotated::new(Tags(PairList(
1181            vec![Annotated::new(TagEntry(
1182                Annotated::new("foobar".to_owned()),
1183                Annotated::new("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx".to_owned()),
1184            )), Annotated::new(TagEntry(
1185                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1186                Annotated::new("bar".to_owned()),
1187            ))]),
1188        )),
1189        ..Event::default()
1190    });
1191
1192        let mut processor = TrimmingProcessor::new();
1193        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1194
1195        insta::assert_debug_snapshot!(get_value!(event.tags!), @r###"
1196        Tags(
1197            PairList(
1198                [
1199                    TagEntry(
1200                        "foobar",
1201                        Annotated(
1202                            "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
1203                            Meta {
1204                                remarks: [
1205                                    Remark {
1206                                        ty: Substituted,
1207                                        rule_id: "!limit",
1208                                        range: Some(
1209                                            (
1210                                                197,
1211                                                200,
1212                                            ),
1213                                        ),
1214                                    },
1215                                ],
1216                                errors: [],
1217                                original_length: Some(
1218                                    203,
1219                                ),
1220                                original_value: None,
1221                            },
1222                        ),
1223                    ),
1224                    TagEntry(
1225                        Annotated(
1226                            "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo...",
1227                            Meta {
1228                                remarks: [
1229                                    Remark {
1230                                        ty: Substituted,
1231                                        rule_id: "!limit",
1232                                        range: Some(
1233                                            (
1234                                                197,
1235                                                200,
1236                                            ),
1237                                        ),
1238                                    },
1239                                ],
1240                                errors: [],
1241                                original_length: Some(
1242                                    203,
1243                                ),
1244                                original_value: None,
1245                            },
1246                        ),
1247                        "bar",
1248                    ),
1249                ],
1250            ),
1251        )
1252        "###);
1253    }
1254
1255    #[test]
1256    fn test_fixed_item_size() {
1257        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1258        struct TestObject {
1259            #[metastructure(max_bytes = 28)]
1260            inner: Annotated<TestObjectInner>,
1261        }
1262        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1263        struct TestObjectInner {
1264            #[metastructure(max_chars = 10, trim = true)]
1265            body: Annotated<String>,
1266            // This should neither be trimmed nor factor into size calculations.
1267            #[metastructure(trim = false, bytes_size = "always_zero")]
1268            number: Annotated<u64>,
1269            // This should count as 10B.
1270            #[metastructure(trim = false, bytes_size = 10)]
1271            other_number: Annotated<u64>,
1272            #[metastructure(trim = true)]
1273            footer: Annotated<String>,
1274        }
1275
1276        fn always_zero(_state: &ProcessingState) -> Option<usize> {
1277            Some(0)
1278        }
1279
1280        let mut object = Annotated::new(TestObject {
1281            inner: Annotated::new(TestObjectInner {
1282                body: Annotated::new("Longer than 10 chars".to_owned()),
1283                number: Annotated::new(13),
1284                other_number: Annotated::new(12),
1285                footer: Annotated::new("There should only be 'Th...' left".to_owned()),
1286            }),
1287        });
1288
1289        let mut processor = TrimmingProcessor::new();
1290        processor::process_value(&mut object, &mut processor, ProcessingState::root()).unwrap();
1291
1292        // * `body` gets trimmed to 13B (10 chars + `...`)
1293        // * `number` counts as 0B
1294        // * `other_number` counts as 10B
1295        // That leaves 5B for the `footer`.
1296        insta::assert_ron_snapshot!(SerializableAnnotated(&object), @r###"
1297        {
1298          "inner": {
1299            "body": "Longer ...",
1300            "number": 13,
1301            "other_number": 12,
1302            "footer": "Th...",
1303          },
1304          "_meta": {
1305            "inner": {
1306              "body": {
1307                "": Meta(Some(MetaInner(
1308                  rem: [
1309                    [
1310                      "!limit",
1311                      s,
1312                      7,
1313                      10,
1314                    ],
1315                  ],
1316                  len: Some(20),
1317                ))),
1318              },
1319              "footer": {
1320                "": Meta(Some(MetaInner(
1321                  rem: [
1322                    [
1323                      "!limit",
1324                      s,
1325                      2,
1326                      5,
1327                    ],
1328                  ],
1329                  len: Some(33),
1330                ))),
1331              },
1332            },
1333          },
1334        }
1335        "###);
1336    }
1337}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
