relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.max_bytes(),
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        // If our current depth is the one where we found a bag_size attribute, this means we
97        // are done processing a databag. Pop the bag size state.
98        self.size_state
99            .pop_if(|size_state| state.depth() == size_state.encountered_at_depth);
100
101        // After processing a value, update the remaining bag sizes. We have a separate if-let
102        // here in case somebody defines nested databags (a struct with bag_size that contains
103        // another struct with a different bag_size), in case we just exited a databag we want
104        // to update the bag_size_state of the outer databag with the remaining size.
105        //
106        // This also has to happen after string trimming, which is why it's running in
107        // after_process.
108        if state.entered_anything() && !self.size_state.is_empty() {
109            // Do not subtract if state is from newtype struct.
110            let item_length = state
111                .bytes_size()
112                .unwrap_or_else(|| relay_protocol::estimate_size_flat(value) + 1);
113            for size_state in self.size_state.iter_mut() {
114                size_state.size_remaining = size_state
115                    .size_remaining
116                    .map(|size| size.saturating_sub(item_length));
117            }
118        }
119
120        Ok(())
121    }
122
123    fn process_string(
124        &mut self,
125        value: &mut String,
126        meta: &mut Meta,
127        state: &ProcessingState<'_>,
128    ) -> ProcessingResult {
129        if let Some(max_chars) = state.max_chars() {
130            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
131        }
132
133        if !state.attrs().trim {
134            return Ok(());
135        }
136
137        if let Some(size_remaining) = self.remaining_size() {
138            trim_string(value, meta, size_remaining, 0);
139        }
140
141        Ok(())
142    }
143
144    fn process_array<T>(
145        &mut self,
146        value: &mut Array<T>,
147        meta: &mut Meta,
148        state: &ProcessingState<'_>,
149    ) -> ProcessingResult
150    where
151        T: ProcessValue,
152    {
153        if !state.attrs().trim {
154            return Ok(());
155        }
156
157        // If we need to check the bag size, then we go down a different path
158        if !self.size_state.is_empty() {
159            let original_length = value.len();
160
161            if self.should_remove_container(value, state) {
162                return Err(ProcessingAction::DeleteValueHard);
163            }
164
165            let mut split_index = None;
166            for (index, item) in value.iter_mut().enumerate() {
167                if self.remaining_size() == Some(0) {
168                    split_index = Some(index);
169                    break;
170                }
171
172                let item_state = state.enter_index(index, None, ValueType::for_field(item));
173                processor::process_value(item, self, &item_state)?;
174            }
175
176            if let Some(split_index) = split_index {
177                let _ = value.split_off(split_index);
178            }
179
180            if value.len() != original_length {
181                meta.set_original_length(Some(original_length));
182            }
183        } else {
184            value.process_child_values(self, state)?;
185        }
186
187        Ok(())
188    }
189
190    fn process_object<T>(
191        &mut self,
192        value: &mut Object<T>,
193        meta: &mut Meta,
194        state: &ProcessingState<'_>,
195    ) -> ProcessingResult
196    where
197        T: ProcessValue,
198    {
199        if !state.attrs().trim {
200            return Ok(());
201        }
202
203        // If we need to check the bag size, then we go down a different path
204        if !self.size_state.is_empty() {
205            let original_length = value.len();
206
207            if self.should_remove_container(value, state) {
208                return Err(ProcessingAction::DeleteValueHard);
209            }
210
211            let mut split_key = None;
212            for (key, item) in value.iter_mut() {
213                if self.remaining_size() == Some(0) {
214                    split_key = Some(key.to_owned());
215                    break;
216                }
217
218                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
219                processor::process_value(item, self, &item_state)?;
220            }
221
222            if let Some(split_key) = split_key {
223                let _ = value.split_off(&split_key);
224            }
225
226            if value.len() != original_length {
227                meta.set_original_length(Some(original_length));
228            }
229        } else {
230            value.process_child_values(self, state)?;
231        }
232
233        Ok(())
234    }
235
236    fn process_value(
237        &mut self,
238        value: &mut Value,
239        _meta: &mut Meta,
240        state: &ProcessingState<'_>,
241    ) -> ProcessingResult {
242        if !state.attrs().trim {
243            return Ok(());
244        }
245
246        match value {
247            Value::Array(_) | Value::Object(_) => {
248                if self.remaining_depth(state) == Some(1)
249                    && let Ok(x) = serde_json::to_string(&value)
250                {
251                    // Error case should not be possible
252                    *value = Value::String(x);
253                }
254            }
255            _ => (),
256        }
257
258        value.process_child_values(self, state)?;
259        Ok(())
260    }
261
262    fn process_replay(
263        &mut self,
264        replay: &mut Replay,
265        _: &mut Meta,
266        state: &ProcessingState<'_>,
267    ) -> ProcessingResult {
268        replay.process_child_values(self, state)
269    }
270
271    fn process_raw_stacktrace(
272        &mut self,
273        stacktrace: &mut RawStacktrace,
274        _meta: &mut Meta,
275        state: &ProcessingState<'_>,
276    ) -> ProcessingResult {
277        if !state.attrs().trim {
278            return Ok(());
279        }
280
281        processor::apply(&mut stacktrace.frames, |frames, meta| {
282            enforce_frame_hard_limit(frames, meta, 200, 50);
283            Ok(())
284        })?;
285
286        stacktrace.process_child_values(self, state)?;
287
288        processor::apply(&mut stacktrace.frames, |frames, _meta| {
289            slim_frame_data(frames, 50);
290            Ok(())
291        })?;
292
293        Ok(())
294    }
295}
296
297/// Trims the string to the given maximum length and updates meta data.
298pub(crate) fn trim_string(
299    value: &mut String,
300    meta: &mut Meta,
301    max_chars: usize,
302    max_chars_allowance: usize,
303) {
304    let hard_limit = max_chars + max_chars_allowance;
305
306    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
307        return;
308    }
309
310    processor::process_chunked_value(value, meta, |chunks| {
311        let mut length = 0;
312        let mut new_chunks = vec![];
313
314        for chunk in chunks {
315            let chunk_chars = chunk.count();
316
317            // if the entire chunk fits, just put it in
318            if length + chunk_chars < max_chars {
319                new_chunks.push(chunk);
320                length += chunk_chars;
321                continue;
322            }
323
324            match chunk {
325                // if there is enough space for this chunk and the 3 character
326                // ellipsis marker we can push the remaining chunk
327                Chunk::Redaction { .. } => {
328                    if length + chunk_chars + 3 < hard_limit {
329                        new_chunks.push(chunk);
330                    }
331                }
332
333                // if this is a text chunk, we can put the remaining characters in.
334                Chunk::Text { text } => {
335                    let mut remaining = String::new();
336                    for c in text.chars() {
337                        if length + 3 < max_chars {
338                            remaining.push(c);
339                        } else {
340                            break;
341                        }
342                        length += 1;
343                    }
344
345                    new_chunks.push(Chunk::Text {
346                        text: Cow::Owned(remaining),
347                    });
348                }
349            }
350
351            new_chunks.push(Chunk::Redaction {
352                text: Cow::Borrowed("..."),
353                rule_id: Cow::Borrowed("!limit"),
354                ty: RemarkType::Substituted,
355            });
356            break;
357        }
358
359        new_chunks
360    });
361}
362
363/// Trim down the frame list to a hard limit.
364///
365/// The total limit is `recent_frames` + `old_frames`.
366/// `recent_frames` is the number of frames to keep from the beginning of the list,
367/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
368/// the oldest frames up the stack.
369///
370/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
371/// the recursion.
372fn enforce_frame_hard_limit(
373    frames: &mut Array<Frame>,
374    meta: &mut Meta,
375    recent_frames: usize,
376    old_frames: usize,
377) {
378    let original_length = frames.len();
379    let limit = recent_frames + old_frames;
380    if original_length > limit {
381        meta.set_original_length(Some(original_length));
382        let _ = frames.drain(old_frames..original_length - recent_frames);
383    }
384}
385
386/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
387///
388/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
389fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
390    let frames_len = frames.len();
391
392    if frames_len <= frame_allowance {
393        return;
394    }
395
396    // Avoid ownership issues by only storing indices
397    let mut app_frame_indices = Vec::with_capacity(frames_len);
398    let mut system_frame_indices = Vec::with_capacity(frames_len);
399
400    for (i, frame) in frames.iter().enumerate() {
401        if let Some(frame) = frame.value() {
402            match frame.in_app.value() {
403                Some(true) => app_frame_indices.push(i),
404                _ => system_frame_indices.push(i),
405            }
406        }
407    }
408
409    let app_count = app_frame_indices.len();
410    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
411    let system_frames_to_remove = system_frame_indices
412        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
413        .unwrap_or(&[]);
414
415    let remaining = frames_len
416        .saturating_sub(frame_allowance)
417        .saturating_sub(system_frames_to_remove.len());
418    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
419    let app_frames_to_remove = app_frame_indices
420        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
421        .unwrap_or(&[]);
422
423    // TODO: Which annotation to set?
424
425    for i in system_frames_to_remove.iter().chain(app_frames_to_remove) {
426        if let Some(frame) = frames.get_mut(*i)
427            && let Some(ref mut frame) = frame.value_mut().as_mut()
428        {
429            frame.vars = Annotated::empty();
430            frame.pre_context = Annotated::empty();
431            frame.post_context = Annotated::empty();
432        }
433    }
434}
435
436#[cfg(test)]
437mod tests {
438    use std::iter::repeat_n;
439
440    use crate::MaxChars;
441    use chrono::DateTime;
442    use relay_event_schema::protocol::{
443        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, FlagsContext, PairList,
444        SentryTags, Span, SpanData, SpanId, TagEntry, Tags, Timestamp, TraceId, Values,
445    };
446    use relay_protocol::{FromValue, IntoValue, Map, Remark, SerializableAnnotated, get_value};
447    use similar_asserts::assert_eq;
448
449    use super::*;
450
451    #[test]
452    fn test_string_trimming() {
453        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
454        processor::apply(&mut value, |v, m| {
455            trim_string(v, m, 20, 0);
456            Ok(())
457        })
458        .unwrap();
459
460        assert_eq!(
461            value,
462            Annotated(Some("This is my long s...".into()), {
463                let mut meta = Meta::default();
464                meta.add_remark(Remark {
465                    ty: RemarkType::Substituted,
466                    rule_id: "!limit".to_owned(),
467                    range: Some((17, 20)),
468                });
469                meta.set_original_length(Some(46));
470                meta
471            })
472        );
473    }
474
475    #[test]
476    fn test_basic_trimming() {
477        let mut processor = TrimmingProcessor::new();
478
479        let mut event = Annotated::new(Event {
480            logger: Annotated::new("x".repeat(300)),
481            ..Default::default()
482        });
483
484        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
485
486        let mut expected = Annotated::new("x".repeat(300));
487        processor::apply(&mut expected, |v, m| {
488            trim_string(v, m, MaxChars::Logger.limit(), 0);
489            Ok(())
490        })
491        .unwrap();
492
493        assert_eq!(event.value().unwrap().logger, expected);
494    }
495
496    #[test]
497    fn test_max_char_allowance() {
498        let string = "This string requires some allowance to fit!";
499        let mut value = Annotated::new(string.to_owned()); // len == 43
500        processor::apply(&mut value, |v, m| {
501            trim_string(v, m, 40, 5);
502            Ok(())
503        })
504        .unwrap();
505
506        assert_eq!(value, Annotated::new(string.to_owned()));
507    }
508
509    #[test]
510    fn test_databag_stripping() {
511        let mut processor = TrimmingProcessor::new();
512
513        fn make_nested_object(depth: usize) -> Annotated<Value> {
514            if depth == 0 {
515                return Annotated::new(Value::String("max depth".to_owned()));
516            }
517            let mut rv = Object::new();
518            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
519            Annotated::new(Value::Object(rv))
520        }
521
522        let databag = Annotated::new({
523            let mut map = Object::new();
524            map.insert(
525                "key_1".to_owned(),
526                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
527            );
528            map.insert(
529                "key_2".to_owned(),
530                make_nested_object(8).map_value(ExtraValue),
531            );
532            map.insert(
533                "key_3".to_owned(),
534                // innermost key (string) is entering json stringify codepath
535                make_nested_object(5).map_value(ExtraValue),
536            );
537            map
538        });
539        let mut event = Annotated::new(Event {
540            extra: databag,
541            ..Default::default()
542        });
543
544        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
545        let stripped_extra = &event.value().unwrap().extra;
546        let json = stripped_extra.to_json_pretty().unwrap();
547
548        assert_eq!(
549            json,
550            r#"{
551  "key_1": "value 1",
552  "key_2": {
553    "key8": {
554      "key7": {
555        "key6": {
556          "key5": {
557            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
558          }
559        }
560      }
561    }
562  },
563  "key_3": {
564    "key5": {
565      "key4": {
566        "key3": {
567          "key2": {
568            "key1": "max depth"
569          }
570        }
571      }
572    }
573  }
574}"#
575        );
576    }
577
578    #[test]
579    fn test_databag_array_stripping() {
580        let mut processor = TrimmingProcessor::new();
581
582        let databag = Annotated::new({
583            let mut map = Object::new();
584            for idx in 0..100 {
585                map.insert(
586                    format!("key_{idx}"),
587                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
588                );
589            }
590            map
591        });
592        let mut event = Annotated::new(Event {
593            extra: databag,
594            ..Default::default()
595        });
596
597        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
598        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
599
600        insta::assert_ron_snapshot!(stripped_extra);
601    }
602
603    /// Tests that a trimming a string takes a lower outer limit into account.
604    #[test]
605    fn test_string_trimming_limits() {
606        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
607        struct Outer {
608            #[metastructure(max_bytes = 10)]
609            inner: Annotated<Inner>,
610        }
611
612        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
613        struct Inner {
614            #[metastructure(max_bytes = 20)]
615            innerer: Annotated<String>,
616        }
617
618        let mut processor = TrimmingProcessor::new();
619
620        let mut outer = Annotated::new({
621            Outer {
622                inner: Annotated::new(Inner {
623                    innerer: Annotated::new("This string is 28 bytes long".into()),
624                }),
625            }
626        });
627
628        processor::process_value(&mut outer, &mut processor, ProcessingState::root()).unwrap();
629        let stripped = SerializableAnnotated(&outer);
630
631        insta::assert_ron_snapshot!(stripped, @r###"
632        {
633          "inner": {
634            "innerer": "This st...",
635          },
636          "_meta": {
637            "inner": {
638              "innerer": {
639                "": Meta(Some(MetaInner(
640                  rem: [
641                    [
642                      "!limit",
643                      s,
644                      7,
645                      10,
646                    ],
647                  ],
648                  len: Some(28),
649                ))),
650              },
651            },
652          },
653        }
654        "###);
655    }
656
657    #[test]
658    fn test_tags_stripping() {
659        let mut processor = TrimmingProcessor::new();
660
661        let mut event = Annotated::new(Event {
662            tags: Annotated::new(Tags(
663                vec![Annotated::new(TagEntry(
664                    Annotated::new("x".repeat(300)),
665                    Annotated::new("x".repeat(300)),
666                ))]
667                .into(),
668            )),
669            ..Default::default()
670        });
671
672        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
673        let json = event
674            .value()
675            .unwrap()
676            .tags
677            .payload_to_json_pretty()
678            .unwrap();
679
680        assert_eq!(
681            json,
682            r#"[
683  [
684    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
685    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
686  ]
687]"#
688        );
689    }
690
691    #[test]
692    fn test_databag_state_leak() {
693        let event = Annotated::new(Event {
694            breadcrumbs: Annotated::new(Values::new(
695                repeat_n(
696                    Annotated::new(Breadcrumb {
697                        data: {
698                            let mut map = Map::new();
699                            map.insert(
700                                "spamspamspam".to_owned(),
701                                Annotated::new(Value::String("blablabla".to_owned())),
702                            );
703                            Annotated::new(map)
704                        },
705                        ..Default::default()
706                    }),
707                    200,
708                )
709                .collect(),
710            )),
711            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
712                ty: Annotated::new("TypeError".to_owned()),
713                value: Annotated::new("important error message".to_owned().into()),
714                stacktrace: Annotated::new(
715                    RawStacktrace {
716                        frames: Annotated::new(
717                            repeat_n(
718                                Annotated::new(Frame {
719                                    function: Annotated::new("importantFunctionName".to_owned()),
720                                    symbol: Annotated::new("important_symbol".to_owned()),
721                                    ..Default::default()
722                                }),
723                                200,
724                            )
725                            .collect(),
726                        ),
727                        ..Default::default()
728                    }
729                    .into(),
730                ),
731                ..Default::default()
732            })])),
733            ..Default::default()
734        });
735
736        let mut processor = TrimmingProcessor::new();
737        let mut stripped_event = event.clone();
738        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
739            .unwrap();
740
741        assert_eq!(
742            event.to_json_pretty().unwrap(),
743            stripped_event.to_json_pretty().unwrap()
744        );
745    }
746
747    #[test]
748    fn test_custom_context_trimming() {
749        let mut contexts = Contexts::new();
750        for i in 1..2 {
751            contexts.insert(format!("despacito{i}"), {
752                let mut context = Object::new();
753                context.insert(
754                    "foo".to_owned(),
755                    Annotated::new(Value::String("a".repeat(4000))),
756                );
757                context.insert(
758                    "bar".to_owned(),
759                    Annotated::new(Value::String("a".repeat(5000))),
760                );
761                Context::Other(context)
762            });
763        }
764
765        let mut contexts = Annotated::new(contexts);
766        let mut processor = TrimmingProcessor::new();
767        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
768
769        let contexts = contexts.value().unwrap();
770        for i in 1..2 {
771            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
772                Context::Other(x) => x,
773                _ => panic!("Context has changed type!"),
774            };
775
776            assert_eq!(
777                other
778                    .get("bar")
779                    .unwrap()
780                    .value()
781                    .unwrap()
782                    .as_str()
783                    .unwrap()
784                    .len(),
785                5000
786            );
787            assert_eq!(
788                other
789                    .get("foo")
790                    .unwrap()
791                    .value()
792                    .unwrap()
793                    .as_str()
794                    .unwrap()
795                    .len(),
796                3189
797            );
798        }
799    }
800
801    #[test]
802    fn test_extra_trimming_long_arrays() {
803        let mut extra = Object::new();
804        extra.insert("foo".to_owned(), {
805            Annotated::new(ExtraValue(Value::Array(
806                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
807            )))
808        });
809
810        let mut event = Annotated::new(Event {
811            extra: Annotated::new(extra),
812            ..Default::default()
813        });
814
815        let mut processor = TrimmingProcessor::new();
816        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
817
818        let arr = match event
819            .value()
820            .unwrap()
821            .extra
822            .value()
823            .unwrap()
824            .get("foo")
825            .unwrap()
826            .value()
827            .unwrap()
828        {
829            ExtraValue(Value::Array(x)) => x,
830            x => panic!("Wrong type: {x:?}"),
831        };
832
833        // this is larger / 2 for the extra value
834        assert_eq!(arr.len(), 8192);
835    }
836
837    // TODO(ja): Enable this test
838    // #[test]
839    // fn test_newtypes_do_not_add_to_depth() {
840    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
841    //     struct WrappedString(String);
842
843    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
844    //     struct StructChild2 {
845    //         inner: Annotated<WrappedString>,
846    //     }
847
848    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
849    //     struct StructChild {
850    //         inner: Annotated<StructChild2>,
851    //     }
852
853    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
854    //     struct Struct {
855    //         #[metastructure(bag_size = "small")]
856    //         inner: Annotated<StructChild>,
857    //     }
858
859    //     let mut value = Annotated::new(Struct {
860    //         inner: Annotated::new(StructChild {
861    //             inner: Annotated::new(StructChild2 {
862    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
863    //             }),
864    //         }),
865    //     });
866
867    //     let mut processor = TrimmingProcessor::new();
868    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
869
870    //     // Ensure stack does not leak with newtypes
871    //     assert!(processor.bag_size_state.is_empty());
872
873    //     assert_eq!(
874    //         value.to_json().unwrap(),
875    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
876    //     );
877    // }
878
879    #[test]
880    fn test_frameqty_equals_limit() {
881        fn create_frame(filename: &str) -> Annotated<Frame> {
882            Annotated::new(Frame {
883                filename: Annotated::new(filename.into()),
884                ..Default::default()
885            })
886        }
887
888        let mut frames = Annotated::new(vec![
889            create_frame("foo3.py"),
890            create_frame("foo4.py"),
891            create_frame("foo5.py"),
892        ]);
893
894        processor::apply(&mut frames, |f, m| {
895            enforce_frame_hard_limit(f, m, 3, 0);
896            Ok(())
897        })
898        .unwrap();
899
900        processor::apply(&mut frames, |f, m| {
901            enforce_frame_hard_limit(f, m, 1, 2);
902            Ok(())
903        })
904        .unwrap();
905
906        // original_length isn't set, when limit is equal to length, as no trimming took place.
907        assert!(frames.meta().original_length().is_none());
908    }
909
910    #[test]
911    fn test_frame_hard_limit() {
912        fn create_frame(filename: &str) -> Annotated<Frame> {
913            Annotated::new(Frame {
914                filename: Annotated::new(filename.into()),
915                ..Default::default()
916            })
917        }
918
919        let mut frames = Annotated::new(vec![
920            create_frame("foo1.py"),
921            create_frame("foo2.py"),
922            create_frame("foo3.py"),
923            create_frame("foo4.py"),
924            create_frame("foo5.py"),
925        ]);
926
927        processor::apply(&mut frames, |f, m| {
928            enforce_frame_hard_limit(f, m, 3, 0);
929            Ok(())
930        })
931        .unwrap();
932
933        let mut expected_meta = Meta::default();
934        expected_meta.set_original_length(Some(5));
935
936        assert_eq!(
937            frames,
938            Annotated(
939                Some(vec![
940                    create_frame("foo3.py"),
941                    create_frame("foo4.py"),
942                    create_frame("foo5.py"),
943                ]),
944                expected_meta
945            )
946        );
947    }
948
949    #[test]
950    fn test_frame_hard_limit_recent_old() {
951        fn create_frame(filename: &str) -> Annotated<Frame> {
952            Annotated::new(Frame {
953                filename: Annotated::new(filename.into()),
954                ..Default::default()
955            })
956        }
957
958        let mut frames = Annotated::new(vec![
959            create_frame("foo1.py"),
960            create_frame("foo2.py"),
961            create_frame("foo3.py"),
962            create_frame("foo4.py"),
963            create_frame("foo5.py"),
964        ]);
965
966        processor::apply(&mut frames, |f, m| {
967            enforce_frame_hard_limit(f, m, 2, 1);
968            Ok(())
969        })
970        .unwrap();
971
972        let mut expected_meta = Meta::default();
973        expected_meta.set_original_length(Some(5));
974
975        assert_eq!(
976            frames,
977            Annotated(
978                Some(vec![
979                    create_frame("foo1.py"),
980                    create_frame("foo4.py"),
981                    create_frame("foo5.py"),
982                ]),
983                expected_meta
984            )
985        );
986    }
987
988    #[test]
989    fn test_slim_frame_data_under_max() {
990        let mut frames = vec![Annotated::new(Frame {
991            filename: Annotated::new("foo".into()),
992            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
993            context_line: Annotated::new("b".to_owned()),
994            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
995            ..Default::default()
996        })];
997
998        let old_frames = frames.clone();
999        slim_frame_data(&mut frames, 4);
1000
1001        assert_eq!(frames, old_frames);
1002    }
1003
1004    #[test]
1005    fn test_slim_frame_data_over_max() {
1006        let mut frames = vec![];
1007
1008        for n in 0..5 {
1009            frames.push(Annotated::new(Frame {
1010                filename: Annotated::new(format!("foo {n}").into()),
1011                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1012                context_line: Annotated::new("b".to_owned()),
1013                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1014                ..Default::default()
1015            }));
1016        }
1017
1018        slim_frame_data(&mut frames, 4);
1019
1020        let expected = vec![
1021            Annotated::new(Frame {
1022                filename: Annotated::new("foo 0".into()),
1023                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1024                context_line: Annotated::new("b".to_owned()),
1025                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1026                ..Default::default()
1027            }),
1028            Annotated::new(Frame {
1029                filename: Annotated::new("foo 1".into()),
1030                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1031                context_line: Annotated::new("b".to_owned()),
1032                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1033                ..Default::default()
1034            }),
1035            Annotated::new(Frame {
1036                filename: Annotated::new("foo 2".into()),
1037                context_line: Annotated::new("b".to_owned()),
1038                ..Default::default()
1039            }),
1040            Annotated::new(Frame {
1041                filename: Annotated::new("foo 3".into()),
1042                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1043                context_line: Annotated::new("b".to_owned()),
1044                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1045                ..Default::default()
1046            }),
1047            Annotated::new(Frame {
1048                filename: Annotated::new("foo 4".into()),
1049                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1050                context_line: Annotated::new("b".to_owned()),
1051                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1052                ..Default::default()
1053            }),
1054        ];
1055
1056        assert_eq!(frames, expected);
1057    }
1058
1059    #[test]
1060    fn test_too_many_spans_trimmed() {
1061        let span = Span {
1062            platform: Annotated::new("a".repeat(1024 * 90)),
1063            sentry_tags: Annotated::new(SentryTags {
1064                release: Annotated::new("b".repeat(1024 * 100)),
1065                ..Default::default()
1066            }),
1067            ..Default::default()
1068        };
1069        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1070            .take(10)
1071            .collect();
1072
1073        let mut event = Annotated::new(Event {
1074            spans: Annotated::new(spans.clone()),
1075            ..Default::default()
1076        });
1077
1078        let mut processor = TrimmingProcessor::new();
1079        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1080
1081        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1082        assert_eq!(trimmed_spans.len(), 5);
1083
1084        // The actual spans were not touched:
1085        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1086    }
1087
1088    #[test]
1089    fn test_span_data_not_partially_trimmed() {
1090        let span_data = SpanData::from([(
1091            "large_attribute".to_owned(),
1092            Annotated::new(Value::String("a".repeat(100 * 1024))),
1093        )]);
1094        let span = Span {
1095            data: Annotated::new(span_data.clone()),
1096            ..Default::default()
1097        };
1098        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1099            .take(10)
1100            .collect();
1101
1102        let mut event = Annotated::new(Event {
1103            spans: Annotated::new(spans.clone()),
1104            ..Default::default()
1105        });
1106
1107        let mut processor = TrimmingProcessor::new();
1108        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1109
1110        let trimmed = event.value().unwrap().spans.value().unwrap();
1111        assert!(trimmed.len() < spans.len());
1112        assert!(trimmed.iter().all(|span| {
1113            span.value()
1114                .and_then(|span| span.data.value())
1115                .is_some_and(|data| data == &span_data)
1116        }));
1117    }
1118
1119    #[test]
1120    fn test_untrimmable_fields() {
1121        let original_description = "a".repeat(819163);
1122        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1123        let mut event = Annotated::new(Event {
1124            spans: Annotated::new(vec![
1125                Span {
1126                    description: original_description.clone().into(),
1127                    ..Default::default()
1128                }
1129                .into(),
1130                Span {
1131                    trace_id: original_trace_id.into(),
1132                    ..Default::default()
1133                }
1134                .into(),
1135            ]),
1136            ..Default::default()
1137        });
1138
1139        let mut processor = TrimmingProcessor::new();
1140        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1141
1142        assert_eq!(
1143            get_value!(event.spans[0].description!),
1144            &original_description
1145        );
1146        // Trace ID would be trimmed without `trim = "false"`
1147        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1148    }
1149
1150    #[test]
1151    fn test_untrimmable_fields_drop() {
1152        let original_description = "a".repeat(819164);
1153        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1154        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1155        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1156        let original_op = "e".repeat(129);
1157
1158        let mut event = Annotated::new(Event {
1159            spans: Annotated::new(vec![
1160                Span {
1161                    description: original_description.clone().into(),
1162                    ..Default::default()
1163                }
1164                .into(),
1165                Span {
1166                    span_id: original_span_id.into(),
1167                    trace_id: original_trace_id.into(),
1168                    segment_id: original_segment_id.into(),
1169                    is_segment: false.into(),
1170                    op: original_op.clone().into(),
1171                    start_timestamp: Timestamp(
1172                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1173                            .unwrap()
1174                            .into(),
1175                    )
1176                    .into(),
1177                    timestamp: Timestamp(
1178                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1179                            .unwrap()
1180                            .into(),
1181                    )
1182                    .into(),
1183                    ..Default::default()
1184                }
1185                .into(),
1186            ]),
1187            ..Default::default()
1188        });
1189
1190        let mut processor = TrimmingProcessor::new();
1191        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1192
1193        assert_eq!(
1194            get_value!(event.spans[0].description!),
1195            &original_description
1196        );
1197        // These fields would be dropped without `trim = "false"`
1198        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1199        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1200        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1201        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1202        // span.op is trimmed to its max_chars, but not dropped:
1203        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1204        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1205        assert!(get_value!(event.spans[1].timestamp).is_some());
1206    }
1207
1208    #[test]
1209    fn test_too_long_tags() {
1210        let mut event = Annotated::new(Event {
1211        tags: Annotated::new(Tags(PairList(
1212            vec![Annotated::new(TagEntry(
1213                Annotated::new("foobar".to_owned()),
1214                Annotated::new("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx".to_owned()),
1215            )), Annotated::new(TagEntry(
1216                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1217                Annotated::new("bar".to_owned()),
1218            ))]),
1219        )),
1220        ..Event::default()
1221    });
1222
1223        let mut processor = TrimmingProcessor::new();
1224        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1225
1226        insta::assert_debug_snapshot!(get_value!(event.tags!), @r###"
1227        Tags(
1228            PairList(
1229                [
1230                    TagEntry(
1231                        "foobar",
1232                        Annotated(
1233                            "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
1234                            Meta {
1235                                remarks: [
1236                                    Remark {
1237                                        ty: Substituted,
1238                                        rule_id: "!limit",
1239                                        range: Some(
1240                                            (
1241                                                197,
1242                                                200,
1243                                            ),
1244                                        ),
1245                                    },
1246                                ],
1247                                errors: [],
1248                                original_length: Some(
1249                                    203,
1250                                ),
1251                                original_value: None,
1252                            },
1253                        ),
1254                    ),
1255                    TagEntry(
1256                        Annotated(
1257                            "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo...",
1258                            Meta {
1259                                remarks: [
1260                                    Remark {
1261                                        ty: Substituted,
1262                                        rule_id: "!limit",
1263                                        range: Some(
1264                                            (
1265                                                197,
1266                                                200,
1267                                            ),
1268                                        ),
1269                                    },
1270                                ],
1271                                errors: [],
1272                                original_length: Some(
1273                                    203,
1274                                ),
1275                                original_value: None,
1276                            },
1277                        ),
1278                        "bar",
1279                    ),
1280                ],
1281            ),
1282        )
1283        "###);
1284    }
1285
1286    #[test]
1287    fn test_fixed_item_size() {
1288        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1289        struct TestObject {
1290            #[metastructure(max_bytes = 28)]
1291            inner: Annotated<TestObjectInner>,
1292        }
1293        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1294        struct TestObjectInner {
1295            #[metastructure(max_chars = 10, trim = true)]
1296            body: Annotated<String>,
1297            // This should neither be trimmed nor factor into size calculations.
1298            #[metastructure(trim = false, bytes_size = "always_zero")]
1299            number: Annotated<u64>,
1300            // This should count as 10B.
1301            #[metastructure(trim = false, bytes_size = 10)]
1302            other_number: Annotated<u64>,
1303            #[metastructure(trim = true)]
1304            footer: Annotated<String>,
1305        }
1306
1307        fn always_zero(_state: &ProcessingState) -> Option<usize> {
1308            Some(0)
1309        }
1310
1311        let mut object = Annotated::new(TestObject {
1312            inner: Annotated::new(TestObjectInner {
1313                body: Annotated::new("Longer than 10 chars".to_owned()),
1314                number: Annotated::new(13),
1315                other_number: Annotated::new(12),
1316                footer: Annotated::new("There should only be 'Th...' left".to_owned()),
1317            }),
1318        });
1319
1320        let mut processor = TrimmingProcessor::new();
1321        processor::process_value(&mut object, &mut processor, ProcessingState::root()).unwrap();
1322
1323        // * `body` gets trimmed to 13B (10 chars + `...`)
1324        // * `number` counts as 0B
1325        // * `other_number` counts as 10B
1326        // That leaves 5B for the `footer`.
1327        insta::assert_ron_snapshot!(SerializableAnnotated(&object), @r###"
1328        {
1329          "inner": {
1330            "body": "Longer ...",
1331            "number": 13,
1332            "other_number": 12,
1333            "footer": "Th...",
1334          },
1335          "_meta": {
1336            "inner": {
1337              "body": {
1338                "": Meta(Some(MetaInner(
1339                  rem: [
1340                    [
1341                      "!limit",
1342                      s,
1343                      7,
1344                      10,
1345                    ],
1346                  ],
1347                  len: Some(20),
1348                ))),
1349              },
1350              "footer": {
1351                "": Meta(Some(MetaInner(
1352                  rem: [
1353                    [
1354                      "!limit",
1355                      s,
1356                      2,
1357                      5,
1358                    ],
1359                  ],
1360                  len: Some(33),
1361                ))),
1362              },
1363            },
1364          },
1365        }
1366        "###);
1367    }
1368
1369    #[test]
1370    fn test_flags_context_trimming() {
1371        let original_flags_count = 1_000;
1372        let values: Vec<_> = (0..original_flags_count)
1373            .map(|i| {
1374                serde_json::json!({
1375                    "flag": format!("feature.flag.{i}"),
1376                    "result": "x".repeat(500),
1377                })
1378            })
1379            .collect();
1380        let json = serde_json::json!({
1381            "contexts": {
1382                "flags": {
1383                    "values": values,
1384                },
1385                "my_custom_context": {
1386                    "foo": "x".repeat(10_000)
1387                }
1388            },
1389        })
1390        .to_string();
1391        let mut event = Annotated::<Event>::from_json(&json).unwrap();
1392
1393        let mut processor = TrimmingProcessor::new();
1394        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1395
1396        let contexts = get_value!(event.contexts!);
1397
1398        // Make sure flags contexts has its own limit applied.
1399        let values = &contexts.get::<FlagsContext>().unwrap().values;
1400
1401        assert_eq!(values.value().unwrap().len(), 292);
1402        assert_eq!(values.meta().original_length(), Some(original_flags_count));
1403
1404        // Make sure the custom context is trimmed to 8192.
1405        let custom = match contexts.get_key("my_custom_context").unwrap() {
1406            Context::Other(custom) => custom,
1407            _ => unreachable!(),
1408        };
1409        assert_eq!(custom["foo"].value().unwrap().as_str().unwrap().len(), 8192);
1410        assert_eq!(custom["foo"].meta().original_length(), Some(10_000));
1411    }
1412}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
