relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.max_bytes(),
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        // If our current depth is the one where we found a bag_size attribute, this means we
97        // are done processing a databag. Pop the bag size state.
98        self.size_state
99            .pop_if(|size_state| state.depth() == size_state.encountered_at_depth);
100
101        // After processing a value, update the remaining bag sizes. We have a separate if-let
102        // here in case somebody defines nested databags (a struct with bag_size that contains
103        // another struct with a different bag_size), in case we just exited a databag we want
104        // to update the bag_size_state of the outer databag with the remaining size.
105        //
106        // This also has to happen after string trimming, which is why it's running in
107        // after_process.
108        if state.entered_anything() && !self.size_state.is_empty() {
109            // Do not subtract if state is from newtype struct.
110            let item_length = state
111                .bytes_size()
112                .unwrap_or_else(|| relay_protocol::estimate_size_flat(value) + 1);
113            for size_state in self.size_state.iter_mut() {
114                size_state.size_remaining = size_state
115                    .size_remaining
116                    .map(|size| size.saturating_sub(item_length));
117            }
118        }
119
120        Ok(())
121    }
122
123    fn process_string(
124        &mut self,
125        value: &mut String,
126        meta: &mut Meta,
127        state: &ProcessingState<'_>,
128    ) -> ProcessingResult {
129        if let Some(max_chars) = state.max_chars() {
130            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
131        }
132
133        if !state.attrs().trim {
134            return Ok(());
135        }
136
137        if let Some(size_remaining) = self.remaining_size() {
138            trim_string(value, meta, size_remaining, 0);
139        }
140
141        Ok(())
142    }
143
144    fn process_array<T>(
145        &mut self,
146        value: &mut Array<T>,
147        meta: &mut Meta,
148        state: &ProcessingState<'_>,
149    ) -> ProcessingResult
150    where
151        T: ProcessValue,
152    {
153        if !state.attrs().trim {
154            return Ok(());
155        }
156
157        // If we need to check the bag size, then we go down a different path
158        if !self.size_state.is_empty() {
159            let original_length = value.len();
160
161            if self.should_remove_container(value, state) {
162                return Err(ProcessingAction::DeleteValueHard);
163            }
164
165            let mut split_index = None;
166            for (index, item) in value.iter_mut().enumerate() {
167                if self.remaining_size() == Some(0) {
168                    split_index = Some(index);
169                    break;
170                }
171
172                let item_state = state.enter_index(index, None, ValueType::for_field(item));
173                processor::process_value(item, self, &item_state)?;
174            }
175
176            if let Some(split_index) = split_index {
177                let _ = value.split_off(split_index);
178            }
179
180            if value.len() != original_length {
181                meta.set_original_length(Some(original_length));
182            }
183        } else {
184            value.process_child_values(self, state)?;
185        }
186
187        Ok(())
188    }
189
190    fn process_object<T>(
191        &mut self,
192        value: &mut Object<T>,
193        meta: &mut Meta,
194        state: &ProcessingState<'_>,
195    ) -> ProcessingResult
196    where
197        T: ProcessValue,
198    {
199        if !state.attrs().trim {
200            return Ok(());
201        }
202
203        // If we need to check the bag size, then we go down a different path
204        if !self.size_state.is_empty() {
205            let original_length = value.len();
206
207            if self.should_remove_container(value, state) {
208                return Err(ProcessingAction::DeleteValueHard);
209            }
210
211            let mut split_key = None;
212            for (key, item) in value.iter_mut() {
213                if self.remaining_size() == Some(0) {
214                    split_key = Some(key.to_owned());
215                    break;
216                }
217
218                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
219                processor::process_value(item, self, &item_state)?;
220            }
221
222            if let Some(split_key) = split_key {
223                let _ = value.split_off(&split_key);
224            }
225
226            if value.len() != original_length {
227                meta.set_original_length(Some(original_length));
228            }
229        } else {
230            value.process_child_values(self, state)?;
231        }
232
233        Ok(())
234    }
235
236    fn process_value(
237        &mut self,
238        value: &mut Value,
239        _meta: &mut Meta,
240        state: &ProcessingState<'_>,
241    ) -> ProcessingResult {
242        if !state.attrs().trim {
243            return Ok(());
244        }
245
246        match value {
247            Value::Array(_) | Value::Object(_) => {
248                if self.remaining_depth(state) == Some(1)
249                    && let Ok(x) = serde_json::to_string(&value)
250                {
251                    // Error case should not be possible
252                    *value = Value::String(x);
253                }
254            }
255            _ => (),
256        }
257
258        value.process_child_values(self, state)?;
259        Ok(())
260    }
261
262    fn process_replay(
263        &mut self,
264        replay: &mut Replay,
265        _: &mut Meta,
266        state: &ProcessingState<'_>,
267    ) -> ProcessingResult {
268        replay.process_child_values(self, state)
269    }
270
271    fn process_raw_stacktrace(
272        &mut self,
273        stacktrace: &mut RawStacktrace,
274        _meta: &mut Meta,
275        state: &ProcessingState<'_>,
276    ) -> ProcessingResult {
277        if !state.attrs().trim {
278            return Ok(());
279        }
280
281        processor::apply(&mut stacktrace.frames, |frames, meta| {
282            enforce_frame_hard_limit(frames, meta, 200, 50);
283            Ok(())
284        })?;
285
286        stacktrace.process_child_values(self, state)?;
287
288        processor::apply(&mut stacktrace.frames, |frames, _meta| {
289            slim_frame_data(frames, 50);
290            Ok(())
291        })?;
292
293        Ok(())
294    }
295}
296
297/// Trims the string to the given maximum length and updates meta data.
298pub(crate) fn trim_string(
299    value: &mut String,
300    meta: &mut Meta,
301    max_chars: usize,
302    max_chars_allowance: usize,
303) {
304    let hard_limit = max_chars + max_chars_allowance;
305
306    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
307        return;
308    }
309
310    processor::process_chunked_value(value, meta, |chunks| {
311        let mut length = 0;
312        let mut new_chunks = vec![];
313
314        for chunk in chunks {
315            let chunk_chars = chunk.count();
316
317            // if the entire chunk fits, just put it in
318            if length + chunk_chars < max_chars {
319                new_chunks.push(chunk);
320                length += chunk_chars;
321                continue;
322            }
323
324            match chunk {
325                // if there is enough space for this chunk and the 3 character
326                // ellipsis marker we can push the remaining chunk
327                Chunk::Redaction { .. } => {
328                    if length + chunk_chars + 3 < hard_limit {
329                        new_chunks.push(chunk);
330                    }
331                }
332
333                // if this is a text chunk, we can put the remaining characters in.
334                Chunk::Text { text } => {
335                    let mut remaining = String::new();
336                    for c in text.chars() {
337                        if length + 3 < max_chars {
338                            remaining.push(c);
339                        } else {
340                            break;
341                        }
342                        length += 1;
343                    }
344
345                    new_chunks.push(Chunk::Text {
346                        text: Cow::Owned(remaining),
347                    });
348                }
349            }
350
351            new_chunks.push(Chunk::Redaction {
352                text: Cow::Borrowed("..."),
353                rule_id: Cow::Borrowed("!limit"),
354                ty: RemarkType::Substituted,
355            });
356            break;
357        }
358
359        new_chunks
360    });
361}
362
363/// Trim down the frame list to a hard limit.
364///
365/// The total limit is `recent_frames` + `old_frames`.
366/// `recent_frames` is the number of frames to keep from the beginning of the list,
367/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
368/// the oldest frames up the stack.
369///
370/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
371/// the recursion.
372fn enforce_frame_hard_limit(
373    frames: &mut Array<Frame>,
374    meta: &mut Meta,
375    recent_frames: usize,
376    old_frames: usize,
377) {
378    let original_length = frames.len();
379    let limit = recent_frames + old_frames;
380    if original_length > limit {
381        meta.set_original_length(Some(original_length));
382        let _ = frames.drain(old_frames..original_length - recent_frames);
383    }
384}
385
386/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
387///
388/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
389fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
390    let frames_len = frames.len();
391
392    if frames_len <= frame_allowance {
393        return;
394    }
395
396    // Avoid ownership issues by only storing indices
397    let mut app_frame_indices = Vec::with_capacity(frames_len);
398    let mut system_frame_indices = Vec::with_capacity(frames_len);
399
400    for (i, frame) in frames.iter().enumerate() {
401        if let Some(frame) = frame.value() {
402            match frame.in_app.value() {
403                Some(true) => app_frame_indices.push(i),
404                _ => system_frame_indices.push(i),
405            }
406        }
407    }
408
409    let app_count = app_frame_indices.len();
410    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
411    let system_frames_to_remove = system_frame_indices
412        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
413        .unwrap_or(&[]);
414
415    let remaining = frames_len
416        .saturating_sub(frame_allowance)
417        .saturating_sub(system_frames_to_remove.len());
418    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
419    let app_frames_to_remove = app_frame_indices
420        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
421        .unwrap_or(&[]);
422
423    // TODO: Which annotation to set?
424
425    let top_frame_index = frames_len.saturating_sub(1);
426    for i in system_frames_to_remove
427        .iter()
428        .chain(app_frames_to_remove)
429        .filter(|&&i| i != top_frame_index)
430    {
431        if let Some(frame) = frames.get_mut(*i)
432            && let Some(ref mut frame) = frame.value_mut().as_mut()
433        {
434            frame.vars = Annotated::empty();
435            frame.pre_context = Annotated::empty();
436            frame.post_context = Annotated::empty();
437        }
438    }
439}
440
441#[cfg(test)]
442mod tests {
443    use std::iter::repeat_n;
444
445    use crate::MaxChars;
446    use chrono::DateTime;
447    use relay_event_schema::protocol::{
448        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, FlagsContext, PairList,
449        SentryTags, Span, SpanData, SpanId, TagEntry, Tags, Timestamp, TraceId, Values,
450    };
451    use relay_protocol::{FromValue, IntoValue, Map, Remark, SerializableAnnotated, get_value};
452    use similar_asserts::assert_eq;
453
454    use super::*;
455
456    #[test]
457    fn test_string_trimming() {
458        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
459        processor::apply(&mut value, |v, m| {
460            trim_string(v, m, 20, 0);
461            Ok(())
462        })
463        .unwrap();
464
465        assert_eq!(
466            value,
467            Annotated(Some("This is my long s...".into()), {
468                let mut meta = Meta::default();
469                meta.add_remark(Remark {
470                    ty: RemarkType::Substituted,
471                    rule_id: "!limit".to_owned(),
472                    range: Some((17, 20)),
473                });
474                meta.set_original_length(Some(46));
475                meta
476            })
477        );
478    }
479
480    #[test]
481    fn test_basic_trimming() {
482        let mut processor = TrimmingProcessor::new();
483
484        let mut event = Annotated::new(Event {
485            logger: Annotated::new("x".repeat(300)),
486            ..Default::default()
487        });
488
489        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
490
491        let mut expected = Annotated::new("x".repeat(300));
492        processor::apply(&mut expected, |v, m| {
493            trim_string(v, m, MaxChars::Logger.limit(), 0);
494            Ok(())
495        })
496        .unwrap();
497
498        assert_eq!(event.value().unwrap().logger, expected);
499    }
500
501    #[test]
502    fn test_max_char_allowance() {
503        let string = "This string requires some allowance to fit!";
504        let mut value = Annotated::new(string.to_owned()); // len == 43
505        processor::apply(&mut value, |v, m| {
506            trim_string(v, m, 40, 5);
507            Ok(())
508        })
509        .unwrap();
510
511        assert_eq!(value, Annotated::new(string.to_owned()));
512    }
513
514    #[test]
515    fn test_databag_stripping() {
516        let mut processor = TrimmingProcessor::new();
517
518        fn make_nested_object(depth: usize) -> Annotated<Value> {
519            if depth == 0 {
520                return Annotated::new(Value::String("max depth".to_owned()));
521            }
522            let mut rv = Object::new();
523            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
524            Annotated::new(Value::Object(rv))
525        }
526
527        let databag = Annotated::new({
528            let mut map = Object::new();
529            map.insert(
530                "key_1".to_owned(),
531                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
532            );
533            map.insert(
534                "key_2".to_owned(),
535                make_nested_object(8).map_value(ExtraValue),
536            );
537            map.insert(
538                "key_3".to_owned(),
539                // innermost key (string) is entering json stringify codepath
540                make_nested_object(5).map_value(ExtraValue),
541            );
542            map
543        });
544        let mut event = Annotated::new(Event {
545            extra: databag,
546            ..Default::default()
547        });
548
549        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
550        let stripped_extra = &event.value().unwrap().extra;
551        let json = stripped_extra.to_json_pretty().unwrap();
552
553        assert_eq!(
554            json,
555            r#"{
556  "key_1": "value 1",
557  "key_2": {
558    "key8": {
559      "key7": {
560        "key6": {
561          "key5": {
562            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
563          }
564        }
565      }
566    }
567  },
568  "key_3": {
569    "key5": {
570      "key4": {
571        "key3": {
572          "key2": {
573            "key1": "max depth"
574          }
575        }
576      }
577    }
578  }
579}"#
580        );
581    }
582
583    #[test]
584    fn test_databag_array_stripping() {
585        let mut processor = TrimmingProcessor::new();
586
587        let databag = Annotated::new({
588            let mut map = Object::new();
589            for idx in 0..100 {
590                map.insert(
591                    format!("key_{idx}"),
592                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
593                );
594            }
595            map
596        });
597        let mut event = Annotated::new(Event {
598            extra: databag,
599            ..Default::default()
600        });
601
602        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
603        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
604
605        insta::assert_ron_snapshot!(stripped_extra);
606    }
607
608    /// Tests that a trimming a string takes a lower outer limit into account.
609    #[test]
610    fn test_string_trimming_limits() {
611        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
612        struct Outer {
613            #[metastructure(max_bytes = 10)]
614            inner: Annotated<Inner>,
615        }
616
617        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
618        struct Inner {
619            #[metastructure(max_bytes = 20)]
620            innerer: Annotated<String>,
621        }
622
623        let mut processor = TrimmingProcessor::new();
624
625        let mut outer = Annotated::new({
626            Outer {
627                inner: Annotated::new(Inner {
628                    innerer: Annotated::new("This string is 28 bytes long".into()),
629                }),
630            }
631        });
632
633        processor::process_value(&mut outer, &mut processor, ProcessingState::root()).unwrap();
634        let stripped = SerializableAnnotated(&outer);
635
636        insta::assert_ron_snapshot!(stripped, @r###"
637        {
638          "inner": {
639            "innerer": "This st...",
640          },
641          "_meta": {
642            "inner": {
643              "innerer": {
644                "": Meta(Some(MetaInner(
645                  rem: [
646                    [
647                      "!limit",
648                      s,
649                      7,
650                      10,
651                    ],
652                  ],
653                  len: Some(28),
654                ))),
655              },
656            },
657          },
658        }
659        "###);
660    }
661
662    #[test]
663    fn test_tags_stripping() {
664        let mut processor = TrimmingProcessor::new();
665
666        let mut event = Annotated::new(Event {
667            tags: Annotated::new(Tags(
668                vec![Annotated::new(TagEntry(
669                    Annotated::new("x".repeat(300)),
670                    Annotated::new("x".repeat(300)),
671                ))]
672                .into(),
673            )),
674            ..Default::default()
675        });
676
677        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
678        let json = event
679            .value()
680            .unwrap()
681            .tags
682            .payload_to_json_pretty()
683            .unwrap();
684
685        assert_eq!(
686            json,
687            r#"[
688  [
689    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
690    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
691  ]
692]"#
693        );
694    }
695
696    #[test]
697    fn test_databag_state_leak() {
698        let event = Annotated::new(Event {
699            breadcrumbs: Annotated::new(Values::new(
700                repeat_n(
701                    Annotated::new(Breadcrumb {
702                        data: {
703                            let mut map = Map::new();
704                            map.insert(
705                                "spamspamspam".to_owned(),
706                                Annotated::new(Value::String("blablabla".to_owned())),
707                            );
708                            Annotated::new(map)
709                        },
710                        ..Default::default()
711                    }),
712                    200,
713                )
714                .collect(),
715            )),
716            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
717                ty: Annotated::new("TypeError".to_owned()),
718                value: Annotated::new("important error message".to_owned().into()),
719                stacktrace: Annotated::new(
720                    RawStacktrace {
721                        frames: Annotated::new(
722                            repeat_n(
723                                Annotated::new(Frame {
724                                    function: Annotated::new("importantFunctionName".to_owned()),
725                                    symbol: Annotated::new("important_symbol".to_owned()),
726                                    ..Default::default()
727                                }),
728                                200,
729                            )
730                            .collect(),
731                        ),
732                        ..Default::default()
733                    }
734                    .into(),
735                ),
736                ..Default::default()
737            })])),
738            ..Default::default()
739        });
740
741        let mut processor = TrimmingProcessor::new();
742        let mut stripped_event = event.clone();
743        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
744            .unwrap();
745
746        assert_eq!(
747            event.to_json_pretty().unwrap(),
748            stripped_event.to_json_pretty().unwrap()
749        );
750    }
751
752    #[test]
753    fn test_custom_context_trimming() {
754        let mut contexts = Contexts::new();
755        for i in 1..2 {
756            contexts.insert(format!("despacito{i}"), {
757                let mut context = Object::new();
758                context.insert(
759                    "foo".to_owned(),
760                    Annotated::new(Value::String("a".repeat(4000))),
761                );
762                context.insert(
763                    "bar".to_owned(),
764                    Annotated::new(Value::String("a".repeat(5000))),
765                );
766                Context::Other(context)
767            });
768        }
769
770        let mut contexts = Annotated::new(contexts);
771        let mut processor = TrimmingProcessor::new();
772        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
773
774        let contexts = contexts.value().unwrap();
775        for i in 1..2 {
776            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
777                Context::Other(x) => x,
778                _ => panic!("Context has changed type!"),
779            };
780
781            assert_eq!(
782                other
783                    .get("bar")
784                    .unwrap()
785                    .value()
786                    .unwrap()
787                    .as_str()
788                    .unwrap()
789                    .len(),
790                5000
791            );
792            assert_eq!(
793                other
794                    .get("foo")
795                    .unwrap()
796                    .value()
797                    .unwrap()
798                    .as_str()
799                    .unwrap()
800                    .len(),
801                3189
802            );
803        }
804    }
805
806    #[test]
807    fn test_extra_trimming_long_arrays() {
808        let mut extra = Object::new();
809        extra.insert("foo".to_owned(), {
810            Annotated::new(ExtraValue(Value::Array(
811                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
812            )))
813        });
814
815        let mut event = Annotated::new(Event {
816            extra: Annotated::new(extra),
817            ..Default::default()
818        });
819
820        let mut processor = TrimmingProcessor::new();
821        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
822
823        let arr = match event
824            .value()
825            .unwrap()
826            .extra
827            .value()
828            .unwrap()
829            .get("foo")
830            .unwrap()
831            .value()
832            .unwrap()
833        {
834            ExtraValue(Value::Array(x)) => x,
835            x => panic!("Wrong type: {x:?}"),
836        };
837
838        // this is larger / 2 for the extra value
839        assert_eq!(arr.len(), 8192);
840    }
841
842    // TODO(ja): Enable this test
843    // #[test]
844    // fn test_newtypes_do_not_add_to_depth() {
845    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
846    //     struct WrappedString(String);
847
848    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
849    //     struct StructChild2 {
850    //         inner: Annotated<WrappedString>,
851    //     }
852
853    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
854    //     struct StructChild {
855    //         inner: Annotated<StructChild2>,
856    //     }
857
858    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
859    //     struct Struct {
860    //         #[metastructure(bag_size = "small")]
861    //         inner: Annotated<StructChild>,
862    //     }
863
864    //     let mut value = Annotated::new(Struct {
865    //         inner: Annotated::new(StructChild {
866    //             inner: Annotated::new(StructChild2 {
867    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
868    //             }),
869    //         }),
870    //     });
871
872    //     let mut processor = TrimmingProcessor::new();
873    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
874
875    //     // Ensure stack does not leak with newtypes
876    //     assert!(processor.bag_size_state.is_empty());
877
878    //     assert_eq!(
879    //         value.to_json().unwrap(),
880    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
881    //     );
882    // }
883
884    #[test]
885    fn test_frameqty_equals_limit() {
886        fn create_frame(filename: &str) -> Annotated<Frame> {
887            Annotated::new(Frame {
888                filename: Annotated::new(filename.into()),
889                ..Default::default()
890            })
891        }
892
893        let mut frames = Annotated::new(vec![
894            create_frame("foo3.py"),
895            create_frame("foo4.py"),
896            create_frame("foo5.py"),
897        ]);
898
899        processor::apply(&mut frames, |f, m| {
900            enforce_frame_hard_limit(f, m, 3, 0);
901            Ok(())
902        })
903        .unwrap();
904
905        processor::apply(&mut frames, |f, m| {
906            enforce_frame_hard_limit(f, m, 1, 2);
907            Ok(())
908        })
909        .unwrap();
910
911        // original_length isn't set, when limit is equal to length, as no trimming took place.
912        assert!(frames.meta().original_length().is_none());
913    }
914
915    #[test]
916    fn test_frame_hard_limit() {
917        fn create_frame(filename: &str) -> Annotated<Frame> {
918            Annotated::new(Frame {
919                filename: Annotated::new(filename.into()),
920                ..Default::default()
921            })
922        }
923
924        let mut frames = Annotated::new(vec![
925            create_frame("foo1.py"),
926            create_frame("foo2.py"),
927            create_frame("foo3.py"),
928            create_frame("foo4.py"),
929            create_frame("foo5.py"),
930        ]);
931
932        processor::apply(&mut frames, |f, m| {
933            enforce_frame_hard_limit(f, m, 3, 0);
934            Ok(())
935        })
936        .unwrap();
937
938        let mut expected_meta = Meta::default();
939        expected_meta.set_original_length(Some(5));
940
941        assert_eq!(
942            frames,
943            Annotated(
944                Some(vec![
945                    create_frame("foo3.py"),
946                    create_frame("foo4.py"),
947                    create_frame("foo5.py"),
948                ]),
949                expected_meta
950            )
951        );
952    }
953
954    #[test]
955    fn test_frame_hard_limit_recent_old() {
956        fn create_frame(filename: &str) -> Annotated<Frame> {
957            Annotated::new(Frame {
958                filename: Annotated::new(filename.into()),
959                ..Default::default()
960            })
961        }
962
963        let mut frames = Annotated::new(vec![
964            create_frame("foo1.py"),
965            create_frame("foo2.py"),
966            create_frame("foo3.py"),
967            create_frame("foo4.py"),
968            create_frame("foo5.py"),
969        ]);
970
971        processor::apply(&mut frames, |f, m| {
972            enforce_frame_hard_limit(f, m, 2, 1);
973            Ok(())
974        })
975        .unwrap();
976
977        let mut expected_meta = Meta::default();
978        expected_meta.set_original_length(Some(5));
979
980        assert_eq!(
981            frames,
982            Annotated(
983                Some(vec![
984                    create_frame("foo1.py"),
985                    create_frame("foo4.py"),
986                    create_frame("foo5.py"),
987                ]),
988                expected_meta
989            )
990        );
991    }
992
993    #[test]
994    fn test_slim_frame_data_under_max() {
995        let mut frames = vec![Annotated::new(Frame {
996            filename: Annotated::new("foo".into()),
997            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
998            context_line: Annotated::new("b".to_owned()),
999            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1000            ..Default::default()
1001        })];
1002
1003        let old_frames = frames.clone();
1004        slim_frame_data(&mut frames, 4);
1005
1006        assert_eq!(frames, old_frames);
1007    }
1008
1009    #[test]
1010    fn test_slim_frame_data_over_max() {
1011        let mut frames = vec![];
1012
1013        for n in 0..5 {
1014            frames.push(Annotated::new(Frame {
1015                filename: Annotated::new(format!("foo {n}").into()),
1016                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1017                context_line: Annotated::new("b".to_owned()),
1018                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1019                ..Default::default()
1020            }));
1021        }
1022
1023        slim_frame_data(&mut frames, 4);
1024
1025        let expected = vec![
1026            Annotated::new(Frame {
1027                filename: Annotated::new("foo 0".into()),
1028                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1029                context_line: Annotated::new("b".to_owned()),
1030                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1031                ..Default::default()
1032            }),
1033            Annotated::new(Frame {
1034                filename: Annotated::new("foo 1".into()),
1035                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1036                context_line: Annotated::new("b".to_owned()),
1037                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1038                ..Default::default()
1039            }),
1040            Annotated::new(Frame {
1041                filename: Annotated::new("foo 2".into()),
1042                context_line: Annotated::new("b".to_owned()),
1043                ..Default::default()
1044            }),
1045            Annotated::new(Frame {
1046                filename: Annotated::new("foo 3".into()),
1047                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1048                context_line: Annotated::new("b".to_owned()),
1049                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1050                ..Default::default()
1051            }),
1052            Annotated::new(Frame {
1053                filename: Annotated::new("foo 4".into()),
1054                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1055                context_line: Annotated::new("b".to_owned()),
1056                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1057                ..Default::default()
1058            }),
1059        ];
1060
1061        assert_eq!(frames, expected);
1062    }
1063
1064    #[test]
1065    fn test_slim_frame_data_does_not_trim_top_frame_metadata() {
1066        let mut frames: Array<Frame> = (0..50)
1067            .map(|n| {
1068                Annotated::new(Frame {
1069                    filename: Annotated::new(format!("system {n}").into()),
1070                    ..Default::default()
1071                })
1072            })
1073            .collect();
1074        frames.push(Annotated::new(Frame {
1075            filename: Annotated::new("raising".into()),
1076            pre_context: Annotated::new(vec![Annotated::new("before".to_owned())]),
1077            context_line: Annotated::new("current".to_owned()),
1078            post_context: Annotated::new(vec![Annotated::new("after".to_owned())]),
1079            vars: Annotated::new({
1080                let mut vars = Object::new();
1081                vars.insert("local".to_owned(), Annotated::new("value".into()));
1082                vars.into()
1083            }),
1084            in_app: Annotated::new(true),
1085            ..Default::default()
1086        }));
1087
1088        slim_frame_data(&mut frames, 50);
1089
1090        let top_frame = frames.last().unwrap().value().unwrap();
1091        assert!(top_frame.vars.value().is_some());
1092        assert!(top_frame.pre_context.value().is_some());
1093        assert!(top_frame.context_line.value().is_some());
1094        assert!(top_frame.post_context.value().is_some());
1095    }
1096
1097    #[test]
1098    fn test_too_many_spans_trimmed() {
1099        let span = Span {
1100            platform: Annotated::new("a".repeat(1024 * 90)),
1101            sentry_tags: Annotated::new(SentryTags {
1102                release: Annotated::new("b".repeat(1024 * 100)),
1103                ..Default::default()
1104            }),
1105            ..Default::default()
1106        };
1107        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1108            .take(10)
1109            .collect();
1110
1111        let mut event = Annotated::new(Event {
1112            spans: Annotated::new(spans.clone()),
1113            ..Default::default()
1114        });
1115
1116        let mut processor = TrimmingProcessor::new();
1117        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1118
1119        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1120        assert_eq!(trimmed_spans.len(), 5);
1121
1122        // The actual spans were not touched:
1123        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1124    }
1125
1126    #[test]
1127    fn test_span_data_not_partially_trimmed() {
1128        let span_data = SpanData::from([(
1129            "large_attribute".to_owned(),
1130            Annotated::new(Value::String("a".repeat(100 * 1024))),
1131        )]);
1132        let span = Span {
1133            data: Annotated::new(span_data.clone()),
1134            ..Default::default()
1135        };
1136        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1137            .take(10)
1138            .collect();
1139
1140        let mut event = Annotated::new(Event {
1141            spans: Annotated::new(spans.clone()),
1142            ..Default::default()
1143        });
1144
1145        let mut processor = TrimmingProcessor::new();
1146        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1147
1148        let trimmed = event.value().unwrap().spans.value().unwrap();
1149        assert!(trimmed.len() < spans.len());
1150        assert!(trimmed.iter().all(|span| {
1151            span.value()
1152                .and_then(|span| span.data.value())
1153                .is_some_and(|data| data == &span_data)
1154        }));
1155    }
1156
1157    #[test]
1158    fn test_untrimmable_fields() {
1159        let original_description = "a".repeat(819163);
1160        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1161        let mut event = Annotated::new(Event {
1162            spans: Annotated::new(vec![
1163                Span {
1164                    description: original_description.clone().into(),
1165                    ..Default::default()
1166                }
1167                .into(),
1168                Span {
1169                    trace_id: original_trace_id.into(),
1170                    ..Default::default()
1171                }
1172                .into(),
1173            ]),
1174            ..Default::default()
1175        });
1176
1177        let mut processor = TrimmingProcessor::new();
1178        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1179
1180        assert_eq!(
1181            get_value!(event.spans[0].description!),
1182            &original_description
1183        );
1184        // Trace ID would be trimmed without `trim = "false"`
1185        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1186    }
1187
1188    #[test]
1189    fn test_untrimmable_fields_drop() {
1190        let original_description = "a".repeat(819164);
1191        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1192        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1193        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1194        let original_op = "e".repeat(129);
1195
1196        let mut event = Annotated::new(Event {
1197            spans: Annotated::new(vec![
1198                Span {
1199                    description: original_description.clone().into(),
1200                    ..Default::default()
1201                }
1202                .into(),
1203                Span {
1204                    span_id: original_span_id.into(),
1205                    trace_id: original_trace_id.into(),
1206                    segment_id: original_segment_id.into(),
1207                    is_segment: false.into(),
1208                    op: original_op.clone().into(),
1209                    start_timestamp: Timestamp(
1210                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1211                            .unwrap()
1212                            .into(),
1213                    )
1214                    .into(),
1215                    timestamp: Timestamp(
1216                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1217                            .unwrap()
1218                            .into(),
1219                    )
1220                    .into(),
1221                    ..Default::default()
1222                }
1223                .into(),
1224            ]),
1225            ..Default::default()
1226        });
1227
1228        let mut processor = TrimmingProcessor::new();
1229        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1230
1231        assert_eq!(
1232            get_value!(event.spans[0].description!),
1233            &original_description
1234        );
1235        // These fields would be dropped without `trim = "false"`
1236        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1237        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1238        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1239        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1240        // span.op is trimmed to its max_chars, but not dropped:
1241        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1242        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1243        assert!(get_value!(event.spans[1].timestamp).is_some());
1244    }
1245
1246    #[test]
1247    fn test_too_long_tags() {
1248        let mut event = Annotated::new(Event {
1249        tags: Annotated::new(Tags(PairList(
1250            vec![Annotated::new(TagEntry(
1251                Annotated::new("foobar".to_owned()),
1252                Annotated::new("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx".to_owned()),
1253            )), Annotated::new(TagEntry(
1254                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1255                Annotated::new("bar".to_owned()),
1256            ))]),
1257        )),
1258        ..Event::default()
1259    });
1260
1261        let mut processor = TrimmingProcessor::new();
1262        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1263
1264        insta::assert_debug_snapshot!(get_value!(event.tags!), @r###"
1265        Tags(
1266            PairList(
1267                [
1268                    TagEntry(
1269                        "foobar",
1270                        Annotated(
1271                            "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
1272                            Meta {
1273                                remarks: [
1274                                    Remark {
1275                                        ty: Substituted,
1276                                        rule_id: "!limit",
1277                                        range: Some(
1278                                            (
1279                                                197,
1280                                                200,
1281                                            ),
1282                                        ),
1283                                    },
1284                                ],
1285                                errors: [],
1286                                original_length: Some(
1287                                    203,
1288                                ),
1289                                original_value: None,
1290                            },
1291                        ),
1292                    ),
1293                    TagEntry(
1294                        Annotated(
1295                            "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo...",
1296                            Meta {
1297                                remarks: [
1298                                    Remark {
1299                                        ty: Substituted,
1300                                        rule_id: "!limit",
1301                                        range: Some(
1302                                            (
1303                                                197,
1304                                                200,
1305                                            ),
1306                                        ),
1307                                    },
1308                                ],
1309                                errors: [],
1310                                original_length: Some(
1311                                    203,
1312                                ),
1313                                original_value: None,
1314                            },
1315                        ),
1316                        "bar",
1317                    ),
1318                ],
1319            ),
1320        )
1321        "###);
1322    }
1323
1324    #[test]
1325    fn test_fixed_item_size() {
1326        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1327        struct TestObject {
1328            #[metastructure(max_bytes = 28)]
1329            inner: Annotated<TestObjectInner>,
1330        }
1331        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1332        struct TestObjectInner {
1333            #[metastructure(max_chars = 10, trim = true)]
1334            body: Annotated<String>,
1335            // This should neither be trimmed nor factor into size calculations.
1336            #[metastructure(trim = false, bytes_size = "always_zero")]
1337            number: Annotated<u64>,
1338            // This should count as 10B.
1339            #[metastructure(trim = false, bytes_size = 10)]
1340            other_number: Annotated<u64>,
1341            #[metastructure(trim = true)]
1342            footer: Annotated<String>,
1343        }
1344
1345        fn always_zero(_state: &ProcessingState) -> Option<usize> {
1346            Some(0)
1347        }
1348
1349        let mut object = Annotated::new(TestObject {
1350            inner: Annotated::new(TestObjectInner {
1351                body: Annotated::new("Longer than 10 chars".to_owned()),
1352                number: Annotated::new(13),
1353                other_number: Annotated::new(12),
1354                footer: Annotated::new("There should only be 'Th...' left".to_owned()),
1355            }),
1356        });
1357
1358        let mut processor = TrimmingProcessor::new();
1359        processor::process_value(&mut object, &mut processor, ProcessingState::root()).unwrap();
1360
1361        // * `body` gets trimmed to 13B (10 chars + `...`)
1362        // * `number` counts as 0B
1363        // * `other_number` counts as 10B
1364        // That leaves 5B for the `footer`.
1365        insta::assert_ron_snapshot!(SerializableAnnotated(&object), @r###"
1366        {
1367          "inner": {
1368            "body": "Longer ...",
1369            "number": 13,
1370            "other_number": 12,
1371            "footer": "Th...",
1372          },
1373          "_meta": {
1374            "inner": {
1375              "body": {
1376                "": Meta(Some(MetaInner(
1377                  rem: [
1378                    [
1379                      "!limit",
1380                      s,
1381                      7,
1382                      10,
1383                    ],
1384                  ],
1385                  len: Some(20),
1386                ))),
1387              },
1388              "footer": {
1389                "": Meta(Some(MetaInner(
1390                  rem: [
1391                    [
1392                      "!limit",
1393                      s,
1394                      2,
1395                      5,
1396                    ],
1397                  ],
1398                  len: Some(33),
1399                ))),
1400              },
1401            },
1402          },
1403        }
1404        "###);
1405    }
1406
1407    #[test]
1408    fn test_flags_context_trimming() {
1409        let original_flags_count = 1_000;
1410        let values: Vec<_> = (0..original_flags_count)
1411            .map(|i| {
1412                serde_json::json!({
1413                    "flag": format!("feature.flag.{i}"),
1414                    "result": "x".repeat(500),
1415                })
1416            })
1417            .collect();
1418        let json = serde_json::json!({
1419            "contexts": {
1420                "flags": {
1421                    "values": values,
1422                },
1423                "my_custom_context": {
1424                    "foo": "x".repeat(10_000)
1425                }
1426            },
1427        })
1428        .to_string();
1429        let mut event = Annotated::<Event>::from_json(&json).unwrap();
1430
1431        let mut processor = TrimmingProcessor::new();
1432        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1433
1434        let contexts = get_value!(event.contexts!);
1435
1436        // Make sure flags contexts has its own limit applied.
1437        let values = &contexts.get::<FlagsContext>().unwrap().values;
1438
1439        assert_eq!(values.value().unwrap().len(), 584);
1440        assert_eq!(values.meta().original_length(), Some(original_flags_count));
1441
1442        // Make sure the custom context is trimmed to 8192.
1443        let custom = match contexts.get_key("my_custom_context").unwrap() {
1444            Context::Other(custom) => custom,
1445            _ => unreachable!(),
1446        };
1447        assert_eq!(custom["foo"].value().unwrap().as_str().unwrap().len(), 8192);
1448        assert_eq!(custom["foo"].meta().original_length(), Some(10_000));
1449    }
1450}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
