relay_event_normalization/
trimming.rs

1use std::borrow::Cow;
2
3use relay_event_schema::processor::{
4    self, Chunk, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor,
5    ValueType,
6};
7use relay_event_schema::protocol::{Frame, RawStacktrace, Replay};
8use relay_protocol::{Annotated, Array, Empty, Meta, Object, RemarkType, Value};
9
10#[derive(Clone, Debug)]
11struct SizeState {
12    max_depth: Option<usize>,
13    encountered_at_depth: usize,
14    size_remaining: Option<usize>,
15}
16
17/// Limits properties to a maximum size and depth.
18#[derive(Default)]
19pub struct TrimmingProcessor {
20    size_state: Vec<SizeState>,
21}
22
23impl TrimmingProcessor {
24    /// Creates a new trimming processor.
25    pub fn new() -> Self {
26        Self::default()
27    }
28
29    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
30        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
31        // null, ...]`, making it take up more space.
32        self.remaining_depth(state) == Some(1) && !value.is_empty()
33    }
34
35    #[inline]
36    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
37        self.size_state
38            .iter()
39            .filter_map(|size_state| {
40                // The current depth in the entire event payload minus the depth at which we found the
41                // max_depth attribute is the depth where we are at in the property.
42                let current_depth = state.depth() - size_state.encountered_at_depth;
43                size_state
44                    .max_depth
45                    .map(|max_depth| max_depth.saturating_sub(current_depth))
46            })
47            .min()
48    }
49
50    #[inline]
51    fn remaining_size(&self) -> Option<usize> {
52        self.size_state
53            .iter()
54            .filter_map(|x| x.size_remaining)
55            .min()
56    }
57}
58
59impl Processor for TrimmingProcessor {
60    fn before_process<T: ProcessValue>(
61        &mut self,
62        _: Option<&T>,
63        _: &mut Meta,
64        state: &ProcessingState<'_>,
65    ) -> ProcessingResult {
66        // If we encounter a max_bytes or max_depth attribute it
67        // resets the size and depth that is permitted below it.
68        // XXX(iker): test setting only one of the two attributes.
69        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
70            self.size_state.push(SizeState {
71                size_remaining: state.max_bytes(),
72                encountered_at_depth: state.depth(),
73                max_depth: state.attrs().max_depth,
74            });
75        }
76
77        if state.attrs().trim {
78            if self.remaining_size() == Some(0) {
79                // TODO: Create remarks (ensure they do not bloat event)
80                return Err(ProcessingAction::DeleteValueHard);
81            }
82            if self.remaining_depth(state) == Some(0) {
83                // TODO: Create remarks (ensure they do not bloat event)
84                return Err(ProcessingAction::DeleteValueHard);
85            }
86        }
87        Ok(())
88    }
89
90    fn after_process<T: ProcessValue>(
91        &mut self,
92        value: Option<&T>,
93        _: &mut Meta,
94        state: &ProcessingState<'_>,
95    ) -> ProcessingResult {
96        // If our current depth is the one where we found a bag_size attribute, this means we
97        // are done processing a databag. Pop the bag size state.
98        self.size_state
99            .pop_if(|size_state| state.depth() == size_state.encountered_at_depth);
100
101        // After processing a value, update the remaining bag sizes. We have a separate if-let
102        // here in case somebody defines nested databags (a struct with bag_size that contains
103        // another struct with a different bag_size), in case we just exited a databag we want
104        // to update the bag_size_state of the outer databag with the remaining size.
105        //
106        // This also has to happen after string trimming, which is why it's running in
107        // after_process.
108        if state.entered_anything() && !self.size_state.is_empty() {
109            // Do not subtract if state is from newtype struct.
110            let item_length = state
111                .bytes_size()
112                .unwrap_or_else(|| relay_protocol::estimate_size_flat(value) + 1);
113            for size_state in self.size_state.iter_mut() {
114                size_state.size_remaining = size_state
115                    .size_remaining
116                    .map(|size| size.saturating_sub(item_length));
117            }
118        }
119
120        Ok(())
121    }
122
123    fn process_string(
124        &mut self,
125        value: &mut String,
126        meta: &mut Meta,
127        state: &ProcessingState<'_>,
128    ) -> ProcessingResult {
129        if let Some(max_chars) = state.max_chars() {
130            trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
131        }
132
133        if !state.attrs().trim {
134            return Ok(());
135        }
136
137        if let Some(size_remaining) = self.remaining_size() {
138            trim_string(value, meta, size_remaining, 0);
139        }
140
141        Ok(())
142    }
143
144    fn process_array<T>(
145        &mut self,
146        value: &mut Array<T>,
147        meta: &mut Meta,
148        state: &ProcessingState<'_>,
149    ) -> ProcessingResult
150    where
151        T: ProcessValue,
152    {
153        if !state.attrs().trim {
154            return Ok(());
155        }
156
157        // If we need to check the bag size, then we go down a different path
158        if !self.size_state.is_empty() {
159            let original_length = value.len();
160
161            if self.should_remove_container(value, state) {
162                return Err(ProcessingAction::DeleteValueHard);
163            }
164
165            let mut split_index = None;
166            for (index, item) in value.iter_mut().enumerate() {
167                if self.remaining_size() == Some(0) {
168                    split_index = Some(index);
169                    break;
170                }
171
172                let item_state = state.enter_index(index, None, ValueType::for_field(item));
173                processor::process_value(item, self, &item_state)?;
174            }
175
176            if let Some(split_index) = split_index {
177                let _ = value.split_off(split_index);
178            }
179
180            if value.len() != original_length {
181                meta.set_original_length(Some(original_length));
182            }
183        } else {
184            value.process_child_values(self, state)?;
185        }
186
187        Ok(())
188    }
189
190    fn process_object<T>(
191        &mut self,
192        value: &mut Object<T>,
193        meta: &mut Meta,
194        state: &ProcessingState<'_>,
195    ) -> ProcessingResult
196    where
197        T: ProcessValue,
198    {
199        if !state.attrs().trim {
200            return Ok(());
201        }
202
203        // If we need to check the bag size, then we go down a different path
204        if !self.size_state.is_empty() {
205            let original_length = value.len();
206
207            if self.should_remove_container(value, state) {
208                return Err(ProcessingAction::DeleteValueHard);
209            }
210
211            let mut split_key = None;
212            for (key, item) in value.iter_mut() {
213                if self.remaining_size() == Some(0) {
214                    split_key = Some(key.to_owned());
215                    break;
216                }
217
218                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
219                processor::process_value(item, self, &item_state)?;
220            }
221
222            if let Some(split_key) = split_key {
223                let _ = value.split_off(&split_key);
224            }
225
226            if value.len() != original_length {
227                meta.set_original_length(Some(original_length));
228            }
229        } else {
230            value.process_child_values(self, state)?;
231        }
232
233        Ok(())
234    }
235
236    fn process_value(
237        &mut self,
238        value: &mut Value,
239        _meta: &mut Meta,
240        state: &ProcessingState<'_>,
241    ) -> ProcessingResult {
242        if !state.attrs().trim {
243            return Ok(());
244        }
245
246        match value {
247            Value::Array(_) | Value::Object(_) => {
248                if self.remaining_depth(state) == Some(1)
249                    && let Ok(x) = serde_json::to_string(&value)
250                {
251                    // Error case should not be possible
252                    *value = Value::String(x);
253                }
254            }
255            _ => (),
256        }
257
258        value.process_child_values(self, state)?;
259        Ok(())
260    }
261
262    fn process_replay(
263        &mut self,
264        replay: &mut Replay,
265        _: &mut Meta,
266        state: &ProcessingState<'_>,
267    ) -> ProcessingResult {
268        replay.process_child_values(self, state)
269    }
270
271    fn process_raw_stacktrace(
272        &mut self,
273        stacktrace: &mut RawStacktrace,
274        _meta: &mut Meta,
275        state: &ProcessingState<'_>,
276    ) -> ProcessingResult {
277        if !state.attrs().trim {
278            return Ok(());
279        }
280
281        processor::apply(&mut stacktrace.frames, |frames, meta| {
282            enforce_frame_hard_limit(
283                frames,
284                meta,
285                RawStacktrace::MAX_NEWEST_FRAMES,
286                RawStacktrace::MAX_OLDEST_FRAMES,
287            );
288            Ok(())
289        })?;
290
291        stacktrace.process_child_values(self, state)?;
292
293        processor::apply(&mut stacktrace.frames, |frames, _meta| {
294            slim_frame_data(frames, 50);
295            Ok(())
296        })?;
297
298        Ok(())
299    }
300}
301
302/// Trims the string to the given maximum length and updates meta data.
303pub(crate) fn trim_string(
304    value: &mut String,
305    meta: &mut Meta,
306    max_chars: usize,
307    max_chars_allowance: usize,
308) {
309    let hard_limit = max_chars + max_chars_allowance;
310
311    if bytecount::num_chars(value.as_bytes()) <= hard_limit {
312        return;
313    }
314
315    processor::process_chunked_value(value, meta, |chunks| {
316        let mut length = 0;
317        let mut new_chunks = vec![];
318
319        for chunk in chunks {
320            let chunk_chars = chunk.count();
321
322            // if the entire chunk fits, just put it in
323            if length + chunk_chars < max_chars {
324                new_chunks.push(chunk);
325                length += chunk_chars;
326                continue;
327            }
328
329            match chunk {
330                // if there is enough space for this chunk and the 3 character
331                // ellipsis marker we can push the remaining chunk
332                Chunk::Redaction { .. } => {
333                    if length + chunk_chars + 3 < hard_limit {
334                        new_chunks.push(chunk);
335                    }
336                }
337
338                // if this is a text chunk, we can put the remaining characters in.
339                Chunk::Text { text } => {
340                    let mut remaining = String::new();
341                    for c in text.chars() {
342                        if length + 3 < max_chars {
343                            remaining.push(c);
344                        } else {
345                            break;
346                        }
347                        length += 1;
348                    }
349
350                    new_chunks.push(Chunk::Text {
351                        text: Cow::Owned(remaining),
352                    });
353                }
354            }
355
356            new_chunks.push(Chunk::Redaction {
357                text: Cow::Borrowed("..."),
358                rule_id: Cow::Borrowed("!limit"),
359                ty: RemarkType::Substituted,
360            });
361            break;
362        }
363
364        new_chunks
365    });
366}
367
368/// Trim down the frame list to a hard limit.
369///
370/// The total limit is `recent_frames` + `old_frames`.
371/// `recent_frames` is the number of frames to keep from the beginning of the list,
372/// the most recent stack frames, `old_frames` is the last at the end of the list of frames,
373/// the oldest frames up the stack.
374///
375/// It makes sense to keep some of the old frames in recursion cases to see what actually caused
376/// the recursion.
377fn enforce_frame_hard_limit(
378    frames: &mut Array<Frame>,
379    meta: &mut Meta,
380    recent_frames: usize,
381    old_frames: usize,
382) {
383    let original_length = frames.len();
384    let limit = recent_frames + old_frames;
385    if original_length > limit {
386        meta.set_original_length(Some(original_length));
387        let _ = frames.drain(old_frames..original_length - recent_frames);
388    }
389}
390
391/// Remove excess metadata for middle frames which go beyond `frame_allowance`.
392///
393/// This is supposed to be equivalent to `slim_frame_data` in Sentry.
394fn slim_frame_data(frames: &mut Array<Frame>, frame_allowance: usize) {
395    let frames_len = frames.len();
396
397    if frames_len <= frame_allowance {
398        return;
399    }
400
401    // Avoid ownership issues by only storing indices
402    let mut app_frame_indices = Vec::with_capacity(frames_len);
403    let mut system_frame_indices = Vec::with_capacity(frames_len);
404
405    for (i, frame) in frames.iter().enumerate() {
406        if let Some(frame) = frame.value() {
407            match frame.in_app.value() {
408                Some(true) => app_frame_indices.push(i),
409                _ => system_frame_indices.push(i),
410            }
411        }
412    }
413
414    let app_count = app_frame_indices.len();
415    let system_allowance_half = frame_allowance.saturating_sub(app_count) / 2;
416    let system_frames_to_remove = system_frame_indices
417        .get(system_allowance_half..system_frame_indices.len() - system_allowance_half)
418        .unwrap_or(&[]);
419
420    let remaining = frames_len
421        .saturating_sub(frame_allowance)
422        .saturating_sub(system_frames_to_remove.len());
423    let app_allowance_half = app_count.saturating_sub(remaining) / 2;
424    let app_frames_to_remove = app_frame_indices
425        .get(app_allowance_half..app_frame_indices.len() - app_allowance_half)
426        .unwrap_or(&[]);
427
428    // TODO: Which annotation to set?
429
430    let top_frame_index = frames_len.saturating_sub(1);
431    for i in system_frames_to_remove
432        .iter()
433        .chain(app_frames_to_remove)
434        .filter(|&&i| i != top_frame_index)
435    {
436        if let Some(frame) = frames.get_mut(*i)
437            && let Some(ref mut frame) = frame.value_mut().as_mut()
438        {
439            frame.vars = Annotated::empty();
440            frame.pre_context = Annotated::empty();
441            frame.post_context = Annotated::empty();
442        }
443    }
444}
445
446#[cfg(test)]
447mod tests {
448    use std::iter::repeat_n;
449
450    use crate::MaxChars;
451    use chrono::DateTime;
452    use relay_event_schema::protocol::{
453        Breadcrumb, Context, Contexts, Event, Exception, ExtraValue, FlagsContext, PairList,
454        SentryTags, Span, SpanData, SpanId, TagEntry, Tags, Timestamp, TraceId, Values,
455    };
456    use relay_protocol::{FromValue, IntoValue, Map, Remark, SerializableAnnotated, get_value};
457    use similar_asserts::assert_eq;
458
459    use super::*;
460
461    #[test]
462    fn test_string_trimming() {
463        let mut value = Annotated::new("This is my long string I want to have trimmed!".to_owned());
464        processor::apply(&mut value, |v, m| {
465            trim_string(v, m, 20, 0);
466            Ok(())
467        })
468        .unwrap();
469
470        assert_eq!(
471            value,
472            Annotated(Some("This is my long s...".into()), {
473                let mut meta = Meta::default();
474                meta.add_remark(Remark {
475                    ty: RemarkType::Substituted,
476                    rule_id: "!limit".to_owned(),
477                    range: Some((17, 20)),
478                });
479                meta.set_original_length(Some(46));
480                meta
481            })
482        );
483    }
484
485    #[test]
486    fn test_basic_trimming() {
487        let mut processor = TrimmingProcessor::new();
488
489        let mut event = Annotated::new(Event {
490            logger: Annotated::new("x".repeat(300)),
491            ..Default::default()
492        });
493
494        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
495
496        let mut expected = Annotated::new("x".repeat(300));
497        processor::apply(&mut expected, |v, m| {
498            trim_string(v, m, MaxChars::Logger.limit(), 0);
499            Ok(())
500        })
501        .unwrap();
502
503        assert_eq!(event.value().unwrap().logger, expected);
504    }
505
506    #[test]
507    fn test_max_char_allowance() {
508        let string = "This string requires some allowance to fit!";
509        let mut value = Annotated::new(string.to_owned()); // len == 43
510        processor::apply(&mut value, |v, m| {
511            trim_string(v, m, 40, 5);
512            Ok(())
513        })
514        .unwrap();
515
516        assert_eq!(value, Annotated::new(string.to_owned()));
517    }
518
519    #[test]
520    fn test_databag_stripping() {
521        let mut processor = TrimmingProcessor::new();
522
523        fn make_nested_object(depth: usize) -> Annotated<Value> {
524            if depth == 0 {
525                return Annotated::new(Value::String("max depth".to_owned()));
526            }
527            let mut rv = Object::new();
528            rv.insert(format!("key{depth}"), make_nested_object(depth - 1));
529            Annotated::new(Value::Object(rv))
530        }
531
532        let databag = Annotated::new({
533            let mut map = Object::new();
534            map.insert(
535                "key_1".to_owned(),
536                Annotated::new(ExtraValue(Value::String("value 1".to_owned()))),
537            );
538            map.insert(
539                "key_2".to_owned(),
540                make_nested_object(8).map_value(ExtraValue),
541            );
542            map.insert(
543                "key_3".to_owned(),
544                // innermost key (string) is entering json stringify codepath
545                make_nested_object(5).map_value(ExtraValue),
546            );
547            map
548        });
549        let mut event = Annotated::new(Event {
550            extra: databag,
551            ..Default::default()
552        });
553
554        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
555        let stripped_extra = &event.value().unwrap().extra;
556        let json = stripped_extra.to_json_pretty().unwrap();
557
558        assert_eq!(
559            json,
560            r#"{
561  "key_1": "value 1",
562  "key_2": {
563    "key8": {
564      "key7": {
565        "key6": {
566          "key5": {
567            "key4": "{\"key3\":{\"key2\":{\"key1\":\"max depth\"}}}"
568          }
569        }
570      }
571    }
572  },
573  "key_3": {
574    "key5": {
575      "key4": {
576        "key3": {
577          "key2": {
578            "key1": "max depth"
579          }
580        }
581      }
582    }
583  }
584}"#
585        );
586    }
587
588    #[test]
589    fn test_databag_array_stripping() {
590        let mut processor = TrimmingProcessor::new();
591
592        let databag = Annotated::new({
593            let mut map = Object::new();
594            for idx in 0..100 {
595                map.insert(
596                    format!("key_{idx}"),
597                    Annotated::new(ExtraValue(Value::String("x".repeat(50000)))),
598                );
599            }
600            map
601        });
602        let mut event = Annotated::new(Event {
603            extra: databag,
604            ..Default::default()
605        });
606
607        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
608        let stripped_extra = SerializableAnnotated(&event.value().unwrap().extra);
609
610        insta::assert_ron_snapshot!(stripped_extra);
611    }
612
613    /// Tests that a trimming a string takes a lower outer limit into account.
614    #[test]
615    fn test_string_trimming_limits() {
616        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
617        struct Outer {
618            #[metastructure(max_bytes = 10)]
619            inner: Annotated<Inner>,
620        }
621
622        #[derive(ProcessValue, IntoValue, FromValue, Empty, Debug, Clone)]
623        struct Inner {
624            #[metastructure(max_bytes = 20)]
625            innerer: Annotated<String>,
626        }
627
628        let mut processor = TrimmingProcessor::new();
629
630        let mut outer = Annotated::new({
631            Outer {
632                inner: Annotated::new(Inner {
633                    innerer: Annotated::new("This string is 28 bytes long".into()),
634                }),
635            }
636        });
637
638        processor::process_value(&mut outer, &mut processor, ProcessingState::root()).unwrap();
639        let stripped = SerializableAnnotated(&outer);
640
641        insta::assert_ron_snapshot!(stripped, @r###"
642        {
643          "inner": {
644            "innerer": "This st...",
645          },
646          "_meta": {
647            "inner": {
648              "innerer": {
649                "": Meta(Some(MetaInner(
650                  rem: [
651                    [
652                      "!limit",
653                      s,
654                      7,
655                      10,
656                    ],
657                  ],
658                  len: Some(28),
659                ))),
660              },
661            },
662          },
663        }
664        "###);
665    }
666
667    #[test]
668    fn test_tags_stripping() {
669        let mut processor = TrimmingProcessor::new();
670
671        let mut event = Annotated::new(Event {
672            tags: Annotated::new(Tags(
673                vec![Annotated::new(TagEntry(
674                    Annotated::new("x".repeat(300)),
675                    Annotated::new("x".repeat(300)),
676                ))]
677                .into(),
678            )),
679            ..Default::default()
680        });
681
682        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
683        let json = event
684            .value()
685            .unwrap()
686            .tags
687            .payload_to_json_pretty()
688            .unwrap();
689
690        assert_eq!(
691            json,
692            r#"[
693  [
694    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
695    "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx..."
696  ]
697]"#
698        );
699    }
700
701    #[test]
702    fn test_databag_state_leak() {
703        let event = Annotated::new(Event {
704            breadcrumbs: Annotated::new(Values::new(
705                repeat_n(
706                    Annotated::new(Breadcrumb {
707                        data: {
708                            let mut map = Map::new();
709                            map.insert(
710                                "spamspamspam".to_owned(),
711                                Annotated::new(Value::String("blablabla".to_owned())),
712                            );
713                            Annotated::new(map)
714                        },
715                        ..Default::default()
716                    }),
717                    200,
718                )
719                .collect(),
720            )),
721            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
722                ty: Annotated::new("TypeError".to_owned()),
723                value: Annotated::new("important error message".to_owned().into()),
724                stacktrace: Annotated::new(
725                    RawStacktrace {
726                        frames: Annotated::new(
727                            repeat_n(
728                                Annotated::new(Frame {
729                                    function: Annotated::new("importantFunctionName".to_owned()),
730                                    symbol: Annotated::new("important_symbol".to_owned()),
731                                    ..Default::default()
732                                }),
733                                200,
734                            )
735                            .collect(),
736                        ),
737                        ..Default::default()
738                    }
739                    .into(),
740                ),
741                ..Default::default()
742            })])),
743            ..Default::default()
744        });
745
746        let mut processor = TrimmingProcessor::new();
747        let mut stripped_event = event.clone();
748        processor::process_value(&mut stripped_event, &mut processor, ProcessingState::root())
749            .unwrap();
750
751        assert_eq!(
752            event.to_json_pretty().unwrap(),
753            stripped_event.to_json_pretty().unwrap()
754        );
755    }
756
757    #[test]
758    fn test_custom_context_trimming() {
759        let mut contexts = Contexts::new();
760        for i in 1..2 {
761            contexts.insert(format!("despacito{i}"), {
762                let mut context = Object::new();
763                context.insert(
764                    "foo".to_owned(),
765                    Annotated::new(Value::String("a".repeat(4000))),
766                );
767                context.insert(
768                    "bar".to_owned(),
769                    Annotated::new(Value::String("a".repeat(5000))),
770                );
771                Context::Other(context)
772            });
773        }
774
775        let mut contexts = Annotated::new(contexts);
776        let mut processor = TrimmingProcessor::new();
777        processor::process_value(&mut contexts, &mut processor, ProcessingState::root()).unwrap();
778
779        let contexts = contexts.value().unwrap();
780        for i in 1..2 {
781            let other = match contexts.get_key(format!("despacito{i}")).unwrap() {
782                Context::Other(x) => x,
783                _ => panic!("Context has changed type!"),
784            };
785
786            assert_eq!(
787                other
788                    .get("bar")
789                    .unwrap()
790                    .value()
791                    .unwrap()
792                    .as_str()
793                    .unwrap()
794                    .len(),
795                5000
796            );
797            assert_eq!(
798                other
799                    .get("foo")
800                    .unwrap()
801                    .value()
802                    .unwrap()
803                    .as_str()
804                    .unwrap()
805                    .len(),
806                3189
807            );
808        }
809    }
810
811    #[test]
812    fn test_extra_trimming_long_arrays() {
813        let mut extra = Object::new();
814        extra.insert("foo".to_owned(), {
815            Annotated::new(ExtraValue(Value::Array(
816                repeat_n(Annotated::new(Value::U64(1)), 200_000).collect(),
817            )))
818        });
819
820        let mut event = Annotated::new(Event {
821            extra: Annotated::new(extra),
822            ..Default::default()
823        });
824
825        let mut processor = TrimmingProcessor::new();
826        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
827
828        let arr = match event
829            .value()
830            .unwrap()
831            .extra
832            .value()
833            .unwrap()
834            .get("foo")
835            .unwrap()
836            .value()
837            .unwrap()
838        {
839            ExtraValue(Value::Array(x)) => x,
840            x => panic!("Wrong type: {x:?}"),
841        };
842
843        // this is larger / 2 for the extra value
844        assert_eq!(arr.len(), 8192);
845    }
846
847    // TODO(ja): Enable this test
848    // #[test]
849    // fn test_newtypes_do_not_add_to_depth() {
850    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
851    //     struct WrappedString(String);
852
853    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
854    //     struct StructChild2 {
855    //         inner: Annotated<WrappedString>,
856    //     }
857
858    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
859    //     struct StructChild {
860    //         inner: Annotated<StructChild2>,
861    //     }
862
863    //     #[derive(Debug, Clone, FromValue, IntoValue, ProcessValue, Empty)]
864    //     struct Struct {
865    //         #[metastructure(bag_size = "small")]
866    //         inner: Annotated<StructChild>,
867    //     }
868
869    //     let mut value = Annotated::new(Struct {
870    //         inner: Annotated::new(StructChild {
871    //             inner: Annotated::new(StructChild2 {
872    //                 inner: Annotated::new(WrappedString("hi".to_owned())),
873    //             }),
874    //         }),
875    //     });
876
877    //     let mut processor = TrimmingProcessor::new();
878    //     process_value(&mut value, &mut processor, ProcessingState::root()).unwrap();
879
880    //     // Ensure stack does not leak with newtypes
881    //     assert!(processor.bag_size_state.is_empty());
882
883    //     assert_eq!(
884    //         value.to_json().unwrap(),
885    //         r#"{"inner":{"inner":{"inner":"hi"}}}"#
886    //     );
887    // }
888
889    #[test]
890    fn test_frameqty_equals_limit() {
891        fn create_frame(filename: &str) -> Annotated<Frame> {
892            Annotated::new(Frame {
893                filename: Annotated::new(filename.into()),
894                ..Default::default()
895            })
896        }
897
898        let mut frames = Annotated::new(vec![
899            create_frame("foo3.py"),
900            create_frame("foo4.py"),
901            create_frame("foo5.py"),
902        ]);
903
904        processor::apply(&mut frames, |f, m| {
905            enforce_frame_hard_limit(f, m, 3, 0);
906            Ok(())
907        })
908        .unwrap();
909
910        processor::apply(&mut frames, |f, m| {
911            enforce_frame_hard_limit(f, m, 1, 2);
912            Ok(())
913        })
914        .unwrap();
915
916        // original_length isn't set, when limit is equal to length, as no trimming took place.
917        assert!(frames.meta().original_length().is_none());
918    }
919
920    #[test]
921    fn test_frame_hard_limit() {
922        fn create_frame(filename: &str) -> Annotated<Frame> {
923            Annotated::new(Frame {
924                filename: Annotated::new(filename.into()),
925                ..Default::default()
926            })
927        }
928
929        let mut frames = Annotated::new(vec![
930            create_frame("foo1.py"),
931            create_frame("foo2.py"),
932            create_frame("foo3.py"),
933            create_frame("foo4.py"),
934            create_frame("foo5.py"),
935        ]);
936
937        processor::apply(&mut frames, |f, m| {
938            enforce_frame_hard_limit(f, m, 3, 0);
939            Ok(())
940        })
941        .unwrap();
942
943        let mut expected_meta = Meta::default();
944        expected_meta.set_original_length(Some(5));
945
946        assert_eq!(
947            frames,
948            Annotated(
949                Some(vec![
950                    create_frame("foo3.py"),
951                    create_frame("foo4.py"),
952                    create_frame("foo5.py"),
953                ]),
954                expected_meta
955            )
956        );
957    }
958
959    #[test]
960    fn test_frame_hard_limit_recent_old() {
961        fn create_frame(filename: &str) -> Annotated<Frame> {
962            Annotated::new(Frame {
963                filename: Annotated::new(filename.into()),
964                ..Default::default()
965            })
966        }
967
968        let mut frames = Annotated::new(vec![
969            create_frame("foo1.py"),
970            create_frame("foo2.py"),
971            create_frame("foo3.py"),
972            create_frame("foo4.py"),
973            create_frame("foo5.py"),
974        ]);
975
976        processor::apply(&mut frames, |f, m| {
977            enforce_frame_hard_limit(f, m, 2, 1);
978            Ok(())
979        })
980        .unwrap();
981
982        let mut expected_meta = Meta::default();
983        expected_meta.set_original_length(Some(5));
984
985        assert_eq!(
986            frames,
987            Annotated(
988                Some(vec![
989                    create_frame("foo1.py"),
990                    create_frame("foo4.py"),
991                    create_frame("foo5.py"),
992                ]),
993                expected_meta
994            )
995        );
996    }
997
998    #[test]
999    fn test_slim_frame_data_under_max() {
1000        let mut frames = vec![Annotated::new(Frame {
1001            filename: Annotated::new("foo".into()),
1002            pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1003            context_line: Annotated::new("b".to_owned()),
1004            post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1005            ..Default::default()
1006        })];
1007
1008        let old_frames = frames.clone();
1009        slim_frame_data(&mut frames, 4);
1010
1011        assert_eq!(frames, old_frames);
1012    }
1013
1014    #[test]
1015    fn test_slim_frame_data_over_max() {
1016        let mut frames = vec![];
1017
1018        for n in 0..5 {
1019            frames.push(Annotated::new(Frame {
1020                filename: Annotated::new(format!("foo {n}").into()),
1021                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1022                context_line: Annotated::new("b".to_owned()),
1023                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1024                ..Default::default()
1025            }));
1026        }
1027
1028        slim_frame_data(&mut frames, 4);
1029
1030        let expected = vec![
1031            Annotated::new(Frame {
1032                filename: Annotated::new("foo 0".into()),
1033                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1034                context_line: Annotated::new("b".to_owned()),
1035                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1036                ..Default::default()
1037            }),
1038            Annotated::new(Frame {
1039                filename: Annotated::new("foo 1".into()),
1040                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1041                context_line: Annotated::new("b".to_owned()),
1042                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1043                ..Default::default()
1044            }),
1045            Annotated::new(Frame {
1046                filename: Annotated::new("foo 2".into()),
1047                context_line: Annotated::new("b".to_owned()),
1048                ..Default::default()
1049            }),
1050            Annotated::new(Frame {
1051                filename: Annotated::new("foo 3".into()),
1052                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1053                context_line: Annotated::new("b".to_owned()),
1054                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1055                ..Default::default()
1056            }),
1057            Annotated::new(Frame {
1058                filename: Annotated::new("foo 4".into()),
1059                pre_context: Annotated::new(vec![Annotated::new("a".to_owned())]),
1060                context_line: Annotated::new("b".to_owned()),
1061                post_context: Annotated::new(vec![Annotated::new("c".to_owned())]),
1062                ..Default::default()
1063            }),
1064        ];
1065
1066        assert_eq!(frames, expected);
1067    }
1068
1069    #[test]
1070    fn test_slim_frame_data_does_not_trim_top_frame_metadata() {
1071        let mut frames: Array<Frame> = (0..50)
1072            .map(|n| {
1073                Annotated::new(Frame {
1074                    filename: Annotated::new(format!("system {n}").into()),
1075                    ..Default::default()
1076                })
1077            })
1078            .collect();
1079        frames.push(Annotated::new(Frame {
1080            filename: Annotated::new("raising".into()),
1081            pre_context: Annotated::new(vec![Annotated::new("before".to_owned())]),
1082            context_line: Annotated::new("current".to_owned()),
1083            post_context: Annotated::new(vec![Annotated::new("after".to_owned())]),
1084            vars: Annotated::new({
1085                let mut vars = Object::new();
1086                vars.insert("local".to_owned(), Annotated::new("value".into()));
1087                vars.into()
1088            }),
1089            in_app: Annotated::new(true),
1090            ..Default::default()
1091        }));
1092
1093        slim_frame_data(&mut frames, 50);
1094
1095        let top_frame = frames.last().unwrap().value().unwrap();
1096        assert!(top_frame.vars.value().is_some());
1097        assert!(top_frame.pre_context.value().is_some());
1098        assert!(top_frame.context_line.value().is_some());
1099        assert!(top_frame.post_context.value().is_some());
1100    }
1101
1102    #[test]
1103    fn test_too_many_spans_trimmed() {
1104        let span = Span {
1105            platform: Annotated::new("a".repeat(1024 * 90)),
1106            sentry_tags: Annotated::new(SentryTags {
1107                release: Annotated::new("b".repeat(1024 * 100)),
1108                ..Default::default()
1109            }),
1110            ..Default::default()
1111        };
1112        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1113            .take(10)
1114            .collect();
1115
1116        let mut event = Annotated::new(Event {
1117            spans: Annotated::new(spans.clone()),
1118            ..Default::default()
1119        });
1120
1121        let mut processor = TrimmingProcessor::new();
1122        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1123
1124        let trimmed_spans = event.0.unwrap().spans.0.unwrap();
1125        assert_eq!(trimmed_spans.len(), 5);
1126
1127        // The actual spans were not touched:
1128        assert_eq!(trimmed_spans.as_slice(), &spans[0..5]);
1129    }
1130
1131    #[test]
1132    fn test_span_data_not_partially_trimmed() {
1133        let span_data = SpanData::from([(
1134            "large_attribute".to_owned(),
1135            Annotated::new(Value::String("a".repeat(100 * 1024))),
1136        )]);
1137        let span = Span {
1138            data: Annotated::new(span_data.clone()),
1139            ..Default::default()
1140        };
1141        let spans: Vec<_> = std::iter::repeat_with(|| Annotated::new(span.clone()))
1142            .take(10)
1143            .collect();
1144
1145        let mut event = Annotated::new(Event {
1146            spans: Annotated::new(spans.clone()),
1147            ..Default::default()
1148        });
1149
1150        let mut processor = TrimmingProcessor::new();
1151        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1152
1153        let trimmed = event.value().unwrap().spans.value().unwrap();
1154        assert!(trimmed.len() < spans.len());
1155        assert!(trimmed.iter().all(|span| {
1156            span.value()
1157                .and_then(|span| span.data.value())
1158                .is_some_and(|data| data == &span_data)
1159        }));
1160    }
1161
1162    #[test]
1163    fn test_untrimmable_fields() {
1164        let original_description = "a".repeat(819163);
1165        let original_trace_id: TraceId = "b".repeat(32).parse().unwrap();
1166        let mut event = Annotated::new(Event {
1167            spans: Annotated::new(vec![
1168                Span {
1169                    description: original_description.clone().into(),
1170                    ..Default::default()
1171                }
1172                .into(),
1173                Span {
1174                    trace_id: original_trace_id.into(),
1175                    ..Default::default()
1176                }
1177                .into(),
1178            ]),
1179            ..Default::default()
1180        });
1181
1182        let mut processor = TrimmingProcessor::new();
1183        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1184
1185        assert_eq!(
1186            get_value!(event.spans[0].description!),
1187            &original_description
1188        );
1189        // Trace ID would be trimmed without `trim = "false"`
1190        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1191    }
1192
1193    #[test]
1194    fn test_untrimmable_fields_drop() {
1195        let original_description = "a".repeat(819164);
1196        let original_span_id: SpanId = "b".repeat(16).parse().unwrap();
1197        let original_trace_id: TraceId = "c".repeat(32).parse().unwrap();
1198        let original_segment_id: SpanId = "d".repeat(16).parse().unwrap();
1199        let original_op = "e".repeat(129);
1200
1201        let mut event = Annotated::new(Event {
1202            spans: Annotated::new(vec![
1203                Span {
1204                    description: original_description.clone().into(),
1205                    ..Default::default()
1206                }
1207                .into(),
1208                Span {
1209                    span_id: original_span_id.into(),
1210                    trace_id: original_trace_id.into(),
1211                    segment_id: original_segment_id.into(),
1212                    is_segment: false.into(),
1213                    op: original_op.clone().into(),
1214                    start_timestamp: Timestamp(
1215                        DateTime::parse_from_rfc3339("1996-12-19T16:39:57Z")
1216                            .unwrap()
1217                            .into(),
1218                    )
1219                    .into(),
1220                    timestamp: Timestamp(
1221                        DateTime::parse_from_rfc3339("1996-12-19T16:39:58Z")
1222                            .unwrap()
1223                            .into(),
1224                    )
1225                    .into(),
1226                    ..Default::default()
1227                }
1228                .into(),
1229            ]),
1230            ..Default::default()
1231        });
1232
1233        let mut processor = TrimmingProcessor::new();
1234        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1235
1236        assert_eq!(
1237            get_value!(event.spans[0].description!),
1238            &original_description
1239        );
1240        // These fields would be dropped without `trim = "false"`
1241        assert_eq!(get_value!(event.spans[1].span_id!), &original_span_id);
1242        assert_eq!(get_value!(event.spans[1].trace_id!), &original_trace_id);
1243        assert_eq!(get_value!(event.spans[1].segment_id!), &original_segment_id);
1244        assert_eq!(get_value!(event.spans[1].is_segment!), &false);
1245        // span.op is trimmed to its max_chars, but not dropped:
1246        assert_eq!(get_value!(event.spans[1].op!).len(), 128);
1247        assert!(get_value!(event.spans[1].start_timestamp).is_some());
1248        assert!(get_value!(event.spans[1].timestamp).is_some());
1249    }
1250
1251    #[test]
1252    fn test_too_long_tags() {
1253        let mut event = Annotated::new(Event {
1254        tags: Annotated::new(Tags(PairList(
1255            vec![Annotated::new(TagEntry(
1256                Annotated::new("foobar".to_owned()),
1257                Annotated::new("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx".to_owned()),
1258            )), Annotated::new(TagEntry(
1259                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1260                Annotated::new("bar".to_owned()),
1261            ))]),
1262        )),
1263        ..Event::default()
1264    });
1265
1266        let mut processor = TrimmingProcessor::new();
1267        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1268
1269        insta::assert_debug_snapshot!(get_value!(event.tags!), @r###"
1270        Tags(
1271            PairList(
1272                [
1273                    TagEntry(
1274                        "foobar",
1275                        Annotated(
1276                            "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx...",
1277                            Meta {
1278                                remarks: [
1279                                    Remark {
1280                                        ty: Substituted,
1281                                        rule_id: "!limit",
1282                                        range: Some(
1283                                            (
1284                                                197,
1285                                                200,
1286                                            ),
1287                                        ),
1288                                    },
1289                                ],
1290                                errors: [],
1291                                original_length: Some(
1292                                    203,
1293                                ),
1294                                original_value: None,
1295                            },
1296                        ),
1297                    ),
1298                    TagEntry(
1299                        Annotated(
1300                            "foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo...",
1301                            Meta {
1302                                remarks: [
1303                                    Remark {
1304                                        ty: Substituted,
1305                                        rule_id: "!limit",
1306                                        range: Some(
1307                                            (
1308                                                197,
1309                                                200,
1310                                            ),
1311                                        ),
1312                                    },
1313                                ],
1314                                errors: [],
1315                                original_length: Some(
1316                                    203,
1317                                ),
1318                                original_value: None,
1319                            },
1320                        ),
1321                        "bar",
1322                    ),
1323                ],
1324            ),
1325        )
1326        "###);
1327    }
1328
1329    #[test]
1330    fn test_fixed_item_size() {
1331        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1332        struct TestObject {
1333            #[metastructure(max_bytes = 28)]
1334            inner: Annotated<TestObjectInner>,
1335        }
1336        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1337        struct TestObjectInner {
1338            #[metastructure(max_chars = 10, trim = true)]
1339            body: Annotated<String>,
1340            // This should neither be trimmed nor factor into size calculations.
1341            #[metastructure(trim = false, bytes_size = "always_zero")]
1342            number: Annotated<u64>,
1343            // This should count as 10B.
1344            #[metastructure(trim = false, bytes_size = 10)]
1345            other_number: Annotated<u64>,
1346            #[metastructure(trim = true)]
1347            footer: Annotated<String>,
1348        }
1349
1350        fn always_zero(_state: &ProcessingState) -> Option<usize> {
1351            Some(0)
1352        }
1353
1354        let mut object = Annotated::new(TestObject {
1355            inner: Annotated::new(TestObjectInner {
1356                body: Annotated::new("Longer than 10 chars".to_owned()),
1357                number: Annotated::new(13),
1358                other_number: Annotated::new(12),
1359                footer: Annotated::new("There should only be 'Th...' left".to_owned()),
1360            }),
1361        });
1362
1363        let mut processor = TrimmingProcessor::new();
1364        processor::process_value(&mut object, &mut processor, ProcessingState::root()).unwrap();
1365
1366        // * `body` gets trimmed to 13B (10 chars + `...`)
1367        // * `number` counts as 0B
1368        // * `other_number` counts as 10B
1369        // That leaves 5B for the `footer`.
1370        insta::assert_ron_snapshot!(SerializableAnnotated(&object), @r###"
1371        {
1372          "inner": {
1373            "body": "Longer ...",
1374            "number": 13,
1375            "other_number": 12,
1376            "footer": "Th...",
1377          },
1378          "_meta": {
1379            "inner": {
1380              "body": {
1381                "": Meta(Some(MetaInner(
1382                  rem: [
1383                    [
1384                      "!limit",
1385                      s,
1386                      7,
1387                      10,
1388                    ],
1389                  ],
1390                  len: Some(20),
1391                ))),
1392              },
1393              "footer": {
1394                "": Meta(Some(MetaInner(
1395                  rem: [
1396                    [
1397                      "!limit",
1398                      s,
1399                      2,
1400                      5,
1401                    ],
1402                  ],
1403                  len: Some(33),
1404                ))),
1405              },
1406            },
1407          },
1408        }
1409        "###);
1410    }
1411
1412    #[test]
1413    fn test_flags_context_trimming() {
1414        let original_flags_count = 1_000;
1415        let values: Vec<_> = (0..original_flags_count)
1416            .map(|i| {
1417                serde_json::json!({
1418                    "flag": format!("feature.flag.{i}"),
1419                    "result": "x".repeat(500),
1420                })
1421            })
1422            .collect();
1423        let json = serde_json::json!({
1424            "contexts": {
1425                "flags": {
1426                    "values": values,
1427                },
1428                "my_custom_context": {
1429                    "foo": "x".repeat(10_000)
1430                }
1431            },
1432        })
1433        .to_string();
1434        let mut event = Annotated::<Event>::from_json(&json).unwrap();
1435
1436        let mut processor = TrimmingProcessor::new();
1437        processor::process_value(&mut event, &mut processor, ProcessingState::root()).unwrap();
1438
1439        let contexts = get_value!(event.contexts!);
1440
1441        // Make sure flags contexts has its own limit applied.
1442        let values = &contexts.get::<FlagsContext>().unwrap().values;
1443
1444        assert_eq!(values.value().unwrap().len(), 584);
1445        assert_eq!(values.meta().original_length(), Some(original_flags_count));
1446
1447        // Make sure the custom context is trimmed to 8192.
1448        let custom = match contexts.get_key("my_custom_context").unwrap() {
1449            Context::Other(custom) => custom,
1450            _ => unreachable!(),
1451        };
1452        assert_eq!(custom["foo"].value().unwrap().as_str().unwrap().len(), 8192);
1453        assert_eq!(custom["foo"].meta().original_length(), Some(10_000));
1454    }
1455}
relay_event_normalization/trimming.rs

relay_event_normalization/
trimming.rs
