relay_event_normalization/eap/
trimming.rs

1use std::cmp::Ordering;
2use std::ops::Bound;
3
4use relay_event_schema::processor::{
5    self, ProcessValue, ProcessingAction, ProcessingResult, ProcessingState, Processor, ValueType,
6};
7use relay_event_schema::protocol::Attributes;
8use relay_protocol::{Array, Empty, Meta, Object};
9
10use crate::eap::size;
11
12#[derive(Clone, Debug)]
13struct SizeState {
14    max_depth: Option<usize>,
15    encountered_at_depth: usize,
16    size_remaining: Option<usize>,
17}
18
19/// The action to take when deleting a value.
20#[derive(Debug, Clone, Copy)]
21enum DeleteAction {
22    /// Delete the value without leaving a trace.
23    Hard,
24    /// Delete the value and leave a remark with the given rule ID.
25    WithRemark(&'static str),
26}
27
28impl From<DeleteAction> for ProcessingAction {
29    fn from(action: DeleteAction) -> Self {
30        match action {
31            DeleteAction::Hard => ProcessingAction::DeleteValueHard,
32            DeleteAction::WithRemark(rule_id) => ProcessingAction::DeleteValueWithRemark(rule_id),
33        }
34    }
35}
36
37/// Processor for trimming EAP items (logs, V2 spans).
38///
39/// This primarily differs from the regular [`TrimmingProcessor`](crate::trimming::TrimmingProcessor)
40/// in the handling of [`Attributes`]. This processor handles attributes as follows:
41/// 1. Sort them by combined key and value size, where the key size is just the string length
42///    and the value size is given by [`size::attribute_size`].
43/// 2. Trim attributes one by one. Key lengths are counted for the attribute's size, but keys
44///    aren't trimmed—if a key is too long, the attribute is simply discarded.
45/// 3. If we run out of space, all subsequent attributes are discarded.
46///
47/// This means that large attributes will be trimmed or discarded before small ones.
48#[derive(Default)]
49pub struct TrimmingProcessor {
50    size_state: Vec<SizeState>,
51    removed_key_byte_budget: usize,
52}
53
54impl TrimmingProcessor {
55    /// Creates a new trimming processor.
56    pub fn new(removed_key_byte_budget: usize) -> Self {
57        Self {
58            size_state: Default::default(),
59            removed_key_byte_budget,
60        }
61    }
62
63    fn should_remove_container<T: Empty>(&self, value: &T, state: &ProcessingState<'_>) -> bool {
64        // Heuristic to avoid trimming a value like `[1, 1, 1, 1, ...]` into `[null, null, null,
65        // null, ...]`, making it take up more space.
66        self.remaining_depth(state) == Some(1) && !value.is_empty()
67    }
68
69    #[inline]
70    fn remaining_size(&self) -> Option<usize> {
71        self.size_state
72            .iter()
73            .filter_map(|x| x.size_remaining)
74            .min()
75    }
76
77    #[inline]
78    fn remaining_depth(&self, state: &ProcessingState<'_>) -> Option<usize> {
79        self.size_state
80            .iter()
81            .filter_map(|size_state| {
82                // The current depth in the entire event payload minus the depth at which we found the
83                // max_depth attribute is the depth where we are at in the property.
84                let current_depth = state.depth() - size_state.encountered_at_depth;
85                size_state
86                    .max_depth
87                    .map(|max_depth| max_depth.saturating_sub(current_depth))
88            })
89            .min()
90    }
91
92    fn consume_size(&mut self, state: Option<&ProcessingState>, default: usize) {
93        let size = state.and_then(|s| s.bytes_size()).unwrap_or(default);
94        for remaining in self
95            .size_state
96            .iter_mut()
97            .filter_map(|state| state.size_remaining.as_mut())
98        {
99            *remaining = remaining.saturating_sub(size);
100        }
101    }
102
103    /// Returns a [`DeleteAction`] for removing the given key.
104    ///
105    /// If there is enough `removed_key_byte_budget` left to accomodate the key,
106    /// this will be [`DeleteAction::WithRemark`] (which causes a remark to be left).
107    /// Otherwise, it will be [`DeleteAction::Hard`] (the key is removed without a trace).
108    fn delete_value(&mut self, key: Option<&str>) -> DeleteAction {
109        let len = key.map_or(0, |key| key.len());
110        if len <= self.removed_key_byte_budget {
111            self.removed_key_byte_budget -= len;
112            DeleteAction::WithRemark("trimmed")
113        } else {
114            DeleteAction::Hard
115        }
116    }
117}
118
119impl Processor for TrimmingProcessor {
120    fn before_process<T: ProcessValue>(
121        &mut self,
122        _: Option<&T>,
123        _: &mut Meta,
124        state: &ProcessingState<'_>,
125    ) -> ProcessingResult {
126        // If we encounter a max_bytes or max_depth attribute it
127        // resets the size and depth that is permitted below it.
128        if state.max_bytes().is_some() || state.attrs().max_depth.is_some() {
129            self.size_state.push(SizeState {
130                size_remaining: state.max_bytes(),
131                encountered_at_depth: state.depth(),
132                max_depth: state.attrs().max_depth,
133            });
134        }
135
136        if state.attrs().trim {
137            let key = state.keys().next();
138            if self.remaining_size() == Some(0) {
139                return Err(self.delete_value(key).into());
140            }
141            if self.remaining_depth(state) == Some(0) {
142                return Err(self.delete_value(key).into());
143            }
144        }
145        Ok(())
146    }
147
148    fn after_process<T: ProcessValue>(
149        &mut self,
150        _value: Option<&T>,
151        _: &mut Meta,
152        state: &ProcessingState<'_>,
153    ) -> ProcessingResult {
154        // If our current depth is the one where we found a bag_size attribute, this means we
155        // are done processing a databag. Pop the bag size state.
156        self.size_state
157            .pop_if(|size_state| state.depth() == size_state.encountered_at_depth);
158
159        // The general `TrimmingProcessor` counts consumed sizes at this point. We can't do this generically
160        // because we want to count sizes using `size::attribute_size` for attribute values. Therefore, the
161        // size accounting needs to happen in the processing functions themselves.
162
163        Ok(())
164    }
165    fn process_u64(
166        &mut self,
167        _value: &mut u64,
168        _meta: &mut Meta,
169        state: &ProcessingState<'_>,
170    ) -> ProcessingResult {
171        self.consume_size(Some(state), 8);
172        Ok(())
173    }
174
175    fn process_i64(
176        &mut self,
177        _value: &mut i64,
178        _meta: &mut Meta,
179        state: &ProcessingState<'_>,
180    ) -> ProcessingResult {
181        self.consume_size(Some(state), 8);
182        Ok(())
183    }
184
185    fn process_f64(
186        &mut self,
187        _value: &mut f64,
188        _meta: &mut Meta,
189        state: &ProcessingState<'_>,
190    ) -> ProcessingResult {
191        self.consume_size(Some(state), 8);
192        Ok(())
193    }
194
195    fn process_bool(
196        &mut self,
197        _value: &mut bool,
198        _meta: &mut Meta,
199        state: &ProcessingState<'_>,
200    ) -> ProcessingResult {
201        self.consume_size(Some(state), 1);
202        Ok(())
203    }
204
205    fn process_string(
206        &mut self,
207        value: &mut String,
208        meta: &mut Meta,
209        state: &ProcessingState<'_>,
210    ) -> ProcessingResult {
211        if let Some(max_chars) = state.max_chars() {
212            crate::trimming::trim_string(value, meta, max_chars, state.attrs().max_chars_allowance);
213        }
214
215        if !state.attrs().trim {
216            self.consume_size(Some(state), value.len());
217            return Ok(());
218        }
219
220        if let Some(size_remaining) = self.remaining_size() {
221            crate::trimming::trim_string(value, meta, size_remaining, 0);
222        }
223
224        self.consume_size(Some(state), value.len());
225
226        Ok(())
227    }
228
229    fn process_array<T>(
230        &mut self,
231        value: &mut Array<T>,
232        meta: &mut Meta,
233        state: &ProcessingState<'_>,
234    ) -> ProcessingResult
235    where
236        T: ProcessValue,
237    {
238        if !state.attrs().trim {
239            return Ok(());
240        }
241
242        // If we need to check the bag size, then we go down a different path
243        if !self.size_state.is_empty() {
244            let original_length = value.len();
245
246            if self.should_remove_container(value, state) {
247                return Err(ProcessingAction::DeleteValueHard);
248            }
249
250            let mut split_index = None;
251            for (index, item) in value.iter_mut().enumerate() {
252                if self.remaining_size() == Some(0) {
253                    split_index = Some(index);
254                    break;
255                }
256
257                let item_state = state.enter_index(index, None, ValueType::for_field(item));
258                processor::process_value(item, self, &item_state)?;
259            }
260
261            if let Some(split_index) = split_index {
262                let mut i = split_index;
263
264                for item in &mut value[split_index..] {
265                    match self.delete_value(None) {
266                        DeleteAction::Hard => break,
267                        DeleteAction::WithRemark(rule_id) => {
268                            processor::delete_with_remark(item, rule_id)
269                        }
270                    }
271
272                    i += 1;
273                }
274
275                let _ = value.split_off(i);
276            }
277
278            if value.len() != original_length {
279                meta.set_original_length(Some(original_length));
280            }
281        } else {
282            value.process_child_values(self, state)?;
283        }
284
285        Ok(())
286    }
287
288    fn process_object<T>(
289        &mut self,
290        value: &mut Object<T>,
291        meta: &mut Meta,
292        state: &ProcessingState<'_>,
293    ) -> ProcessingResult
294    where
295        T: ProcessValue,
296    {
297        if !state.attrs().trim {
298            return Ok(());
299        }
300
301        // If we need to check the bag size, then we go down a different path
302        if !self.size_state.is_empty() {
303            let original_length = value.len();
304
305            if self.should_remove_container(value, state) {
306                return Err(ProcessingAction::DeleteValueHard);
307            }
308
309            let mut split_key = None;
310            for (key, item) in value.iter_mut() {
311                if self.remaining_size() == Some(0) {
312                    split_key = Some(key.to_owned());
313                    break;
314                }
315
316                let item_state = state.enter_borrowed(key, None, ValueType::for_field(item));
317                processor::process_value(item, self, &item_state)?;
318            }
319
320            if let Some(split_key) = split_key {
321                let mut i = split_key.as_str();
322
323                // Morally this is just `range_mut(split_key.as_str()..)`, but that doesn't work for type
324                // inference reasons.
325                for (key, value) in value
326                    .range_mut::<str, _>((Bound::Included(split_key.as_str()), Bound::Unbounded))
327                {
328                    i = key.as_str();
329
330                    match self.delete_value(Some(key.as_ref())) {
331                        DeleteAction::Hard => break,
332                        DeleteAction::WithRemark(rule_id) => {
333                            processor::delete_with_remark(value, rule_id)
334                        }
335                    }
336                }
337
338                let split_key = i.to_owned();
339                let _ = value.split_off(&split_key);
340            }
341
342            if value.len() != original_length {
343                meta.set_original_length(Some(original_length));
344            }
345        } else {
346            value.process_child_values(self, state)?;
347        }
348
349        Ok(())
350    }
351
352    fn process_attributes(
353        &mut self,
354        attributes: &mut Attributes,
355        meta: &mut Meta,
356        state: &ProcessingState,
357    ) -> ProcessingResult {
358        if !state.attrs().trim {
359            return Ok(());
360        }
361
362        // This counts the lengths of all attribute keys regardless of whether
363        // the attribute itself is valid or invalid. Strictly speaking, this is
364        // inconsistent with the trimming logic, which only counts keys of valid
365        // attributes. However, this value is only used to set the `original_value`
366        // on the attributes collection for documentation purposes, we accept this
367        // discrepancy for now. In any case this is fine to change.
368        let original_length = size::attributes_size(attributes);
369
370        // Sort attributes by key + value size so small attributes are more likely to be preserved.
371        // Attributes with missing values will be sorted at the beginning.
372        let inner = std::mem::take(&mut attributes.0);
373        let mut sorted: Vec<_> = inner.into_iter().collect();
374        sorted.sort_by(
375            |(k1, v1), (k2, v2)| match (v1.value().is_some(), v2.value().is_some()) {
376                (false, false) => k1.len().cmp(&k2.len()),
377                (false, true) => Ordering::Less,
378                (true, false) => Ordering::Greater,
379                (true, true) => (k1.len() + size::attribute_size(v1))
380                    .cmp(&(k2.len() + size::attribute_size(v2))),
381            },
382        );
383
384        // Drop keys without values once we run out of
385        // `removed_key_budget`.
386        sorted.retain(|(k, v)| {
387            if v.value().is_some() {
388                return true;
389            }
390
391            match self.delete_value(Some(k)) {
392                DeleteAction::Hard => false,
393                DeleteAction::WithRemark(_) => true,
394            }
395        });
396
397        let mut split_idx = None;
398        for (idx, (key, value)) in sorted.iter_mut().enumerate() {
399            if value.value().is_none() {
400                // Keys without values were already treated in the `retain` above.
401                // In any case, we don't want such keys to be counted against the
402                // trimming size budget.
403                continue;
404            }
405            if let Some(remaining) = self.remaining_size()
406                && remaining < key.len()
407            {
408                split_idx = Some(idx);
409                break;
410            }
411
412            self.consume_size(None, key.len());
413
414            let value_state = state.enter_borrowed(key, None, ValueType::for_field(value));
415            processor::process_value(value, self, &value_state)?;
416        }
417
418        if let Some(split_idx) = split_idx {
419            let mut i = split_idx;
420
421            for (key, value) in &mut sorted[split_idx..] {
422                match self.delete_value(Some(key.as_ref())) {
423                    DeleteAction::Hard => break,
424                    DeleteAction::WithRemark(rule_id) => {
425                        processor::delete_with_remark(value, rule_id)
426                    }
427                }
428
429                i += 1;
430            }
431
432            let _ = sorted.split_off(i);
433        }
434
435        attributes.0 = sorted.into_iter().collect();
436
437        let new_size = size::attributes_size(attributes);
438        if new_size != original_length {
439            meta.set_original_length(Some(original_length));
440        }
441
442        Ok(())
443    }
444}
445
446#[cfg(test)]
447mod tests {
448    use relay_event_schema::protocol::{AttributeType, AttributeValue};
449    use relay_protocol::{
450        Annotated, FromValue, IntoValue, SerializableAnnotated, Value, assert_annotated_snapshot,
451    };
452
453    use super::*;
454
455    #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
456    struct TestObject {
457        #[metastructure(max_chars = 10, trim = true)]
458        body: Annotated<String>,
459        // This should neither be trimmed nor factor into size calculations.
460        #[metastructure(trim = false, bytes_size = 0)]
461        number: Annotated<u64>,
462        // This should count as 10B.
463        #[metastructure(trim = false, bytes_size = 10)]
464        other_number: Annotated<u64>,
465        #[metastructure(max_bytes = 40, trim = true)]
466        attributes: Annotated<Attributes>,
467        #[metastructure(trim = true)]
468        footer: Annotated<String>,
469    }
470
471    #[test]
472    fn test_split_on_string() {
473        let mut attributes = Attributes::new();
474
475        attributes.insert("small", 17); // 13B
476        attributes.insert("medium string", "This string should be trimmed"); // 42B
477        attributes.insert("attribute is very large and should be removed", true); // 47B
478
479        let mut value = Annotated::new(TestObject {
480            attributes: Annotated::new(attributes),
481            number: Annotated::empty(),
482            other_number: Annotated::empty(),
483            body: Annotated::new("This is longer than allowed".to_owned()),
484            footer: Annotated::empty(),
485        });
486
487        let mut processor = TrimmingProcessor::new(100);
488
489        let state = ProcessingState::new_root(Default::default(), []);
490        processor::process_value(&mut value, &mut processor, &state).unwrap();
491
492        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
493        {
494          "body": "This is...",
495          "attributes": {
496            "attribute is very large and should be removed": null,
497            "medium string": {
498              "type": "string",
499              "value": "This string..."
500            },
501            "small": {
502              "type": "integer",
503              "value": 17
504            }
505          },
506          "_meta": {
507            "attributes": {
508              "": {
509                "len": 101
510              },
511              "attribute is very large and should be removed": {
512                "": {
513                  "rem": [
514                    [
515                      "trimmed",
516                      "x"
517                    ]
518                  ]
519                }
520              },
521              "medium string": {
522                "value": {
523                  "": {
524                    "rem": [
525                      [
526                        "!limit",
527                        "s",
528                        11,
529                        14
530                      ]
531                    ],
532                    "len": 29
533                  }
534                }
535              }
536            },
537            "body": {
538              "": {
539                "rem": [
540                  [
541                    "!limit",
542                    "s",
543                    7,
544                    10
545                  ]
546                ],
547                "len": 27
548              }
549            }
550          }
551        }
552        "###);
553    }
554
555    #[test]
556    fn test_one_byte_left() {
557        let mut attributes = Attributes::new();
558
559        // First attribute + key of second attribute is 39B, leaving exactly one
560        // byte for the second attribute's value.
561        attributes.insert("small attribute", 17); // 23B
562        attributes.insert("medium attribute", "This string should be trimmed"); // 45B
563
564        let mut value = Annotated::new(TestObject {
565            attributes: Annotated::new(attributes),
566            number: Annotated::empty(),
567            other_number: Annotated::empty(),
568            body: Annotated::new("This is longer than allowed".to_owned()),
569            footer: Annotated::empty(),
570        });
571
572        let mut processor = TrimmingProcessor::new(100);
573
574        let state = ProcessingState::new_root(Default::default(), []);
575        processor::process_value(&mut value, &mut processor, &state).unwrap();
576
577        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
578        {
579          "body": "This is...",
580          "attributes": {
581            "medium attribute": {
582              "type": "string",
583              "value": "..."
584            },
585            "small attribute": {
586              "type": "integer",
587              "value": 17
588            }
589          },
590          "_meta": {
591            "attributes": {
592              "": {
593                "len": 68
594              },
595              "medium attribute": {
596                "value": {
597                  "": {
598                    "rem": [
599                      [
600                        "!limit",
601                        "s",
602                        0,
603                        3
604                      ]
605                    ],
606                    "len": 29
607                  }
608                }
609              }
610            },
611            "body": {
612              "": {
613                "rem": [
614                  [
615                    "!limit",
616                    "s",
617                    7,
618                    10
619                  ]
620                ],
621                "len": 27
622              }
623            }
624          }
625        }
626        "###);
627    }
628
629    #[test]
630    fn test_overaccept_number() {
631        let mut attributes = Attributes::new();
632
633        // The attribute size would get used up by the value of "attribute with long name".
634        // Nevertheless, we accept this attribute, thereby overaccepting 5B.
635        attributes.insert("small", "abcdefgh"); // 5 + 8 = 13B
636        attributes.insert("attribute with long name", 71); // 24 + 8 = 32B
637        attributes.insert("attribute is very large and should be removed", true); // 46 + 1 = 47B
638
639        let mut value = Annotated::new(TestObject {
640            attributes: Annotated::new(attributes),
641            number: Annotated::empty(),
642            other_number: Annotated::empty(),
643            body: Annotated::new("This is longer than allowed".to_owned()),
644            footer: Annotated::empty(),
645        });
646
647        let mut processor = TrimmingProcessor::new(100);
648
649        let state = ProcessingState::new_root(Default::default(), []);
650        processor::process_value(&mut value, &mut processor, &state).unwrap();
651
652        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
653        {
654          "body": "This is...",
655          "attributes": {
656            "attribute is very large and should be removed": null,
657            "attribute with long name": {
658              "type": "integer",
659              "value": 71
660            },
661            "small": {
662              "type": "string",
663              "value": "abcdefgh"
664            }
665          },
666          "_meta": {
667            "attributes": {
668              "": {
669                "len": 91
670              },
671              "attribute is very large and should be removed": {
672                "": {
673                  "rem": [
674                    [
675                      "trimmed",
676                      "x"
677                    ]
678                  ]
679                }
680              }
681            },
682            "body": {
683              "": {
684                "rem": [
685                  [
686                    "!limit",
687                    "s",
688                    7,
689                    10
690                  ]
691                ],
692                "len": 27
693              }
694            }
695          }
696        }
697        "###);
698    }
699
700    #[test]
701    fn test_max_item_size() {
702        let mut attributes = Attributes::new();
703
704        attributes.insert("small", 17); // 13B
705        attributes.insert("medium string", "This string should be trimmed"); // 42B
706        attributes.insert("attribute is very large and should be removed", true); // 47B
707
708        let mut value = Annotated::new(TestObject {
709            attributes: Annotated::new(attributes),
710            number: Annotated::new(0),
711            other_number: Annotated::new(0),
712            body: Annotated::new("Short".to_owned()),
713            footer: Annotated::new("Hello World".to_owned()),
714        });
715
716        let mut processor = TrimmingProcessor::new(100);
717
718        // The `body` takes up 5B, `other_number` 10B, the `"small"` attribute 13B, and the key "medium string" another 13B.
719        // That leaves 9B for the string's value.
720        // Note that the `number` field doesn't take up any size.
721        // The `"footer"` is removed because it comes after the attributes and there's no space left.
722        let state = ProcessingState::root_builder().max_bytes(50).build();
723        processor::process_value(&mut value, &mut processor, &state).unwrap();
724
725        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
726        {
727          "body": "Short",
728          "number": 0,
729          "other_number": 0,
730          "attributes": {
731            "attribute is very large and should be removed": null,
732            "medium string": {
733              "type": "string",
734              "value": "This s..."
735            },
736            "small": {
737              "type": "integer",
738              "value": 17
739            }
740          },
741          "footer": null,
742          "_meta": {
743            "attributes": {
744              "": {
745                "len": 101
746              },
747              "attribute is very large and should be removed": {
748                "": {
749                  "rem": [
750                    [
751                      "trimmed",
752                      "x"
753                    ]
754                  ]
755                }
756              },
757              "medium string": {
758                "value": {
759                  "": {
760                    "rem": [
761                      [
762                        "!limit",
763                        "s",
764                        6,
765                        9
766                      ]
767                    ],
768                    "len": 29
769                  }
770                }
771              }
772            },
773            "footer": {
774              "": {
775                "rem": [
776                  [
777                    "trimmed",
778                    "x"
779                  ]
780                ]
781              }
782            }
783          }
784        }
785        "###);
786    }
787
788    #[test]
789    fn test_array_attribute() {
790        let mut attributes = Attributes::new();
791
792        let array = vec![
793            Annotated::new("first string".into()),
794            Annotated::new("second string".into()),
795            Annotated::new("another string".into()),
796            Annotated::new("last string".into()),
797        ];
798
799        attributes.insert(
800            "array",
801            AttributeValue {
802                ty: Annotated::new(AttributeType::Array),
803                value: Annotated::new(Value::Array(array)),
804            },
805        );
806
807        let mut value = Annotated::new(TestObject {
808            attributes: Annotated::new(attributes),
809            number: Annotated::empty(),
810            other_number: Annotated::empty(),
811            body: Annotated::new("Short".to_owned()),
812            footer: Annotated::empty(),
813        });
814
815        let mut processor = TrimmingProcessor::new(100);
816        let state = ProcessingState::new_root(Default::default(), []);
817        processor::process_value(&mut value, &mut processor, &state).unwrap();
818
819        // The key `"array"` and the first and second array value take up 5 + 12 + 13 = 30B in total,
820        // leaving 10B for the third array value and nothing for the last.
821        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
822        {
823          "body": "Short",
824          "attributes": {
825            "array": {
826              "type": "array",
827              "value": [
828                "first string",
829                "second string",
830                "another...",
831                null
832              ]
833            }
834          },
835          "_meta": {
836            "attributes": {
837              "": {
838                "len": 55
839              },
840              "array": {
841                "value": {
842                  "2": {
843                    "": {
844                      "rem": [
845                        [
846                          "!limit",
847                          "s",
848                          7,
849                          10
850                        ]
851                      ],
852                      "len": 14
853                    }
854                  },
855                  "3": {
856                    "": {
857                      "rem": [
858                        [
859                          "trimmed",
860                          "x"
861                        ]
862                      ]
863                    }
864                  }
865                }
866              }
867            }
868          }
869        }
870        "###);
871    }
872
873    #[test]
874    fn test_oversized_key_does_not_consume_global_limit() {
875        let mut attributes = Attributes::new();
876        attributes.insert("a", 1); // 9B 
877        attributes.insert("this_key_is_exactly_35_chars_long!!", true); // 35B key + 1B = 36B
878
879        let mut value = Annotated::new(TestObject {
880            body: Annotated::new("Hi".to_owned()), // 2B
881            number: Annotated::new(0),
882            other_number: Annotated::empty(),
883            attributes: Annotated::new(attributes),
884            footer: Annotated::new("Hello World".to_owned()), // 11B
885        });
886
887        let mut processor = TrimmingProcessor::new(100);
888        let state = ProcessingState::root_builder().max_bytes(30).build();
889        processor::process_value(&mut value, &mut processor, &state).unwrap();
890
891        insta::assert_json_snapshot!(SerializableAnnotated(&value), @r###"
892        {
893          "body": "Hi",
894          "number": 0,
895          "attributes": {
896            "a": {
897              "type": "integer",
898              "value": 1
899            },
900            "this_key_is_exactly_35_chars_long!!": null
901          },
902          "footer": "Hello World",
903          "_meta": {
904            "attributes": {
905              "": {
906                "len": 45
907              },
908              "this_key_is_exactly_35_chars_long!!": {
909                "": {
910                  "rem": [
911                    [
912                      "trimmed",
913                      "x"
914                    ]
915                  ]
916                }
917              }
918            }
919          }
920        }
921        "###);
922    }
923
924    #[test]
925    fn test_invalid_values() {
926        let mut attributes = Attributes::new();
927        attributes.insert("small", 17); // 13B
928        attributes.insert("medium string", "This string should be trimmed"); // 42B
929        attributes.insert("attribute is very large and should be removed", true); // 47B
930        // Manually insert "broken" attributes. We have enough `removed_key_byte_budget`
931        // for the first but not the second.
932        attributes
933            .0
934            .insert("removed attribute".to_owned(), Annotated::empty());
935        attributes
936            .0
937            .insert("another removed attribute".to_owned(), Annotated::empty());
938
939        let mut attributes = Annotated::new(attributes);
940
941        let state = ProcessingState::root_builder().max_bytes(40).build();
942        processor::process_value(&mut attributes, &mut TrimmingProcessor::new(20), &state).unwrap();
943        let attributes_after_trimming = attributes.clone();
944        processor::process_value(&mut attributes, &mut TrimmingProcessor::new(20), &state).unwrap();
945
946        assert_eq!(
947            &attributes, &attributes_after_trimming,
948            "trimming should be idempotent"
949        );
950
951        insta::assert_json_snapshot!(SerializableAnnotated(&attributes), @r###"
952        {
953          "medium string": {
954            "type": "string",
955            "value": "This string..."
956          },
957          "removed attribute": null,
958          "small": {
959            "type": "integer",
960            "value": 17
961          },
962          "_meta": {
963            "": {
964              "len": 143
965            },
966            "medium string": {
967              "value": {
968                "": {
969                  "rem": [
970                    [
971                      "!limit",
972                      "s",
973                      11,
974                      14
975                    ]
976                  ],
977                  "len": 29
978                }
979              }
980            }
981          }
982        }
983        "###);
984    }
985
986    #[test]
987    fn test_tuple_inner_trim_settings() {
988        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
989        struct Outer {
990            inner: Annotated<TestTuple>,
991        }
992
993        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
994        struct TestTuple(#[metastructure(max_bytes = 10, trim = true)] String);
995
996        let mut value = Annotated::new(Outer {
997            inner: Annotated::new(TestTuple("This is longer than allowed".to_owned())),
998        });
999
1000        let mut processor = TrimmingProcessor::new(100);
1001        let state = ProcessingState::new_root(Default::default(), []);
1002        processor::process_value(&mut value, &mut processor, &state).unwrap();
1003
1004        assert_annotated_snapshot!(value, @r#"
1005        {
1006          "inner": "This is...",
1007          "_meta": {
1008            "inner": {
1009              "": {
1010                "rem": [
1011                  [
1012                    "!limit",
1013                    "s",
1014                    7,
1015                    10
1016                  ]
1017                ],
1018                "len": 27
1019              }
1020            }
1021          }
1022        }
1023        "#);
1024    }
1025
1026    #[test]
1027    fn test_enum_inner_trim_settings() {
1028        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1029        struct Outer {
1030            inner: Annotated<TestEnum>,
1031        }
1032
1033        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1034        enum TestEnum {
1035            Inner(#[metastructure(max_bytes = 10, trim = true)] TestEnumInner),
1036            #[metastructure(fallback_variant)]
1037            Other(#[metastructure(max_bytes = 10, trim = true)] Object<Value>),
1038        }
1039
1040        #[derive(Debug, Clone, Empty, IntoValue, FromValue, ProcessValue)]
1041        struct TestEnumInner {
1042            value: Annotated<String>,
1043        }
1044
1045        let mut value = Annotated::new(Outer {
1046            inner: Annotated::new(TestEnum::Inner(TestEnumInner {
1047                value: Annotated::new("This is longer than allowed".to_owned()),
1048            })),
1049        });
1050
1051        let mut processor = TrimmingProcessor::new(100);
1052        let state = ProcessingState::new_root(Default::default(), []);
1053        processor::process_value(&mut value, &mut processor, &state).unwrap();
1054
1055        assert_annotated_snapshot!(value, @r#"
1056        {
1057          "inner": {
1058            "value": "This is...",
1059            "type": "inner"
1060          },
1061          "_meta": {
1062            "inner": {
1063              "value": {
1064                "": {
1065                  "rem": [
1066                    [
1067                      "!limit",
1068                      "s",
1069                      7,
1070                      10
1071                    ]
1072                  ],
1073                  "len": 27
1074                }
1075              }
1076            }
1077          }
1078        }
1079        "#);
1080
1081        let mut value = Annotated::new(Outer {
1082            inner: Annotated::new(TestEnum::Other({
1083                let mut other = Object::new();
1084                other.insert(
1085                    "foo".to_owned(),
1086                    Value::String("This is longer than allowed".to_owned()).into(),
1087                );
1088                other
1089            })),
1090        });
1091
1092        let mut processor = TrimmingProcessor::new(100);
1093        let state = ProcessingState::new_root(Default::default(), []);
1094        processor::process_value(&mut value, &mut processor, &state).unwrap();
1095
1096        assert_annotated_snapshot!(value, @r#"
1097        {
1098          "inner": {
1099            "foo": "This is..."
1100          },
1101          "_meta": {
1102            "inner": {
1103              "foo": {
1104                "": {
1105                  "rem": [
1106                    [
1107                      "!limit",
1108                      "s",
1109                      7,
1110                      10
1111                    ]
1112                  ],
1113                  "len": 27
1114                }
1115              }
1116            }
1117          }
1118        }
1119        "#);
1120    }
1121}
relay_event_normalization/eap/trimming.rs

relay_event_normalization/eap/
trimming.rs
