relay_event_normalization/normalize/
mod.rs

1use std::collections::HashMap;
2use std::hash::Hash;
3
4use relay_base_schema::metrics::MetricUnit;
5use relay_common::glob2::LazyGlob;
6use relay_event_schema::protocol::{Event, VALID_PLATFORMS};
7use relay_protocol::{FiniteF64, RuleCondition};
8use serde::{Deserialize, Serialize};
9
10pub mod breakdowns;
11pub mod contexts;
12pub mod nel;
13pub mod request;
14pub mod span;
15pub mod user_agent;
16pub mod utils;
17
18/// Defines a builtin measurement.
19#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
20#[serde(default, rename_all = "camelCase")]
21pub struct BuiltinMeasurementKey {
22    name: String,
23    unit: MetricUnit,
24    #[serde(skip_serializing_if = "is_false")]
25    allow_negative: bool,
26}
27
28fn is_false(b: &bool) -> bool {
29    !b
30}
31
32impl BuiltinMeasurementKey {
33    /// Creates a new [`BuiltinMeasurementKey`].
34    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
35        Self {
36            name: name.into(),
37            unit,
38            allow_negative: false,
39        }
40    }
41
42    /// Returns the name of the built in measurement key.
43    pub fn name(&self) -> &str {
44        &self.name
45    }
46
47    /// Returns the unit of the built in measurement key.
48    pub fn unit(&self) -> &MetricUnit {
49        &self.unit
50    }
51
52    /// Return true if the built in measurement key allows negative values.
53    pub fn allow_negative(&self) -> &bool {
54        &self.allow_negative
55    }
56}
57
58/// Configuration for measurements normalization.
59#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
60#[serde(default, rename_all = "camelCase")]
61pub struct MeasurementsConfig {
62    /// A list of measurements that are built-in and are not subject to custom measurement limits.
63    #[serde(default, skip_serializing_if = "Vec::is_empty")]
64    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
65
66    /// The maximum number of measurements allowed per event that are not known measurements.
67    pub max_custom_measurements: usize,
68}
69
70impl MeasurementsConfig {
71    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
72    /// for every measurement-mri.
73    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
74}
75
76/// Returns `true` if the given platform string is a known platform identifier.
77///
78/// See [`VALID_PLATFORMS`] for a list of all known platforms.
79pub fn is_valid_platform(platform: &str) -> bool {
80    VALID_PLATFORMS.contains(&platform)
81}
82
83/// Replaces snake_case app start spans op with dot.case op.
84///
85/// This is done for the affected React Native SDK versions (from 3 to 4.4).
86pub fn normalize_app_start_spans(event: &mut Event) {
87    if !event.sdk_name().eq("sentry.javascript.react-native")
88        || !(event.sdk_version().starts_with("4.4")
89            || event.sdk_version().starts_with("4.3")
90            || event.sdk_version().starts_with("4.2")
91            || event.sdk_version().starts_with("4.1")
92            || event.sdk_version().starts_with("4.0")
93            || event.sdk_version().starts_with('3'))
94    {
95        return;
96    }
97
98    if let Some(spans) = event.spans.value_mut() {
99        for span in spans {
100            if let Some(span) = span.value_mut() {
101                if let Some(op) = span.op.value() {
102                    if op == "app_start_cold" {
103                        span.op.set_value(Some("app.start.cold".to_owned()));
104                        break;
105                    } else if op == "app_start_warm" {
106                        span.op.set_value(Some("app.start.warm".to_owned()));
107                        break;
108                    }
109                }
110            }
111        }
112    }
113}
114
115/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
116/// the merging logic.
117#[derive(Clone, Debug)]
118pub struct CombinedMeasurementsConfig<'a> {
119    project: Option<&'a MeasurementsConfig>,
120    global: Option<&'a MeasurementsConfig>,
121}
122
123impl<'a> CombinedMeasurementsConfig<'a> {
124    /// Constructor for [`CombinedMeasurementsConfig`].
125    pub fn new(
126        project: Option<&'a MeasurementsConfig>,
127        global: Option<&'a MeasurementsConfig>,
128    ) -> Self {
129        CombinedMeasurementsConfig { project, global }
130    }
131
132    /// Returns an iterator over the merged builtin measurement keys.
133    ///
134    /// Items from the project config are prioritized over global config, and
135    /// there are no duplicates.
136    pub fn builtin_measurement_keys(
137        &'a self,
138    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
139        let project = self
140            .project
141            .map(|p| p.builtin_measurements.as_slice())
142            .unwrap_or_default();
143
144        let global = self
145            .global
146            .map(|g| g.builtin_measurements.as_slice())
147            .unwrap_or_default();
148
149        project
150            .iter()
151            .chain(global.iter().filter(|key| !project.contains(key)))
152    }
153
154    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
155    /// global level. If both of them are available, it will choose the most restrictive.
156    pub fn max_custom_measurements(&'a self) -> Option<usize> {
157        match (&self.project, &self.global) {
158            (None, None) => None,
159            (None, Some(global)) => Some(global.max_custom_measurements),
160            (Some(project), None) => Some(project.max_custom_measurements),
161            (Some(project), Some(global)) => Some(std::cmp::min(
162                project.max_custom_measurements,
163                global.max_custom_measurements,
164            )),
165        }
166    }
167}
168
169/// Defines a weighted component for a performance score.
170///
171/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
172/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
173#[derive(Debug, Default, Clone, Serialize, Deserialize)]
174pub struct PerformanceScoreWeightedComponent {
175    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
176    /// profile will be discarded.
177    pub measurement: String,
178    /// Weight [0,1.0] of this component in the performance score
179    pub weight: FiniteF64,
180    /// p10 used to define the log-normal for calculation
181    pub p10: FiniteF64,
182    /// Median used to define the log-normal for calculation
183    pub p50: FiniteF64,
184    /// Whether the measurement is optional. If the measurement is missing, performance score processing
185    /// may still continue, and the weight will be 0.
186    #[serde(default)]
187    pub optional: bool,
188}
189
190/// Defines a profile for performance score.
191///
192/// A profile contains weights for a score of 100% and match against an event using a condition.
193/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
194#[derive(Debug, Default, Clone, Serialize, Deserialize)]
195#[serde(rename_all = "camelCase")]
196pub struct PerformanceScoreProfile {
197    /// Name of the profile, used for debugging and faceting multiple profiles
198    pub name: Option<String>,
199    /// Score components
200    #[serde(default, skip_serializing_if = "Vec::is_empty")]
201    pub score_components: Vec<PerformanceScoreWeightedComponent>,
202    /// See [`RuleCondition`] for all available options to specify and combine conditions.
203    pub condition: Option<RuleCondition>,
204    /// The version of the profile, used to isolate changes to score calculations.
205    #[serde(skip_serializing_if = "Option::is_none")]
206    pub version: Option<String>,
207}
208
209/// Defines the performance configuration for the project.
210///
211/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
212/// specific conditions.
213#[derive(Debug, Default, Clone, Serialize, Deserialize)]
214pub struct PerformanceScoreConfig {
215    /// List of performance profiles, only the first with matching conditions will be applied.
216    #[serde(default, skip_serializing_if = "Vec::is_empty")]
217    pub profiles: Vec<PerformanceScoreProfile>,
218}
219
220/// A mapping of AI model types (like GPT-4) to their respective costs.
221///
222/// This struct supports multiple versions with different cost structures:
223/// - Version 1: Array-based costs with glob pattern matching for model IDs (uses `costs` field)
224/// - Version 2: Dictionary-based costs with exact model ID keys and granular token pricing (uses `models` field)
225///
226/// Example V1 JSON:
227/// ```json
228/// {
229///   "version": 1,
230///   "costs": [
231///     {
232///       "modelId": "gpt-4*",
233///       "forCompletion": false,
234///       "costPer1kTokens": 0.03
235///     }
236///   ]
237/// }
238/// ```
239///
240/// Example V2 JSON:
241/// ```json
242/// {
243///   "version": 2,
244///   "models": {
245///     "gpt-4": {
246///       "inputPerToken": 0.03,
247///       "outputPerToken": 0.06,
248///       "outputReasoningPerToken": 0.12,
249///       "inputCachedPerToken": 0.015
250///     }
251///   }
252/// }
253/// ```
254#[derive(Clone, Default, Debug, Serialize, Deserialize)]
255#[serde(rename_all = "camelCase")]
256pub struct ModelCosts {
257    /// The version of the model cost struct
258    pub version: u16,
259
260    /// The mappings of model ID => cost (used in version 1)
261    #[serde(default, skip_serializing_if = "Vec::is_empty")]
262    pub costs: Vec<ModelCost>,
263
264    /// The mappings of model ID => cost as a dictionary (version 2)
265    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
266    pub models: HashMap<String, ModelCostV2>,
267}
268
269impl ModelCosts {
270    const MAX_SUPPORTED_VERSION: u16 = 2;
271
272    /// `true` if the model costs are empty and the version is supported.
273    pub fn is_empty(&self) -> bool {
274        (self.costs.is_empty() && self.models.is_empty()) || !self.is_enabled()
275    }
276
277    /// `false` if measurement and metrics extraction should be skipped.
278    pub fn is_enabled(&self) -> bool {
279        self.version > 0 && self.version <= ModelCosts::MAX_SUPPORTED_VERSION
280    }
281
282    /// Gets the cost per token, if defined for the given model.
283    pub fn cost_per_token(&self, model_id: &str) -> Option<ModelCostV2> {
284        match self.version {
285            1 => {
286                let input_cost = self.costs.iter().find(|cost| cost.matches(model_id, false));
287                let output_cost = self.costs.iter().find(|cost| cost.matches(model_id, true));
288
289                // V1 costs were defined per 1k tokens, so we need to convert to per token.
290                if input_cost.is_some() || output_cost.is_some() {
291                    Some(ModelCostV2 {
292                        input_per_token: input_cost.map_or(0.0, |c| c.cost_per_1k_tokens / 1000.0),
293                        output_per_token: output_cost
294                            .map_or(0.0, |c| c.cost_per_1k_tokens / 1000.0),
295                        output_reasoning_per_token: 0.0, // in v1 this info is not available
296                        input_cached_per_token: 0.0,     // in v1 this info is not available
297                    })
298                } else {
299                    None
300                }
301            }
302            2 => self.models.get(model_id).copied(),
303            _ => None,
304        }
305    }
306}
307
308/// A mapping of AI model types (like GPT-4) to their respective costs.
309#[derive(Clone, Debug, Serialize, Deserialize)]
310#[serde(rename_all = "camelCase")]
311pub struct ModelCost {
312    pub(crate) model_id: LazyGlob,
313    pub(crate) for_completion: bool,
314    pub(crate) cost_per_1k_tokens: f64,
315}
316
317impl ModelCost {
318    /// `true` if this cost definition matches the given model.
319    pub fn matches(&self, model_id: &str, for_completion: bool) -> bool {
320        self.for_completion == for_completion && self.model_id.compiled().is_match(model_id)
321    }
322}
323
324/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
325/// Version 1 had some limitations, so we're moving to a more flexible format.
326#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
327#[serde(rename_all = "camelCase")]
328pub struct ModelCostV2 {
329    /// The cost per input token
330    pub input_per_token: f64,
331    /// The cost per output token
332    pub output_per_token: f64,
333    /// The cost per output reasoning token
334    pub output_reasoning_per_token: f64,
335    /// The cost per input cached token
336    pub input_cached_per_token: f64,
337}
338
339#[cfg(test)]
340mod tests {
341    use chrono::{TimeZone, Utc};
342    use insta::{assert_debug_snapshot, assert_json_snapshot};
343    use itertools::Itertools;
344    use relay_base_schema::events::EventType;
345    use relay_base_schema::metrics::DurationUnit;
346    use relay_base_schema::spans::SpanStatus;
347    use relay_event_schema::protocol::{
348        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, EventId, Exception,
349        Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace, ReplayContext,
350        Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
351    };
352    use relay_protocol::{
353        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
354        assert_annotated_snapshot, get_path, get_value,
355    };
356    use serde_json::json;
357    use similar_asserts::assert_eq;
358    use uuid::Uuid;
359
360    use crate::stacktrace::normalize_non_raw_frame;
361    use crate::validation::validate_event;
362    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
363
364    use super::*;
365
366    /// Test that integer versions are handled correctly in the struct format
367    #[test]
368    fn test_model_cost_version_sent_as_number() {
369        // Test integer version 1
370        let original = r#"{"version":1,"costs":[{"modelId":"babbage-002.ft","forCompletion":false,"costPer1kTokens":0.0016}]}"#;
371        let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
372        assert_debug_snapshot!(
373            deserialized,
374            @r#"
375        ModelCosts {
376            version: 1,
377            costs: [
378                ModelCost {
379                    model_id: LazyGlob("babbage-002.ft"),
380                    for_completion: false,
381                    cost_per_1k_tokens: 0.0016,
382                },
383            ],
384            models: {},
385        }
386        "#,
387        );
388
389        // Test integer version 2
390        let original_v2 = r#"{"version":2,"models":{"gpt-4":{"inputPerToken":0.03,"outputPerToken":0.06,"outputReasoningPerToken":0.12,"inputCachedPerToken":0.015}}}"#;
391        let deserialized_v2: ModelCosts = serde_json::from_str(original_v2).unwrap();
392        assert_debug_snapshot!(
393            deserialized_v2,
394            @r###"
395            ModelCosts {
396                version: 2,
397                costs: [],
398                models: {
399                    "gpt-4": ModelCostV2 {
400                        input_per_token: 0.03,
401                        output_per_token: 0.06,
402                        output_reasoning_per_token: 0.12,
403                        input_cached_per_token: 0.015,
404                    },
405                },
406            }
407            "###,
408        );
409
410        // Test unknown integer version
411        let original_unknown = r#"{"version":99,"costs":[]}"#;
412        let deserialized_unknown: ModelCosts = serde_json::from_str(original_unknown).unwrap();
413        assert_eq!(deserialized_unknown.version, 99);
414        assert!(!deserialized_unknown.is_enabled());
415    }
416
417    #[test]
418    fn test_model_cost_config_v1() {
419        let original = r#"{"version":1,"costs":[{"modelId":"babbage-002.ft","forCompletion":false,"costPer1kTokens":0.0016}]}"#;
420        let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
421        assert_debug_snapshot!(deserialized, @r###"
422        ModelCosts {
423            version: 1,
424            costs: [
425                ModelCost {
426                    model_id: LazyGlob("babbage-002.ft"),
427                    for_completion: false,
428                    cost_per_1k_tokens: 0.0016,
429                },
430            ],
431            models: {},
432        }
433        "###);
434
435        let serialized = serde_json::to_string(&deserialized).unwrap();
436        assert_eq!(&serialized, original);
437    }
438
439    #[test]
440    fn test_model_cost_config_v2() {
441        let original = r#"{"version":2,"models":{"gpt-4":{"inputPerToken":0.03,"outputPerToken":0.06,"outputReasoningPerToken":0.12,"inputCachedPerToken":0.015}}}"#;
442        let deserialized: ModelCosts = serde_json::from_str(original).unwrap();
443        assert_debug_snapshot!(deserialized, @r###"
444        ModelCosts {
445            version: 2,
446            costs: [],
447            models: {
448                "gpt-4": ModelCostV2 {
449                    input_per_token: 0.03,
450                    output_per_token: 0.06,
451                    output_reasoning_per_token: 0.12,
452                    input_cached_per_token: 0.015,
453                },
454            },
455        }
456        "###);
457
458        let serialized = serde_json::to_string(&deserialized).unwrap();
459        assert_eq!(&serialized, original);
460    }
461
462    #[test]
463    fn test_model_cost_functionality_v1_only_input_tokens() {
464        // Test V1 functionality
465        let v1_config = ModelCosts {
466            version: 1,
467            costs: vec![ModelCost {
468                model_id: LazyGlob::new("gpt-4*"),
469                for_completion: false,
470                cost_per_1k_tokens: 0.03,
471            }],
472            models: HashMap::new(),
473        };
474        assert!(v1_config.is_enabled());
475        let costs = v1_config.cost_per_token("gpt-4-turbo").unwrap();
476        assert_eq!(costs.input_per_token * 1000.0, 0.03); // multiplying by 1000 to avoid floating point errors
477        assert_eq!(costs.output_per_token, 0.0); // output tokens are not defined
478    }
479
480    #[test]
481    fn test_model_cost_functionality_v1() {
482        let v1_config = ModelCosts {
483            version: 1,
484            costs: vec![
485                ModelCost {
486                    model_id: LazyGlob::new("gpt-4*"),
487                    for_completion: false,
488                    cost_per_1k_tokens: 0.03,
489                },
490                ModelCost {
491                    model_id: LazyGlob::new("gpt-4*"),
492                    for_completion: true,
493                    cost_per_1k_tokens: 0.06,
494                },
495            ],
496            models: HashMap::new(),
497        };
498        assert!(v1_config.is_enabled());
499        let costs = v1_config.cost_per_token("gpt-4").unwrap();
500        assert_eq!(costs.input_per_token * 1000.0, 0.03); // multiplying by 1000 to avoid floating point errors
501        assert_eq!(costs.output_per_token * 1000.0, 0.06); // multiplying by 1000 to avoid floating point errors
502    }
503
504    #[test]
505    fn test_model_cost_functionality_v2() {
506        // Test V2 functionality
507        let mut models_map = HashMap::new();
508        models_map.insert(
509            "gpt-4".to_owned(),
510            ModelCostV2 {
511                input_per_token: 0.03,
512                output_per_token: 0.06,
513                output_reasoning_per_token: 0.12,
514                input_cached_per_token: 0.015,
515            },
516        );
517        let v2_config = ModelCosts {
518            version: 2,
519            costs: vec![],
520            models: models_map,
521        };
522        assert!(v2_config.is_enabled());
523        let cost = v2_config.cost_per_token("gpt-4").unwrap();
524        assert_eq!(
525            cost,
526            ModelCostV2 {
527                input_per_token: 0.03,
528                output_per_token: 0.06,
529                output_reasoning_per_token: 0.12,
530                input_cached_per_token: 0.015,
531            }
532        );
533    }
534
535    #[test]
536    fn test_model_cost_unknown_version() {
537        // Test that unknown versions are handled properly
538        let unknown_version_json = r#"{"version":3,"models":{"some-model":{"inputPerToken":0.01,"outputPerToken":0.02,"outputReasoningPerToken":0.03,"inputCachedPerToken":0.005}}}"#;
539        let deserialized: ModelCosts = serde_json::from_str(unknown_version_json).unwrap();
540        assert_eq!(deserialized.version, 3);
541        assert!(!deserialized.is_enabled());
542        assert_eq!(deserialized.cost_per_token("some-model"), None);
543
544        // Test version 0 (invalid)
545        let version_zero_json = r#"{"version":0,"models":{}}"#;
546        let deserialized: ModelCosts = serde_json::from_str(version_zero_json).unwrap();
547        assert_eq!(deserialized.version, 0);
548        assert!(!deserialized.is_enabled());
549    }
550
551    #[test]
552    fn test_merge_builtin_measurement_keys() {
553        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
554        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
555        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
556
557        let proj = MeasurementsConfig {
558            builtin_measurements: vec![foo.clone(), bar.clone()],
559            max_custom_measurements: 4,
560        };
561
562        let glob = MeasurementsConfig {
563            // The 'bar' here will be ignored since it's a duplicate from the project level.
564            builtin_measurements: vec![baz.clone(), bar.clone()],
565            max_custom_measurements: 4,
566        };
567        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
568
569        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
570
571        assert_eq!(keys, vec![&foo, &bar, &baz]);
572    }
573
574    #[test]
575    fn test_max_custom_measurement() {
576        // Empty configs will return a None value for max measurements.
577        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
578        assert!(dynamic_config.max_custom_measurements().is_none());
579
580        let proj = MeasurementsConfig {
581            builtin_measurements: vec![],
582            max_custom_measurements: 3,
583        };
584
585        let glob = MeasurementsConfig {
586            builtin_measurements: vec![],
587            max_custom_measurements: 4,
588        };
589
590        // If only project level measurement config is there, return its max custom measurement variable.
591        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
592        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
593
594        // Same logic for when only global level measurement config exists.
595        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
596        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
597
598        // If both is available, pick the smallest number.
599        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
600        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
601    }
602
603    #[test]
604    fn test_geo_from_ip_address() {
605        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
606
607        let json = r#"{
608            "user": {
609                "ip_address": "2.125.160.216"
610            }
611        }"#;
612        let mut event = Annotated::<Event>::from_json(json).unwrap();
613
614        normalize_event(
615            &mut event,
616            &NormalizationConfig {
617                geoip_lookup: Some(&lookup),
618                ..Default::default()
619            },
620        );
621
622        let expected = Annotated::new(Geo {
623            country_code: Annotated::new("GB".to_owned()),
624            city: Annotated::new("Boxford".to_owned()),
625            subdivision: Annotated::new("England".to_owned()),
626            region: Annotated::new("United Kingdom".to_owned()),
627            ..Geo::default()
628        });
629        assert_eq!(get_value!(event.user!).geo, expected);
630    }
631
632    #[test]
633    fn test_user_ip_from_remote_addr() {
634        let mut event = Annotated::new(Event {
635            request: Annotated::from(Request {
636                env: Annotated::new({
637                    let mut map = Object::new();
638                    map.insert(
639                        "REMOTE_ADDR".to_owned(),
640                        Annotated::new(Value::String("2.125.160.216".to_owned())),
641                    );
642                    map
643                }),
644                ..Request::default()
645            }),
646            platform: Annotated::new("javascript".to_owned()),
647            ..Event::default()
648        });
649
650        normalize_event(&mut event, &NormalizationConfig::default());
651
652        let ip_addr = get_value!(event.user.ip_address!);
653        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
654    }
655
656    #[test]
657    fn test_user_ip_from_invalid_remote_addr() {
658        let mut event = Annotated::new(Event {
659            request: Annotated::from(Request {
660                env: Annotated::new({
661                    let mut map = Object::new();
662                    map.insert(
663                        "REMOTE_ADDR".to_owned(),
664                        Annotated::new(Value::String("whoops".to_owned())),
665                    );
666                    map
667                }),
668                ..Request::default()
669            }),
670            platform: Annotated::new("javascript".to_owned()),
671            ..Event::default()
672        });
673
674        normalize_event(&mut event, &NormalizationConfig::default());
675
676        assert_eq!(Annotated::empty(), event.value().unwrap().user);
677    }
678
679    #[test]
680    fn test_user_ip_from_client_ip_without_auto() {
681        let mut event = Annotated::new(Event {
682            platform: Annotated::new("javascript".to_owned()),
683            ..Default::default()
684        });
685
686        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
687
688        normalize_event(
689            &mut event,
690            &NormalizationConfig {
691                client_ip: Some(&ip_address),
692                infer_ip_address: true,
693                ..Default::default()
694            },
695        );
696
697        let ip_addr = get_value!(event.user.ip_address!);
698        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
699    }
700
701    #[test]
702    fn test_user_ip_from_client_ip_with_auto() {
703        let mut event = Annotated::new(Event {
704            user: Annotated::new(User {
705                ip_address: Annotated::new(IpAddr::auto()),
706                ..Default::default()
707            }),
708            ..Default::default()
709        });
710
711        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
712
713        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
714        normalize_event(
715            &mut event,
716            &NormalizationConfig {
717                client_ip: Some(&ip_address),
718                geoip_lookup: Some(&geo),
719                infer_ip_address: true,
720                ..Default::default()
721            },
722        );
723
724        let user = get_value!(event.user!);
725        let ip_addr = user.ip_address.value().expect("ip address missing");
726
727        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
728        assert!(user.geo.value().is_some());
729    }
730
731    #[test]
732    fn test_user_ip_from_client_ip_without_appropriate_platform() {
733        let mut event = Annotated::new(Event::default());
734
735        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
736        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
737        normalize_event(
738            &mut event,
739            &NormalizationConfig {
740                client_ip: Some(&ip_address),
741                geoip_lookup: Some(&geo),
742                ..Default::default()
743            },
744        );
745
746        let user = get_value!(event.user!);
747        assert!(user.ip_address.value().is_none());
748        assert!(user.geo.value().is_some());
749    }
750
751    #[test]
752    fn test_geo_present_if_ip_inferring_disabled() {
753        let mut event = Annotated::new(Event {
754            user: Annotated::new(User {
755                ip_address: Annotated::new(IpAddr::auto()),
756                ..Default::default()
757            }),
758            ..Default::default()
759        });
760
761        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
762        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
763
764        normalize_event(
765            &mut event,
766            &NormalizationConfig {
767                client_ip: Some(&ip_address),
768                geoip_lookup: Some(&geo),
769                infer_ip_address: false,
770                ..Default::default()
771            },
772        );
773
774        let user = get_value!(event.user!);
775        assert!(user.ip_address.value().unwrap().is_auto());
776        assert!(user.geo.value().is_some());
777    }
778
779    #[test]
780    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
781        let mut event = Annotated::new(Event {
782            user: Annotated::new(User {
783                ip_address: Annotated::new(IpAddr::auto()),
784                ..Default::default()
785            }),
786            ..Default::default()
787        });
788
789        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
790        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
791
792        normalize_event(
793            &mut event,
794            &NormalizationConfig {
795                client_ip: Some(&ip_address),
796                geoip_lookup: Some(&geo),
797                infer_ip_address: true,
798                ..Default::default()
799            },
800        );
801
802        let user = get_value!(event.user!);
803        assert_eq!(
804            user.ip_address.value(),
805            Some(&IpAddr::parse("2.125.160.216").unwrap())
806        );
807        assert!(user.geo.value().is_some());
808    }
809
810    #[test]
811    fn test_event_level_defaulted() {
812        let mut event = Annotated::new(Event::default());
813        normalize_event(&mut event, &NormalizationConfig::default());
814        assert_eq!(get_value!(event.level), Some(&Level::Error));
815    }
816
817    #[test]
818    fn test_transaction_level_untouched() {
819        let mut event = Annotated::new(Event {
820            ty: Annotated::new(EventType::Transaction),
821            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
822            start_timestamp: Annotated::new(
823                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
824            ),
825            contexts: {
826                let mut contexts = Contexts::new();
827                contexts.add(TraceContext {
828                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
829                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
830                    op: Annotated::new("http.server".to_owned()),
831                    ..Default::default()
832                });
833                Annotated::new(contexts)
834            },
835            ..Event::default()
836        });
837        normalize_event(&mut event, &NormalizationConfig::default());
838        assert_eq!(get_value!(event.level), Some(&Level::Info));
839    }
840
841    #[test]
842    fn test_environment_tag_is_moved() {
843        let mut event = Annotated::new(Event {
844            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
845                Annotated::new("environment".to_owned()),
846                Annotated::new("despacito".to_owned()),
847            ))]))),
848            ..Event::default()
849        });
850
851        normalize_event(&mut event, &NormalizationConfig::default());
852
853        let event = event.value().unwrap();
854
855        assert_eq!(event.environment.as_str(), Some("despacito"));
856        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
857    }
858
859    #[test]
860    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
861        let mut event = Annotated::new(Event {
862            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
863                Annotated::new("environment".to_owned()),
864                Annotated::new("despacito".to_owned()),
865            ))]))),
866            environment: Annotated::new("".to_owned()),
867            ..Event::default()
868        });
869
870        normalize_event(&mut event, &NormalizationConfig::default());
871
872        let event = event.value().unwrap();
873
874        assert_eq!(event.environment.as_str(), Some("despacito"));
875        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
876    }
877
878    #[test]
879    fn test_empty_environment_is_removed() {
880        let mut event = Annotated::new(Event {
881            environment: Annotated::new("".to_owned()),
882            ..Event::default()
883        });
884
885        normalize_event(&mut event, &NormalizationConfig::default());
886        assert_eq!(get_value!(event.environment), None);
887    }
888    #[test]
889    fn test_replay_id_added_from_dsc() {
890        let replay_id = Uuid::new_v4();
891        let mut event = Annotated::new(Event {
892            contexts: Annotated::new(Contexts(Object::new())),
893            ..Event::default()
894        });
895        normalize_event(
896            &mut event,
897            &NormalizationConfig {
898                replay_id: Some(replay_id),
899                ..Default::default()
900            },
901        );
902
903        let event = event.value().unwrap();
904
905        assert_eq!(event.contexts, {
906            let mut contexts = Contexts::new();
907            contexts.add(ReplayContext {
908                replay_id: Annotated::new(EventId(replay_id)),
909                other: Object::default(),
910            });
911            Annotated::new(contexts)
912        })
913    }
914
915    #[test]
916    fn test_none_environment_errors() {
917        let mut event = Annotated::new(Event {
918            environment: Annotated::new("none".to_owned()),
919            ..Event::default()
920        });
921
922        normalize_event(&mut event, &NormalizationConfig::default());
923
924        let environment = get_path!(event.environment!);
925        let expected_original = &Value::String("none".to_owned());
926
927        assert_eq!(
928            environment.meta().iter_errors().collect::<Vec<&Error>>(),
929            vec![&Error::new(ErrorKind::InvalidData)],
930        );
931        assert_eq!(
932            environment.meta().original_value().unwrap(),
933            expected_original
934        );
935        assert_eq!(environment.value(), None);
936    }
937
938    #[test]
939    fn test_invalid_release_removed() {
940        let mut event = Annotated::new(Event {
941            release: Annotated::new(LenientString("Latest".to_owned())),
942            ..Event::default()
943        });
944
945        normalize_event(&mut event, &NormalizationConfig::default());
946
947        let release = get_path!(event.release!);
948        let expected_original = &Value::String("Latest".to_owned());
949
950        assert_eq!(
951            release.meta().iter_errors().collect::<Vec<&Error>>(),
952            vec![&Error::new(ErrorKind::InvalidData)],
953        );
954        assert_eq!(release.meta().original_value().unwrap(), expected_original);
955        assert_eq!(release.value(), None);
956    }
957
958    #[test]
959    fn test_top_level_keys_moved_into_tags() {
960        let mut event = Annotated::new(Event {
961            server_name: Annotated::new("foo".to_owned()),
962            site: Annotated::new("foo".to_owned()),
963            tags: Annotated::new(Tags(PairList(vec![
964                Annotated::new(TagEntry(
965                    Annotated::new("site".to_owned()),
966                    Annotated::new("old".to_owned()),
967                )),
968                Annotated::new(TagEntry(
969                    Annotated::new("server_name".to_owned()),
970                    Annotated::new("old".to_owned()),
971                )),
972            ]))),
973            ..Event::default()
974        });
975
976        normalize_event(&mut event, &NormalizationConfig::default());
977
978        assert_eq!(get_value!(event.site), None);
979        assert_eq!(get_value!(event.server_name), None);
980
981        assert_eq!(
982            get_value!(event.tags!),
983            &Tags(PairList(vec![
984                Annotated::new(TagEntry(
985                    Annotated::new("site".to_owned()),
986                    Annotated::new("foo".to_owned()),
987                )),
988                Annotated::new(TagEntry(
989                    Annotated::new("server_name".to_owned()),
990                    Annotated::new("foo".to_owned()),
991                )),
992            ]))
993        );
994    }
995
996    #[test]
997    fn test_internal_tags_removed() {
998        let mut event = Annotated::new(Event {
999            tags: Annotated::new(Tags(PairList(vec![
1000                Annotated::new(TagEntry(
1001                    Annotated::new("release".to_owned()),
1002                    Annotated::new("foo".to_owned()),
1003                )),
1004                Annotated::new(TagEntry(
1005                    Annotated::new("dist".to_owned()),
1006                    Annotated::new("foo".to_owned()),
1007                )),
1008                Annotated::new(TagEntry(
1009                    Annotated::new("user".to_owned()),
1010                    Annotated::new("foo".to_owned()),
1011                )),
1012                Annotated::new(TagEntry(
1013                    Annotated::new("filename".to_owned()),
1014                    Annotated::new("foo".to_owned()),
1015                )),
1016                Annotated::new(TagEntry(
1017                    Annotated::new("function".to_owned()),
1018                    Annotated::new("foo".to_owned()),
1019                )),
1020                Annotated::new(TagEntry(
1021                    Annotated::new("something".to_owned()),
1022                    Annotated::new("else".to_owned()),
1023                )),
1024            ]))),
1025            ..Event::default()
1026        });
1027
1028        normalize_event(&mut event, &NormalizationConfig::default());
1029
1030        assert_eq!(get_value!(event.tags!).len(), 1);
1031    }
1032
1033    #[test]
1034    fn test_empty_tags_removed() {
1035        let mut event = Annotated::new(Event {
1036            tags: Annotated::new(Tags(PairList(vec![
1037                Annotated::new(TagEntry(
1038                    Annotated::new("".to_owned()),
1039                    Annotated::new("foo".to_owned()),
1040                )),
1041                Annotated::new(TagEntry(
1042                    Annotated::new("foo".to_owned()),
1043                    Annotated::new("".to_owned()),
1044                )),
1045                Annotated::new(TagEntry(
1046                    Annotated::new("something".to_owned()),
1047                    Annotated::new("else".to_owned()),
1048                )),
1049            ]))),
1050            ..Event::default()
1051        });
1052
1053        normalize_event(&mut event, &NormalizationConfig::default());
1054
1055        assert_eq!(
1056            get_value!(event.tags!),
1057            &Tags(PairList(vec![
1058                Annotated::new(TagEntry(
1059                    Annotated::from_error(Error::nonempty(), None),
1060                    Annotated::new("foo".to_owned()),
1061                )),
1062                Annotated::new(TagEntry(
1063                    Annotated::new("foo".to_owned()),
1064                    Annotated::from_error(Error::nonempty(), None),
1065                )),
1066                Annotated::new(TagEntry(
1067                    Annotated::new("something".to_owned()),
1068                    Annotated::new("else".to_owned()),
1069                )),
1070            ]))
1071        );
1072    }
1073
1074    #[test]
1075    fn test_tags_deduplicated() {
1076        let mut event = Annotated::new(Event {
1077            tags: Annotated::new(Tags(PairList(vec![
1078                Annotated::new(TagEntry(
1079                    Annotated::new("foo".to_owned()),
1080                    Annotated::new("1".to_owned()),
1081                )),
1082                Annotated::new(TagEntry(
1083                    Annotated::new("bar".to_owned()),
1084                    Annotated::new("1".to_owned()),
1085                )),
1086                Annotated::new(TagEntry(
1087                    Annotated::new("foo".to_owned()),
1088                    Annotated::new("2".to_owned()),
1089                )),
1090                Annotated::new(TagEntry(
1091                    Annotated::new("bar".to_owned()),
1092                    Annotated::new("2".to_owned()),
1093                )),
1094                Annotated::new(TagEntry(
1095                    Annotated::new("foo".to_owned()),
1096                    Annotated::new("3".to_owned()),
1097                )),
1098            ]))),
1099            ..Event::default()
1100        });
1101
1102        normalize_event(&mut event, &NormalizationConfig::default());
1103
1104        // should keep the first occurrence of every tag
1105        assert_eq!(
1106            get_value!(event.tags!),
1107            &Tags(PairList(vec![
1108                Annotated::new(TagEntry(
1109                    Annotated::new("foo".to_owned()),
1110                    Annotated::new("1".to_owned()),
1111                )),
1112                Annotated::new(TagEntry(
1113                    Annotated::new("bar".to_owned()),
1114                    Annotated::new("1".to_owned()),
1115                )),
1116            ]))
1117        );
1118    }
1119
1120    #[test]
1121    fn test_transaction_status_defaulted_to_unknown() {
1122        let mut object = Object::new();
1123        let trace_context = TraceContext {
1124            // We assume the status to be null.
1125            status: Annotated::empty(),
1126            ..TraceContext::default()
1127        };
1128        object.insert(
1129            "trace".to_owned(),
1130            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1131        );
1132
1133        let mut event = Annotated::new(Event {
1134            contexts: Annotated::new(Contexts(object)),
1135            ..Event::default()
1136        });
1137        normalize_event(&mut event, &NormalizationConfig::default());
1138
1139        let event = event.value().unwrap();
1140
1141        let event_trace_context = event.context::<TraceContext>().unwrap();
1142        assert_eq!(
1143            event_trace_context.status,
1144            Annotated::new(SpanStatus::Unknown)
1145        )
1146    }
1147
1148    #[test]
1149    fn test_unknown_debug_image() {
1150        let mut event = Annotated::new(Event {
1151            debug_meta: Annotated::new(DebugMeta {
1152                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1153                ..DebugMeta::default()
1154            }),
1155            ..Event::default()
1156        });
1157
1158        normalize_event(&mut event, &NormalizationConfig::default());
1159
1160        assert_eq!(
1161            get_path!(event.debug_meta!),
1162            &Annotated::new(DebugMeta {
1163                images: Annotated::new(vec![Annotated::from_error(
1164                    Error::invalid("unsupported debug image type"),
1165                    Some(Value::Object(Object::default())),
1166                )]),
1167                ..DebugMeta::default()
1168            })
1169        );
1170    }
1171
1172    #[test]
1173    fn test_context_line_default() {
1174        let mut frame = Annotated::new(Frame {
1175            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1176            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1177            ..Frame::default()
1178        });
1179
1180        normalize_non_raw_frame(&mut frame);
1181
1182        let frame = frame.value().unwrap();
1183        assert_eq!(frame.context_line.as_str(), Some(""));
1184    }
1185
1186    #[test]
1187    fn test_context_line_retain() {
1188        let mut frame = Annotated::new(Frame {
1189            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1190            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1191            context_line: Annotated::new("some line".to_owned()),
1192            ..Frame::default()
1193        });
1194
1195        normalize_non_raw_frame(&mut frame);
1196
1197        let frame = frame.value().unwrap();
1198        assert_eq!(frame.context_line.as_str(), Some("some line"));
1199    }
1200
1201    #[test]
1202    fn test_frame_null_context_lines() {
1203        let mut frame = Annotated::new(Frame {
1204            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1205            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1206            ..Frame::default()
1207        });
1208
1209        normalize_non_raw_frame(&mut frame);
1210
1211        assert_eq!(
1212            *get_value!(frame.pre_context!),
1213            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1214        );
1215        assert_eq!(
1216            *get_value!(frame.post_context!),
1217            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1218        );
1219    }
1220
1221    #[test]
1222    fn test_too_long_tags() {
1223        let mut event = Annotated::new(Event {
1224        tags: Annotated::new(Tags(PairList(
1225            vec![Annotated::new(TagEntry(
1226                Annotated::new("foobar".to_owned()),
1227                Annotated::new("...........................................................................................................................................................................................................".to_owned()),
1228            )), Annotated::new(TagEntry(
1229                Annotated::new("foooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooooo".to_owned()),
1230                Annotated::new("bar".to_owned()),
1231            ))]),
1232        )),
1233        ..Event::default()
1234    });
1235
1236        normalize_event(&mut event, &NormalizationConfig::default());
1237
1238        assert_eq!(
1239            get_value!(event.tags!),
1240            &Tags(PairList(vec![
1241                Annotated::new(TagEntry(
1242                    Annotated::new("foobar".to_owned()),
1243                    Annotated::from_error(Error::new(ErrorKind::ValueTooLong), None),
1244                )),
1245                Annotated::new(TagEntry(
1246                    Annotated::from_error(Error::new(ErrorKind::ValueTooLong), None),
1247                    Annotated::new("bar".to_owned()),
1248                )),
1249            ]))
1250        );
1251    }
1252
1253    #[test]
1254    fn test_too_long_distribution() {
1255        let json = r#"{
1256  "event_id": "52df9022835246eeb317dbd739ccd059",
1257  "fingerprint": [
1258    "{{ default }}"
1259  ],
1260  "platform": "other",
1261  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1262}"#;
1263
1264        let mut event = Annotated::<Event>::from_json(json).unwrap();
1265
1266        normalize_event(&mut event, &NormalizationConfig::default());
1267
1268        let dist = &event.value().unwrap().dist;
1269        let result = &Annotated::<String>::from_error(
1270            Error::new(ErrorKind::ValueTooLong),
1271            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1272        );
1273        assert_eq!(dist, result);
1274    }
1275
1276    #[test]
1277    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1278        let mut event = Annotated::new(Event {
1279            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1280                ty: Annotated::new("FooDivisionError".to_owned()),
1281                value: Annotated::new("hi".to_owned().into()),
1282                ..Exception::default()
1283            })])),
1284            stacktrace: Annotated::new(
1285                RawStacktrace {
1286                    frames: Annotated::new(vec![Annotated::new(Frame {
1287                        module: Annotated::new("MyModule".to_owned()),
1288                        filename: Annotated::new("MyFilename".into()),
1289                        function: Annotated::new("Void FooBar()".to_owned()),
1290                        ..Frame::default()
1291                    })]),
1292                    ..RawStacktrace::default()
1293                }
1294                .into(),
1295            ),
1296            ..Event::default()
1297        });
1298
1299        normalize_event(&mut event, &NormalizationConfig::default());
1300
1301        assert_eq!(
1302            get_value!(event.exceptions.values[0].stacktrace!),
1303            &Stacktrace(RawStacktrace {
1304                frames: Annotated::new(vec![Annotated::new(Frame {
1305                    module: Annotated::new("MyModule".to_owned()),
1306                    filename: Annotated::new("MyFilename".into()),
1307                    abs_path: Annotated::new("MyFilename".into()),
1308                    function: Annotated::new("Void FooBar()".to_owned()),
1309                    ..Frame::default()
1310                })]),
1311                ..RawStacktrace::default()
1312            })
1313        );
1314    }
1315
1316    #[test]
1317    fn test_parses_sdk_info_from_header() {
1318        let mut event = Annotated::new(Event::default());
1319
1320        normalize_event(
1321            &mut event,
1322            &NormalizationConfig {
1323                client: Some("_fooBar/0.0.0".to_owned()),
1324                ..Default::default()
1325            },
1326        );
1327
1328        assert_eq!(
1329            get_path!(event.client_sdk!),
1330            &Annotated::new(ClientSdkInfo {
1331                name: Annotated::new("_fooBar".to_owned()),
1332                version: Annotated::new("0.0.0".to_owned()),
1333                ..ClientSdkInfo::default()
1334            })
1335        );
1336    }
1337
1338    #[test]
1339    fn test_discards_received() {
1340        let mut event = Annotated::new(Event {
1341            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1342            ..Default::default()
1343        });
1344
1345        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1346        normalize_event(&mut event, &NormalizationConfig::default());
1347
1348        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1349    }
1350
1351    #[test]
1352    fn test_grouping_config() {
1353        let mut event = Annotated::new(Event {
1354            logentry: Annotated::from(LogEntry {
1355                message: Annotated::new("Hello World!".to_owned().into()),
1356                ..Default::default()
1357            }),
1358            ..Default::default()
1359        });
1360
1361        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1362        normalize_event(
1363            &mut event,
1364            &NormalizationConfig {
1365                grouping_config: Some(json!({
1366                    "id": "legacy:1234-12-12".to_owned(),
1367                })),
1368                ..Default::default()
1369            },
1370        );
1371
1372        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1373            ".event_id" => "[event-id]",
1374            ".received" => "[received]",
1375            ".timestamp" => "[timestamp]"
1376        }, @r###"
1377        {
1378          "event_id": "[event-id]",
1379          "level": "error",
1380          "type": "default",
1381          "logentry": {
1382            "formatted": "Hello World!",
1383          },
1384          "logger": "",
1385          "platform": "other",
1386          "timestamp": "[timestamp]",
1387          "received": "[received]",
1388          "grouping_config": {
1389            "id": "legacy:1234-12-12",
1390          },
1391        }
1392        "###);
1393    }
1394
1395    #[test]
1396    fn test_logentry_error() {
1397        let json = r#"
1398{
1399    "event_id": "74ad1301f4df489ead37d757295442b1",
1400    "timestamp": 1668148328.308933,
1401    "received": 1668148328.308933,
1402    "level": "error",
1403    "platform": "python",
1404    "logentry": {
1405        "params": [
1406            "bogus"
1407        ],
1408        "formatted": 42
1409    }
1410}
1411"#;
1412        let mut event = Annotated::<Event>::from_json(json).unwrap();
1413
1414        normalize_event(&mut event, &NormalizationConfig::default());
1415
1416        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1417        {
1418          "event_id": "74ad1301f4df489ead37d757295442b1",
1419          "level": "error",
1420          "type": "default",
1421          "logentry": null,
1422          "logger": "",
1423          "platform": "python",
1424          "timestamp": 1668148328.308933,
1425          "received": "[received]",
1426          "_meta": {
1427            "logentry": {
1428              "": {
1429                "err": [
1430                  [
1431                    "invalid_data",
1432                    {
1433                      "reason": "no message present"
1434                    }
1435                  ]
1436                ],
1437                "val": {
1438                  "formatted": null,
1439                  "message": null,
1440                  "params": [
1441                    "bogus"
1442                  ]
1443                }
1444              }
1445            }
1446          }
1447        }
1448        "###)
1449    }
1450
1451    #[test]
1452    fn test_future_timestamp() {
1453        let mut event = Annotated::new(Event {
1454            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1455            ..Default::default()
1456        });
1457
1458        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1459        let max_secs_in_past = Some(30 * 24 * 3600);
1460        let max_secs_in_future = Some(60);
1461
1462        validate_event(
1463            &mut event,
1464            &EventValidationConfig {
1465                received_at,
1466                max_secs_in_past,
1467                max_secs_in_future,
1468                is_validated: false,
1469                ..Default::default()
1470            },
1471        )
1472        .unwrap();
1473        normalize_event(&mut event, &NormalizationConfig::default());
1474
1475        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1476        ".event_id" => "[event-id]",
1477    }, @r###"
1478        {
1479          "event_id": "[event-id]",
1480          "level": "error",
1481          "type": "default",
1482          "logger": "",
1483          "platform": "other",
1484          "timestamp": 946857600.0,
1485          "received": 946857600.0,
1486          "_meta": {
1487            "timestamp": {
1488              "": Meta(Some(MetaInner(
1489                err: [
1490                  [
1491                    "future_timestamp",
1492                    {
1493                      "sdk_time": "2000-01-03T00:02:00+00:00",
1494                      "server_time": "2000-01-03T00:00:00+00:00",
1495                    },
1496                  ],
1497                ],
1498              ))),
1499            },
1500          },
1501        }
1502        "###);
1503    }
1504
1505    #[test]
1506    fn test_past_timestamp() {
1507        let mut event = Annotated::new(Event {
1508            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1509            ..Default::default()
1510        });
1511
1512        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1513        let max_secs_in_past = Some(30 * 24 * 3600);
1514        let max_secs_in_future = Some(60);
1515
1516        validate_event(
1517            &mut event,
1518            &EventValidationConfig {
1519                received_at,
1520                max_secs_in_past,
1521                max_secs_in_future,
1522                is_validated: false,
1523                ..Default::default()
1524            },
1525        )
1526        .unwrap();
1527        normalize_event(&mut event, &NormalizationConfig::default());
1528
1529        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1530        ".event_id" => "[event-id]",
1531    }, @r###"
1532        {
1533          "event_id": "[event-id]",
1534          "level": "error",
1535          "type": "default",
1536          "logger": "",
1537          "platform": "other",
1538          "timestamp": 952041600.0,
1539          "received": 952041600.0,
1540          "_meta": {
1541            "timestamp": {
1542              "": Meta(Some(MetaInner(
1543                err: [
1544                  [
1545                    "past_timestamp",
1546                    {
1547                      "sdk_time": "2000-01-03T00:00:00+00:00",
1548                      "server_time": "2000-03-03T00:00:00+00:00",
1549                    },
1550                  ],
1551                ],
1552              ))),
1553            },
1554          },
1555        }
1556        "###);
1557    }
1558
1559    #[test]
1560    fn test_normalize_logger_empty() {
1561        let mut event = Event::from_value(
1562            serde_json::json!({
1563                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1564                "platform": "java",
1565                "logger": "",
1566            })
1567            .into(),
1568        );
1569
1570        normalize_event(&mut event, &NormalizationConfig::default());
1571        assert_annotated_snapshot!(event);
1572    }
1573
1574    #[test]
1575    fn test_normalize_logger_trimmed() {
1576        let mut event = Event::from_value(
1577            serde_json::json!({
1578                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1579                "platform": "java",
1580                "logger": " \t  \t   ",
1581            })
1582            .into(),
1583        );
1584
1585        normalize_event(&mut event, &NormalizationConfig::default());
1586        assert_annotated_snapshot!(event);
1587    }
1588
1589    #[test]
1590    fn test_normalize_logger_short_no_trimming() {
1591        let mut event = Event::from_value(
1592            serde_json::json!({
1593                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1594                "platform": "java",
1595                "logger": "my.short-logger.isnt_trimmed",
1596            })
1597            .into(),
1598        );
1599
1600        normalize_event(&mut event, &NormalizationConfig::default());
1601        assert_annotated_snapshot!(event);
1602    }
1603
1604    #[test]
1605    fn test_normalize_logger_exact_length() {
1606        let mut event = Event::from_value(
1607            serde_json::json!({
1608                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1609                "platform": "java",
1610                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1611            })
1612            .into(),
1613        );
1614
1615        normalize_event(&mut event, &NormalizationConfig::default());
1616        assert_annotated_snapshot!(event);
1617    }
1618
1619    #[test]
1620    fn test_normalize_logger_too_long_single_word() {
1621        let mut event = Event::from_value(
1622            serde_json::json!({
1623                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1624                "platform": "java",
1625                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1626            })
1627            .into(),
1628        );
1629
1630        normalize_event(&mut event, &NormalizationConfig::default());
1631        assert_annotated_snapshot!(event);
1632    }
1633
1634    #[test]
1635    fn test_normalize_logger_word_trimmed_at_max() {
1636        let mut event = Event::from_value(
1637            serde_json::json!({
1638                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1639                "platform": "java",
1640                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1641            })
1642            .into(),
1643        );
1644
1645        normalize_event(&mut event, &NormalizationConfig::default());
1646        assert_annotated_snapshot!(event);
1647    }
1648
1649    #[test]
1650    fn test_normalize_logger_word_trimmed_before_max() {
1651        // This test verifies the "smart" trimming on words -- the logger name
1652        // should be cut before the max limit, removing entire words.
1653        let mut event = Event::from_value(
1654            serde_json::json!({
1655                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1656                "platform": "java",
1657                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1658            })
1659            .into(),
1660        );
1661
1662        normalize_event(&mut event, &NormalizationConfig::default());
1663        assert_annotated_snapshot!(event);
1664    }
1665
1666    #[test]
1667    fn test_normalize_logger_word_leading_dots() {
1668        let mut event = Event::from_value(
1669            serde_json::json!({
1670                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1671                "platform": "java",
1672                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1673            })
1674            .into(),
1675        );
1676
1677        normalize_event(&mut event, &NormalizationConfig::default());
1678        assert_annotated_snapshot!(event);
1679    }
1680
1681    #[test]
1682    fn test_normalization_is_idempotent() {
1683        // get an event, normalize it. the result of that must be the same as normalizing it once more
1684        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1685        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1686        let mut event = Annotated::new(Event {
1687            ty: Annotated::new(EventType::Transaction),
1688            transaction: Annotated::new("/".to_owned()),
1689            timestamp: Annotated::new(end.into()),
1690            start_timestamp: Annotated::new(start.into()),
1691            contexts: {
1692                let mut contexts = Contexts::new();
1693                contexts.add(TraceContext {
1694                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1695                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1696                    op: Annotated::new("http.server".to_owned()),
1697                    ..Default::default()
1698                });
1699                Annotated::new(contexts)
1700            },
1701            spans: Annotated::new(vec![Annotated::new(Span {
1702                timestamp: Annotated::new(
1703                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1704                ),
1705                start_timestamp: Annotated::new(
1706                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1707                ),
1708                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1709                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1710
1711                ..Default::default()
1712            })]),
1713            ..Default::default()
1714        });
1715
1716        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1717            relay_event_schema::processor::apply(event, |e, _m| {
1718                e.received = Annotated::empty();
1719                Ok(())
1720            })
1721            .unwrap();
1722            event
1723        }
1724
1725        normalize_event(&mut event, &NormalizationConfig::default());
1726        let first = remove_received_from_event(&mut event.clone())
1727            .to_json()
1728            .unwrap();
1729        // Expected some fields (such as timestamps) exist after first normalization.
1730
1731        normalize_event(&mut event, &NormalizationConfig::default());
1732        let second = remove_received_from_event(&mut event.clone())
1733            .to_json()
1734            .unwrap();
1735        assert_eq!(&first, &second, "idempotency check failed");
1736
1737        normalize_event(&mut event, &NormalizationConfig::default());
1738        let third = remove_received_from_event(&mut event.clone())
1739            .to_json()
1740            .unwrap();
1741        assert_eq!(&second, &third, "idempotency check failed");
1742    }
1743
1744    /// Validate full normalization is idempotent.
1745    ///
1746    /// Both PoPs and processing relays will temporarily run full normalization
1747    /// in events, during the rollout of running normalization once in internal
1748    /// relays.
1749    // TODO(iker): remove this test after the rollout is done.
1750    #[test]
1751    fn test_full_normalization_is_idempotent() {
1752        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1753        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1754        let mut event = Annotated::new(Event {
1755            ty: Annotated::new(EventType::Transaction),
1756            transaction: Annotated::new("/".to_owned()),
1757            timestamp: Annotated::new(end.into()),
1758            start_timestamp: Annotated::new(start.into()),
1759            contexts: {
1760                let mut contexts = Contexts::new();
1761                contexts.add(TraceContext {
1762                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1763                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1764                    op: Annotated::new("http.server".to_owned()),
1765                    ..Default::default()
1766                });
1767                Annotated::new(contexts)
1768            },
1769            spans: Annotated::new(vec![Annotated::new(Span {
1770                timestamp: Annotated::new(
1771                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1772                ),
1773                start_timestamp: Annotated::new(
1774                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1775                ),
1776                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1777                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1778
1779                ..Default::default()
1780            })]),
1781            ..Default::default()
1782        });
1783
1784        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1785            relay_event_schema::processor::apply(event, |e, _m| {
1786                e.received = Annotated::empty();
1787                Ok(())
1788            })
1789            .unwrap();
1790            event
1791        }
1792
1793        let full_normalization_config = NormalizationConfig {
1794            is_renormalize: false,
1795            enable_trimming: true,
1796            remove_other: true,
1797            emit_event_errors: true,
1798            ..Default::default()
1799        };
1800
1801        normalize_event(&mut event, &full_normalization_config);
1802        let first = remove_received_from_event(&mut event.clone())
1803            .to_json()
1804            .unwrap();
1805        // Expected some fields (such as timestamps) exist after first normalization.
1806
1807        normalize_event(&mut event, &full_normalization_config);
1808        let second = remove_received_from_event(&mut event.clone())
1809            .to_json()
1810            .unwrap();
1811        assert_eq!(&first, &second, "idempotency check failed");
1812
1813        normalize_event(&mut event, &full_normalization_config);
1814        let third = remove_received_from_event(&mut event.clone())
1815            .to_json()
1816            .unwrap();
1817        assert_eq!(&second, &third, "idempotency check failed");
1818    }
1819
1820    #[test]
1821    fn test_normalize_validates_spans() {
1822        let event = Annotated::<Event>::from_json(
1823            r#"
1824            {
1825                "type": "transaction",
1826                "transaction": "/",
1827                "timestamp": 946684810.0,
1828                "start_timestamp": 946684800.0,
1829                "contexts": {
1830                    "trace": {
1831                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1832                    "span_id": "fa90fdead5f74053",
1833                    "op": "http.server",
1834                    "type": "trace"
1835                    }
1836                },
1837                "spans": []
1838            }
1839            "#,
1840        )
1841        .unwrap();
1842
1843        // All incomplete spans should be caught by normalization:
1844        for span in [
1845            r#"null"#,
1846            r#"{
1847              "timestamp": 946684810.0,
1848              "start_timestamp": 946684900.0,
1849              "span_id": "fa90fdead5f74053",
1850              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1851            }"#,
1852            r#"{
1853              "timestamp": 946684810.0,
1854              "span_id": "fa90fdead5f74053",
1855              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1856            }"#,
1857            r#"{
1858              "timestamp": 946684810.0,
1859              "start_timestamp": 946684800.0,
1860              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1861            }"#,
1862            r#"{
1863              "timestamp": 946684810.0,
1864              "start_timestamp": 946684800.0,
1865              "span_id": "fa90fdead5f74053"
1866            }"#,
1867        ] {
1868            let mut modified_event = event.clone();
1869            let event_ref = modified_event.value_mut().as_mut().unwrap();
1870            event_ref
1871                .spans
1872                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1873
1874            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1875
1876            assert!(res.is_err(), "{span:?}");
1877        }
1878    }
1879
1880    #[test]
1881    fn test_normalization_respects_is_renormalize() {
1882        let mut event = Annotated::<Event>::from_json(
1883            r#"
1884            {
1885                "type": "default",
1886                "tags": [["environment", "some_environment"]]
1887            }
1888            "#,
1889        )
1890        .unwrap();
1891
1892        normalize_event(
1893            &mut event,
1894            &NormalizationConfig {
1895                is_renormalize: true,
1896                ..Default::default()
1897            },
1898        );
1899
1900        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1901        Tags(
1902            PairList(
1903                [
1904                    TagEntry(
1905                        "environment",
1906                        "some_environment",
1907                    ),
1908                ],
1909            ),
1910        )
1911        "###);
1912    }
1913
1914    #[test]
1915    fn test_geo_in_normalize() {
1916        let mut event = Annotated::<Event>::from_json(
1917            r#"
1918            {
1919                "type": "transaction",
1920                "transaction": "/foo/",
1921                "timestamp": 946684810.0,
1922                "start_timestamp": 946684800.0,
1923                "contexts": {
1924                    "trace": {
1925                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1926                        "span_id": "fa90fdead5f74053",
1927                        "op": "http.server",
1928                        "type": "trace"
1929                    }
1930                },
1931                "transaction_info": {
1932                    "source": "url"
1933                },
1934                "user": {
1935                    "ip_address": "2.125.160.216"
1936                }
1937            }
1938            "#,
1939        )
1940        .unwrap();
1941
1942        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1943
1944        // Extract user's geo information before normalization.
1945        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1946        assert!(user_geo.is_none());
1947
1948        normalize_event(
1949            &mut event,
1950            &NormalizationConfig {
1951                geoip_lookup: Some(&lookup),
1952                ..Default::default()
1953            },
1954        );
1955
1956        // Extract user's geo information after normalization.
1957        let user_geo = event
1958            .value()
1959            .unwrap()
1960            .user
1961            .value()
1962            .unwrap()
1963            .geo
1964            .value()
1965            .unwrap();
1966
1967        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1968        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1969    }
1970
1971    #[test]
1972    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1973        let mut event = Event {
1974            spans: Annotated::new(vec![Annotated::new(Span {
1975                op: Annotated::new("app_start_cold".to_owned()),
1976                ..Default::default()
1977            })]),
1978            client_sdk: Annotated::new(ClientSdkInfo {
1979                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1980                version: Annotated::new("4.5.0".to_owned()),
1981                ..Default::default()
1982            }),
1983            ..Default::default()
1984        };
1985        normalize_app_start_spans(&mut event);
1986        assert_debug_snapshot!(event.spans, @r###"
1987        [
1988            Span {
1989                timestamp: ~,
1990                start_timestamp: ~,
1991                exclusive_time: ~,
1992                op: "app_start_cold",
1993                span_id: ~,
1994                parent_span_id: ~,
1995                trace_id: ~,
1996                segment_id: ~,
1997                is_segment: ~,
1998                is_remote: ~,
1999                status: ~,
2000                description: ~,
2001                tags: ~,
2002                origin: ~,
2003                profile_id: ~,
2004                data: ~,
2005                links: ~,
2006                sentry_tags: ~,
2007                received: ~,
2008                measurements: ~,
2009                platform: ~,
2010                was_transaction: ~,
2011                kind: ~,
2012                _performance_issues_spans: ~,
2013                other: {},
2014            },
2015        ]
2016        "###);
2017    }
2018
2019    #[test]
2020    fn test_normalize_app_start_cold_spans_for_react_native() {
2021        let mut event = Event {
2022            spans: Annotated::new(vec![Annotated::new(Span {
2023                op: Annotated::new("app_start_cold".to_owned()),
2024                ..Default::default()
2025            })]),
2026            client_sdk: Annotated::new(ClientSdkInfo {
2027                name: Annotated::new("sentry.javascript.react-native".to_owned()),
2028                version: Annotated::new("4.4.0".to_owned()),
2029                ..Default::default()
2030            }),
2031            ..Default::default()
2032        };
2033        normalize_app_start_spans(&mut event);
2034        assert_debug_snapshot!(event.spans, @r###"
2035        [
2036            Span {
2037                timestamp: ~,
2038                start_timestamp: ~,
2039                exclusive_time: ~,
2040                op: "app.start.cold",
2041                span_id: ~,
2042                parent_span_id: ~,
2043                trace_id: ~,
2044                segment_id: ~,
2045                is_segment: ~,
2046                is_remote: ~,
2047                status: ~,
2048                description: ~,
2049                tags: ~,
2050                origin: ~,
2051                profile_id: ~,
2052                data: ~,
2053                links: ~,
2054                sentry_tags: ~,
2055                received: ~,
2056                measurements: ~,
2057                platform: ~,
2058                was_transaction: ~,
2059                kind: ~,
2060                _performance_issues_spans: ~,
2061                other: {},
2062            },
2063        ]
2064        "###);
2065    }
2066
2067    #[test]
2068    fn test_normalize_app_start_warm_spans_for_react_native() {
2069        let mut event = Event {
2070            spans: Annotated::new(vec![Annotated::new(Span {
2071                op: Annotated::new("app_start_warm".to_owned()),
2072                ..Default::default()
2073            })]),
2074            client_sdk: Annotated::new(ClientSdkInfo {
2075                name: Annotated::new("sentry.javascript.react-native".to_owned()),
2076                version: Annotated::new("4.4.0".to_owned()),
2077                ..Default::default()
2078            }),
2079            ..Default::default()
2080        };
2081        normalize_app_start_spans(&mut event);
2082        assert_debug_snapshot!(event.spans, @r###"
2083        [
2084            Span {
2085                timestamp: ~,
2086                start_timestamp: ~,
2087                exclusive_time: ~,
2088                op: "app.start.warm",
2089                span_id: ~,
2090                parent_span_id: ~,
2091                trace_id: ~,
2092                segment_id: ~,
2093                is_segment: ~,
2094                is_remote: ~,
2095                status: ~,
2096                description: ~,
2097                tags: ~,
2098                origin: ~,
2099                profile_id: ~,
2100                data: ~,
2101                links: ~,
2102                sentry_tags: ~,
2103                received: ~,
2104                measurements: ~,
2105                platform: ~,
2106                was_transaction: ~,
2107                kind: ~,
2108                _performance_issues_spans: ~,
2109                other: {},
2110            },
2111        ]
2112        "###);
2113    }
2114}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
