relay_event_normalization/normalize/
mod.rs

1use std::hash::Hash;
2use std::{collections::HashMap, sync::LazyLock};
3
4use regex::Regex;
5use relay_base_schema::metrics::MetricUnit;
6use relay_event_schema::protocol::VALID_PLATFORMS;
7use relay_pattern::{Gas, Pattern};
8use relay_protocol::{FiniteF64, RuleCondition};
9use serde::{Deserialize, Serialize};
10
11pub mod breakdowns;
12pub mod contexts;
13pub mod integrity;
14pub mod nel;
15pub mod request;
16pub mod span;
17pub mod user_agent;
18pub mod utils;
19
20/// Defines a builtin measurement.
21#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
22#[serde(default, rename_all = "camelCase")]
23pub struct BuiltinMeasurementKey {
24    name: String,
25    unit: MetricUnit,
26    #[serde(skip_serializing_if = "is_false")]
27    allow_negative: bool,
28}
29
30fn is_false(b: &bool) -> bool {
31    !b
32}
33
34impl BuiltinMeasurementKey {
35    /// Creates a new [`BuiltinMeasurementKey`].
36    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
37        Self {
38            name: name.into(),
39            unit,
40            allow_negative: false,
41        }
42    }
43
44    /// Returns the name of the built in measurement key.
45    pub fn name(&self) -> &str {
46        &self.name
47    }
48
49    /// Returns the unit of the built in measurement key.
50    pub fn unit(&self) -> &MetricUnit {
51        &self.unit
52    }
53
54    /// Return true if the built in measurement key allows negative values.
55    pub fn allow_negative(&self) -> &bool {
56        &self.allow_negative
57    }
58}
59
60/// Configuration for measurements normalization.
61#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
62#[serde(default, rename_all = "camelCase")]
63pub struct MeasurementsConfig {
64    /// A list of measurements that are built-in and are not subject to custom measurement limits.
65    #[serde(default, skip_serializing_if = "Vec::is_empty")]
66    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
67
68    /// The maximum number of measurements allowed per event that are not known measurements.
69    pub max_custom_measurements: usize,
70}
71
72impl MeasurementsConfig {
73    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
74    /// for every measurement-mri.
75    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
76}
77
78/// Returns `true` if the given platform string is a known platform identifier.
79///
80/// See [`VALID_PLATFORMS`] for a list of all known platforms.
81pub fn is_valid_platform(platform: &str) -> bool {
82    VALID_PLATFORMS.contains(&platform)
83}
84
85/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
86/// the merging logic.
87#[derive(Copy, Clone, Debug)]
88pub struct CombinedMeasurementsConfig<'a> {
89    project: Option<&'a MeasurementsConfig>,
90    global: Option<&'a MeasurementsConfig>,
91}
92
93impl<'a> CombinedMeasurementsConfig<'a> {
94    /// Constructor for [`CombinedMeasurementsConfig`].
95    pub fn new(
96        project: Option<&'a MeasurementsConfig>,
97        global: Option<&'a MeasurementsConfig>,
98    ) -> Self {
99        CombinedMeasurementsConfig { project, global }
100    }
101
102    /// Returns an iterator over the merged builtin measurement keys.
103    ///
104    /// Items from the project config are prioritized over global config, and
105    /// there are no duplicates.
106    pub fn builtin_measurement_keys(
107        &'a self,
108    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
109        let project = self
110            .project
111            .map(|p| p.builtin_measurements.as_slice())
112            .unwrap_or_default();
113
114        let global = self
115            .global
116            .map(|g| g.builtin_measurements.as_slice())
117            .unwrap_or_default();
118
119        project
120            .iter()
121            .chain(global.iter().filter(|key| !project.contains(key)))
122    }
123
124    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
125    /// global level. If both of them are available, it will choose the most restrictive.
126    pub fn max_custom_measurements(&'a self) -> Option<usize> {
127        match (&self.project, &self.global) {
128            (None, None) => None,
129            (None, Some(global)) => Some(global.max_custom_measurements),
130            (Some(project), None) => Some(project.max_custom_measurements),
131            (Some(project), Some(global)) => Some(std::cmp::min(
132                project.max_custom_measurements,
133                global.max_custom_measurements,
134            )),
135        }
136    }
137}
138
139/// Defines a weighted component for a performance score.
140///
141/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
142/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
143#[derive(Debug, Default, Clone, Serialize, Deserialize)]
144pub struct PerformanceScoreWeightedComponent {
145    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
146    /// profile will be discarded.
147    pub measurement: String,
148    /// Weight [0,1.0] of this component in the performance score
149    pub weight: FiniteF64,
150    /// p10 used to define the log-normal for calculation
151    pub p10: FiniteF64,
152    /// Median used to define the log-normal for calculation
153    pub p50: FiniteF64,
154    /// Whether the measurement is optional. If the measurement is missing, performance score processing
155    /// may still continue, and the weight will be 0.
156    #[serde(default)]
157    pub optional: bool,
158}
159
160/// Defines a profile for performance score.
161///
162/// A profile contains weights for a score of 100% and match against an event using a condition.
163/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
164#[derive(Debug, Default, Clone, Serialize, Deserialize)]
165#[serde(rename_all = "camelCase")]
166pub struct PerformanceScoreProfile {
167    /// Name of the profile, used for debugging and faceting multiple profiles
168    pub name: Option<String>,
169    /// Score components
170    #[serde(default, skip_serializing_if = "Vec::is_empty")]
171    pub score_components: Vec<PerformanceScoreWeightedComponent>,
172    /// See [`RuleCondition`] for all available options to specify and combine conditions.
173    pub condition: Option<RuleCondition>,
174    /// The version of the profile, used to isolate changes to score calculations.
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub version: Option<String>,
177}
178
179/// Defines the performance configuration for the project.
180///
181/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
182/// specific conditions.
183#[derive(Debug, Default, Clone, Serialize, Deserialize)]
184pub struct PerformanceScoreConfig {
185    /// List of performance profiles, only the first with matching conditions will be applied.
186    #[serde(default, skip_serializing_if = "Vec::is_empty")]
187    pub profiles: Vec<PerformanceScoreProfile>,
188}
189
190/// Regex that matches version and/or date patterns at the end of a model name.
191///
192/// Examples matched:
193/// - "-20250805-v1:0" in "claude-opus-4-1-20250805-v1:0" (both)
194/// - "-20250514-v1:0" in "claude-sonnet-4-20250514-v1:0" (both)
195/// - "_v2" in "model_v2" (version only)
196/// - "-v1.0" in "gpt-4-v1.0" (version only)
197/// - "-20250522" in "claude-4-sonnet-20250522" (date only)
198/// - "-2025-06-10" in "o3-pro-2025-06-10" (date only)
199/// - "@20241022" in "claude-3-5-haiku@20241022" (date only)
200static VERSION_AND_DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
201    Regex::new(
202        r"(?x)
203        (
204            ([-_@])                                    # Required separator before date (-, _, or @)
205            (\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8})          # Date: YYYY-MM-DD/YYYY/MM/DD/YYYY.MM.DD or YYYYMMDD
206        )?                                              # Date is optional
207        (
208            [-_]                                        # Required separator before version (- or _)
209            v\d+[:.]?\d*                                # Version: v1, v1.0, v1:0
210            ([-:].*)?                                   # Optional trailing content
211        )?                                              # Version is optional
212        $  # Must be at the end of the string
213        ",
214    )
215    .unwrap()
216});
217
218/// Normalizes an AI model name by stripping date and version patterns.
219///
220/// This function converts specific model versions into normalized names suitable for matching
221/// against cost configurations, ensuring that different versions of the same model can share
222/// cost information.
223///
224/// The normalization strips version and/or date patterns from the end of the string in a single pass.
225///
226/// Examples:
227/// - "claude-4-sonnet-20250522" -> "claude-4-sonnet"
228/// - "o3-pro-2025-06-10" -> "o3-pro"
229/// - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku"
230/// - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1"
231/// - "us.anthropic.claude-sonnet-4-20250514-v1:0" -> "us.anthropic.claude-sonnet-4"
232/// - "gpt-4-v1.0" -> "gpt-4"
233/// - "gpt-4-v1:0" -> "gpt-4"
234/// - "model_v2" -> "model"
235///
236/// Version patterns:
237/// - Must start with 'v': v1, v1.0, v1:0 (requires -, _ separator)
238///
239/// Date patterns:
240/// - YYYYMMDD: 20250522
241/// - YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD: 2025-06-10
242///
243/// Args:
244///     model_id: The original model ID
245///
246/// Returns:
247///     The normalized model name without date and version patterns
248fn normalize_ai_model_name(model_id: &str) -> &str {
249    if let Some(captures) = VERSION_AND_DATE_REGEX.captures(model_id) {
250        let match_idx = captures.get(0).map(|m| m.start()).unwrap_or(model_id.len());
251        &model_id[..match_idx]
252    } else {
253        model_id
254    }
255}
256
257/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
258#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
259#[serde(rename_all = "camelCase")]
260pub struct ModelCostV2 {
261    /// The cost per input token
262    pub input_per_token: f64,
263    /// The cost per output token
264    pub output_per_token: f64,
265    /// The cost per output reasoning token
266    pub output_reasoning_per_token: f64,
267    /// The cost per input cached token
268    pub input_cached_per_token: f64,
269    /// The cost per input cache write token
270    pub input_cache_write_per_token: f64,
271}
272
273/// Metadata for AI models including costs and context size.
274///
275/// Example JSON:
276/// ```json
277/// {
278///   "version": 1,
279///   "models": {
280///     "gpt-4": {
281///       "costs": {
282///         "inputPerToken": 0.0000003,
283///         "outputPerToken": 0.00000165,
284///         "outputReasoningPerToken": 0.0,
285///         "inputCachedPerToken": 0.0000015,
286///         "inputCacheWritePerToken": 0.00001875
287///       },
288///       "contextSize": 1000000
289///     }
290///   }
291/// }
292/// ```
293#[derive(Clone, Default, Debug, Serialize, Deserialize)]
294#[serde(rename_all = "camelCase")]
295pub struct ModelMetadata {
296    /// The version of the model metadata struct.
297    pub version: u16,
298
299    /// The mappings of model ID => metadata as a dictionary.
300    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
301    pub models: HashMap<ModelPattern, ModelMetadataEntry>,
302}
303
304impl ModelMetadata {
305    const SUPPORTED_VERSION: u16 = 1;
306
307    /// `true` if the model metadata is empty or the version is unsupported.
308    pub fn is_empty(&self) -> bool {
309        self.models.is_empty() || !self.is_enabled()
310    }
311
312    /// `false` if the version is unsupported.
313    pub fn is_enabled(&self) -> bool {
314        self.version == Self::SUPPORTED_VERSION
315    }
316
317    /// Gets the cost per token for a given model, if defined.
318    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
319        self.get(model_id).and_then(|entry| entry.costs.as_ref())
320    }
321
322    /// Gets the context window size for a given model, if defined.
323    ///
324    /// Returns `None` for a zero context size, as it is not a meaningful value.
325    pub fn context_size(&self, model_id: &str) -> Option<u64> {
326        self.get(model_id)
327            .and_then(|entry| entry.context_size)
328            .filter(|&size| size > 0)
329    }
330
331    /// Gets the metadata for a given model, if defined.
332    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
333        if !self.is_enabled() {
334            return None;
335        }
336
337        let normalized_model_id = normalize_ai_model_name(model_id);
338
339        // First try exact match.
340        if let Some(value) = self.models.get(normalized_model_id) {
341            return Some(value);
342        }
343
344        // Fall back to glob matching.
345        self.models.iter().find_map(|(key, value)| {
346            if key.is_match(normalized_model_id) {
347                Some(value)
348            } else {
349                None
350            }
351        })
352    }
353}
354
355/// A [`Pattern`] which remembers its original pattern and can be stored in a hash-map.
356#[derive(Debug, Clone)]
357pub struct ModelPattern {
358    original: Box<str>,
359    pattern: Pattern,
360}
361
362impl ModelPattern {
363    /// Returns `true` if the pattern matches the passed string.
364    pub fn is_match(&self, haystack: &str) -> bool {
365        // Patterns are expected to be safe and not user controlled.
366        self.pattern.is_match(haystack, Gas::MAX).unwrap_or(false)
367    }
368}
369
370impl std::cmp::PartialEq for ModelPattern {
371    fn eq(&self, other: &Self) -> bool {
372        self.original == other.original
373    }
374}
375
376impl std::cmp::Eq for ModelPattern {}
377
378impl std::hash::Hash for ModelPattern {
379    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
380        self.original.hash(state);
381    }
382}
383
384impl std::borrow::Borrow<str> for ModelPattern {
385    fn borrow(&self) -> &str {
386        &self.original
387    }
388}
389
390impl std::str::FromStr for ModelPattern {
391    type Err = relay_pattern::Error;
392
393    fn from_str(s: &str) -> Result<Self, Self::Err> {
394        Ok(Self {
395            original: s.into(),
396            pattern: Pattern::new(s)?,
397        })
398    }
399}
400
401impl<'de> serde::de::Deserialize<'de> for ModelPattern {
402    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
403    where
404        D: serde::Deserializer<'de>,
405    {
406        let original = Box::<str>::deserialize(deserializer)?;
407        let pattern = Pattern::new(&original).map_err(serde::de::Error::custom)?;
408
409        Ok(Self { original, pattern })
410    }
411}
412
413impl serde::ser::Serialize for ModelPattern {
414    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
415    where
416        S: serde::Serializer,
417    {
418        serializer.serialize_str(&self.original)
419    }
420}
421
422/// Metadata for a single AI model.
423#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
424#[serde(rename_all = "camelCase")]
425pub struct ModelMetadataEntry {
426    /// Token costs for this model.
427    #[serde(default, skip_serializing_if = "Option::is_none")]
428    pub costs: Option<ModelCostV2>,
429
430    /// The context window size in tokens.
431    #[serde(default, skip_serializing_if = "Option::is_none")]
432    pub context_size: Option<u64>,
433}
434
435#[cfg(test)]
436mod tests {
437    use chrono::{TimeZone, Utc};
438    use insta::{assert_debug_snapshot, assert_json_snapshot};
439    use itertools::Itertools;
440    use relay_base_schema::events::EventType;
441    use relay_base_schema::metrics::DurationUnit;
442    use relay_base_schema::spans::SpanStatus;
443    use relay_event_schema::protocol::{
444        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, Event, EventId,
445        Exception, Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace,
446        ReplayContext, Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
447    };
448    use relay_protocol::{
449        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
450        assert_annotated_snapshot, get_path, get_value,
451    };
452    use serde_json::json;
453    use similar_asserts::assert_eq;
454    use uuid::Uuid;
455
456    use crate::stacktrace::normalize_non_raw_frame;
457    use crate::validation::validate_event;
458    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
459
460    use super::*;
461
462    #[test]
463    fn test_normalize_ai_model_name() {
464        // Test date patterns
465        assert_eq!(
466            normalize_ai_model_name("claude-4-sonnet-20250522"),
467            "claude-4-sonnet"
468        );
469        assert_eq!(normalize_ai_model_name("o3-pro-2025-06-10"), "o3-pro");
470        assert_eq!(
471            normalize_ai_model_name("claude-3-5-haiku@20241022"),
472            "claude-3-5-haiku"
473        );
474
475        // Test date with version patterns
476        assert_eq!(
477            normalize_ai_model_name("claude-opus-4-1-20250805-v1:0"),
478            "claude-opus-4-1"
479        );
480        assert_eq!(
481            normalize_ai_model_name("us.anthropic.claude-sonnet-4-20250514-v1:0"),
482            "us.anthropic.claude-sonnet-4"
483        );
484
485        // Test standalone version patterns with 'v' prefix
486        assert_eq!(normalize_ai_model_name("gpt-4-v1.0"), "gpt-4");
487        assert_eq!(normalize_ai_model_name("gpt-4-v1:0"), "gpt-4");
488        assert_eq!(normalize_ai_model_name("gpt-4-v1"), "gpt-4");
489        assert_eq!(normalize_ai_model_name("model_v2"), "model");
490
491        // Test that version patterns WITHOUT 'v' prefix are NOT stripped
492        assert_eq!(normalize_ai_model_name("gpt-4.5"), "gpt-4.5");
493
494        // Test that if version without 'v' comes after date, nothing is stripped
495        // (date regex requires end of string, so date won't match if followed by version)
496        assert_eq!(
497            normalize_ai_model_name("anthropic.claude-3-5-sonnet-20241022-v2:0"),
498            "anthropic.claude-3-5-sonnet"
499        );
500
501        // Test no pattern (should return original)
502        assert_eq!(normalize_ai_model_name("gpt-4"), "gpt-4");
503        assert_eq!(normalize_ai_model_name("claude-3-opus"), "claude-3-opus");
504    }
505
506    #[test]
507    fn test_merge_builtin_measurement_keys() {
508        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
509        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
510        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
511
512        let proj = MeasurementsConfig {
513            builtin_measurements: vec![foo.clone(), bar.clone()],
514            max_custom_measurements: 4,
515        };
516
517        let glob = MeasurementsConfig {
518            // The 'bar' here will be ignored since it's a duplicate from the project level.
519            builtin_measurements: vec![baz.clone(), bar.clone()],
520            max_custom_measurements: 4,
521        };
522        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
523
524        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
525
526        assert_eq!(keys, vec![&foo, &bar, &baz]);
527    }
528
529    #[test]
530    fn test_max_custom_measurement() {
531        // Empty configs will return a None value for max measurements.
532        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
533        assert!(dynamic_config.max_custom_measurements().is_none());
534
535        let proj = MeasurementsConfig {
536            builtin_measurements: vec![],
537            max_custom_measurements: 3,
538        };
539
540        let glob = MeasurementsConfig {
541            builtin_measurements: vec![],
542            max_custom_measurements: 4,
543        };
544
545        // If only project level measurement config is there, return its max custom measurement variable.
546        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
547        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
548
549        // Same logic for when only global level measurement config exists.
550        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
551        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
552
553        // If both is available, pick the smallest number.
554        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
555        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
556    }
557
558    #[test]
559    fn test_geo_from_ip_address() {
560        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
561
562        let json = r#"{
563            "user": {
564                "ip_address": "2.125.160.216"
565            }
566        }"#;
567        let mut event = Annotated::<Event>::from_json(json).unwrap();
568
569        normalize_event(
570            &mut event,
571            &NormalizationConfig {
572                geoip_lookup: Some(&lookup),
573                ..Default::default()
574            },
575        );
576
577        let expected = Annotated::new(Geo {
578            country_code: Annotated::new("GB".to_owned()),
579            city: Annotated::new("Boxford".to_owned()),
580            subdivision: Annotated::new("England".to_owned()),
581            region: Annotated::new("United Kingdom".to_owned()),
582            ..Geo::default()
583        });
584        assert_eq!(get_value!(event.user!).geo, expected);
585    }
586
587    #[test]
588    fn test_user_ip_from_remote_addr() {
589        let mut event = Annotated::new(Event {
590            request: Annotated::from(Request {
591                env: Annotated::new({
592                    let mut map = Object::new();
593                    map.insert(
594                        "REMOTE_ADDR".to_owned(),
595                        Annotated::new(Value::String("2.125.160.216".to_owned())),
596                    );
597                    map
598                }),
599                ..Request::default()
600            }),
601            platform: Annotated::new("javascript".to_owned()),
602            ..Event::default()
603        });
604
605        normalize_event(&mut event, &NormalizationConfig::default());
606
607        let ip_addr = get_value!(event.user.ip_address!);
608        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
609    }
610
611    #[test]
612    fn test_user_ip_from_invalid_remote_addr() {
613        let mut event = Annotated::new(Event {
614            request: Annotated::from(Request {
615                env: Annotated::new({
616                    let mut map = Object::new();
617                    map.insert(
618                        "REMOTE_ADDR".to_owned(),
619                        Annotated::new(Value::String("whoops".to_owned())),
620                    );
621                    map
622                }),
623                ..Request::default()
624            }),
625            platform: Annotated::new("javascript".to_owned()),
626            ..Event::default()
627        });
628
629        normalize_event(&mut event, &NormalizationConfig::default());
630
631        assert_eq!(Annotated::empty(), event.value().unwrap().user);
632    }
633
634    #[test]
635    fn test_user_ip_from_client_ip_without_auto() {
636        let mut event = Annotated::new(Event {
637            platform: Annotated::new("javascript".to_owned()),
638            ..Default::default()
639        });
640
641        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
642
643        normalize_event(
644            &mut event,
645            &NormalizationConfig {
646                client_ip: Some(&ip_address),
647                infer_ip_address: true,
648                ..Default::default()
649            },
650        );
651
652        let ip_addr = get_value!(event.user.ip_address!);
653        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
654    }
655
656    #[test]
657    fn test_user_ip_from_client_ip_with_auto() {
658        let mut event = Annotated::new(Event {
659            user: Annotated::new(User {
660                ip_address: Annotated::new(IpAddr::auto()),
661                ..Default::default()
662            }),
663            ..Default::default()
664        });
665
666        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
667
668        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
669        normalize_event(
670            &mut event,
671            &NormalizationConfig {
672                client_ip: Some(&ip_address),
673                geoip_lookup: Some(&geo),
674                infer_ip_address: true,
675                ..Default::default()
676            },
677        );
678
679        let user = get_value!(event.user!);
680        let ip_addr = user.ip_address.value().expect("ip address missing");
681
682        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
683        assert!(user.geo.value().is_some());
684    }
685
686    #[test]
687    fn test_user_ip_from_client_ip_without_appropriate_platform() {
688        let mut event = Annotated::new(Event::default());
689
690        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
691        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
692        normalize_event(
693            &mut event,
694            &NormalizationConfig {
695                client_ip: Some(&ip_address),
696                geoip_lookup: Some(&geo),
697                ..Default::default()
698            },
699        );
700
701        let user = get_value!(event.user!);
702        assert!(user.ip_address.value().is_none());
703        assert!(user.geo.value().is_some());
704    }
705
706    #[test]
707    fn test_geo_present_if_ip_inferring_disabled() {
708        let mut event = Annotated::new(Event {
709            user: Annotated::new(User {
710                ip_address: Annotated::new(IpAddr::auto()),
711                ..Default::default()
712            }),
713            ..Default::default()
714        });
715
716        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
717        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
718
719        normalize_event(
720            &mut event,
721            &NormalizationConfig {
722                client_ip: Some(&ip_address),
723                geoip_lookup: Some(&geo),
724                infer_ip_address: false,
725                ..Default::default()
726            },
727        );
728
729        let user = get_value!(event.user!);
730        assert!(user.ip_address.value().unwrap().is_auto());
731        assert!(user.geo.value().is_some());
732    }
733
734    #[test]
735    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
736        let mut event = Annotated::new(Event {
737            user: Annotated::new(User {
738                ip_address: Annotated::new(IpAddr::auto()),
739                ..Default::default()
740            }),
741            ..Default::default()
742        });
743
744        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
745        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
746
747        normalize_event(
748            &mut event,
749            &NormalizationConfig {
750                client_ip: Some(&ip_address),
751                geoip_lookup: Some(&geo),
752                infer_ip_address: true,
753                ..Default::default()
754            },
755        );
756
757        let user = get_value!(event.user!);
758        assert_eq!(
759            user.ip_address.value(),
760            Some(&IpAddr::parse("2.125.160.216").unwrap())
761        );
762        assert!(user.geo.value().is_some());
763    }
764
765    #[test]
766    fn test_event_level_defaulted() {
767        let mut event = Annotated::new(Event::default());
768        normalize_event(&mut event, &NormalizationConfig::default());
769        assert_eq!(get_value!(event.level), Some(&Level::Error));
770    }
771
772    #[test]
773    fn test_transaction_level_untouched() {
774        let mut event = Annotated::new(Event {
775            ty: Annotated::new(EventType::Transaction),
776            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
777            start_timestamp: Annotated::new(
778                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
779            ),
780            contexts: {
781                let mut contexts = Contexts::new();
782                contexts.add(TraceContext {
783                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
784                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
785                    op: Annotated::new("http.server".to_owned()),
786                    ..Default::default()
787                });
788                Annotated::new(contexts)
789            },
790            ..Event::default()
791        });
792        normalize_event(&mut event, &NormalizationConfig::default());
793        assert_eq!(get_value!(event.level), Some(&Level::Info));
794    }
795
796    #[test]
797    fn test_environment_tag_is_moved() {
798        let mut event = Annotated::new(Event {
799            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
800                Annotated::new("environment".to_owned()),
801                Annotated::new("despacito".to_owned()),
802            ))]))),
803            ..Event::default()
804        });
805
806        normalize_event(&mut event, &NormalizationConfig::default());
807
808        let event = event.value().unwrap();
809
810        assert_eq!(event.environment.as_str(), Some("despacito"));
811        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
812    }
813
814    #[test]
815    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
816        let mut event = Annotated::new(Event {
817            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
818                Annotated::new("environment".to_owned()),
819                Annotated::new("despacito".to_owned()),
820            ))]))),
821            environment: Annotated::new("".to_owned()),
822            ..Event::default()
823        });
824
825        normalize_event(&mut event, &NormalizationConfig::default());
826
827        let event = event.value().unwrap();
828
829        assert_eq!(event.environment.as_str(), Some("despacito"));
830        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
831    }
832
833    #[test]
834    fn test_empty_environment_is_removed() {
835        let mut event = Annotated::new(Event {
836            environment: Annotated::new("".to_owned()),
837            ..Event::default()
838        });
839
840        normalize_event(&mut event, &NormalizationConfig::default());
841        assert_eq!(get_value!(event.environment), None);
842    }
843    #[test]
844    fn test_replay_id_added_from_dsc() {
845        let replay_id = Uuid::new_v4();
846        let mut event = Annotated::new(Event {
847            contexts: Annotated::new(Contexts(Object::new())),
848            ..Event::default()
849        });
850        normalize_event(
851            &mut event,
852            &NormalizationConfig {
853                replay_id: Some(replay_id),
854                ..Default::default()
855            },
856        );
857
858        let event = event.value().unwrap();
859
860        assert_eq!(event.contexts, {
861            let mut contexts = Contexts::new();
862            contexts.add(ReplayContext {
863                replay_id: Annotated::new(EventId(replay_id)),
864                other: Object::default(),
865            });
866            Annotated::new(contexts)
867        })
868    }
869
870    #[test]
871    fn test_none_environment_errors() {
872        let mut event = Annotated::new(Event {
873            environment: Annotated::new("none".to_owned()),
874            ..Event::default()
875        });
876
877        normalize_event(&mut event, &NormalizationConfig::default());
878
879        let environment = get_path!(event.environment!);
880        let expected_original = &Value::String("none".to_owned());
881
882        assert_eq!(
883            environment.meta().iter_errors().collect::<Vec<&Error>>(),
884            vec![&Error::new(ErrorKind::InvalidData)],
885        );
886        assert_eq!(
887            environment.meta().original_value().unwrap(),
888            expected_original
889        );
890        assert_eq!(environment.value(), None);
891    }
892
893    #[test]
894    fn test_invalid_release_removed() {
895        let mut event = Annotated::new(Event {
896            release: Annotated::new(LenientString("Latest".to_owned())),
897            ..Event::default()
898        });
899
900        normalize_event(&mut event, &NormalizationConfig::default());
901
902        let release = get_path!(event.release!);
903        let expected_original = &Value::String("Latest".to_owned());
904
905        assert_eq!(
906            release.meta().iter_errors().collect::<Vec<&Error>>(),
907            vec![&Error::new(ErrorKind::InvalidData)],
908        );
909        assert_eq!(release.meta().original_value().unwrap(), expected_original);
910        assert_eq!(release.value(), None);
911    }
912
913    #[test]
914    fn test_top_level_keys_moved_into_tags() {
915        let mut event = Annotated::new(Event {
916            server_name: Annotated::new("foo".to_owned()),
917            site: Annotated::new("foo".to_owned()),
918            tags: Annotated::new(Tags(PairList(vec![
919                Annotated::new(TagEntry(
920                    Annotated::new("site".to_owned()),
921                    Annotated::new("old".to_owned()),
922                )),
923                Annotated::new(TagEntry(
924                    Annotated::new("server_name".to_owned()),
925                    Annotated::new("old".to_owned()),
926                )),
927            ]))),
928            ..Event::default()
929        });
930
931        normalize_event(&mut event, &NormalizationConfig::default());
932
933        assert_eq!(get_value!(event.site), None);
934        assert_eq!(get_value!(event.server_name), None);
935
936        assert_eq!(
937            get_value!(event.tags!),
938            &Tags(PairList(vec![
939                Annotated::new(TagEntry(
940                    Annotated::new("site".to_owned()),
941                    Annotated::new("foo".to_owned()),
942                )),
943                Annotated::new(TagEntry(
944                    Annotated::new("server_name".to_owned()),
945                    Annotated::new("foo".to_owned()),
946                )),
947            ]))
948        );
949    }
950
951    #[test]
952    fn test_internal_tags_removed() {
953        let mut event = Annotated::new(Event {
954            tags: Annotated::new(Tags(PairList(vec![
955                Annotated::new(TagEntry(
956                    Annotated::new("release".to_owned()),
957                    Annotated::new("foo".to_owned()),
958                )),
959                Annotated::new(TagEntry(
960                    Annotated::new("dist".to_owned()),
961                    Annotated::new("foo".to_owned()),
962                )),
963                Annotated::new(TagEntry(
964                    Annotated::new("user".to_owned()),
965                    Annotated::new("foo".to_owned()),
966                )),
967                Annotated::new(TagEntry(
968                    Annotated::new("filename".to_owned()),
969                    Annotated::new("foo".to_owned()),
970                )),
971                Annotated::new(TagEntry(
972                    Annotated::new("function".to_owned()),
973                    Annotated::new("foo".to_owned()),
974                )),
975                Annotated::new(TagEntry(
976                    Annotated::new("something".to_owned()),
977                    Annotated::new("else".to_owned()),
978                )),
979            ]))),
980            ..Event::default()
981        });
982
983        normalize_event(&mut event, &NormalizationConfig::default());
984
985        assert_eq!(get_value!(event.tags!).len(), 1);
986    }
987
988    #[test]
989    fn test_empty_tags_removed() {
990        let mut event = Annotated::new(Event {
991            tags: Annotated::new(Tags(PairList(vec![
992                Annotated::new(TagEntry(
993                    Annotated::new("".to_owned()),
994                    Annotated::new("foo".to_owned()),
995                )),
996                Annotated::new(TagEntry(
997                    Annotated::new("foo".to_owned()),
998                    Annotated::new("".to_owned()),
999                )),
1000                Annotated::new(TagEntry(
1001                    Annotated::new("something".to_owned()),
1002                    Annotated::new("else".to_owned()),
1003                )),
1004            ]))),
1005            ..Event::default()
1006        });
1007
1008        normalize_event(&mut event, &NormalizationConfig::default());
1009
1010        assert_eq!(
1011            get_value!(event.tags!),
1012            &Tags(PairList(vec![
1013                Annotated::new(TagEntry(
1014                    Annotated::from_error(Error::nonempty(), None),
1015                    Annotated::new("foo".to_owned()),
1016                )),
1017                Annotated::new(TagEntry(
1018                    Annotated::new("foo".to_owned()),
1019                    Annotated::from_error(Error::nonempty(), None),
1020                )),
1021                Annotated::new(TagEntry(
1022                    Annotated::new("something".to_owned()),
1023                    Annotated::new("else".to_owned()),
1024                )),
1025            ]))
1026        );
1027    }
1028
1029    #[test]
1030    fn test_tags_deduplicated() {
1031        let mut event = Annotated::new(Event {
1032            tags: Annotated::new(Tags(PairList(vec![
1033                Annotated::new(TagEntry(
1034                    Annotated::new("foo".to_owned()),
1035                    Annotated::new("1".to_owned()),
1036                )),
1037                Annotated::new(TagEntry(
1038                    Annotated::new("bar".to_owned()),
1039                    Annotated::new("1".to_owned()),
1040                )),
1041                Annotated::new(TagEntry(
1042                    Annotated::new("foo".to_owned()),
1043                    Annotated::new("2".to_owned()),
1044                )),
1045                Annotated::new(TagEntry(
1046                    Annotated::new("bar".to_owned()),
1047                    Annotated::new("2".to_owned()),
1048                )),
1049                Annotated::new(TagEntry(
1050                    Annotated::new("foo".to_owned()),
1051                    Annotated::new("3".to_owned()),
1052                )),
1053            ]))),
1054            ..Event::default()
1055        });
1056
1057        normalize_event(&mut event, &NormalizationConfig::default());
1058
1059        // should keep the first occurrence of every tag
1060        assert_eq!(
1061            get_value!(event.tags!),
1062            &Tags(PairList(vec![
1063                Annotated::new(TagEntry(
1064                    Annotated::new("foo".to_owned()),
1065                    Annotated::new("1".to_owned()),
1066                )),
1067                Annotated::new(TagEntry(
1068                    Annotated::new("bar".to_owned()),
1069                    Annotated::new("1".to_owned()),
1070                )),
1071            ]))
1072        );
1073    }
1074
1075    #[test]
1076    fn test_transaction_status_defaulted_to_unknown() {
1077        let mut object = Object::new();
1078        let trace_context = TraceContext {
1079            // We assume the status to be null.
1080            status: Annotated::empty(),
1081            ..TraceContext::default()
1082        };
1083        object.insert(
1084            "trace".to_owned(),
1085            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1086        );
1087
1088        let mut event = Annotated::new(Event {
1089            contexts: Annotated::new(Contexts(object)),
1090            ..Event::default()
1091        });
1092        normalize_event(&mut event, &NormalizationConfig::default());
1093
1094        let event = event.value().unwrap();
1095
1096        let event_trace_context = event.context::<TraceContext>().unwrap();
1097        assert_eq!(
1098            event_trace_context.status,
1099            Annotated::new(SpanStatus::Unknown)
1100        )
1101    }
1102
1103    #[test]
1104    fn test_unknown_debug_image() {
1105        let mut event = Annotated::new(Event {
1106            debug_meta: Annotated::new(DebugMeta {
1107                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1108                ..DebugMeta::default()
1109            }),
1110            ..Event::default()
1111        });
1112
1113        normalize_event(&mut event, &NormalizationConfig::default());
1114
1115        assert_eq!(
1116            get_path!(event.debug_meta!),
1117            &Annotated::new(DebugMeta {
1118                images: Annotated::new(vec![Annotated::from_error(
1119                    Error::invalid("unsupported debug image type"),
1120                    Some(Value::Object(Object::default())),
1121                )]),
1122                ..DebugMeta::default()
1123            })
1124        );
1125    }
1126
1127    #[test]
1128    fn test_context_line_default() {
1129        let mut frame = Annotated::new(Frame {
1130            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1131            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1132            ..Frame::default()
1133        });
1134
1135        normalize_non_raw_frame(&mut frame);
1136
1137        let frame = frame.value().unwrap();
1138        assert_eq!(frame.context_line.as_str(), Some(""));
1139    }
1140
1141    #[test]
1142    fn test_context_line_retain() {
1143        let mut frame = Annotated::new(Frame {
1144            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1145            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1146            context_line: Annotated::new("some line".to_owned()),
1147            ..Frame::default()
1148        });
1149
1150        normalize_non_raw_frame(&mut frame);
1151
1152        let frame = frame.value().unwrap();
1153        assert_eq!(frame.context_line.as_str(), Some("some line"));
1154    }
1155
1156    #[test]
1157    fn test_frame_null_context_lines() {
1158        let mut frame = Annotated::new(Frame {
1159            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1160            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1161            ..Frame::default()
1162        });
1163
1164        normalize_non_raw_frame(&mut frame);
1165
1166        assert_eq!(
1167            *get_value!(frame.pre_context!),
1168            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1169        );
1170        assert_eq!(
1171            *get_value!(frame.post_context!),
1172            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1173        );
1174    }
1175
1176    #[test]
1177    fn test_too_long_distribution() {
1178        let json = r#"{
1179  "event_id": "52df9022835246eeb317dbd739ccd059",
1180  "fingerprint": [
1181    "{{ default }}"
1182  ],
1183  "platform": "other",
1184  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1185}"#;
1186
1187        let mut event = Annotated::<Event>::from_json(json).unwrap();
1188
1189        normalize_event(&mut event, &NormalizationConfig::default());
1190
1191        let dist = &event.value().unwrap().dist;
1192        let result = &Annotated::<String>::from_error(
1193            Error::new(ErrorKind::ValueTooLong),
1194            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1195        );
1196        assert_eq!(dist, result);
1197    }
1198
1199    #[test]
1200    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1201        let mut event = Annotated::new(Event {
1202            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1203                ty: Annotated::new("FooDivisionError".to_owned()),
1204                value: Annotated::new("hi".to_owned().into()),
1205                ..Exception::default()
1206            })])),
1207            stacktrace: Annotated::new(
1208                RawStacktrace {
1209                    frames: Annotated::new(vec![Annotated::new(Frame {
1210                        module: Annotated::new("MyModule".to_owned()),
1211                        filename: Annotated::new("MyFilename".into()),
1212                        function: Annotated::new("Void FooBar()".to_owned()),
1213                        ..Frame::default()
1214                    })]),
1215                    ..RawStacktrace::default()
1216                }
1217                .into(),
1218            ),
1219            ..Event::default()
1220        });
1221
1222        normalize_event(&mut event, &NormalizationConfig::default());
1223
1224        assert_eq!(
1225            get_value!(event.exceptions.values[0].stacktrace!),
1226            &Stacktrace(RawStacktrace {
1227                frames: Annotated::new(vec![Annotated::new(Frame {
1228                    module: Annotated::new("MyModule".to_owned()),
1229                    filename: Annotated::new("MyFilename".into()),
1230                    abs_path: Annotated::new("MyFilename".into()),
1231                    function: Annotated::new("Void FooBar()".to_owned()),
1232                    ..Frame::default()
1233                })]),
1234                ..RawStacktrace::default()
1235            })
1236        );
1237    }
1238
1239    #[test]
1240    fn test_parses_sdk_info_from_header() {
1241        let mut event = Annotated::new(Event::default());
1242
1243        normalize_event(
1244            &mut event,
1245            &NormalizationConfig {
1246                client: Some("_fooBar/0.0.0".to_owned()),
1247                ..Default::default()
1248            },
1249        );
1250
1251        assert_eq!(
1252            get_path!(event.client_sdk!),
1253            &Annotated::new(ClientSdkInfo {
1254                name: Annotated::new("_fooBar".to_owned()),
1255                version: Annotated::new("0.0.0".to_owned()),
1256                ..ClientSdkInfo::default()
1257            })
1258        );
1259    }
1260
1261    #[test]
1262    fn test_discards_received() {
1263        let mut event = Annotated::new(Event {
1264            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1265            ..Default::default()
1266        });
1267
1268        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1269        normalize_event(&mut event, &NormalizationConfig::default());
1270
1271        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1272    }
1273
1274    #[test]
1275    fn test_grouping_config() {
1276        let mut event = Annotated::new(Event {
1277            logentry: Annotated::from(LogEntry {
1278                message: Annotated::new("Hello World!".to_owned().into()),
1279                ..Default::default()
1280            }),
1281            ..Default::default()
1282        });
1283
1284        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1285        normalize_event(
1286            &mut event,
1287            &NormalizationConfig {
1288                grouping_config: Some(json!({
1289                    "id": "legacy:1234-12-12".to_owned(),
1290                })),
1291                ..Default::default()
1292            },
1293        );
1294
1295        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1296            ".event_id" => "[event-id]",
1297            ".received" => "[received]",
1298            ".timestamp" => "[timestamp]"
1299        }, @r###"
1300        {
1301          "event_id": "[event-id]",
1302          "level": "error",
1303          "type": "default",
1304          "logentry": {
1305            "formatted": "Hello World!",
1306          },
1307          "logger": "",
1308          "platform": "other",
1309          "timestamp": "[timestamp]",
1310          "received": "[received]",
1311          "grouping_config": {
1312            "id": "legacy:1234-12-12",
1313          },
1314        }
1315        "###);
1316    }
1317
1318    #[test]
1319    fn test_logentry_error() {
1320        let json = r#"
1321{
1322    "event_id": "74ad1301f4df489ead37d757295442b1",
1323    "timestamp": 1668148328.308933,
1324    "received": 1668148328.308933,
1325    "level": "error",
1326    "platform": "python",
1327    "logentry": {
1328        "params": [
1329            "bogus"
1330        ],
1331        "formatted": 42
1332    }
1333}
1334"#;
1335        let mut event = Annotated::<Event>::from_json(json).unwrap();
1336
1337        normalize_event(&mut event, &NormalizationConfig::default());
1338
1339        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1340        {
1341          "event_id": "74ad1301f4df489ead37d757295442b1",
1342          "level": "error",
1343          "type": "default",
1344          "logentry": null,
1345          "logger": "",
1346          "platform": "python",
1347          "timestamp": 1668148328.308933,
1348          "received": "[received]",
1349          "_meta": {
1350            "logentry": {
1351              "": {
1352                "err": [
1353                  [
1354                    "invalid_data",
1355                    {
1356                      "reason": "no message present"
1357                    }
1358                  ]
1359                ],
1360                "val": {
1361                  "formatted": null,
1362                  "message": null,
1363                  "params": [
1364                    "bogus"
1365                  ]
1366                }
1367              }
1368            }
1369          }
1370        }
1371        "###);
1372    }
1373
1374    #[test]
1375    fn test_future_timestamp() {
1376        let mut event = Annotated::new(Event {
1377            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1378            ..Default::default()
1379        });
1380
1381        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1382        let max_secs_in_past = Some(30 * 24 * 3600);
1383        let max_secs_in_future = Some(60);
1384
1385        validate_event(
1386            &mut event,
1387            &EventValidationConfig {
1388                received_at,
1389                max_secs_in_past,
1390                max_secs_in_future,
1391                is_validated: false,
1392                ..Default::default()
1393            },
1394        )
1395        .unwrap();
1396        normalize_event(&mut event, &NormalizationConfig::default());
1397
1398        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1399        ".event_id" => "[event-id]",
1400    }, @r###"
1401        {
1402          "event_id": "[event-id]",
1403          "level": "error",
1404          "type": "default",
1405          "logger": "",
1406          "platform": "other",
1407          "timestamp": 946857600.0,
1408          "received": 946857600.0,
1409          "_meta": {
1410            "timestamp": {
1411              "": Meta(Some(MetaInner(
1412                err: [
1413                  [
1414                    "future_timestamp",
1415                    {
1416                      "sdk_time": "2000-01-03T00:02:00+00:00",
1417                      "server_time": "2000-01-03T00:00:00+00:00",
1418                    },
1419                  ],
1420                ],
1421              ))),
1422            },
1423          },
1424        }
1425        "###);
1426    }
1427
1428    #[test]
1429    fn test_past_timestamp() {
1430        let mut event = Annotated::new(Event {
1431            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1432            ..Default::default()
1433        });
1434
1435        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1436        let max_secs_in_past = Some(30 * 24 * 3600);
1437        let max_secs_in_future = Some(60);
1438
1439        validate_event(
1440            &mut event,
1441            &EventValidationConfig {
1442                received_at,
1443                max_secs_in_past,
1444                max_secs_in_future,
1445                is_validated: false,
1446                ..Default::default()
1447            },
1448        )
1449        .unwrap();
1450        normalize_event(&mut event, &NormalizationConfig::default());
1451
1452        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1453        ".event_id" => "[event-id]",
1454    }, @r###"
1455        {
1456          "event_id": "[event-id]",
1457          "level": "error",
1458          "type": "default",
1459          "logger": "",
1460          "platform": "other",
1461          "timestamp": 952041600.0,
1462          "received": 952041600.0,
1463          "_meta": {
1464            "timestamp": {
1465              "": Meta(Some(MetaInner(
1466                err: [
1467                  [
1468                    "past_timestamp",
1469                    {
1470                      "sdk_time": "2000-01-03T00:00:00+00:00",
1471                      "server_time": "2000-03-03T00:00:00+00:00",
1472                    },
1473                  ],
1474                ],
1475              ))),
1476            },
1477          },
1478        }
1479        "###);
1480    }
1481
1482    #[test]
1483    fn test_normalize_logger_empty() {
1484        let mut event = Event::from_value(
1485            serde_json::json!({
1486                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1487                "platform": "java",
1488                "logger": "",
1489            })
1490            .into(),
1491        );
1492
1493        normalize_event(&mut event, &NormalizationConfig::default());
1494        assert_annotated_snapshot!(event);
1495    }
1496
1497    #[test]
1498    fn test_normalize_logger_trimmed() {
1499        let mut event = Event::from_value(
1500            serde_json::json!({
1501                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1502                "platform": "java",
1503                "logger": " \t  \t   ",
1504            })
1505            .into(),
1506        );
1507
1508        normalize_event(&mut event, &NormalizationConfig::default());
1509        assert_annotated_snapshot!(event);
1510    }
1511
1512    #[test]
1513    fn test_normalize_logger_short_no_trimming() {
1514        let mut event = Event::from_value(
1515            serde_json::json!({
1516                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1517                "platform": "java",
1518                "logger": "my.short-logger.isnt_trimmed",
1519            })
1520            .into(),
1521        );
1522
1523        normalize_event(&mut event, &NormalizationConfig::default());
1524        assert_annotated_snapshot!(event);
1525    }
1526
1527    #[test]
1528    fn test_normalize_logger_exact_length() {
1529        let mut event = Event::from_value(
1530            serde_json::json!({
1531                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1532                "platform": "java",
1533                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1534            })
1535            .into(),
1536        );
1537
1538        normalize_event(&mut event, &NormalizationConfig::default());
1539        assert_annotated_snapshot!(event);
1540    }
1541
1542    #[test]
1543    fn test_normalize_logger_too_long_single_word() {
1544        let mut event = Event::from_value(
1545            serde_json::json!({
1546                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1547                "platform": "java",
1548                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1549            })
1550            .into(),
1551        );
1552
1553        normalize_event(&mut event, &NormalizationConfig::default());
1554        assert_annotated_snapshot!(event);
1555    }
1556
1557    #[test]
1558    fn test_normalize_logger_word_trimmed_at_max() {
1559        let mut event = Event::from_value(
1560            serde_json::json!({
1561                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1562                "platform": "java",
1563                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1564            })
1565            .into(),
1566        );
1567
1568        normalize_event(&mut event, &NormalizationConfig::default());
1569        assert_annotated_snapshot!(event);
1570    }
1571
1572    #[test]
1573    fn test_normalize_logger_word_trimmed_before_max() {
1574        // This test verifies the "smart" trimming on words -- the logger name
1575        // should be cut before the max limit, removing entire words.
1576        let mut event = Event::from_value(
1577            serde_json::json!({
1578                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1579                "platform": "java",
1580                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1581            })
1582            .into(),
1583        );
1584
1585        normalize_event(&mut event, &NormalizationConfig::default());
1586        assert_annotated_snapshot!(event);
1587    }
1588
1589    #[test]
1590    fn test_normalize_logger_word_leading_dots() {
1591        let mut event = Event::from_value(
1592            serde_json::json!({
1593                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1594                "platform": "java",
1595                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1596            })
1597            .into(),
1598        );
1599
1600        normalize_event(&mut event, &NormalizationConfig::default());
1601        assert_annotated_snapshot!(event);
1602    }
1603
1604    #[test]
1605    fn test_normalization_is_idempotent() {
1606        // get an event, normalize it. the result of that must be the same as normalizing it once more
1607        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1608        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1609        let mut event = Annotated::new(Event {
1610            ty: Annotated::new(EventType::Transaction),
1611            transaction: Annotated::new("/".to_owned()),
1612            timestamp: Annotated::new(end.into()),
1613            start_timestamp: Annotated::new(start.into()),
1614            contexts: {
1615                let mut contexts = Contexts::new();
1616                contexts.add(TraceContext {
1617                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1618                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1619                    op: Annotated::new("http.server".to_owned()),
1620                    ..Default::default()
1621                });
1622                Annotated::new(contexts)
1623            },
1624            spans: Annotated::new(vec![Annotated::new(Span {
1625                timestamp: Annotated::new(
1626                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1627                ),
1628                start_timestamp: Annotated::new(
1629                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1630                ),
1631                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1632                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1633
1634                ..Default::default()
1635            })]),
1636            ..Default::default()
1637        });
1638
1639        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1640            relay_event_schema::processor::apply(event, |e, _m| {
1641                e.received = Annotated::empty();
1642                Ok(())
1643            })
1644            .unwrap();
1645            event
1646        }
1647
1648        normalize_event(&mut event, &NormalizationConfig::default());
1649        let first = remove_received_from_event(&mut event.clone())
1650            .to_json()
1651            .unwrap();
1652        // Expected some fields (such as timestamps) exist after first normalization.
1653
1654        normalize_event(&mut event, &NormalizationConfig::default());
1655        let second = remove_received_from_event(&mut event.clone())
1656            .to_json()
1657            .unwrap();
1658        assert_eq!(&first, &second, "idempotency check failed");
1659
1660        normalize_event(&mut event, &NormalizationConfig::default());
1661        let third = remove_received_from_event(&mut event.clone())
1662            .to_json()
1663            .unwrap();
1664        assert_eq!(&second, &third, "idempotency check failed");
1665    }
1666
1667    /// Validate full normalization is idempotent.
1668    ///
1669    /// Both PoPs and processing relays will temporarily run full normalization
1670    /// in events, during the rollout of running normalization once in internal
1671    /// relays.
1672    // TODO(iker): remove this test after the rollout is done.
1673    #[test]
1674    fn test_full_normalization_is_idempotent() {
1675        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1676        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1677        let mut event = Annotated::new(Event {
1678            ty: Annotated::new(EventType::Transaction),
1679            transaction: Annotated::new("/".to_owned()),
1680            timestamp: Annotated::new(end.into()),
1681            start_timestamp: Annotated::new(start.into()),
1682            contexts: {
1683                let mut contexts = Contexts::new();
1684                contexts.add(TraceContext {
1685                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1686                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1687                    op: Annotated::new("http.server".to_owned()),
1688                    ..Default::default()
1689                });
1690                Annotated::new(contexts)
1691            },
1692            spans: Annotated::new(vec![Annotated::new(Span {
1693                timestamp: Annotated::new(
1694                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1695                ),
1696                start_timestamp: Annotated::new(
1697                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1698                ),
1699                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1700                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1701
1702                ..Default::default()
1703            })]),
1704            ..Default::default()
1705        });
1706
1707        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1708            relay_event_schema::processor::apply(event, |e, _m| {
1709                e.received = Annotated::empty();
1710                Ok(())
1711            })
1712            .unwrap();
1713            event
1714        }
1715
1716        let full_normalization_config = NormalizationConfig {
1717            is_renormalize: false,
1718            enable_trimming: true,
1719            remove_other: true,
1720            emit_event_errors: true,
1721            ..Default::default()
1722        };
1723
1724        normalize_event(&mut event, &full_normalization_config);
1725        let first = remove_received_from_event(&mut event.clone())
1726            .to_json()
1727            .unwrap();
1728        // Expected some fields (such as timestamps) exist after first normalization.
1729
1730        normalize_event(&mut event, &full_normalization_config);
1731        let second = remove_received_from_event(&mut event.clone())
1732            .to_json()
1733            .unwrap();
1734        assert_eq!(&first, &second, "idempotency check failed");
1735
1736        normalize_event(&mut event, &full_normalization_config);
1737        let third = remove_received_from_event(&mut event.clone())
1738            .to_json()
1739            .unwrap();
1740        assert_eq!(&second, &third, "idempotency check failed");
1741    }
1742
1743    #[test]
1744    fn test_normalize_validates_spans() {
1745        let event = Annotated::<Event>::from_json(
1746            r#"
1747            {
1748                "type": "transaction",
1749                "transaction": "/",
1750                "timestamp": 946684810.0,
1751                "start_timestamp": 946684800.0,
1752                "contexts": {
1753                    "trace": {
1754                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1755                    "span_id": "fa90fdead5f74053",
1756                    "op": "http.server",
1757                    "type": "trace"
1758                    }
1759                },
1760                "spans": []
1761            }
1762            "#,
1763        )
1764        .unwrap();
1765
1766        // All incomplete spans should be caught by normalization:
1767        for span in [
1768            r#"null"#,
1769            r#"{
1770              "timestamp": 946684810.0,
1771              "start_timestamp": 946684900.0,
1772              "span_id": "fa90fdead5f74053",
1773              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1774            }"#,
1775            r#"{
1776              "timestamp": 946684810.0,
1777              "span_id": "fa90fdead5f74053",
1778              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1779            }"#,
1780            r#"{
1781              "timestamp": 946684810.0,
1782              "start_timestamp": 946684800.0,
1783              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1784            }"#,
1785            r#"{
1786              "timestamp": 946684810.0,
1787              "start_timestamp": 946684800.0,
1788              "span_id": "fa90fdead5f74053"
1789            }"#,
1790        ] {
1791            let mut modified_event = event.clone();
1792            let event_ref = modified_event.value_mut().as_mut().unwrap();
1793            event_ref
1794                .spans
1795                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1796
1797            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1798
1799            assert!(res.is_err(), "{span:?}");
1800        }
1801    }
1802
1803    #[test]
1804    fn test_normalization_respects_is_renormalize() {
1805        let mut event = Annotated::<Event>::from_json(
1806            r#"
1807            {
1808                "type": "default",
1809                "tags": [["environment", "some_environment"]]
1810            }
1811            "#,
1812        )
1813        .unwrap();
1814
1815        normalize_event(
1816            &mut event,
1817            &NormalizationConfig {
1818                is_renormalize: true,
1819                ..Default::default()
1820            },
1821        );
1822
1823        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1824        Tags(
1825            PairList(
1826                [
1827                    TagEntry(
1828                        "environment",
1829                        "some_environment",
1830                    ),
1831                ],
1832            ),
1833        )
1834        "###);
1835    }
1836
1837    #[test]
1838    fn test_geo_in_normalize() {
1839        let mut event = Annotated::<Event>::from_json(
1840            r#"
1841            {
1842                "type": "transaction",
1843                "transaction": "/foo/",
1844                "timestamp": 946684810.0,
1845                "start_timestamp": 946684800.0,
1846                "contexts": {
1847                    "trace": {
1848                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1849                        "span_id": "fa90fdead5f74053",
1850                        "op": "http.server",
1851                        "type": "trace"
1852                    }
1853                },
1854                "transaction_info": {
1855                    "source": "url"
1856                },
1857                "user": {
1858                    "ip_address": "2.125.160.216"
1859                }
1860            }
1861            "#,
1862        )
1863        .unwrap();
1864
1865        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1866
1867        // Extract user's geo information before normalization.
1868        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1869        assert!(user_geo.is_none());
1870
1871        normalize_event(
1872            &mut event,
1873            &NormalizationConfig {
1874                geoip_lookup: Some(&lookup),
1875                ..Default::default()
1876            },
1877        );
1878
1879        // Extract user's geo information after normalization.
1880        let user_geo = event
1881            .value()
1882            .unwrap()
1883            .user
1884            .value()
1885            .unwrap()
1886            .geo
1887            .value()
1888            .unwrap();
1889
1890        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1891        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1892    }
1893
1894    #[test]
1895    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1896        let mut event = Event {
1897            spans: Annotated::new(vec![Annotated::new(Span {
1898                op: Annotated::new("app_start_cold".to_owned()),
1899                ..Default::default()
1900            })]),
1901            client_sdk: Annotated::new(ClientSdkInfo {
1902                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1903                version: Annotated::new("4.5.0".to_owned()),
1904                ..Default::default()
1905            }),
1906            ..Default::default()
1907        };
1908        span::normalize_app_start_spans(&mut event);
1909        assert_debug_snapshot!(event.spans, @r#"
1910        [
1911            Span {
1912                timestamp: ~,
1913                start_timestamp: ~,
1914                exclusive_time: ~,
1915                op: "app_start_cold",
1916                span_id: ~,
1917                parent_span_id: ~,
1918                trace_id: ~,
1919                segment_id: ~,
1920                is_segment: ~,
1921                is_remote: ~,
1922                status: ~,
1923                description: ~,
1924                tags: ~,
1925                origin: ~,
1926                profile_id: ~,
1927                data: ~,
1928                links: ~,
1929                sentry_tags: ~,
1930                received: ~,
1931                measurements: ~,
1932                platform: ~,
1933                was_transaction: ~,
1934                kind: ~,
1935                other: {},
1936            },
1937        ]
1938        "#);
1939    }
1940
1941    #[test]
1942    fn test_normalize_app_start_cold_spans_for_react_native() {
1943        let mut event = Event {
1944            spans: Annotated::new(vec![Annotated::new(Span {
1945                op: Annotated::new("app_start_cold".to_owned()),
1946                ..Default::default()
1947            })]),
1948            client_sdk: Annotated::new(ClientSdkInfo {
1949                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1950                version: Annotated::new("4.4.0".to_owned()),
1951                ..Default::default()
1952            }),
1953            ..Default::default()
1954        };
1955        span::normalize_app_start_spans(&mut event);
1956        assert_debug_snapshot!(event.spans, @r#"
1957        [
1958            Span {
1959                timestamp: ~,
1960                start_timestamp: ~,
1961                exclusive_time: ~,
1962                op: "app.start.cold",
1963                span_id: ~,
1964                parent_span_id: ~,
1965                trace_id: ~,
1966                segment_id: ~,
1967                is_segment: ~,
1968                is_remote: ~,
1969                status: ~,
1970                description: ~,
1971                tags: ~,
1972                origin: ~,
1973                profile_id: ~,
1974                data: ~,
1975                links: ~,
1976                sentry_tags: ~,
1977                received: ~,
1978                measurements: ~,
1979                platform: ~,
1980                was_transaction: ~,
1981                kind: ~,
1982                other: {},
1983            },
1984        ]
1985        "#);
1986    }
1987
1988    #[test]
1989    fn test_normalize_app_start_warm_spans_for_react_native() {
1990        let mut event = Event {
1991            spans: Annotated::new(vec![Annotated::new(Span {
1992                op: Annotated::new("app_start_warm".to_owned()),
1993                ..Default::default()
1994            })]),
1995            client_sdk: Annotated::new(ClientSdkInfo {
1996                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1997                version: Annotated::new("4.4.0".to_owned()),
1998                ..Default::default()
1999            }),
2000            ..Default::default()
2001        };
2002        span::normalize_app_start_spans(&mut event);
2003        assert_debug_snapshot!(event.spans, @r#"
2004        [
2005            Span {
2006                timestamp: ~,
2007                start_timestamp: ~,
2008                exclusive_time: ~,
2009                op: "app.start.warm",
2010                span_id: ~,
2011                parent_span_id: ~,
2012                trace_id: ~,
2013                segment_id: ~,
2014                is_segment: ~,
2015                is_remote: ~,
2016                status: ~,
2017                description: ~,
2018                tags: ~,
2019                origin: ~,
2020                profile_id: ~,
2021                data: ~,
2022                links: ~,
2023                sentry_tags: ~,
2024                received: ~,
2025                measurements: ~,
2026                platform: ~,
2027                was_transaction: ~,
2028                kind: ~,
2029                other: {},
2030            },
2031        ]
2032        "#);
2033    }
2034}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
