relay_event_normalization/normalize/
mod.rs

1use std::hash::Hash;
2use std::{collections::HashMap, sync::LazyLock};
3
4use regex::Regex;
5use relay_base_schema::metrics::MetricUnit;
6use relay_base_schema::project::ProjectId;
7use relay_event_schema::protocol::VALID_PLATFORMS;
8use relay_pattern::Pattern;
9use relay_protocol::{FiniteF64, RuleCondition};
10use relay_sampling::DynamicSamplingContext;
11use serde::{Deserialize, Serialize};
12
13pub mod breakdowns;
14pub mod contexts;
15pub mod nel;
16pub mod request;
17pub mod span;
18pub mod user_agent;
19pub mod utils;
20
21/// Defines a builtin measurement.
22#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
23#[serde(default, rename_all = "camelCase")]
24pub struct BuiltinMeasurementKey {
25    name: String,
26    unit: MetricUnit,
27    #[serde(skip_serializing_if = "is_false")]
28    allow_negative: bool,
29}
30
31fn is_false(b: &bool) -> bool {
32    !b
33}
34
35impl BuiltinMeasurementKey {
36    /// Creates a new [`BuiltinMeasurementKey`].
37    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
38        Self {
39            name: name.into(),
40            unit,
41            allow_negative: false,
42        }
43    }
44
45    /// Returns the name of the built in measurement key.
46    pub fn name(&self) -> &str {
47        &self.name
48    }
49
50    /// Returns the unit of the built in measurement key.
51    pub fn unit(&self) -> &MetricUnit {
52        &self.unit
53    }
54
55    /// Return true if the built in measurement key allows negative values.
56    pub fn allow_negative(&self) -> &bool {
57        &self.allow_negative
58    }
59}
60
61/// Configuration for measurements normalization.
62#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
63#[serde(default, rename_all = "camelCase")]
64pub struct MeasurementsConfig {
65    /// A list of measurements that are built-in and are not subject to custom measurement limits.
66    #[serde(default, skip_serializing_if = "Vec::is_empty")]
67    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
68
69    /// The maximum number of measurements allowed per event that are not known measurements.
70    pub max_custom_measurements: usize,
71}
72
73impl MeasurementsConfig {
74    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
75    /// for every measurement-mri.
76    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
77}
78
79/// Returns `true` if the given platform string is a known platform identifier.
80///
81/// See [`VALID_PLATFORMS`] for a list of all known platforms.
82pub fn is_valid_platform(platform: &str) -> bool {
83    VALID_PLATFORMS.contains(&platform)
84}
85
86/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
87/// the merging logic.
88#[derive(Copy, Clone, Debug)]
89pub struct CombinedMeasurementsConfig<'a> {
90    project: Option<&'a MeasurementsConfig>,
91    global: Option<&'a MeasurementsConfig>,
92}
93
94impl<'a> CombinedMeasurementsConfig<'a> {
95    /// Constructor for [`CombinedMeasurementsConfig`].
96    pub fn new(
97        project: Option<&'a MeasurementsConfig>,
98        global: Option<&'a MeasurementsConfig>,
99    ) -> Self {
100        CombinedMeasurementsConfig { project, global }
101    }
102
103    /// Returns an iterator over the merged builtin measurement keys.
104    ///
105    /// Items from the project config are prioritized over global config, and
106    /// there are no duplicates.
107    pub fn builtin_measurement_keys(
108        &'a self,
109    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
110        let project = self
111            .project
112            .map(|p| p.builtin_measurements.as_slice())
113            .unwrap_or_default();
114
115        let global = self
116            .global
117            .map(|g| g.builtin_measurements.as_slice())
118            .unwrap_or_default();
119
120        project
121            .iter()
122            .chain(global.iter().filter(|key| !project.contains(key)))
123    }
124
125    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
126    /// global level. If both of them are available, it will choose the most restrictive.
127    pub fn max_custom_measurements(&'a self) -> Option<usize> {
128        match (&self.project, &self.global) {
129            (None, None) => None,
130            (None, Some(global)) => Some(global.max_custom_measurements),
131            (Some(project), None) => Some(project.max_custom_measurements),
132            (Some(project), Some(global)) => Some(std::cmp::min(
133                project.max_custom_measurements,
134                global.max_custom_measurements,
135            )),
136        }
137    }
138}
139
140/// Defines a weighted component for a performance score.
141///
142/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
143/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
144#[derive(Debug, Default, Clone, Serialize, Deserialize)]
145pub struct PerformanceScoreWeightedComponent {
146    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
147    /// profile will be discarded.
148    pub measurement: String,
149    /// Weight [0,1.0] of this component in the performance score
150    pub weight: FiniteF64,
151    /// p10 used to define the log-normal for calculation
152    pub p10: FiniteF64,
153    /// Median used to define the log-normal for calculation
154    pub p50: FiniteF64,
155    /// Whether the measurement is optional. If the measurement is missing, performance score processing
156    /// may still continue, and the weight will be 0.
157    #[serde(default)]
158    pub optional: bool,
159}
160
161/// Defines a profile for performance score.
162///
163/// A profile contains weights for a score of 100% and match against an event using a condition.
164/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
165#[derive(Debug, Default, Clone, Serialize, Deserialize)]
166#[serde(rename_all = "camelCase")]
167pub struct PerformanceScoreProfile {
168    /// Name of the profile, used for debugging and faceting multiple profiles
169    pub name: Option<String>,
170    /// Score components
171    #[serde(default, skip_serializing_if = "Vec::is_empty")]
172    pub score_components: Vec<PerformanceScoreWeightedComponent>,
173    /// See [`RuleCondition`] for all available options to specify and combine conditions.
174    pub condition: Option<RuleCondition>,
175    /// The version of the profile, used to isolate changes to score calculations.
176    #[serde(skip_serializing_if = "Option::is_none")]
177    pub version: Option<String>,
178}
179
180/// Defines the performance configuration for the project.
181///
182/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
183/// specific conditions.
184#[derive(Debug, Default, Clone, Serialize, Deserialize)]
185pub struct PerformanceScoreConfig {
186    /// List of performance profiles, only the first with matching conditions will be applied.
187    #[serde(default, skip_serializing_if = "Vec::is_empty")]
188    pub profiles: Vec<PerformanceScoreProfile>,
189}
190
191/// Regex that matches version and/or date patterns at the end of a model name.
192///
193/// Examples matched:
194/// - "-20250805-v1:0" in "claude-opus-4-1-20250805-v1:0" (both)
195/// - "-20250514-v1:0" in "claude-sonnet-4-20250514-v1:0" (both)
196/// - "_v2" in "model_v2" (version only)
197/// - "-v1.0" in "gpt-4-v1.0" (version only)
198/// - "-20250522" in "claude-4-sonnet-20250522" (date only)
199/// - "-2025-06-10" in "o3-pro-2025-06-10" (date only)
200/// - "@20241022" in "claude-3-5-haiku@20241022" (date only)
201static VERSION_AND_DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
202    Regex::new(
203        r"(?x)
204        (
205            ([-_@])                                    # Required separator before date (-, _, or @)
206            (\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8})          # Date: YYYY-MM-DD/YYYY/MM/DD/YYYY.MM.DD or YYYYMMDD
207        )?                                              # Date is optional
208        (
209            [-_]                                        # Required separator before version (- or _)
210            v\d+[:.]?\d*                                # Version: v1, v1.0, v1:0
211            ([-:].*)?                                   # Optional trailing content
212        )?                                              # Version is optional
213        $  # Must be at the end of the string
214        ",
215    )
216    .unwrap()
217});
218
219/// Normalizes an AI model name by stripping date and version patterns.
220///
221/// This function converts specific model versions into normalized names suitable for matching
222/// against cost configurations, ensuring that different versions of the same model can share
223/// cost information.
224///
225/// The normalization strips version and/or date patterns from the end of the string in a single pass.
226///
227/// Examples:
228/// - "claude-4-sonnet-20250522" -> "claude-4-sonnet"
229/// - "o3-pro-2025-06-10" -> "o3-pro"
230/// - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku"
231/// - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1"
232/// - "us.anthropic.claude-sonnet-4-20250514-v1:0" -> "us.anthropic.claude-sonnet-4"
233/// - "gpt-4-v1.0" -> "gpt-4"
234/// - "gpt-4-v1:0" -> "gpt-4"
235/// - "model_v2" -> "model"
236///
237/// Version patterns:
238/// - Must start with 'v': v1, v1.0, v1:0 (requires -, _ separator)
239///
240/// Date patterns:
241/// - YYYYMMDD: 20250522
242/// - YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD: 2025-06-10
243///
244/// Args:
245///     model_id: The original model ID
246///
247/// Returns:
248///     The normalized model name without date and version patterns
249fn normalize_ai_model_name(model_id: &str) -> &str {
250    if let Some(captures) = VERSION_AND_DATE_REGEX.captures(model_id) {
251        let match_idx = captures.get(0).map(|m| m.start()).unwrap_or(model_id.len());
252        &model_id[..match_idx]
253    } else {
254        model_id
255    }
256}
257
258/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
259#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
260#[serde(rename_all = "camelCase")]
261pub struct ModelCostV2 {
262    /// The cost per input token
263    pub input_per_token: f64,
264    /// The cost per output token
265    pub output_per_token: f64,
266    /// The cost per output reasoning token
267    pub output_reasoning_per_token: f64,
268    /// The cost per input cached token
269    pub input_cached_per_token: f64,
270    /// The cost per input cache write token
271    pub input_cache_write_per_token: f64,
272}
273
274/// Metadata for AI models including costs and context size.
275///
276/// Example JSON:
277/// ```json
278/// {
279///   "version": 1,
280///   "models": {
281///     "gpt-4": {
282///       "costs": {
283///         "inputPerToken": 0.0000003,
284///         "outputPerToken": 0.00000165,
285///         "outputReasoningPerToken": 0.0,
286///         "inputCachedPerToken": 0.0000015,
287///         "inputCacheWritePerToken": 0.00001875
288///       },
289///       "contextSize": 1000000
290///     }
291///   }
292/// }
293/// ```
294#[derive(Clone, Default, Debug, Serialize, Deserialize)]
295#[serde(rename_all = "camelCase")]
296pub struct ModelMetadata {
297    /// The version of the model metadata struct.
298    pub version: u16,
299
300    /// The mappings of model ID => metadata as a dictionary.
301    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
302    pub models: HashMap<Pattern, ModelMetadataEntry>,
303}
304
305impl ModelMetadata {
306    const SUPPORTED_VERSION: u16 = 1;
307
308    /// `true` if the model metadata is empty or the version is unsupported.
309    pub fn is_empty(&self) -> bool {
310        self.models.is_empty() || !self.is_enabled()
311    }
312
313    /// `false` if the version is unsupported.
314    pub fn is_enabled(&self) -> bool {
315        self.version == Self::SUPPORTED_VERSION
316    }
317
318    /// Gets the cost per token for a given model, if defined.
319    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
320        self.get(model_id).and_then(|entry| entry.costs.as_ref())
321    }
322
323    /// Gets the context window size for a given model, if defined.
324    ///
325    /// Returns `None` for a zero context size, as it is not a meaningful value.
326    pub fn context_size(&self, model_id: &str) -> Option<u64> {
327        self.get(model_id)
328            .and_then(|entry| entry.context_size)
329            .filter(|&size| size > 0)
330    }
331
332    /// Gets the metadata for a given model, if defined.
333    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
334        if !self.is_enabled() {
335            return None;
336        }
337
338        let normalized_model_id = normalize_ai_model_name(model_id);
339
340        // First try exact match.
341        if let Some(value) = self.models.get(normalized_model_id) {
342            return Some(value);
343        }
344
345        // Fall back to glob matching.
346        self.models.iter().find_map(|(key, value)| {
347            if key.is_match(normalized_model_id) {
348                Some(value)
349            } else {
350                None
351            }
352        })
353    }
354}
355
356/// Metadata for a single AI model.
357#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
358#[serde(rename_all = "camelCase")]
359pub struct ModelMetadataEntry {
360    /// Token costs for this model.
361    #[serde(default, skip_serializing_if = "Option::is_none")]
362    pub costs: Option<ModelCostV2>,
363
364    /// The context window size in tokens.
365    #[serde(default, skip_serializing_if = "Option::is_none")]
366    pub context_size: Option<u64>,
367}
368
369/// [`DynamicSamplingContext`] plus additional attributes used for dsc span normalization.
370#[derive(Debug, Clone, Copy)]
371pub struct EnrichedDsc<'a> {
372    /// Dynamic sampling context containing the trace id and root transaction that started the trace.
373    pub dsc: &'a DynamicSamplingContext,
374    /// ID of the project where the trace originated.
375    pub sampling_project_id: ProjectId,
376}
377
378#[cfg(test)]
379mod tests {
380    use chrono::{TimeZone, Utc};
381    use insta::{assert_debug_snapshot, assert_json_snapshot};
382    use itertools::Itertools;
383    use relay_base_schema::events::EventType;
384    use relay_base_schema::metrics::DurationUnit;
385    use relay_base_schema::spans::SpanStatus;
386    use relay_event_schema::protocol::{
387        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, Event, EventId,
388        Exception, Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace,
389        ReplayContext, Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
390    };
391    use relay_protocol::{
392        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
393        assert_annotated_snapshot, get_path, get_value,
394    };
395    use serde_json::json;
396    use similar_asserts::assert_eq;
397    use uuid::Uuid;
398
399    use crate::stacktrace::normalize_non_raw_frame;
400    use crate::validation::validate_event;
401    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
402
403    use super::*;
404
405    #[test]
406    fn test_normalize_ai_model_name() {
407        // Test date patterns
408        assert_eq!(
409            normalize_ai_model_name("claude-4-sonnet-20250522"),
410            "claude-4-sonnet"
411        );
412        assert_eq!(normalize_ai_model_name("o3-pro-2025-06-10"), "o3-pro");
413        assert_eq!(
414            normalize_ai_model_name("claude-3-5-haiku@20241022"),
415            "claude-3-5-haiku"
416        );
417
418        // Test date with version patterns
419        assert_eq!(
420            normalize_ai_model_name("claude-opus-4-1-20250805-v1:0"),
421            "claude-opus-4-1"
422        );
423        assert_eq!(
424            normalize_ai_model_name("us.anthropic.claude-sonnet-4-20250514-v1:0"),
425            "us.anthropic.claude-sonnet-4"
426        );
427
428        // Test standalone version patterns with 'v' prefix
429        assert_eq!(normalize_ai_model_name("gpt-4-v1.0"), "gpt-4");
430        assert_eq!(normalize_ai_model_name("gpt-4-v1:0"), "gpt-4");
431        assert_eq!(normalize_ai_model_name("gpt-4-v1"), "gpt-4");
432        assert_eq!(normalize_ai_model_name("model_v2"), "model");
433
434        // Test that version patterns WITHOUT 'v' prefix are NOT stripped
435        assert_eq!(normalize_ai_model_name("gpt-4.5"), "gpt-4.5");
436
437        // Test that if version without 'v' comes after date, nothing is stripped
438        // (date regex requires end of string, so date won't match if followed by version)
439        assert_eq!(
440            normalize_ai_model_name("anthropic.claude-3-5-sonnet-20241022-v2:0"),
441            "anthropic.claude-3-5-sonnet"
442        );
443
444        // Test no pattern (should return original)
445        assert_eq!(normalize_ai_model_name("gpt-4"), "gpt-4");
446        assert_eq!(normalize_ai_model_name("claude-3-opus"), "claude-3-opus");
447    }
448
449    #[test]
450    fn test_merge_builtin_measurement_keys() {
451        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
452        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
453        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
454
455        let proj = MeasurementsConfig {
456            builtin_measurements: vec![foo.clone(), bar.clone()],
457            max_custom_measurements: 4,
458        };
459
460        let glob = MeasurementsConfig {
461            // The 'bar' here will be ignored since it's a duplicate from the project level.
462            builtin_measurements: vec![baz.clone(), bar.clone()],
463            max_custom_measurements: 4,
464        };
465        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
466
467        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
468
469        assert_eq!(keys, vec![&foo, &bar, &baz]);
470    }
471
472    #[test]
473    fn test_max_custom_measurement() {
474        // Empty configs will return a None value for max measurements.
475        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
476        assert!(dynamic_config.max_custom_measurements().is_none());
477
478        let proj = MeasurementsConfig {
479            builtin_measurements: vec![],
480            max_custom_measurements: 3,
481        };
482
483        let glob = MeasurementsConfig {
484            builtin_measurements: vec![],
485            max_custom_measurements: 4,
486        };
487
488        // If only project level measurement config is there, return its max custom measurement variable.
489        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
490        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
491
492        // Same logic for when only global level measurement config exists.
493        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
494        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
495
496        // If both is available, pick the smallest number.
497        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
498        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
499    }
500
501    #[test]
502    fn test_geo_from_ip_address() {
503        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
504
505        let json = r#"{
506            "user": {
507                "ip_address": "2.125.160.216"
508            }
509        }"#;
510        let mut event = Annotated::<Event>::from_json(json).unwrap();
511
512        normalize_event(
513            &mut event,
514            &NormalizationConfig {
515                geoip_lookup: Some(&lookup),
516                ..Default::default()
517            },
518        );
519
520        let expected = Annotated::new(Geo {
521            country_code: Annotated::new("GB".to_owned()),
522            city: Annotated::new("Boxford".to_owned()),
523            subdivision: Annotated::new("England".to_owned()),
524            region: Annotated::new("United Kingdom".to_owned()),
525            ..Geo::default()
526        });
527        assert_eq!(get_value!(event.user!).geo, expected);
528    }
529
530    #[test]
531    fn test_user_ip_from_remote_addr() {
532        let mut event = Annotated::new(Event {
533            request: Annotated::from(Request {
534                env: Annotated::new({
535                    let mut map = Object::new();
536                    map.insert(
537                        "REMOTE_ADDR".to_owned(),
538                        Annotated::new(Value::String("2.125.160.216".to_owned())),
539                    );
540                    map
541                }),
542                ..Request::default()
543            }),
544            platform: Annotated::new("javascript".to_owned()),
545            ..Event::default()
546        });
547
548        normalize_event(&mut event, &NormalizationConfig::default());
549
550        let ip_addr = get_value!(event.user.ip_address!);
551        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
552    }
553
554    #[test]
555    fn test_user_ip_from_invalid_remote_addr() {
556        let mut event = Annotated::new(Event {
557            request: Annotated::from(Request {
558                env: Annotated::new({
559                    let mut map = Object::new();
560                    map.insert(
561                        "REMOTE_ADDR".to_owned(),
562                        Annotated::new(Value::String("whoops".to_owned())),
563                    );
564                    map
565                }),
566                ..Request::default()
567            }),
568            platform: Annotated::new("javascript".to_owned()),
569            ..Event::default()
570        });
571
572        normalize_event(&mut event, &NormalizationConfig::default());
573
574        assert_eq!(Annotated::empty(), event.value().unwrap().user);
575    }
576
577    #[test]
578    fn test_user_ip_from_client_ip_without_auto() {
579        let mut event = Annotated::new(Event {
580            platform: Annotated::new("javascript".to_owned()),
581            ..Default::default()
582        });
583
584        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
585
586        normalize_event(
587            &mut event,
588            &NormalizationConfig {
589                client_ip: Some(&ip_address),
590                infer_ip_address: true,
591                ..Default::default()
592            },
593        );
594
595        let ip_addr = get_value!(event.user.ip_address!);
596        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
597    }
598
599    #[test]
600    fn test_user_ip_from_client_ip_with_auto() {
601        let mut event = Annotated::new(Event {
602            user: Annotated::new(User {
603                ip_address: Annotated::new(IpAddr::auto()),
604                ..Default::default()
605            }),
606            ..Default::default()
607        });
608
609        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
610
611        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
612        normalize_event(
613            &mut event,
614            &NormalizationConfig {
615                client_ip: Some(&ip_address),
616                geoip_lookup: Some(&geo),
617                infer_ip_address: true,
618                ..Default::default()
619            },
620        );
621
622        let user = get_value!(event.user!);
623        let ip_addr = user.ip_address.value().expect("ip address missing");
624
625        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
626        assert!(user.geo.value().is_some());
627    }
628
629    #[test]
630    fn test_user_ip_from_client_ip_without_appropriate_platform() {
631        let mut event = Annotated::new(Event::default());
632
633        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
634        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
635        normalize_event(
636            &mut event,
637            &NormalizationConfig {
638                client_ip: Some(&ip_address),
639                geoip_lookup: Some(&geo),
640                ..Default::default()
641            },
642        );
643
644        let user = get_value!(event.user!);
645        assert!(user.ip_address.value().is_none());
646        assert!(user.geo.value().is_some());
647    }
648
649    #[test]
650    fn test_geo_present_if_ip_inferring_disabled() {
651        let mut event = Annotated::new(Event {
652            user: Annotated::new(User {
653                ip_address: Annotated::new(IpAddr::auto()),
654                ..Default::default()
655            }),
656            ..Default::default()
657        });
658
659        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
660        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
661
662        normalize_event(
663            &mut event,
664            &NormalizationConfig {
665                client_ip: Some(&ip_address),
666                geoip_lookup: Some(&geo),
667                infer_ip_address: false,
668                ..Default::default()
669            },
670        );
671
672        let user = get_value!(event.user!);
673        assert!(user.ip_address.value().unwrap().is_auto());
674        assert!(user.geo.value().is_some());
675    }
676
677    #[test]
678    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
679        let mut event = Annotated::new(Event {
680            user: Annotated::new(User {
681                ip_address: Annotated::new(IpAddr::auto()),
682                ..Default::default()
683            }),
684            ..Default::default()
685        });
686
687        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
688        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
689
690        normalize_event(
691            &mut event,
692            &NormalizationConfig {
693                client_ip: Some(&ip_address),
694                geoip_lookup: Some(&geo),
695                infer_ip_address: true,
696                ..Default::default()
697            },
698        );
699
700        let user = get_value!(event.user!);
701        assert_eq!(
702            user.ip_address.value(),
703            Some(&IpAddr::parse("2.125.160.216").unwrap())
704        );
705        assert!(user.geo.value().is_some());
706    }
707
708    #[test]
709    fn test_event_level_defaulted() {
710        let mut event = Annotated::new(Event::default());
711        normalize_event(&mut event, &NormalizationConfig::default());
712        assert_eq!(get_value!(event.level), Some(&Level::Error));
713    }
714
715    #[test]
716    fn test_transaction_level_untouched() {
717        let mut event = Annotated::new(Event {
718            ty: Annotated::new(EventType::Transaction),
719            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
720            start_timestamp: Annotated::new(
721                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
722            ),
723            contexts: {
724                let mut contexts = Contexts::new();
725                contexts.add(TraceContext {
726                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
727                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
728                    op: Annotated::new("http.server".to_owned()),
729                    ..Default::default()
730                });
731                Annotated::new(contexts)
732            },
733            ..Event::default()
734        });
735        normalize_event(&mut event, &NormalizationConfig::default());
736        assert_eq!(get_value!(event.level), Some(&Level::Info));
737    }
738
739    #[test]
740    fn test_environment_tag_is_moved() {
741        let mut event = Annotated::new(Event {
742            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
743                Annotated::new("environment".to_owned()),
744                Annotated::new("despacito".to_owned()),
745            ))]))),
746            ..Event::default()
747        });
748
749        normalize_event(&mut event, &NormalizationConfig::default());
750
751        let event = event.value().unwrap();
752
753        assert_eq!(event.environment.as_str(), Some("despacito"));
754        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
755    }
756
757    #[test]
758    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
759        let mut event = Annotated::new(Event {
760            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
761                Annotated::new("environment".to_owned()),
762                Annotated::new("despacito".to_owned()),
763            ))]))),
764            environment: Annotated::new("".to_owned()),
765            ..Event::default()
766        });
767
768        normalize_event(&mut event, &NormalizationConfig::default());
769
770        let event = event.value().unwrap();
771
772        assert_eq!(event.environment.as_str(), Some("despacito"));
773        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
774    }
775
776    #[test]
777    fn test_empty_environment_is_removed() {
778        let mut event = Annotated::new(Event {
779            environment: Annotated::new("".to_owned()),
780            ..Event::default()
781        });
782
783        normalize_event(&mut event, &NormalizationConfig::default());
784        assert_eq!(get_value!(event.environment), None);
785    }
786    #[test]
787    fn test_replay_id_added_from_dsc() {
788        let replay_id = Uuid::new_v4();
789        let mut event = Annotated::new(Event {
790            contexts: Annotated::new(Contexts(Object::new())),
791            ..Event::default()
792        });
793        normalize_event(
794            &mut event,
795            &NormalizationConfig {
796                replay_id: Some(replay_id),
797                ..Default::default()
798            },
799        );
800
801        let event = event.value().unwrap();
802
803        assert_eq!(event.contexts, {
804            let mut contexts = Contexts::new();
805            contexts.add(ReplayContext {
806                replay_id: Annotated::new(EventId(replay_id)),
807                other: Object::default(),
808            });
809            Annotated::new(contexts)
810        })
811    }
812
813    #[test]
814    fn test_none_environment_errors() {
815        let mut event = Annotated::new(Event {
816            environment: Annotated::new("none".to_owned()),
817            ..Event::default()
818        });
819
820        normalize_event(&mut event, &NormalizationConfig::default());
821
822        let environment = get_path!(event.environment!);
823        let expected_original = &Value::String("none".to_owned());
824
825        assert_eq!(
826            environment.meta().iter_errors().collect::<Vec<&Error>>(),
827            vec![&Error::new(ErrorKind::InvalidData)],
828        );
829        assert_eq!(
830            environment.meta().original_value().unwrap(),
831            expected_original
832        );
833        assert_eq!(environment.value(), None);
834    }
835
836    #[test]
837    fn test_invalid_release_removed() {
838        let mut event = Annotated::new(Event {
839            release: Annotated::new(LenientString("Latest".to_owned())),
840            ..Event::default()
841        });
842
843        normalize_event(&mut event, &NormalizationConfig::default());
844
845        let release = get_path!(event.release!);
846        let expected_original = &Value::String("Latest".to_owned());
847
848        assert_eq!(
849            release.meta().iter_errors().collect::<Vec<&Error>>(),
850            vec![&Error::new(ErrorKind::InvalidData)],
851        );
852        assert_eq!(release.meta().original_value().unwrap(), expected_original);
853        assert_eq!(release.value(), None);
854    }
855
856    #[test]
857    fn test_top_level_keys_moved_into_tags() {
858        let mut event = Annotated::new(Event {
859            server_name: Annotated::new("foo".to_owned()),
860            site: Annotated::new("foo".to_owned()),
861            tags: Annotated::new(Tags(PairList(vec![
862                Annotated::new(TagEntry(
863                    Annotated::new("site".to_owned()),
864                    Annotated::new("old".to_owned()),
865                )),
866                Annotated::new(TagEntry(
867                    Annotated::new("server_name".to_owned()),
868                    Annotated::new("old".to_owned()),
869                )),
870            ]))),
871            ..Event::default()
872        });
873
874        normalize_event(&mut event, &NormalizationConfig::default());
875
876        assert_eq!(get_value!(event.site), None);
877        assert_eq!(get_value!(event.server_name), None);
878
879        assert_eq!(
880            get_value!(event.tags!),
881            &Tags(PairList(vec![
882                Annotated::new(TagEntry(
883                    Annotated::new("site".to_owned()),
884                    Annotated::new("foo".to_owned()),
885                )),
886                Annotated::new(TagEntry(
887                    Annotated::new("server_name".to_owned()),
888                    Annotated::new("foo".to_owned()),
889                )),
890            ]))
891        );
892    }
893
894    #[test]
895    fn test_internal_tags_removed() {
896        let mut event = Annotated::new(Event {
897            tags: Annotated::new(Tags(PairList(vec![
898                Annotated::new(TagEntry(
899                    Annotated::new("release".to_owned()),
900                    Annotated::new("foo".to_owned()),
901                )),
902                Annotated::new(TagEntry(
903                    Annotated::new("dist".to_owned()),
904                    Annotated::new("foo".to_owned()),
905                )),
906                Annotated::new(TagEntry(
907                    Annotated::new("user".to_owned()),
908                    Annotated::new("foo".to_owned()),
909                )),
910                Annotated::new(TagEntry(
911                    Annotated::new("filename".to_owned()),
912                    Annotated::new("foo".to_owned()),
913                )),
914                Annotated::new(TagEntry(
915                    Annotated::new("function".to_owned()),
916                    Annotated::new("foo".to_owned()),
917                )),
918                Annotated::new(TagEntry(
919                    Annotated::new("something".to_owned()),
920                    Annotated::new("else".to_owned()),
921                )),
922            ]))),
923            ..Event::default()
924        });
925
926        normalize_event(&mut event, &NormalizationConfig::default());
927
928        assert_eq!(get_value!(event.tags!).len(), 1);
929    }
930
931    #[test]
932    fn test_empty_tags_removed() {
933        let mut event = Annotated::new(Event {
934            tags: Annotated::new(Tags(PairList(vec![
935                Annotated::new(TagEntry(
936                    Annotated::new("".to_owned()),
937                    Annotated::new("foo".to_owned()),
938                )),
939                Annotated::new(TagEntry(
940                    Annotated::new("foo".to_owned()),
941                    Annotated::new("".to_owned()),
942                )),
943                Annotated::new(TagEntry(
944                    Annotated::new("something".to_owned()),
945                    Annotated::new("else".to_owned()),
946                )),
947            ]))),
948            ..Event::default()
949        });
950
951        normalize_event(&mut event, &NormalizationConfig::default());
952
953        assert_eq!(
954            get_value!(event.tags!),
955            &Tags(PairList(vec![
956                Annotated::new(TagEntry(
957                    Annotated::from_error(Error::nonempty(), None),
958                    Annotated::new("foo".to_owned()),
959                )),
960                Annotated::new(TagEntry(
961                    Annotated::new("foo".to_owned()),
962                    Annotated::from_error(Error::nonempty(), None),
963                )),
964                Annotated::new(TagEntry(
965                    Annotated::new("something".to_owned()),
966                    Annotated::new("else".to_owned()),
967                )),
968            ]))
969        );
970    }
971
972    #[test]
973    fn test_tags_deduplicated() {
974        let mut event = Annotated::new(Event {
975            tags: Annotated::new(Tags(PairList(vec![
976                Annotated::new(TagEntry(
977                    Annotated::new("foo".to_owned()),
978                    Annotated::new("1".to_owned()),
979                )),
980                Annotated::new(TagEntry(
981                    Annotated::new("bar".to_owned()),
982                    Annotated::new("1".to_owned()),
983                )),
984                Annotated::new(TagEntry(
985                    Annotated::new("foo".to_owned()),
986                    Annotated::new("2".to_owned()),
987                )),
988                Annotated::new(TagEntry(
989                    Annotated::new("bar".to_owned()),
990                    Annotated::new("2".to_owned()),
991                )),
992                Annotated::new(TagEntry(
993                    Annotated::new("foo".to_owned()),
994                    Annotated::new("3".to_owned()),
995                )),
996            ]))),
997            ..Event::default()
998        });
999
1000        normalize_event(&mut event, &NormalizationConfig::default());
1001
1002        // should keep the first occurrence of every tag
1003        assert_eq!(
1004            get_value!(event.tags!),
1005            &Tags(PairList(vec![
1006                Annotated::new(TagEntry(
1007                    Annotated::new("foo".to_owned()),
1008                    Annotated::new("1".to_owned()),
1009                )),
1010                Annotated::new(TagEntry(
1011                    Annotated::new("bar".to_owned()),
1012                    Annotated::new("1".to_owned()),
1013                )),
1014            ]))
1015        );
1016    }
1017
1018    #[test]
1019    fn test_transaction_status_defaulted_to_unknown() {
1020        let mut object = Object::new();
1021        let trace_context = TraceContext {
1022            // We assume the status to be null.
1023            status: Annotated::empty(),
1024            ..TraceContext::default()
1025        };
1026        object.insert(
1027            "trace".to_owned(),
1028            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1029        );
1030
1031        let mut event = Annotated::new(Event {
1032            contexts: Annotated::new(Contexts(object)),
1033            ..Event::default()
1034        });
1035        normalize_event(&mut event, &NormalizationConfig::default());
1036
1037        let event = event.value().unwrap();
1038
1039        let event_trace_context = event.context::<TraceContext>().unwrap();
1040        assert_eq!(
1041            event_trace_context.status,
1042            Annotated::new(SpanStatus::Unknown)
1043        )
1044    }
1045
1046    #[test]
1047    fn test_unknown_debug_image() {
1048        let mut event = Annotated::new(Event {
1049            debug_meta: Annotated::new(DebugMeta {
1050                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1051                ..DebugMeta::default()
1052            }),
1053            ..Event::default()
1054        });
1055
1056        normalize_event(&mut event, &NormalizationConfig::default());
1057
1058        assert_eq!(
1059            get_path!(event.debug_meta!),
1060            &Annotated::new(DebugMeta {
1061                images: Annotated::new(vec![Annotated::from_error(
1062                    Error::invalid("unsupported debug image type"),
1063                    Some(Value::Object(Object::default())),
1064                )]),
1065                ..DebugMeta::default()
1066            })
1067        );
1068    }
1069
1070    #[test]
1071    fn test_context_line_default() {
1072        let mut frame = Annotated::new(Frame {
1073            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1074            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1075            ..Frame::default()
1076        });
1077
1078        normalize_non_raw_frame(&mut frame);
1079
1080        let frame = frame.value().unwrap();
1081        assert_eq!(frame.context_line.as_str(), Some(""));
1082    }
1083
1084    #[test]
1085    fn test_context_line_retain() {
1086        let mut frame = Annotated::new(Frame {
1087            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1088            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1089            context_line: Annotated::new("some line".to_owned()),
1090            ..Frame::default()
1091        });
1092
1093        normalize_non_raw_frame(&mut frame);
1094
1095        let frame = frame.value().unwrap();
1096        assert_eq!(frame.context_line.as_str(), Some("some line"));
1097    }
1098
1099    #[test]
1100    fn test_frame_null_context_lines() {
1101        let mut frame = Annotated::new(Frame {
1102            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1103            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1104            ..Frame::default()
1105        });
1106
1107        normalize_non_raw_frame(&mut frame);
1108
1109        assert_eq!(
1110            *get_value!(frame.pre_context!),
1111            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1112        );
1113        assert_eq!(
1114            *get_value!(frame.post_context!),
1115            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1116        );
1117    }
1118
1119    #[test]
1120    fn test_too_long_distribution() {
1121        let json = r#"{
1122  "event_id": "52df9022835246eeb317dbd739ccd059",
1123  "fingerprint": [
1124    "{{ default }}"
1125  ],
1126  "platform": "other",
1127  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1128}"#;
1129
1130        let mut event = Annotated::<Event>::from_json(json).unwrap();
1131
1132        normalize_event(&mut event, &NormalizationConfig::default());
1133
1134        let dist = &event.value().unwrap().dist;
1135        let result = &Annotated::<String>::from_error(
1136            Error::new(ErrorKind::ValueTooLong),
1137            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1138        );
1139        assert_eq!(dist, result);
1140    }
1141
1142    #[test]
1143    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1144        let mut event = Annotated::new(Event {
1145            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1146                ty: Annotated::new("FooDivisionError".to_owned()),
1147                value: Annotated::new("hi".to_owned().into()),
1148                ..Exception::default()
1149            })])),
1150            stacktrace: Annotated::new(
1151                RawStacktrace {
1152                    frames: Annotated::new(vec![Annotated::new(Frame {
1153                        module: Annotated::new("MyModule".to_owned()),
1154                        filename: Annotated::new("MyFilename".into()),
1155                        function: Annotated::new("Void FooBar()".to_owned()),
1156                        ..Frame::default()
1157                    })]),
1158                    ..RawStacktrace::default()
1159                }
1160                .into(),
1161            ),
1162            ..Event::default()
1163        });
1164
1165        normalize_event(&mut event, &NormalizationConfig::default());
1166
1167        assert_eq!(
1168            get_value!(event.exceptions.values[0].stacktrace!),
1169            &Stacktrace(RawStacktrace {
1170                frames: Annotated::new(vec![Annotated::new(Frame {
1171                    module: Annotated::new("MyModule".to_owned()),
1172                    filename: Annotated::new("MyFilename".into()),
1173                    abs_path: Annotated::new("MyFilename".into()),
1174                    function: Annotated::new("Void FooBar()".to_owned()),
1175                    ..Frame::default()
1176                })]),
1177                ..RawStacktrace::default()
1178            })
1179        );
1180    }
1181
1182    #[test]
1183    fn test_parses_sdk_info_from_header() {
1184        let mut event = Annotated::new(Event::default());
1185
1186        normalize_event(
1187            &mut event,
1188            &NormalizationConfig {
1189                client: Some("_fooBar/0.0.0".to_owned()),
1190                ..Default::default()
1191            },
1192        );
1193
1194        assert_eq!(
1195            get_path!(event.client_sdk!),
1196            &Annotated::new(ClientSdkInfo {
1197                name: Annotated::new("_fooBar".to_owned()),
1198                version: Annotated::new("0.0.0".to_owned()),
1199                ..ClientSdkInfo::default()
1200            })
1201        );
1202    }
1203
1204    #[test]
1205    fn test_discards_received() {
1206        let mut event = Annotated::new(Event {
1207            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1208            ..Default::default()
1209        });
1210
1211        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1212        normalize_event(&mut event, &NormalizationConfig::default());
1213
1214        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1215    }
1216
1217    #[test]
1218    fn test_grouping_config() {
1219        let mut event = Annotated::new(Event {
1220            logentry: Annotated::from(LogEntry {
1221                message: Annotated::new("Hello World!".to_owned().into()),
1222                ..Default::default()
1223            }),
1224            ..Default::default()
1225        });
1226
1227        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1228        normalize_event(
1229            &mut event,
1230            &NormalizationConfig {
1231                grouping_config: Some(json!({
1232                    "id": "legacy:1234-12-12".to_owned(),
1233                })),
1234                ..Default::default()
1235            },
1236        );
1237
1238        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1239            ".event_id" => "[event-id]",
1240            ".received" => "[received]",
1241            ".timestamp" => "[timestamp]"
1242        }, @r###"
1243        {
1244          "event_id": "[event-id]",
1245          "level": "error",
1246          "type": "default",
1247          "logentry": {
1248            "formatted": "Hello World!",
1249          },
1250          "logger": "",
1251          "platform": "other",
1252          "timestamp": "[timestamp]",
1253          "received": "[received]",
1254          "grouping_config": {
1255            "id": "legacy:1234-12-12",
1256          },
1257        }
1258        "###);
1259    }
1260
1261    #[test]
1262    fn test_logentry_error() {
1263        let json = r#"
1264{
1265    "event_id": "74ad1301f4df489ead37d757295442b1",
1266    "timestamp": 1668148328.308933,
1267    "received": 1668148328.308933,
1268    "level": "error",
1269    "platform": "python",
1270    "logentry": {
1271        "params": [
1272            "bogus"
1273        ],
1274        "formatted": 42
1275    }
1276}
1277"#;
1278        let mut event = Annotated::<Event>::from_json(json).unwrap();
1279
1280        normalize_event(&mut event, &NormalizationConfig::default());
1281
1282        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1283        {
1284          "event_id": "74ad1301f4df489ead37d757295442b1",
1285          "level": "error",
1286          "type": "default",
1287          "logentry": null,
1288          "logger": "",
1289          "platform": "python",
1290          "timestamp": 1668148328.308933,
1291          "received": "[received]",
1292          "_meta": {
1293            "logentry": {
1294              "": {
1295                "err": [
1296                  [
1297                    "invalid_data",
1298                    {
1299                      "reason": "no message present"
1300                    }
1301                  ]
1302                ],
1303                "val": {
1304                  "formatted": null,
1305                  "message": null,
1306                  "params": [
1307                    "bogus"
1308                  ]
1309                }
1310              }
1311            }
1312          }
1313        }
1314        "###)
1315    }
1316
1317    #[test]
1318    fn test_future_timestamp() {
1319        let mut event = Annotated::new(Event {
1320            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1321            ..Default::default()
1322        });
1323
1324        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1325        let max_secs_in_past = Some(30 * 24 * 3600);
1326        let max_secs_in_future = Some(60);
1327
1328        validate_event(
1329            &mut event,
1330            &EventValidationConfig {
1331                received_at,
1332                max_secs_in_past,
1333                max_secs_in_future,
1334                is_validated: false,
1335                ..Default::default()
1336            },
1337        )
1338        .unwrap();
1339        normalize_event(&mut event, &NormalizationConfig::default());
1340
1341        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1342        ".event_id" => "[event-id]",
1343    }, @r###"
1344        {
1345          "event_id": "[event-id]",
1346          "level": "error",
1347          "type": "default",
1348          "logger": "",
1349          "platform": "other",
1350          "timestamp": 946857600.0,
1351          "received": 946857600.0,
1352          "_meta": {
1353            "timestamp": {
1354              "": Meta(Some(MetaInner(
1355                err: [
1356                  [
1357                    "future_timestamp",
1358                    {
1359                      "sdk_time": "2000-01-03T00:02:00+00:00",
1360                      "server_time": "2000-01-03T00:00:00+00:00",
1361                    },
1362                  ],
1363                ],
1364              ))),
1365            },
1366          },
1367        }
1368        "###);
1369    }
1370
1371    #[test]
1372    fn test_past_timestamp() {
1373        let mut event = Annotated::new(Event {
1374            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1375            ..Default::default()
1376        });
1377
1378        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1379        let max_secs_in_past = Some(30 * 24 * 3600);
1380        let max_secs_in_future = Some(60);
1381
1382        validate_event(
1383            &mut event,
1384            &EventValidationConfig {
1385                received_at,
1386                max_secs_in_past,
1387                max_secs_in_future,
1388                is_validated: false,
1389                ..Default::default()
1390            },
1391        )
1392        .unwrap();
1393        normalize_event(&mut event, &NormalizationConfig::default());
1394
1395        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1396        ".event_id" => "[event-id]",
1397    }, @r###"
1398        {
1399          "event_id": "[event-id]",
1400          "level": "error",
1401          "type": "default",
1402          "logger": "",
1403          "platform": "other",
1404          "timestamp": 952041600.0,
1405          "received": 952041600.0,
1406          "_meta": {
1407            "timestamp": {
1408              "": Meta(Some(MetaInner(
1409                err: [
1410                  [
1411                    "past_timestamp",
1412                    {
1413                      "sdk_time": "2000-01-03T00:00:00+00:00",
1414                      "server_time": "2000-03-03T00:00:00+00:00",
1415                    },
1416                  ],
1417                ],
1418              ))),
1419            },
1420          },
1421        }
1422        "###);
1423    }
1424
1425    #[test]
1426    fn test_normalize_logger_empty() {
1427        let mut event = Event::from_value(
1428            serde_json::json!({
1429                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1430                "platform": "java",
1431                "logger": "",
1432            })
1433            .into(),
1434        );
1435
1436        normalize_event(&mut event, &NormalizationConfig::default());
1437        assert_annotated_snapshot!(event);
1438    }
1439
1440    #[test]
1441    fn test_normalize_logger_trimmed() {
1442        let mut event = Event::from_value(
1443            serde_json::json!({
1444                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1445                "platform": "java",
1446                "logger": " \t  \t   ",
1447            })
1448            .into(),
1449        );
1450
1451        normalize_event(&mut event, &NormalizationConfig::default());
1452        assert_annotated_snapshot!(event);
1453    }
1454
1455    #[test]
1456    fn test_normalize_logger_short_no_trimming() {
1457        let mut event = Event::from_value(
1458            serde_json::json!({
1459                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1460                "platform": "java",
1461                "logger": "my.short-logger.isnt_trimmed",
1462            })
1463            .into(),
1464        );
1465
1466        normalize_event(&mut event, &NormalizationConfig::default());
1467        assert_annotated_snapshot!(event);
1468    }
1469
1470    #[test]
1471    fn test_normalize_logger_exact_length() {
1472        let mut event = Event::from_value(
1473            serde_json::json!({
1474                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1475                "platform": "java",
1476                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1477            })
1478            .into(),
1479        );
1480
1481        normalize_event(&mut event, &NormalizationConfig::default());
1482        assert_annotated_snapshot!(event);
1483    }
1484
1485    #[test]
1486    fn test_normalize_logger_too_long_single_word() {
1487        let mut event = Event::from_value(
1488            serde_json::json!({
1489                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1490                "platform": "java",
1491                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1492            })
1493            .into(),
1494        );
1495
1496        normalize_event(&mut event, &NormalizationConfig::default());
1497        assert_annotated_snapshot!(event);
1498    }
1499
1500    #[test]
1501    fn test_normalize_logger_word_trimmed_at_max() {
1502        let mut event = Event::from_value(
1503            serde_json::json!({
1504                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1505                "platform": "java",
1506                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1507            })
1508            .into(),
1509        );
1510
1511        normalize_event(&mut event, &NormalizationConfig::default());
1512        assert_annotated_snapshot!(event);
1513    }
1514
1515    #[test]
1516    fn test_normalize_logger_word_trimmed_before_max() {
1517        // This test verifies the "smart" trimming on words -- the logger name
1518        // should be cut before the max limit, removing entire words.
1519        let mut event = Event::from_value(
1520            serde_json::json!({
1521                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1522                "platform": "java",
1523                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1524            })
1525            .into(),
1526        );
1527
1528        normalize_event(&mut event, &NormalizationConfig::default());
1529        assert_annotated_snapshot!(event);
1530    }
1531
1532    #[test]
1533    fn test_normalize_logger_word_leading_dots() {
1534        let mut event = Event::from_value(
1535            serde_json::json!({
1536                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1537                "platform": "java",
1538                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1539            })
1540            .into(),
1541        );
1542
1543        normalize_event(&mut event, &NormalizationConfig::default());
1544        assert_annotated_snapshot!(event);
1545    }
1546
1547    #[test]
1548    fn test_normalization_is_idempotent() {
1549        // get an event, normalize it. the result of that must be the same as normalizing it once more
1550        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1551        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1552        let mut event = Annotated::new(Event {
1553            ty: Annotated::new(EventType::Transaction),
1554            transaction: Annotated::new("/".to_owned()),
1555            timestamp: Annotated::new(end.into()),
1556            start_timestamp: Annotated::new(start.into()),
1557            contexts: {
1558                let mut contexts = Contexts::new();
1559                contexts.add(TraceContext {
1560                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1561                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1562                    op: Annotated::new("http.server".to_owned()),
1563                    ..Default::default()
1564                });
1565                Annotated::new(contexts)
1566            },
1567            spans: Annotated::new(vec![Annotated::new(Span {
1568                timestamp: Annotated::new(
1569                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1570                ),
1571                start_timestamp: Annotated::new(
1572                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1573                ),
1574                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1575                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1576
1577                ..Default::default()
1578            })]),
1579            ..Default::default()
1580        });
1581
1582        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1583            relay_event_schema::processor::apply(event, |e, _m| {
1584                e.received = Annotated::empty();
1585                Ok(())
1586            })
1587            .unwrap();
1588            event
1589        }
1590
1591        normalize_event(&mut event, &NormalizationConfig::default());
1592        let first = remove_received_from_event(&mut event.clone())
1593            .to_json()
1594            .unwrap();
1595        // Expected some fields (such as timestamps) exist after first normalization.
1596
1597        normalize_event(&mut event, &NormalizationConfig::default());
1598        let second = remove_received_from_event(&mut event.clone())
1599            .to_json()
1600            .unwrap();
1601        assert_eq!(&first, &second, "idempotency check failed");
1602
1603        normalize_event(&mut event, &NormalizationConfig::default());
1604        let third = remove_received_from_event(&mut event.clone())
1605            .to_json()
1606            .unwrap();
1607        assert_eq!(&second, &third, "idempotency check failed");
1608    }
1609
1610    /// Validate full normalization is idempotent.
1611    ///
1612    /// Both PoPs and processing relays will temporarily run full normalization
1613    /// in events, during the rollout of running normalization once in internal
1614    /// relays.
1615    // TODO(iker): remove this test after the rollout is done.
1616    #[test]
1617    fn test_full_normalization_is_idempotent() {
1618        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1619        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1620        let mut event = Annotated::new(Event {
1621            ty: Annotated::new(EventType::Transaction),
1622            transaction: Annotated::new("/".to_owned()),
1623            timestamp: Annotated::new(end.into()),
1624            start_timestamp: Annotated::new(start.into()),
1625            contexts: {
1626                let mut contexts = Contexts::new();
1627                contexts.add(TraceContext {
1628                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1629                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1630                    op: Annotated::new("http.server".to_owned()),
1631                    ..Default::default()
1632                });
1633                Annotated::new(contexts)
1634            },
1635            spans: Annotated::new(vec![Annotated::new(Span {
1636                timestamp: Annotated::new(
1637                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1638                ),
1639                start_timestamp: Annotated::new(
1640                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1641                ),
1642                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1643                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1644
1645                ..Default::default()
1646            })]),
1647            ..Default::default()
1648        });
1649
1650        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1651            relay_event_schema::processor::apply(event, |e, _m| {
1652                e.received = Annotated::empty();
1653                Ok(())
1654            })
1655            .unwrap();
1656            event
1657        }
1658
1659        let full_normalization_config = NormalizationConfig {
1660            is_renormalize: false,
1661            enable_trimming: true,
1662            remove_other: true,
1663            emit_event_errors: true,
1664            ..Default::default()
1665        };
1666
1667        normalize_event(&mut event, &full_normalization_config);
1668        let first = remove_received_from_event(&mut event.clone())
1669            .to_json()
1670            .unwrap();
1671        // Expected some fields (such as timestamps) exist after first normalization.
1672
1673        normalize_event(&mut event, &full_normalization_config);
1674        let second = remove_received_from_event(&mut event.clone())
1675            .to_json()
1676            .unwrap();
1677        assert_eq!(&first, &second, "idempotency check failed");
1678
1679        normalize_event(&mut event, &full_normalization_config);
1680        let third = remove_received_from_event(&mut event.clone())
1681            .to_json()
1682            .unwrap();
1683        assert_eq!(&second, &third, "idempotency check failed");
1684    }
1685
1686    #[test]
1687    fn test_normalize_validates_spans() {
1688        let event = Annotated::<Event>::from_json(
1689            r#"
1690            {
1691                "type": "transaction",
1692                "transaction": "/",
1693                "timestamp": 946684810.0,
1694                "start_timestamp": 946684800.0,
1695                "contexts": {
1696                    "trace": {
1697                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1698                    "span_id": "fa90fdead5f74053",
1699                    "op": "http.server",
1700                    "type": "trace"
1701                    }
1702                },
1703                "spans": []
1704            }
1705            "#,
1706        )
1707        .unwrap();
1708
1709        // All incomplete spans should be caught by normalization:
1710        for span in [
1711            r#"null"#,
1712            r#"{
1713              "timestamp": 946684810.0,
1714              "start_timestamp": 946684900.0,
1715              "span_id": "fa90fdead5f74053",
1716              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1717            }"#,
1718            r#"{
1719              "timestamp": 946684810.0,
1720              "span_id": "fa90fdead5f74053",
1721              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1722            }"#,
1723            r#"{
1724              "timestamp": 946684810.0,
1725              "start_timestamp": 946684800.0,
1726              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1727            }"#,
1728            r#"{
1729              "timestamp": 946684810.0,
1730              "start_timestamp": 946684800.0,
1731              "span_id": "fa90fdead5f74053"
1732            }"#,
1733        ] {
1734            let mut modified_event = event.clone();
1735            let event_ref = modified_event.value_mut().as_mut().unwrap();
1736            event_ref
1737                .spans
1738                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1739
1740            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1741
1742            assert!(res.is_err(), "{span:?}");
1743        }
1744    }
1745
1746    #[test]
1747    fn test_normalization_respects_is_renormalize() {
1748        let mut event = Annotated::<Event>::from_json(
1749            r#"
1750            {
1751                "type": "default",
1752                "tags": [["environment", "some_environment"]]
1753            }
1754            "#,
1755        )
1756        .unwrap();
1757
1758        normalize_event(
1759            &mut event,
1760            &NormalizationConfig {
1761                is_renormalize: true,
1762                ..Default::default()
1763            },
1764        );
1765
1766        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1767        Tags(
1768            PairList(
1769                [
1770                    TagEntry(
1771                        "environment",
1772                        "some_environment",
1773                    ),
1774                ],
1775            ),
1776        )
1777        "###);
1778    }
1779
1780    #[test]
1781    fn test_geo_in_normalize() {
1782        let mut event = Annotated::<Event>::from_json(
1783            r#"
1784            {
1785                "type": "transaction",
1786                "transaction": "/foo/",
1787                "timestamp": 946684810.0,
1788                "start_timestamp": 946684800.0,
1789                "contexts": {
1790                    "trace": {
1791                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1792                        "span_id": "fa90fdead5f74053",
1793                        "op": "http.server",
1794                        "type": "trace"
1795                    }
1796                },
1797                "transaction_info": {
1798                    "source": "url"
1799                },
1800                "user": {
1801                    "ip_address": "2.125.160.216"
1802                }
1803            }
1804            "#,
1805        )
1806        .unwrap();
1807
1808        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1809
1810        // Extract user's geo information before normalization.
1811        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1812        assert!(user_geo.is_none());
1813
1814        normalize_event(
1815            &mut event,
1816            &NormalizationConfig {
1817                geoip_lookup: Some(&lookup),
1818                ..Default::default()
1819            },
1820        );
1821
1822        // Extract user's geo information after normalization.
1823        let user_geo = event
1824            .value()
1825            .unwrap()
1826            .user
1827            .value()
1828            .unwrap()
1829            .geo
1830            .value()
1831            .unwrap();
1832
1833        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1834        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1835    }
1836
1837    #[test]
1838    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1839        let mut event = Event {
1840            spans: Annotated::new(vec![Annotated::new(Span {
1841                op: Annotated::new("app_start_cold".to_owned()),
1842                ..Default::default()
1843            })]),
1844            client_sdk: Annotated::new(ClientSdkInfo {
1845                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1846                version: Annotated::new("4.5.0".to_owned()),
1847                ..Default::default()
1848            }),
1849            ..Default::default()
1850        };
1851        span::normalize_app_start_spans(&mut event);
1852        assert_debug_snapshot!(event.spans, @r#"
1853        [
1854            Span {
1855                timestamp: ~,
1856                start_timestamp: ~,
1857                exclusive_time: ~,
1858                op: "app_start_cold",
1859                span_id: ~,
1860                parent_span_id: ~,
1861                trace_id: ~,
1862                segment_id: ~,
1863                is_segment: ~,
1864                is_remote: ~,
1865                status: ~,
1866                description: ~,
1867                tags: ~,
1868                origin: ~,
1869                profile_id: ~,
1870                data: ~,
1871                links: ~,
1872                sentry_tags: ~,
1873                received: ~,
1874                measurements: ~,
1875                platform: ~,
1876                was_transaction: ~,
1877                kind: ~,
1878                other: {},
1879            },
1880        ]
1881        "#);
1882    }
1883
1884    #[test]
1885    fn test_normalize_app_start_cold_spans_for_react_native() {
1886        let mut event = Event {
1887            spans: Annotated::new(vec![Annotated::new(Span {
1888                op: Annotated::new("app_start_cold".to_owned()),
1889                ..Default::default()
1890            })]),
1891            client_sdk: Annotated::new(ClientSdkInfo {
1892                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1893                version: Annotated::new("4.4.0".to_owned()),
1894                ..Default::default()
1895            }),
1896            ..Default::default()
1897        };
1898        span::normalize_app_start_spans(&mut event);
1899        assert_debug_snapshot!(event.spans, @r#"
1900        [
1901            Span {
1902                timestamp: ~,
1903                start_timestamp: ~,
1904                exclusive_time: ~,
1905                op: "app.start.cold",
1906                span_id: ~,
1907                parent_span_id: ~,
1908                trace_id: ~,
1909                segment_id: ~,
1910                is_segment: ~,
1911                is_remote: ~,
1912                status: ~,
1913                description: ~,
1914                tags: ~,
1915                origin: ~,
1916                profile_id: ~,
1917                data: ~,
1918                links: ~,
1919                sentry_tags: ~,
1920                received: ~,
1921                measurements: ~,
1922                platform: ~,
1923                was_transaction: ~,
1924                kind: ~,
1925                other: {},
1926            },
1927        ]
1928        "#);
1929    }
1930
1931    #[test]
1932    fn test_normalize_app_start_warm_spans_for_react_native() {
1933        let mut event = Event {
1934            spans: Annotated::new(vec![Annotated::new(Span {
1935                op: Annotated::new("app_start_warm".to_owned()),
1936                ..Default::default()
1937            })]),
1938            client_sdk: Annotated::new(ClientSdkInfo {
1939                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1940                version: Annotated::new("4.4.0".to_owned()),
1941                ..Default::default()
1942            }),
1943            ..Default::default()
1944        };
1945        span::normalize_app_start_spans(&mut event);
1946        assert_debug_snapshot!(event.spans, @r#"
1947        [
1948            Span {
1949                timestamp: ~,
1950                start_timestamp: ~,
1951                exclusive_time: ~,
1952                op: "app.start.warm",
1953                span_id: ~,
1954                parent_span_id: ~,
1955                trace_id: ~,
1956                segment_id: ~,
1957                is_segment: ~,
1958                is_remote: ~,
1959                status: ~,
1960                description: ~,
1961                tags: ~,
1962                origin: ~,
1963                profile_id: ~,
1964                data: ~,
1965                links: ~,
1966                sentry_tags: ~,
1967                received: ~,
1968                measurements: ~,
1969                platform: ~,
1970                was_transaction: ~,
1971                kind: ~,
1972                other: {},
1973            },
1974        ]
1975        "#);
1976    }
1977}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
