relay_event_normalization/normalize/
mod.rs

1use std::hash::Hash;
2use std::{collections::HashMap, sync::LazyLock};
3
4use regex::Regex;
5use relay_base_schema::metrics::MetricUnit;
6use relay_event_schema::protocol::VALID_PLATFORMS;
7use relay_pattern::Pattern;
8use relay_protocol::{FiniteF64, RuleCondition};
9use serde::{Deserialize, Serialize};
10
11pub mod breakdowns;
12pub mod contexts;
13pub mod nel;
14pub mod request;
15pub mod span;
16pub mod user_agent;
17pub mod utils;
18
19/// Defines a builtin measurement.
20#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
21#[serde(default, rename_all = "camelCase")]
22pub struct BuiltinMeasurementKey {
23    name: String,
24    unit: MetricUnit,
25    #[serde(skip_serializing_if = "is_false")]
26    allow_negative: bool,
27}
28
29fn is_false(b: &bool) -> bool {
30    !b
31}
32
33impl BuiltinMeasurementKey {
34    /// Creates a new [`BuiltinMeasurementKey`].
35    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
36        Self {
37            name: name.into(),
38            unit,
39            allow_negative: false,
40        }
41    }
42
43    /// Returns the name of the built in measurement key.
44    pub fn name(&self) -> &str {
45        &self.name
46    }
47
48    /// Returns the unit of the built in measurement key.
49    pub fn unit(&self) -> &MetricUnit {
50        &self.unit
51    }
52
53    /// Return true if the built in measurement key allows negative values.
54    pub fn allow_negative(&self) -> &bool {
55        &self.allow_negative
56    }
57}
58
59/// Configuration for measurements normalization.
60#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
61#[serde(default, rename_all = "camelCase")]
62pub struct MeasurementsConfig {
63    /// A list of measurements that are built-in and are not subject to custom measurement limits.
64    #[serde(default, skip_serializing_if = "Vec::is_empty")]
65    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
66
67    /// The maximum number of measurements allowed per event that are not known measurements.
68    pub max_custom_measurements: usize,
69}
70
71impl MeasurementsConfig {
72    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
73    /// for every measurement-mri.
74    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
75}
76
77/// Returns `true` if the given platform string is a known platform identifier.
78///
79/// See [`VALID_PLATFORMS`] for a list of all known platforms.
80pub fn is_valid_platform(platform: &str) -> bool {
81    VALID_PLATFORMS.contains(&platform)
82}
83
84/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
85/// the merging logic.
86#[derive(Copy, Clone, Debug)]
87pub struct CombinedMeasurementsConfig<'a> {
88    project: Option<&'a MeasurementsConfig>,
89    global: Option<&'a MeasurementsConfig>,
90}
91
92impl<'a> CombinedMeasurementsConfig<'a> {
93    /// Constructor for [`CombinedMeasurementsConfig`].
94    pub fn new(
95        project: Option<&'a MeasurementsConfig>,
96        global: Option<&'a MeasurementsConfig>,
97    ) -> Self {
98        CombinedMeasurementsConfig { project, global }
99    }
100
101    /// Returns an iterator over the merged builtin measurement keys.
102    ///
103    /// Items from the project config are prioritized over global config, and
104    /// there are no duplicates.
105    pub fn builtin_measurement_keys(
106        &'a self,
107    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
108        let project = self
109            .project
110            .map(|p| p.builtin_measurements.as_slice())
111            .unwrap_or_default();
112
113        let global = self
114            .global
115            .map(|g| g.builtin_measurements.as_slice())
116            .unwrap_or_default();
117
118        project
119            .iter()
120            .chain(global.iter().filter(|key| !project.contains(key)))
121    }
122
123    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
124    /// global level. If both of them are available, it will choose the most restrictive.
125    pub fn max_custom_measurements(&'a self) -> Option<usize> {
126        match (&self.project, &self.global) {
127            (None, None) => None,
128            (None, Some(global)) => Some(global.max_custom_measurements),
129            (Some(project), None) => Some(project.max_custom_measurements),
130            (Some(project), Some(global)) => Some(std::cmp::min(
131                project.max_custom_measurements,
132                global.max_custom_measurements,
133            )),
134        }
135    }
136}
137
138/// Defines a weighted component for a performance score.
139///
140/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
141/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
142#[derive(Debug, Default, Clone, Serialize, Deserialize)]
143pub struct PerformanceScoreWeightedComponent {
144    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
145    /// profile will be discarded.
146    pub measurement: String,
147    /// Weight [0,1.0] of this component in the performance score
148    pub weight: FiniteF64,
149    /// p10 used to define the log-normal for calculation
150    pub p10: FiniteF64,
151    /// Median used to define the log-normal for calculation
152    pub p50: FiniteF64,
153    /// Whether the measurement is optional. If the measurement is missing, performance score processing
154    /// may still continue, and the weight will be 0.
155    #[serde(default)]
156    pub optional: bool,
157}
158
159/// Defines a profile for performance score.
160///
161/// A profile contains weights for a score of 100% and match against an event using a condition.
162/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
163#[derive(Debug, Default, Clone, Serialize, Deserialize)]
164#[serde(rename_all = "camelCase")]
165pub struct PerformanceScoreProfile {
166    /// Name of the profile, used for debugging and faceting multiple profiles
167    pub name: Option<String>,
168    /// Score components
169    #[serde(default, skip_serializing_if = "Vec::is_empty")]
170    pub score_components: Vec<PerformanceScoreWeightedComponent>,
171    /// See [`RuleCondition`] for all available options to specify and combine conditions.
172    pub condition: Option<RuleCondition>,
173    /// The version of the profile, used to isolate changes to score calculations.
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub version: Option<String>,
176}
177
178/// Defines the performance configuration for the project.
179///
180/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
181/// specific conditions.
182#[derive(Debug, Default, Clone, Serialize, Deserialize)]
183pub struct PerformanceScoreConfig {
184    /// List of performance profiles, only the first with matching conditions will be applied.
185    #[serde(default, skip_serializing_if = "Vec::is_empty")]
186    pub profiles: Vec<PerformanceScoreProfile>,
187}
188
189/// Regex that matches version and/or date patterns at the end of a model name.
190///
191/// Examples matched:
192/// - "-20250805-v1:0" in "claude-opus-4-1-20250805-v1:0" (both)
193/// - "-20250514-v1:0" in "claude-sonnet-4-20250514-v1:0" (both)
194/// - "_v2" in "model_v2" (version only)
195/// - "-v1.0" in "gpt-4-v1.0" (version only)
196/// - "-20250522" in "claude-4-sonnet-20250522" (date only)
197/// - "-2025-06-10" in "o3-pro-2025-06-10" (date only)
198/// - "@20241022" in "claude-3-5-haiku@20241022" (date only)
199static VERSION_AND_DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
200    Regex::new(
201        r"(?x)
202        (
203            ([-_@])                                    # Required separator before date (-, _, or @)
204            (\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8})          # Date: YYYY-MM-DD/YYYY/MM/DD/YYYY.MM.DD or YYYYMMDD
205        )?                                              # Date is optional
206        (
207            [-_]                                        # Required separator before version (- or _)
208            v\d+[:.]?\d*                                # Version: v1, v1.0, v1:0
209            ([-:].*)?                                   # Optional trailing content
210        )?                                              # Version is optional
211        $  # Must be at the end of the string
212        ",
213    )
214    .unwrap()
215});
216
217/// Normalizes an AI model name by stripping date and version patterns.
218///
219/// This function converts specific model versions into normalized names suitable for matching
220/// against cost configurations, ensuring that different versions of the same model can share
221/// cost information.
222///
223/// The normalization strips version and/or date patterns from the end of the string in a single pass.
224///
225/// Examples:
226/// - "claude-4-sonnet-20250522" -> "claude-4-sonnet"
227/// - "o3-pro-2025-06-10" -> "o3-pro"
228/// - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku"
229/// - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1"
230/// - "us.anthropic.claude-sonnet-4-20250514-v1:0" -> "us.anthropic.claude-sonnet-4"
231/// - "gpt-4-v1.0" -> "gpt-4"
232/// - "gpt-4-v1:0" -> "gpt-4"
233/// - "model_v2" -> "model"
234///
235/// Version patterns:
236/// - Must start with 'v': v1, v1.0, v1:0 (requires -, _ separator)
237///
238/// Date patterns:
239/// - YYYYMMDD: 20250522
240/// - YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD: 2025-06-10
241///
242/// Args:
243///     model_id: The original model ID
244///
245/// Returns:
246///     The normalized model name without date and version patterns
247fn normalize_ai_model_name(model_id: &str) -> &str {
248    if let Some(captures) = VERSION_AND_DATE_REGEX.captures(model_id) {
249        let match_idx = captures.get(0).map(|m| m.start()).unwrap_or(model_id.len());
250        &model_id[..match_idx]
251    } else {
252        model_id
253    }
254}
255
256/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
257#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
258#[serde(rename_all = "camelCase")]
259pub struct ModelCostV2 {
260    /// The cost per input token
261    pub input_per_token: f64,
262    /// The cost per output token
263    pub output_per_token: f64,
264    /// The cost per output reasoning token
265    pub output_reasoning_per_token: f64,
266    /// The cost per input cached token
267    pub input_cached_per_token: f64,
268    /// The cost per input cache write token
269    pub input_cache_write_per_token: f64,
270}
271
272/// Metadata for AI models including costs and context size.
273///
274/// Example JSON:
275/// ```json
276/// {
277///   "version": 1,
278///   "models": {
279///     "gpt-4": {
280///       "costs": {
281///         "inputPerToken": 0.0000003,
282///         "outputPerToken": 0.00000165,
283///         "outputReasoningPerToken": 0.0,
284///         "inputCachedPerToken": 0.0000015,
285///         "inputCacheWritePerToken": 0.00001875
286///       },
287///       "contextSize": 1000000
288///     }
289///   }
290/// }
291/// ```
292#[derive(Clone, Default, Debug, Serialize, Deserialize)]
293#[serde(rename_all = "camelCase")]
294pub struct ModelMetadata {
295    /// The version of the model metadata struct.
296    pub version: u16,
297
298    /// The mappings of model ID => metadata as a dictionary.
299    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
300    pub models: HashMap<ModelPattern, ModelMetadataEntry>,
301}
302
303impl ModelMetadata {
304    const SUPPORTED_VERSION: u16 = 1;
305
306    /// `true` if the model metadata is empty or the version is unsupported.
307    pub fn is_empty(&self) -> bool {
308        self.models.is_empty() || !self.is_enabled()
309    }
310
311    /// `false` if the version is unsupported.
312    pub fn is_enabled(&self) -> bool {
313        self.version == Self::SUPPORTED_VERSION
314    }
315
316    /// Gets the cost per token for a given model, if defined.
317    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
318        self.get(model_id).and_then(|entry| entry.costs.as_ref())
319    }
320
321    /// Gets the context window size for a given model, if defined.
322    ///
323    /// Returns `None` for a zero context size, as it is not a meaningful value.
324    pub fn context_size(&self, model_id: &str) -> Option<u64> {
325        self.get(model_id)
326            .and_then(|entry| entry.context_size)
327            .filter(|&size| size > 0)
328    }
329
330    /// Gets the metadata for a given model, if defined.
331    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
332        if !self.is_enabled() {
333            return None;
334        }
335
336        let normalized_model_id = normalize_ai_model_name(model_id);
337
338        // First try exact match.
339        if let Some(value) = self.models.get(normalized_model_id) {
340            return Some(value);
341        }
342
343        // Fall back to glob matching.
344        self.models.iter().find_map(|(key, value)| {
345            if key.is_match(normalized_model_id) {
346                Some(value)
347            } else {
348                None
349            }
350        })
351    }
352}
353
354/// A [`Pattern`] which remembers its original pattern and can be stored in a hash-map.
355#[derive(Debug, Clone)]
356pub struct ModelPattern {
357    original: Box<str>,
358    pattern: Pattern,
359}
360
361impl ModelPattern {
362    /// Returns `true` if the pattern matches the passed string.
363    pub fn is_match(&self, haystack: &str) -> bool {
364        self.pattern.is_match(haystack)
365    }
366}
367
368impl std::cmp::PartialEq for ModelPattern {
369    fn eq(&self, other: &Self) -> bool {
370        self.original == other.original
371    }
372}
373
374impl std::cmp::Eq for ModelPattern {}
375
376impl std::hash::Hash for ModelPattern {
377    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
378        self.original.hash(state);
379    }
380}
381
382impl std::borrow::Borrow<str> for ModelPattern {
383    fn borrow(&self) -> &str {
384        &self.original
385    }
386}
387
388impl std::str::FromStr for ModelPattern {
389    type Err = relay_pattern::Error;
390
391    fn from_str(s: &str) -> Result<Self, Self::Err> {
392        Ok(Self {
393            original: s.into(),
394            pattern: Pattern::new(s)?,
395        })
396    }
397}
398
399impl<'de> serde::de::Deserialize<'de> for ModelPattern {
400    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
401    where
402        D: serde::Deserializer<'de>,
403    {
404        let original = Box::<str>::deserialize(deserializer)?;
405        let pattern = Pattern::new(&original).map_err(serde::de::Error::custom)?;
406
407        Ok(Self { original, pattern })
408    }
409}
410
411impl serde::ser::Serialize for ModelPattern {
412    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
413    where
414        S: serde::Serializer,
415    {
416        serializer.serialize_str(&self.original)
417    }
418}
419
420/// Metadata for a single AI model.
421#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
422#[serde(rename_all = "camelCase")]
423pub struct ModelMetadataEntry {
424    /// Token costs for this model.
425    #[serde(default, skip_serializing_if = "Option::is_none")]
426    pub costs: Option<ModelCostV2>,
427
428    /// The context window size in tokens.
429    #[serde(default, skip_serializing_if = "Option::is_none")]
430    pub context_size: Option<u64>,
431}
432
433#[cfg(test)]
434mod tests {
435    use chrono::{TimeZone, Utc};
436    use insta::{assert_debug_snapshot, assert_json_snapshot};
437    use itertools::Itertools;
438    use relay_base_schema::events::EventType;
439    use relay_base_schema::metrics::DurationUnit;
440    use relay_base_schema::spans::SpanStatus;
441    use relay_event_schema::protocol::{
442        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, Event, EventId,
443        Exception, Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace,
444        ReplayContext, Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
445    };
446    use relay_protocol::{
447        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
448        assert_annotated_snapshot, get_path, get_value,
449    };
450    use serde_json::json;
451    use similar_asserts::assert_eq;
452    use uuid::Uuid;
453
454    use crate::stacktrace::normalize_non_raw_frame;
455    use crate::validation::validate_event;
456    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
457
458    use super::*;
459
460    #[test]
461    fn test_normalize_ai_model_name() {
462        // Test date patterns
463        assert_eq!(
464            normalize_ai_model_name("claude-4-sonnet-20250522"),
465            "claude-4-sonnet"
466        );
467        assert_eq!(normalize_ai_model_name("o3-pro-2025-06-10"), "o3-pro");
468        assert_eq!(
469            normalize_ai_model_name("claude-3-5-haiku@20241022"),
470            "claude-3-5-haiku"
471        );
472
473        // Test date with version patterns
474        assert_eq!(
475            normalize_ai_model_name("claude-opus-4-1-20250805-v1:0"),
476            "claude-opus-4-1"
477        );
478        assert_eq!(
479            normalize_ai_model_name("us.anthropic.claude-sonnet-4-20250514-v1:0"),
480            "us.anthropic.claude-sonnet-4"
481        );
482
483        // Test standalone version patterns with 'v' prefix
484        assert_eq!(normalize_ai_model_name("gpt-4-v1.0"), "gpt-4");
485        assert_eq!(normalize_ai_model_name("gpt-4-v1:0"), "gpt-4");
486        assert_eq!(normalize_ai_model_name("gpt-4-v1"), "gpt-4");
487        assert_eq!(normalize_ai_model_name("model_v2"), "model");
488
489        // Test that version patterns WITHOUT 'v' prefix are NOT stripped
490        assert_eq!(normalize_ai_model_name("gpt-4.5"), "gpt-4.5");
491
492        // Test that if version without 'v' comes after date, nothing is stripped
493        // (date regex requires end of string, so date won't match if followed by version)
494        assert_eq!(
495            normalize_ai_model_name("anthropic.claude-3-5-sonnet-20241022-v2:0"),
496            "anthropic.claude-3-5-sonnet"
497        );
498
499        // Test no pattern (should return original)
500        assert_eq!(normalize_ai_model_name("gpt-4"), "gpt-4");
501        assert_eq!(normalize_ai_model_name("claude-3-opus"), "claude-3-opus");
502    }
503
504    #[test]
505    fn test_merge_builtin_measurement_keys() {
506        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
507        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
508        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
509
510        let proj = MeasurementsConfig {
511            builtin_measurements: vec![foo.clone(), bar.clone()],
512            max_custom_measurements: 4,
513        };
514
515        let glob = MeasurementsConfig {
516            // The 'bar' here will be ignored since it's a duplicate from the project level.
517            builtin_measurements: vec![baz.clone(), bar.clone()],
518            max_custom_measurements: 4,
519        };
520        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
521
522        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
523
524        assert_eq!(keys, vec![&foo, &bar, &baz]);
525    }
526
527    #[test]
528    fn test_max_custom_measurement() {
529        // Empty configs will return a None value for max measurements.
530        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
531        assert!(dynamic_config.max_custom_measurements().is_none());
532
533        let proj = MeasurementsConfig {
534            builtin_measurements: vec![],
535            max_custom_measurements: 3,
536        };
537
538        let glob = MeasurementsConfig {
539            builtin_measurements: vec![],
540            max_custom_measurements: 4,
541        };
542
543        // If only project level measurement config is there, return its max custom measurement variable.
544        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
545        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
546
547        // Same logic for when only global level measurement config exists.
548        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
549        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
550
551        // If both is available, pick the smallest number.
552        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
553        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
554    }
555
556    #[test]
557    fn test_geo_from_ip_address() {
558        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
559
560        let json = r#"{
561            "user": {
562                "ip_address": "2.125.160.216"
563            }
564        }"#;
565        let mut event = Annotated::<Event>::from_json(json).unwrap();
566
567        normalize_event(
568            &mut event,
569            &NormalizationConfig {
570                geoip_lookup: Some(&lookup),
571                ..Default::default()
572            },
573        );
574
575        let expected = Annotated::new(Geo {
576            country_code: Annotated::new("GB".to_owned()),
577            city: Annotated::new("Boxford".to_owned()),
578            subdivision: Annotated::new("England".to_owned()),
579            region: Annotated::new("United Kingdom".to_owned()),
580            ..Geo::default()
581        });
582        assert_eq!(get_value!(event.user!).geo, expected);
583    }
584
585    #[test]
586    fn test_user_ip_from_remote_addr() {
587        let mut event = Annotated::new(Event {
588            request: Annotated::from(Request {
589                env: Annotated::new({
590                    let mut map = Object::new();
591                    map.insert(
592                        "REMOTE_ADDR".to_owned(),
593                        Annotated::new(Value::String("2.125.160.216".to_owned())),
594                    );
595                    map
596                }),
597                ..Request::default()
598            }),
599            platform: Annotated::new("javascript".to_owned()),
600            ..Event::default()
601        });
602
603        normalize_event(&mut event, &NormalizationConfig::default());
604
605        let ip_addr = get_value!(event.user.ip_address!);
606        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
607    }
608
609    #[test]
610    fn test_user_ip_from_invalid_remote_addr() {
611        let mut event = Annotated::new(Event {
612            request: Annotated::from(Request {
613                env: Annotated::new({
614                    let mut map = Object::new();
615                    map.insert(
616                        "REMOTE_ADDR".to_owned(),
617                        Annotated::new(Value::String("whoops".to_owned())),
618                    );
619                    map
620                }),
621                ..Request::default()
622            }),
623            platform: Annotated::new("javascript".to_owned()),
624            ..Event::default()
625        });
626
627        normalize_event(&mut event, &NormalizationConfig::default());
628
629        assert_eq!(Annotated::empty(), event.value().unwrap().user);
630    }
631
632    #[test]
633    fn test_user_ip_from_client_ip_without_auto() {
634        let mut event = Annotated::new(Event {
635            platform: Annotated::new("javascript".to_owned()),
636            ..Default::default()
637        });
638
639        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
640
641        normalize_event(
642            &mut event,
643            &NormalizationConfig {
644                client_ip: Some(&ip_address),
645                infer_ip_address: true,
646                ..Default::default()
647            },
648        );
649
650        let ip_addr = get_value!(event.user.ip_address!);
651        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
652    }
653
654    #[test]
655    fn test_user_ip_from_client_ip_with_auto() {
656        let mut event = Annotated::new(Event {
657            user: Annotated::new(User {
658                ip_address: Annotated::new(IpAddr::auto()),
659                ..Default::default()
660            }),
661            ..Default::default()
662        });
663
664        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
665
666        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
667        normalize_event(
668            &mut event,
669            &NormalizationConfig {
670                client_ip: Some(&ip_address),
671                geoip_lookup: Some(&geo),
672                infer_ip_address: true,
673                ..Default::default()
674            },
675        );
676
677        let user = get_value!(event.user!);
678        let ip_addr = user.ip_address.value().expect("ip address missing");
679
680        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
681        assert!(user.geo.value().is_some());
682    }
683
684    #[test]
685    fn test_user_ip_from_client_ip_without_appropriate_platform() {
686        let mut event = Annotated::new(Event::default());
687
688        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
689        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
690        normalize_event(
691            &mut event,
692            &NormalizationConfig {
693                client_ip: Some(&ip_address),
694                geoip_lookup: Some(&geo),
695                ..Default::default()
696            },
697        );
698
699        let user = get_value!(event.user!);
700        assert!(user.ip_address.value().is_none());
701        assert!(user.geo.value().is_some());
702    }
703
704    #[test]
705    fn test_geo_present_if_ip_inferring_disabled() {
706        let mut event = Annotated::new(Event {
707            user: Annotated::new(User {
708                ip_address: Annotated::new(IpAddr::auto()),
709                ..Default::default()
710            }),
711            ..Default::default()
712        });
713
714        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
715        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
716
717        normalize_event(
718            &mut event,
719            &NormalizationConfig {
720                client_ip: Some(&ip_address),
721                geoip_lookup: Some(&geo),
722                infer_ip_address: false,
723                ..Default::default()
724            },
725        );
726
727        let user = get_value!(event.user!);
728        assert!(user.ip_address.value().unwrap().is_auto());
729        assert!(user.geo.value().is_some());
730    }
731
732    #[test]
733    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
734        let mut event = Annotated::new(Event {
735            user: Annotated::new(User {
736                ip_address: Annotated::new(IpAddr::auto()),
737                ..Default::default()
738            }),
739            ..Default::default()
740        });
741
742        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
743        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
744
745        normalize_event(
746            &mut event,
747            &NormalizationConfig {
748                client_ip: Some(&ip_address),
749                geoip_lookup: Some(&geo),
750                infer_ip_address: true,
751                ..Default::default()
752            },
753        );
754
755        let user = get_value!(event.user!);
756        assert_eq!(
757            user.ip_address.value(),
758            Some(&IpAddr::parse("2.125.160.216").unwrap())
759        );
760        assert!(user.geo.value().is_some());
761    }
762
763    #[test]
764    fn test_event_level_defaulted() {
765        let mut event = Annotated::new(Event::default());
766        normalize_event(&mut event, &NormalizationConfig::default());
767        assert_eq!(get_value!(event.level), Some(&Level::Error));
768    }
769
770    #[test]
771    fn test_transaction_level_untouched() {
772        let mut event = Annotated::new(Event {
773            ty: Annotated::new(EventType::Transaction),
774            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
775            start_timestamp: Annotated::new(
776                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
777            ),
778            contexts: {
779                let mut contexts = Contexts::new();
780                contexts.add(TraceContext {
781                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
782                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
783                    op: Annotated::new("http.server".to_owned()),
784                    ..Default::default()
785                });
786                Annotated::new(contexts)
787            },
788            ..Event::default()
789        });
790        normalize_event(&mut event, &NormalizationConfig::default());
791        assert_eq!(get_value!(event.level), Some(&Level::Info));
792    }
793
794    #[test]
795    fn test_environment_tag_is_moved() {
796        let mut event = Annotated::new(Event {
797            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
798                Annotated::new("environment".to_owned()),
799                Annotated::new("despacito".to_owned()),
800            ))]))),
801            ..Event::default()
802        });
803
804        normalize_event(&mut event, &NormalizationConfig::default());
805
806        let event = event.value().unwrap();
807
808        assert_eq!(event.environment.as_str(), Some("despacito"));
809        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
810    }
811
812    #[test]
813    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
814        let mut event = Annotated::new(Event {
815            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
816                Annotated::new("environment".to_owned()),
817                Annotated::new("despacito".to_owned()),
818            ))]))),
819            environment: Annotated::new("".to_owned()),
820            ..Event::default()
821        });
822
823        normalize_event(&mut event, &NormalizationConfig::default());
824
825        let event = event.value().unwrap();
826
827        assert_eq!(event.environment.as_str(), Some("despacito"));
828        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
829    }
830
831    #[test]
832    fn test_empty_environment_is_removed() {
833        let mut event = Annotated::new(Event {
834            environment: Annotated::new("".to_owned()),
835            ..Event::default()
836        });
837
838        normalize_event(&mut event, &NormalizationConfig::default());
839        assert_eq!(get_value!(event.environment), None);
840    }
841    #[test]
842    fn test_replay_id_added_from_dsc() {
843        let replay_id = Uuid::new_v4();
844        let mut event = Annotated::new(Event {
845            contexts: Annotated::new(Contexts(Object::new())),
846            ..Event::default()
847        });
848        normalize_event(
849            &mut event,
850            &NormalizationConfig {
851                replay_id: Some(replay_id),
852                ..Default::default()
853            },
854        );
855
856        let event = event.value().unwrap();
857
858        assert_eq!(event.contexts, {
859            let mut contexts = Contexts::new();
860            contexts.add(ReplayContext {
861                replay_id: Annotated::new(EventId(replay_id)),
862                other: Object::default(),
863            });
864            Annotated::new(contexts)
865        })
866    }
867
868    #[test]
869    fn test_none_environment_errors() {
870        let mut event = Annotated::new(Event {
871            environment: Annotated::new("none".to_owned()),
872            ..Event::default()
873        });
874
875        normalize_event(&mut event, &NormalizationConfig::default());
876
877        let environment = get_path!(event.environment!);
878        let expected_original = &Value::String("none".to_owned());
879
880        assert_eq!(
881            environment.meta().iter_errors().collect::<Vec<&Error>>(),
882            vec![&Error::new(ErrorKind::InvalidData)],
883        );
884        assert_eq!(
885            environment.meta().original_value().unwrap(),
886            expected_original
887        );
888        assert_eq!(environment.value(), None);
889    }
890
891    #[test]
892    fn test_invalid_release_removed() {
893        let mut event = Annotated::new(Event {
894            release: Annotated::new(LenientString("Latest".to_owned())),
895            ..Event::default()
896        });
897
898        normalize_event(&mut event, &NormalizationConfig::default());
899
900        let release = get_path!(event.release!);
901        let expected_original = &Value::String("Latest".to_owned());
902
903        assert_eq!(
904            release.meta().iter_errors().collect::<Vec<&Error>>(),
905            vec![&Error::new(ErrorKind::InvalidData)],
906        );
907        assert_eq!(release.meta().original_value().unwrap(), expected_original);
908        assert_eq!(release.value(), None);
909    }
910
911    #[test]
912    fn test_top_level_keys_moved_into_tags() {
913        let mut event = Annotated::new(Event {
914            server_name: Annotated::new("foo".to_owned()),
915            site: Annotated::new("foo".to_owned()),
916            tags: Annotated::new(Tags(PairList(vec![
917                Annotated::new(TagEntry(
918                    Annotated::new("site".to_owned()),
919                    Annotated::new("old".to_owned()),
920                )),
921                Annotated::new(TagEntry(
922                    Annotated::new("server_name".to_owned()),
923                    Annotated::new("old".to_owned()),
924                )),
925            ]))),
926            ..Event::default()
927        });
928
929        normalize_event(&mut event, &NormalizationConfig::default());
930
931        assert_eq!(get_value!(event.site), None);
932        assert_eq!(get_value!(event.server_name), None);
933
934        assert_eq!(
935            get_value!(event.tags!),
936            &Tags(PairList(vec![
937                Annotated::new(TagEntry(
938                    Annotated::new("site".to_owned()),
939                    Annotated::new("foo".to_owned()),
940                )),
941                Annotated::new(TagEntry(
942                    Annotated::new("server_name".to_owned()),
943                    Annotated::new("foo".to_owned()),
944                )),
945            ]))
946        );
947    }
948
949    #[test]
950    fn test_internal_tags_removed() {
951        let mut event = Annotated::new(Event {
952            tags: Annotated::new(Tags(PairList(vec![
953                Annotated::new(TagEntry(
954                    Annotated::new("release".to_owned()),
955                    Annotated::new("foo".to_owned()),
956                )),
957                Annotated::new(TagEntry(
958                    Annotated::new("dist".to_owned()),
959                    Annotated::new("foo".to_owned()),
960                )),
961                Annotated::new(TagEntry(
962                    Annotated::new("user".to_owned()),
963                    Annotated::new("foo".to_owned()),
964                )),
965                Annotated::new(TagEntry(
966                    Annotated::new("filename".to_owned()),
967                    Annotated::new("foo".to_owned()),
968                )),
969                Annotated::new(TagEntry(
970                    Annotated::new("function".to_owned()),
971                    Annotated::new("foo".to_owned()),
972                )),
973                Annotated::new(TagEntry(
974                    Annotated::new("something".to_owned()),
975                    Annotated::new("else".to_owned()),
976                )),
977            ]))),
978            ..Event::default()
979        });
980
981        normalize_event(&mut event, &NormalizationConfig::default());
982
983        assert_eq!(get_value!(event.tags!).len(), 1);
984    }
985
986    #[test]
987    fn test_empty_tags_removed() {
988        let mut event = Annotated::new(Event {
989            tags: Annotated::new(Tags(PairList(vec![
990                Annotated::new(TagEntry(
991                    Annotated::new("".to_owned()),
992                    Annotated::new("foo".to_owned()),
993                )),
994                Annotated::new(TagEntry(
995                    Annotated::new("foo".to_owned()),
996                    Annotated::new("".to_owned()),
997                )),
998                Annotated::new(TagEntry(
999                    Annotated::new("something".to_owned()),
1000                    Annotated::new("else".to_owned()),
1001                )),
1002            ]))),
1003            ..Event::default()
1004        });
1005
1006        normalize_event(&mut event, &NormalizationConfig::default());
1007
1008        assert_eq!(
1009            get_value!(event.tags!),
1010            &Tags(PairList(vec![
1011                Annotated::new(TagEntry(
1012                    Annotated::from_error(Error::nonempty(), None),
1013                    Annotated::new("foo".to_owned()),
1014                )),
1015                Annotated::new(TagEntry(
1016                    Annotated::new("foo".to_owned()),
1017                    Annotated::from_error(Error::nonempty(), None),
1018                )),
1019                Annotated::new(TagEntry(
1020                    Annotated::new("something".to_owned()),
1021                    Annotated::new("else".to_owned()),
1022                )),
1023            ]))
1024        );
1025    }
1026
1027    #[test]
1028    fn test_tags_deduplicated() {
1029        let mut event = Annotated::new(Event {
1030            tags: Annotated::new(Tags(PairList(vec![
1031                Annotated::new(TagEntry(
1032                    Annotated::new("foo".to_owned()),
1033                    Annotated::new("1".to_owned()),
1034                )),
1035                Annotated::new(TagEntry(
1036                    Annotated::new("bar".to_owned()),
1037                    Annotated::new("1".to_owned()),
1038                )),
1039                Annotated::new(TagEntry(
1040                    Annotated::new("foo".to_owned()),
1041                    Annotated::new("2".to_owned()),
1042                )),
1043                Annotated::new(TagEntry(
1044                    Annotated::new("bar".to_owned()),
1045                    Annotated::new("2".to_owned()),
1046                )),
1047                Annotated::new(TagEntry(
1048                    Annotated::new("foo".to_owned()),
1049                    Annotated::new("3".to_owned()),
1050                )),
1051            ]))),
1052            ..Event::default()
1053        });
1054
1055        normalize_event(&mut event, &NormalizationConfig::default());
1056
1057        // should keep the first occurrence of every tag
1058        assert_eq!(
1059            get_value!(event.tags!),
1060            &Tags(PairList(vec![
1061                Annotated::new(TagEntry(
1062                    Annotated::new("foo".to_owned()),
1063                    Annotated::new("1".to_owned()),
1064                )),
1065                Annotated::new(TagEntry(
1066                    Annotated::new("bar".to_owned()),
1067                    Annotated::new("1".to_owned()),
1068                )),
1069            ]))
1070        );
1071    }
1072
1073    #[test]
1074    fn test_transaction_status_defaulted_to_unknown() {
1075        let mut object = Object::new();
1076        let trace_context = TraceContext {
1077            // We assume the status to be null.
1078            status: Annotated::empty(),
1079            ..TraceContext::default()
1080        };
1081        object.insert(
1082            "trace".to_owned(),
1083            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1084        );
1085
1086        let mut event = Annotated::new(Event {
1087            contexts: Annotated::new(Contexts(object)),
1088            ..Event::default()
1089        });
1090        normalize_event(&mut event, &NormalizationConfig::default());
1091
1092        let event = event.value().unwrap();
1093
1094        let event_trace_context = event.context::<TraceContext>().unwrap();
1095        assert_eq!(
1096            event_trace_context.status,
1097            Annotated::new(SpanStatus::Unknown)
1098        )
1099    }
1100
1101    #[test]
1102    fn test_unknown_debug_image() {
1103        let mut event = Annotated::new(Event {
1104            debug_meta: Annotated::new(DebugMeta {
1105                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1106                ..DebugMeta::default()
1107            }),
1108            ..Event::default()
1109        });
1110
1111        normalize_event(&mut event, &NormalizationConfig::default());
1112
1113        assert_eq!(
1114            get_path!(event.debug_meta!),
1115            &Annotated::new(DebugMeta {
1116                images: Annotated::new(vec![Annotated::from_error(
1117                    Error::invalid("unsupported debug image type"),
1118                    Some(Value::Object(Object::default())),
1119                )]),
1120                ..DebugMeta::default()
1121            })
1122        );
1123    }
1124
1125    #[test]
1126    fn test_context_line_default() {
1127        let mut frame = Annotated::new(Frame {
1128            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1129            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1130            ..Frame::default()
1131        });
1132
1133        normalize_non_raw_frame(&mut frame);
1134
1135        let frame = frame.value().unwrap();
1136        assert_eq!(frame.context_line.as_str(), Some(""));
1137    }
1138
1139    #[test]
1140    fn test_context_line_retain() {
1141        let mut frame = Annotated::new(Frame {
1142            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1143            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1144            context_line: Annotated::new("some line".to_owned()),
1145            ..Frame::default()
1146        });
1147
1148        normalize_non_raw_frame(&mut frame);
1149
1150        let frame = frame.value().unwrap();
1151        assert_eq!(frame.context_line.as_str(), Some("some line"));
1152    }
1153
1154    #[test]
1155    fn test_frame_null_context_lines() {
1156        let mut frame = Annotated::new(Frame {
1157            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1158            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1159            ..Frame::default()
1160        });
1161
1162        normalize_non_raw_frame(&mut frame);
1163
1164        assert_eq!(
1165            *get_value!(frame.pre_context!),
1166            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1167        );
1168        assert_eq!(
1169            *get_value!(frame.post_context!),
1170            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1171        );
1172    }
1173
1174    #[test]
1175    fn test_too_long_distribution() {
1176        let json = r#"{
1177  "event_id": "52df9022835246eeb317dbd739ccd059",
1178  "fingerprint": [
1179    "{{ default }}"
1180  ],
1181  "platform": "other",
1182  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1183}"#;
1184
1185        let mut event = Annotated::<Event>::from_json(json).unwrap();
1186
1187        normalize_event(&mut event, &NormalizationConfig::default());
1188
1189        let dist = &event.value().unwrap().dist;
1190        let result = &Annotated::<String>::from_error(
1191            Error::new(ErrorKind::ValueTooLong),
1192            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1193        );
1194        assert_eq!(dist, result);
1195    }
1196
1197    #[test]
1198    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1199        let mut event = Annotated::new(Event {
1200            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1201                ty: Annotated::new("FooDivisionError".to_owned()),
1202                value: Annotated::new("hi".to_owned().into()),
1203                ..Exception::default()
1204            })])),
1205            stacktrace: Annotated::new(
1206                RawStacktrace {
1207                    frames: Annotated::new(vec![Annotated::new(Frame {
1208                        module: Annotated::new("MyModule".to_owned()),
1209                        filename: Annotated::new("MyFilename".into()),
1210                        function: Annotated::new("Void FooBar()".to_owned()),
1211                        ..Frame::default()
1212                    })]),
1213                    ..RawStacktrace::default()
1214                }
1215                .into(),
1216            ),
1217            ..Event::default()
1218        });
1219
1220        normalize_event(&mut event, &NormalizationConfig::default());
1221
1222        assert_eq!(
1223            get_value!(event.exceptions.values[0].stacktrace!),
1224            &Stacktrace(RawStacktrace {
1225                frames: Annotated::new(vec![Annotated::new(Frame {
1226                    module: Annotated::new("MyModule".to_owned()),
1227                    filename: Annotated::new("MyFilename".into()),
1228                    abs_path: Annotated::new("MyFilename".into()),
1229                    function: Annotated::new("Void FooBar()".to_owned()),
1230                    ..Frame::default()
1231                })]),
1232                ..RawStacktrace::default()
1233            })
1234        );
1235    }
1236
1237    #[test]
1238    fn test_parses_sdk_info_from_header() {
1239        let mut event = Annotated::new(Event::default());
1240
1241        normalize_event(
1242            &mut event,
1243            &NormalizationConfig {
1244                client: Some("_fooBar/0.0.0".to_owned()),
1245                ..Default::default()
1246            },
1247        );
1248
1249        assert_eq!(
1250            get_path!(event.client_sdk!),
1251            &Annotated::new(ClientSdkInfo {
1252                name: Annotated::new("_fooBar".to_owned()),
1253                version: Annotated::new("0.0.0".to_owned()),
1254                ..ClientSdkInfo::default()
1255            })
1256        );
1257    }
1258
1259    #[test]
1260    fn test_discards_received() {
1261        let mut event = Annotated::new(Event {
1262            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1263            ..Default::default()
1264        });
1265
1266        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1267        normalize_event(&mut event, &NormalizationConfig::default());
1268
1269        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1270    }
1271
1272    #[test]
1273    fn test_grouping_config() {
1274        let mut event = Annotated::new(Event {
1275            logentry: Annotated::from(LogEntry {
1276                message: Annotated::new("Hello World!".to_owned().into()),
1277                ..Default::default()
1278            }),
1279            ..Default::default()
1280        });
1281
1282        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1283        normalize_event(
1284            &mut event,
1285            &NormalizationConfig {
1286                grouping_config: Some(json!({
1287                    "id": "legacy:1234-12-12".to_owned(),
1288                })),
1289                ..Default::default()
1290            },
1291        );
1292
1293        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1294            ".event_id" => "[event-id]",
1295            ".received" => "[received]",
1296            ".timestamp" => "[timestamp]"
1297        }, @r###"
1298        {
1299          "event_id": "[event-id]",
1300          "level": "error",
1301          "type": "default",
1302          "logentry": {
1303            "formatted": "Hello World!",
1304          },
1305          "logger": "",
1306          "platform": "other",
1307          "timestamp": "[timestamp]",
1308          "received": "[received]",
1309          "grouping_config": {
1310            "id": "legacy:1234-12-12",
1311          },
1312        }
1313        "###);
1314    }
1315
1316    #[test]
1317    fn test_logentry_error() {
1318        let json = r#"
1319{
1320    "event_id": "74ad1301f4df489ead37d757295442b1",
1321    "timestamp": 1668148328.308933,
1322    "received": 1668148328.308933,
1323    "level": "error",
1324    "platform": "python",
1325    "logentry": {
1326        "params": [
1327            "bogus"
1328        ],
1329        "formatted": 42
1330    }
1331}
1332"#;
1333        let mut event = Annotated::<Event>::from_json(json).unwrap();
1334
1335        normalize_event(&mut event, &NormalizationConfig::default());
1336
1337        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1338        {
1339          "event_id": "74ad1301f4df489ead37d757295442b1",
1340          "level": "error",
1341          "type": "default",
1342          "logentry": null,
1343          "logger": "",
1344          "platform": "python",
1345          "timestamp": 1668148328.308933,
1346          "received": "[received]",
1347          "_meta": {
1348            "logentry": {
1349              "": {
1350                "err": [
1351                  [
1352                    "invalid_data",
1353                    {
1354                      "reason": "no message present"
1355                    }
1356                  ]
1357                ],
1358                "val": {
1359                  "formatted": null,
1360                  "message": null,
1361                  "params": [
1362                    "bogus"
1363                  ]
1364                }
1365              }
1366            }
1367          }
1368        }
1369        "###);
1370    }
1371
1372    #[test]
1373    fn test_future_timestamp() {
1374        let mut event = Annotated::new(Event {
1375            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1376            ..Default::default()
1377        });
1378
1379        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1380        let max_secs_in_past = Some(30 * 24 * 3600);
1381        let max_secs_in_future = Some(60);
1382
1383        validate_event(
1384            &mut event,
1385            &EventValidationConfig {
1386                received_at,
1387                max_secs_in_past,
1388                max_secs_in_future,
1389                is_validated: false,
1390                ..Default::default()
1391            },
1392        )
1393        .unwrap();
1394        normalize_event(&mut event, &NormalizationConfig::default());
1395
1396        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1397        ".event_id" => "[event-id]",
1398    }, @r###"
1399        {
1400          "event_id": "[event-id]",
1401          "level": "error",
1402          "type": "default",
1403          "logger": "",
1404          "platform": "other",
1405          "timestamp": 946857600.0,
1406          "received": 946857600.0,
1407          "_meta": {
1408            "timestamp": {
1409              "": Meta(Some(MetaInner(
1410                err: [
1411                  [
1412                    "future_timestamp",
1413                    {
1414                      "sdk_time": "2000-01-03T00:02:00+00:00",
1415                      "server_time": "2000-01-03T00:00:00+00:00",
1416                    },
1417                  ],
1418                ],
1419              ))),
1420            },
1421          },
1422        }
1423        "###);
1424    }
1425
1426    #[test]
1427    fn test_past_timestamp() {
1428        let mut event = Annotated::new(Event {
1429            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1430            ..Default::default()
1431        });
1432
1433        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1434        let max_secs_in_past = Some(30 * 24 * 3600);
1435        let max_secs_in_future = Some(60);
1436
1437        validate_event(
1438            &mut event,
1439            &EventValidationConfig {
1440                received_at,
1441                max_secs_in_past,
1442                max_secs_in_future,
1443                is_validated: false,
1444                ..Default::default()
1445            },
1446        )
1447        .unwrap();
1448        normalize_event(&mut event, &NormalizationConfig::default());
1449
1450        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1451        ".event_id" => "[event-id]",
1452    }, @r###"
1453        {
1454          "event_id": "[event-id]",
1455          "level": "error",
1456          "type": "default",
1457          "logger": "",
1458          "platform": "other",
1459          "timestamp": 952041600.0,
1460          "received": 952041600.0,
1461          "_meta": {
1462            "timestamp": {
1463              "": Meta(Some(MetaInner(
1464                err: [
1465                  [
1466                    "past_timestamp",
1467                    {
1468                      "sdk_time": "2000-01-03T00:00:00+00:00",
1469                      "server_time": "2000-03-03T00:00:00+00:00",
1470                    },
1471                  ],
1472                ],
1473              ))),
1474            },
1475          },
1476        }
1477        "###);
1478    }
1479
1480    #[test]
1481    fn test_normalize_logger_empty() {
1482        let mut event = Event::from_value(
1483            serde_json::json!({
1484                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1485                "platform": "java",
1486                "logger": "",
1487            })
1488            .into(),
1489        );
1490
1491        normalize_event(&mut event, &NormalizationConfig::default());
1492        assert_annotated_snapshot!(event);
1493    }
1494
1495    #[test]
1496    fn test_normalize_logger_trimmed() {
1497        let mut event = Event::from_value(
1498            serde_json::json!({
1499                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1500                "platform": "java",
1501                "logger": " \t  \t   ",
1502            })
1503            .into(),
1504        );
1505
1506        normalize_event(&mut event, &NormalizationConfig::default());
1507        assert_annotated_snapshot!(event);
1508    }
1509
1510    #[test]
1511    fn test_normalize_logger_short_no_trimming() {
1512        let mut event = Event::from_value(
1513            serde_json::json!({
1514                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1515                "platform": "java",
1516                "logger": "my.short-logger.isnt_trimmed",
1517            })
1518            .into(),
1519        );
1520
1521        normalize_event(&mut event, &NormalizationConfig::default());
1522        assert_annotated_snapshot!(event);
1523    }
1524
1525    #[test]
1526    fn test_normalize_logger_exact_length() {
1527        let mut event = Event::from_value(
1528            serde_json::json!({
1529                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1530                "platform": "java",
1531                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1532            })
1533            .into(),
1534        );
1535
1536        normalize_event(&mut event, &NormalizationConfig::default());
1537        assert_annotated_snapshot!(event);
1538    }
1539
1540    #[test]
1541    fn test_normalize_logger_too_long_single_word() {
1542        let mut event = Event::from_value(
1543            serde_json::json!({
1544                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1545                "platform": "java",
1546                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1547            })
1548            .into(),
1549        );
1550
1551        normalize_event(&mut event, &NormalizationConfig::default());
1552        assert_annotated_snapshot!(event);
1553    }
1554
1555    #[test]
1556    fn test_normalize_logger_word_trimmed_at_max() {
1557        let mut event = Event::from_value(
1558            serde_json::json!({
1559                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1560                "platform": "java",
1561                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1562            })
1563            .into(),
1564        );
1565
1566        normalize_event(&mut event, &NormalizationConfig::default());
1567        assert_annotated_snapshot!(event);
1568    }
1569
1570    #[test]
1571    fn test_normalize_logger_word_trimmed_before_max() {
1572        // This test verifies the "smart" trimming on words -- the logger name
1573        // should be cut before the max limit, removing entire words.
1574        let mut event = Event::from_value(
1575            serde_json::json!({
1576                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1577                "platform": "java",
1578                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1579            })
1580            .into(),
1581        );
1582
1583        normalize_event(&mut event, &NormalizationConfig::default());
1584        assert_annotated_snapshot!(event);
1585    }
1586
1587    #[test]
1588    fn test_normalize_logger_word_leading_dots() {
1589        let mut event = Event::from_value(
1590            serde_json::json!({
1591                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1592                "platform": "java",
1593                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1594            })
1595            .into(),
1596        );
1597
1598        normalize_event(&mut event, &NormalizationConfig::default());
1599        assert_annotated_snapshot!(event);
1600    }
1601
1602    #[test]
1603    fn test_normalization_is_idempotent() {
1604        // get an event, normalize it. the result of that must be the same as normalizing it once more
1605        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1606        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1607        let mut event = Annotated::new(Event {
1608            ty: Annotated::new(EventType::Transaction),
1609            transaction: Annotated::new("/".to_owned()),
1610            timestamp: Annotated::new(end.into()),
1611            start_timestamp: Annotated::new(start.into()),
1612            contexts: {
1613                let mut contexts = Contexts::new();
1614                contexts.add(TraceContext {
1615                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1616                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1617                    op: Annotated::new("http.server".to_owned()),
1618                    ..Default::default()
1619                });
1620                Annotated::new(contexts)
1621            },
1622            spans: Annotated::new(vec![Annotated::new(Span {
1623                timestamp: Annotated::new(
1624                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1625                ),
1626                start_timestamp: Annotated::new(
1627                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1628                ),
1629                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1630                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1631
1632                ..Default::default()
1633            })]),
1634            ..Default::default()
1635        });
1636
1637        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1638            relay_event_schema::processor::apply(event, |e, _m| {
1639                e.received = Annotated::empty();
1640                Ok(())
1641            })
1642            .unwrap();
1643            event
1644        }
1645
1646        normalize_event(&mut event, &NormalizationConfig::default());
1647        let first = remove_received_from_event(&mut event.clone())
1648            .to_json()
1649            .unwrap();
1650        // Expected some fields (such as timestamps) exist after first normalization.
1651
1652        normalize_event(&mut event, &NormalizationConfig::default());
1653        let second = remove_received_from_event(&mut event.clone())
1654            .to_json()
1655            .unwrap();
1656        assert_eq!(&first, &second, "idempotency check failed");
1657
1658        normalize_event(&mut event, &NormalizationConfig::default());
1659        let third = remove_received_from_event(&mut event.clone())
1660            .to_json()
1661            .unwrap();
1662        assert_eq!(&second, &third, "idempotency check failed");
1663    }
1664
1665    /// Validate full normalization is idempotent.
1666    ///
1667    /// Both PoPs and processing relays will temporarily run full normalization
1668    /// in events, during the rollout of running normalization once in internal
1669    /// relays.
1670    // TODO(iker): remove this test after the rollout is done.
1671    #[test]
1672    fn test_full_normalization_is_idempotent() {
1673        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1674        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1675        let mut event = Annotated::new(Event {
1676            ty: Annotated::new(EventType::Transaction),
1677            transaction: Annotated::new("/".to_owned()),
1678            timestamp: Annotated::new(end.into()),
1679            start_timestamp: Annotated::new(start.into()),
1680            contexts: {
1681                let mut contexts = Contexts::new();
1682                contexts.add(TraceContext {
1683                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1684                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1685                    op: Annotated::new("http.server".to_owned()),
1686                    ..Default::default()
1687                });
1688                Annotated::new(contexts)
1689            },
1690            spans: Annotated::new(vec![Annotated::new(Span {
1691                timestamp: Annotated::new(
1692                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1693                ),
1694                start_timestamp: Annotated::new(
1695                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1696                ),
1697                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1698                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1699
1700                ..Default::default()
1701            })]),
1702            ..Default::default()
1703        });
1704
1705        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1706            relay_event_schema::processor::apply(event, |e, _m| {
1707                e.received = Annotated::empty();
1708                Ok(())
1709            })
1710            .unwrap();
1711            event
1712        }
1713
1714        let full_normalization_config = NormalizationConfig {
1715            is_renormalize: false,
1716            enable_trimming: true,
1717            remove_other: true,
1718            emit_event_errors: true,
1719            ..Default::default()
1720        };
1721
1722        normalize_event(&mut event, &full_normalization_config);
1723        let first = remove_received_from_event(&mut event.clone())
1724            .to_json()
1725            .unwrap();
1726        // Expected some fields (such as timestamps) exist after first normalization.
1727
1728        normalize_event(&mut event, &full_normalization_config);
1729        let second = remove_received_from_event(&mut event.clone())
1730            .to_json()
1731            .unwrap();
1732        assert_eq!(&first, &second, "idempotency check failed");
1733
1734        normalize_event(&mut event, &full_normalization_config);
1735        let third = remove_received_from_event(&mut event.clone())
1736            .to_json()
1737            .unwrap();
1738        assert_eq!(&second, &third, "idempotency check failed");
1739    }
1740
1741    #[test]
1742    fn test_normalize_validates_spans() {
1743        let event = Annotated::<Event>::from_json(
1744            r#"
1745            {
1746                "type": "transaction",
1747                "transaction": "/",
1748                "timestamp": 946684810.0,
1749                "start_timestamp": 946684800.0,
1750                "contexts": {
1751                    "trace": {
1752                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1753                    "span_id": "fa90fdead5f74053",
1754                    "op": "http.server",
1755                    "type": "trace"
1756                    }
1757                },
1758                "spans": []
1759            }
1760            "#,
1761        )
1762        .unwrap();
1763
1764        // All incomplete spans should be caught by normalization:
1765        for span in [
1766            r#"null"#,
1767            r#"{
1768              "timestamp": 946684810.0,
1769              "start_timestamp": 946684900.0,
1770              "span_id": "fa90fdead5f74053",
1771              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1772            }"#,
1773            r#"{
1774              "timestamp": 946684810.0,
1775              "span_id": "fa90fdead5f74053",
1776              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1777            }"#,
1778            r#"{
1779              "timestamp": 946684810.0,
1780              "start_timestamp": 946684800.0,
1781              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1782            }"#,
1783            r#"{
1784              "timestamp": 946684810.0,
1785              "start_timestamp": 946684800.0,
1786              "span_id": "fa90fdead5f74053"
1787            }"#,
1788        ] {
1789            let mut modified_event = event.clone();
1790            let event_ref = modified_event.value_mut().as_mut().unwrap();
1791            event_ref
1792                .spans
1793                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1794
1795            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1796
1797            assert!(res.is_err(), "{span:?}");
1798        }
1799    }
1800
1801    #[test]
1802    fn test_normalization_respects_is_renormalize() {
1803        let mut event = Annotated::<Event>::from_json(
1804            r#"
1805            {
1806                "type": "default",
1807                "tags": [["environment", "some_environment"]]
1808            }
1809            "#,
1810        )
1811        .unwrap();
1812
1813        normalize_event(
1814            &mut event,
1815            &NormalizationConfig {
1816                is_renormalize: true,
1817                ..Default::default()
1818            },
1819        );
1820
1821        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1822        Tags(
1823            PairList(
1824                [
1825                    TagEntry(
1826                        "environment",
1827                        "some_environment",
1828                    ),
1829                ],
1830            ),
1831        )
1832        "###);
1833    }
1834
1835    #[test]
1836    fn test_geo_in_normalize() {
1837        let mut event = Annotated::<Event>::from_json(
1838            r#"
1839            {
1840                "type": "transaction",
1841                "transaction": "/foo/",
1842                "timestamp": 946684810.0,
1843                "start_timestamp": 946684800.0,
1844                "contexts": {
1845                    "trace": {
1846                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1847                        "span_id": "fa90fdead5f74053",
1848                        "op": "http.server",
1849                        "type": "trace"
1850                    }
1851                },
1852                "transaction_info": {
1853                    "source": "url"
1854                },
1855                "user": {
1856                    "ip_address": "2.125.160.216"
1857                }
1858            }
1859            "#,
1860        )
1861        .unwrap();
1862
1863        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1864
1865        // Extract user's geo information before normalization.
1866        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1867        assert!(user_geo.is_none());
1868
1869        normalize_event(
1870            &mut event,
1871            &NormalizationConfig {
1872                geoip_lookup: Some(&lookup),
1873                ..Default::default()
1874            },
1875        );
1876
1877        // Extract user's geo information after normalization.
1878        let user_geo = event
1879            .value()
1880            .unwrap()
1881            .user
1882            .value()
1883            .unwrap()
1884            .geo
1885            .value()
1886            .unwrap();
1887
1888        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1889        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1890    }
1891
1892    #[test]
1893    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1894        let mut event = Event {
1895            spans: Annotated::new(vec![Annotated::new(Span {
1896                op: Annotated::new("app_start_cold".to_owned()),
1897                ..Default::default()
1898            })]),
1899            client_sdk: Annotated::new(ClientSdkInfo {
1900                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1901                version: Annotated::new("4.5.0".to_owned()),
1902                ..Default::default()
1903            }),
1904            ..Default::default()
1905        };
1906        span::normalize_app_start_spans(&mut event);
1907        assert_debug_snapshot!(event.spans, @r#"
1908        [
1909            Span {
1910                timestamp: ~,
1911                start_timestamp: ~,
1912                exclusive_time: ~,
1913                op: "app_start_cold",
1914                span_id: ~,
1915                parent_span_id: ~,
1916                trace_id: ~,
1917                segment_id: ~,
1918                is_segment: ~,
1919                is_remote: ~,
1920                status: ~,
1921                description: ~,
1922                tags: ~,
1923                origin: ~,
1924                profile_id: ~,
1925                data: ~,
1926                links: ~,
1927                sentry_tags: ~,
1928                received: ~,
1929                measurements: ~,
1930                platform: ~,
1931                was_transaction: ~,
1932                kind: ~,
1933                other: {},
1934            },
1935        ]
1936        "#);
1937    }
1938
1939    #[test]
1940    fn test_normalize_app_start_cold_spans_for_react_native() {
1941        let mut event = Event {
1942            spans: Annotated::new(vec![Annotated::new(Span {
1943                op: Annotated::new("app_start_cold".to_owned()),
1944                ..Default::default()
1945            })]),
1946            client_sdk: Annotated::new(ClientSdkInfo {
1947                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1948                version: Annotated::new("4.4.0".to_owned()),
1949                ..Default::default()
1950            }),
1951            ..Default::default()
1952        };
1953        span::normalize_app_start_spans(&mut event);
1954        assert_debug_snapshot!(event.spans, @r#"
1955        [
1956            Span {
1957                timestamp: ~,
1958                start_timestamp: ~,
1959                exclusive_time: ~,
1960                op: "app.start.cold",
1961                span_id: ~,
1962                parent_span_id: ~,
1963                trace_id: ~,
1964                segment_id: ~,
1965                is_segment: ~,
1966                is_remote: ~,
1967                status: ~,
1968                description: ~,
1969                tags: ~,
1970                origin: ~,
1971                profile_id: ~,
1972                data: ~,
1973                links: ~,
1974                sentry_tags: ~,
1975                received: ~,
1976                measurements: ~,
1977                platform: ~,
1978                was_transaction: ~,
1979                kind: ~,
1980                other: {},
1981            },
1982        ]
1983        "#);
1984    }
1985
1986    #[test]
1987    fn test_normalize_app_start_warm_spans_for_react_native() {
1988        let mut event = Event {
1989            spans: Annotated::new(vec![Annotated::new(Span {
1990                op: Annotated::new("app_start_warm".to_owned()),
1991                ..Default::default()
1992            })]),
1993            client_sdk: Annotated::new(ClientSdkInfo {
1994                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1995                version: Annotated::new("4.4.0".to_owned()),
1996                ..Default::default()
1997            }),
1998            ..Default::default()
1999        };
2000        span::normalize_app_start_spans(&mut event);
2001        assert_debug_snapshot!(event.spans, @r#"
2002        [
2003            Span {
2004                timestamp: ~,
2005                start_timestamp: ~,
2006                exclusive_time: ~,
2007                op: "app.start.warm",
2008                span_id: ~,
2009                parent_span_id: ~,
2010                trace_id: ~,
2011                segment_id: ~,
2012                is_segment: ~,
2013                is_remote: ~,
2014                status: ~,
2015                description: ~,
2016                tags: ~,
2017                origin: ~,
2018                profile_id: ~,
2019                data: ~,
2020                links: ~,
2021                sentry_tags: ~,
2022                received: ~,
2023                measurements: ~,
2024                platform: ~,
2025                was_transaction: ~,
2026                kind: ~,
2027                other: {},
2028            },
2029        ]
2030        "#);
2031    }
2032}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
