relay_event_normalization/normalize/
mod.rs

1use std::hash::Hash;
2use std::{collections::HashMap, sync::LazyLock};
3
4use regex::Regex;
5use relay_base_schema::metrics::MetricUnit;
6use relay_event_schema::protocol::VALID_PLATFORMS;
7use relay_pattern::{Gas, Pattern};
8use relay_protocol::{FiniteF64, RuleCondition};
9use serde::{Deserialize, Serialize};
10
11pub mod breakdowns;
12pub mod contexts;
13pub mod nel;
14pub mod request;
15pub mod span;
16pub mod user_agent;
17pub mod utils;
18
19/// Defines a builtin measurement.
20#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
21#[serde(default, rename_all = "camelCase")]
22pub struct BuiltinMeasurementKey {
23    name: String,
24    unit: MetricUnit,
25    #[serde(skip_serializing_if = "is_false")]
26    allow_negative: bool,
27}
28
29fn is_false(b: &bool) -> bool {
30    !b
31}
32
33impl BuiltinMeasurementKey {
34    /// Creates a new [`BuiltinMeasurementKey`].
35    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
36        Self {
37            name: name.into(),
38            unit,
39            allow_negative: false,
40        }
41    }
42
43    /// Returns the name of the built in measurement key.
44    pub fn name(&self) -> &str {
45        &self.name
46    }
47
48    /// Returns the unit of the built in measurement key.
49    pub fn unit(&self) -> &MetricUnit {
50        &self.unit
51    }
52
53    /// Return true if the built in measurement key allows negative values.
54    pub fn allow_negative(&self) -> &bool {
55        &self.allow_negative
56    }
57}
58
59/// Configuration for measurements normalization.
60#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
61#[serde(default, rename_all = "camelCase")]
62pub struct MeasurementsConfig {
63    /// A list of measurements that are built-in and are not subject to custom measurement limits.
64    #[serde(default, skip_serializing_if = "Vec::is_empty")]
65    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
66
67    /// The maximum number of measurements allowed per event that are not known measurements.
68    pub max_custom_measurements: usize,
69}
70
71impl MeasurementsConfig {
72    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
73    /// for every measurement-mri.
74    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
75}
76
77/// Returns `true` if the given platform string is a known platform identifier.
78///
79/// See [`VALID_PLATFORMS`] for a list of all known platforms.
80pub fn is_valid_platform(platform: &str) -> bool {
81    VALID_PLATFORMS.contains(&platform)
82}
83
84/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
85/// the merging logic.
86#[derive(Copy, Clone, Debug)]
87pub struct CombinedMeasurementsConfig<'a> {
88    project: Option<&'a MeasurementsConfig>,
89    global: Option<&'a MeasurementsConfig>,
90}
91
92impl<'a> CombinedMeasurementsConfig<'a> {
93    /// Constructor for [`CombinedMeasurementsConfig`].
94    pub fn new(
95        project: Option<&'a MeasurementsConfig>,
96        global: Option<&'a MeasurementsConfig>,
97    ) -> Self {
98        CombinedMeasurementsConfig { project, global }
99    }
100
101    /// Returns an iterator over the merged builtin measurement keys.
102    ///
103    /// Items from the project config are prioritized over global config, and
104    /// there are no duplicates.
105    pub fn builtin_measurement_keys(
106        &'a self,
107    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
108        let project = self
109            .project
110            .map(|p| p.builtin_measurements.as_slice())
111            .unwrap_or_default();
112
113        let global = self
114            .global
115            .map(|g| g.builtin_measurements.as_slice())
116            .unwrap_or_default();
117
118        project
119            .iter()
120            .chain(global.iter().filter(|key| !project.contains(key)))
121    }
122
123    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
124    /// global level. If both of them are available, it will choose the most restrictive.
125    pub fn max_custom_measurements(&'a self) -> Option<usize> {
126        match (&self.project, &self.global) {
127            (None, None) => None,
128            (None, Some(global)) => Some(global.max_custom_measurements),
129            (Some(project), None) => Some(project.max_custom_measurements),
130            (Some(project), Some(global)) => Some(std::cmp::min(
131                project.max_custom_measurements,
132                global.max_custom_measurements,
133            )),
134        }
135    }
136}
137
138/// Defines a weighted component for a performance score.
139///
140/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
141/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
142#[derive(Debug, Default, Clone, Serialize, Deserialize)]
143pub struct PerformanceScoreWeightedComponent {
144    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
145    /// profile will be discarded.
146    pub measurement: String,
147    /// Weight [0,1.0] of this component in the performance score
148    pub weight: FiniteF64,
149    /// p10 used to define the log-normal for calculation
150    pub p10: FiniteF64,
151    /// Median used to define the log-normal for calculation
152    pub p50: FiniteF64,
153    /// Whether the measurement is optional. If the measurement is missing, performance score processing
154    /// may still continue, and the weight will be 0.
155    #[serde(default)]
156    pub optional: bool,
157}
158
159/// Defines a profile for performance score.
160///
161/// A profile contains weights for a score of 100% and match against an event using a condition.
162/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
163#[derive(Debug, Default, Clone, Serialize, Deserialize)]
164#[serde(rename_all = "camelCase")]
165pub struct PerformanceScoreProfile {
166    /// Name of the profile, used for debugging and faceting multiple profiles
167    pub name: Option<String>,
168    /// Score components
169    #[serde(default, skip_serializing_if = "Vec::is_empty")]
170    pub score_components: Vec<PerformanceScoreWeightedComponent>,
171    /// See [`RuleCondition`] for all available options to specify and combine conditions.
172    pub condition: Option<RuleCondition>,
173    /// The version of the profile, used to isolate changes to score calculations.
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub version: Option<String>,
176}
177
178/// Defines the performance configuration for the project.
179///
180/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
181/// specific conditions.
182#[derive(Debug, Default, Clone, Serialize, Deserialize)]
183pub struct PerformanceScoreConfig {
184    /// List of performance profiles, only the first with matching conditions will be applied.
185    #[serde(default, skip_serializing_if = "Vec::is_empty")]
186    pub profiles: Vec<PerformanceScoreProfile>,
187}
188
189/// Regex that matches version and/or date patterns at the end of a model name.
190///
191/// Examples matched:
192/// - "-20250805-v1:0" in "claude-opus-4-1-20250805-v1:0" (both)
193/// - "-20250514-v1:0" in "claude-sonnet-4-20250514-v1:0" (both)
194/// - "_v2" in "model_v2" (version only)
195/// - "-v1.0" in "gpt-4-v1.0" (version only)
196/// - "-20250522" in "claude-4-sonnet-20250522" (date only)
197/// - "-2025-06-10" in "o3-pro-2025-06-10" (date only)
198/// - "@20241022" in "claude-3-5-haiku@20241022" (date only)
199static VERSION_AND_DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
200    Regex::new(
201        r"(?x)
202        (
203            ([-_@])                                    # Required separator before date (-, _, or @)
204            (\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8})          # Date: YYYY-MM-DD/YYYY/MM/DD/YYYY.MM.DD or YYYYMMDD
205        )?                                              # Date is optional
206        (
207            [-_]                                        # Required separator before version (- or _)
208            v\d+[:.]?\d*                                # Version: v1, v1.0, v1:0
209            ([-:].*)?                                   # Optional trailing content
210        )?                                              # Version is optional
211        $  # Must be at the end of the string
212        ",
213    )
214    .unwrap()
215});
216
217/// Normalizes an AI model name by stripping date and version patterns.
218///
219/// This function converts specific model versions into normalized names suitable for matching
220/// against cost configurations, ensuring that different versions of the same model can share
221/// cost information.
222///
223/// The normalization strips version and/or date patterns from the end of the string in a single pass.
224///
225/// Examples:
226/// - "claude-4-sonnet-20250522" -> "claude-4-sonnet"
227/// - "o3-pro-2025-06-10" -> "o3-pro"
228/// - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku"
229/// - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1"
230/// - "us.anthropic.claude-sonnet-4-20250514-v1:0" -> "us.anthropic.claude-sonnet-4"
231/// - "gpt-4-v1.0" -> "gpt-4"
232/// - "gpt-4-v1:0" -> "gpt-4"
233/// - "model_v2" -> "model"
234///
235/// Version patterns:
236/// - Must start with 'v': v1, v1.0, v1:0 (requires -, _ separator)
237///
238/// Date patterns:
239/// - YYYYMMDD: 20250522
240/// - YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD: 2025-06-10
241///
242/// Args:
243///     model_id: The original model ID
244///
245/// Returns:
246///     The normalized model name without date and version patterns
247fn normalize_ai_model_name(model_id: &str) -> &str {
248    if let Some(captures) = VERSION_AND_DATE_REGEX.captures(model_id) {
249        let match_idx = captures.get(0).map(|m| m.start()).unwrap_or(model_id.len());
250        &model_id[..match_idx]
251    } else {
252        model_id
253    }
254}
255
256/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
257#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
258#[serde(rename_all = "camelCase")]
259pub struct ModelCostV2 {
260    /// The cost per input token
261    pub input_per_token: f64,
262    /// The cost per output token
263    pub output_per_token: f64,
264    /// The cost per output reasoning token
265    pub output_reasoning_per_token: f64,
266    /// The cost per input cached token
267    pub input_cached_per_token: f64,
268    /// The cost per input cache write token
269    pub input_cache_write_per_token: f64,
270}
271
272/// Metadata for AI models including costs and context size.
273///
274/// Example JSON:
275/// ```json
276/// {
277///   "version": 1,
278///   "models": {
279///     "gpt-4": {
280///       "costs": {
281///         "inputPerToken": 0.0000003,
282///         "outputPerToken": 0.00000165,
283///         "outputReasoningPerToken": 0.0,
284///         "inputCachedPerToken": 0.0000015,
285///         "inputCacheWritePerToken": 0.00001875
286///       },
287///       "contextSize": 1000000
288///     }
289///   }
290/// }
291/// ```
292#[derive(Clone, Default, Debug, Serialize, Deserialize)]
293#[serde(rename_all = "camelCase")]
294pub struct ModelMetadata {
295    /// The version of the model metadata struct.
296    pub version: u16,
297
298    /// The mappings of model ID => metadata as a dictionary.
299    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
300    pub models: HashMap<ModelPattern, ModelMetadataEntry>,
301}
302
303impl ModelMetadata {
304    const SUPPORTED_VERSION: u16 = 1;
305
306    /// `true` if the model metadata is empty or the version is unsupported.
307    pub fn is_empty(&self) -> bool {
308        self.models.is_empty() || !self.is_enabled()
309    }
310
311    /// `false` if the version is unsupported.
312    pub fn is_enabled(&self) -> bool {
313        self.version == Self::SUPPORTED_VERSION
314    }
315
316    /// Gets the cost per token for a given model, if defined.
317    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
318        self.get(model_id).and_then(|entry| entry.costs.as_ref())
319    }
320
321    /// Gets the context window size for a given model, if defined.
322    ///
323    /// Returns `None` for a zero context size, as it is not a meaningful value.
324    pub fn context_size(&self, model_id: &str) -> Option<u64> {
325        self.get(model_id)
326            .and_then(|entry| entry.context_size)
327            .filter(|&size| size > 0)
328    }
329
330    /// Gets the metadata for a given model, if defined.
331    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
332        if !self.is_enabled() {
333            return None;
334        }
335
336        let normalized_model_id = normalize_ai_model_name(model_id);
337
338        // First try exact match.
339        if let Some(value) = self.models.get(normalized_model_id) {
340            return Some(value);
341        }
342
343        // Fall back to glob matching.
344        self.models.iter().find_map(|(key, value)| {
345            if key.is_match(normalized_model_id) {
346                Some(value)
347            } else {
348                None
349            }
350        })
351    }
352}
353
354/// A [`Pattern`] which remembers its original pattern and can be stored in a hash-map.
355#[derive(Debug, Clone)]
356pub struct ModelPattern {
357    original: Box<str>,
358    pattern: Pattern,
359}
360
361impl ModelPattern {
362    /// Returns `true` if the pattern matches the passed string.
363    pub fn is_match(&self, haystack: &str) -> bool {
364        // Patterns are expected to be safe and not user controlled.
365        self.pattern.is_match(haystack, Gas::MAX).unwrap_or(false)
366    }
367}
368
369impl std::cmp::PartialEq for ModelPattern {
370    fn eq(&self, other: &Self) -> bool {
371        self.original == other.original
372    }
373}
374
375impl std::cmp::Eq for ModelPattern {}
376
377impl std::hash::Hash for ModelPattern {
378    fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
379        self.original.hash(state);
380    }
381}
382
383impl std::borrow::Borrow<str> for ModelPattern {
384    fn borrow(&self) -> &str {
385        &self.original
386    }
387}
388
389impl std::str::FromStr for ModelPattern {
390    type Err = relay_pattern::Error;
391
392    fn from_str(s: &str) -> Result<Self, Self::Err> {
393        Ok(Self {
394            original: s.into(),
395            pattern: Pattern::new(s)?,
396        })
397    }
398}
399
400impl<'de> serde::de::Deserialize<'de> for ModelPattern {
401    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
402    where
403        D: serde::Deserializer<'de>,
404    {
405        let original = Box::<str>::deserialize(deserializer)?;
406        let pattern = Pattern::new(&original).map_err(serde::de::Error::custom)?;
407
408        Ok(Self { original, pattern })
409    }
410}
411
412impl serde::ser::Serialize for ModelPattern {
413    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
414    where
415        S: serde::Serializer,
416    {
417        serializer.serialize_str(&self.original)
418    }
419}
420
421/// Metadata for a single AI model.
422#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
423#[serde(rename_all = "camelCase")]
424pub struct ModelMetadataEntry {
425    /// Token costs for this model.
426    #[serde(default, skip_serializing_if = "Option::is_none")]
427    pub costs: Option<ModelCostV2>,
428
429    /// The context window size in tokens.
430    #[serde(default, skip_serializing_if = "Option::is_none")]
431    pub context_size: Option<u64>,
432}
433
434#[cfg(test)]
435mod tests {
436    use chrono::{TimeZone, Utc};
437    use insta::{assert_debug_snapshot, assert_json_snapshot};
438    use itertools::Itertools;
439    use relay_base_schema::events::EventType;
440    use relay_base_schema::metrics::DurationUnit;
441    use relay_base_schema::spans::SpanStatus;
442    use relay_event_schema::protocol::{
443        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, Event, EventId,
444        Exception, Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace,
445        ReplayContext, Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
446    };
447    use relay_protocol::{
448        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
449        assert_annotated_snapshot, get_path, get_value,
450    };
451    use serde_json::json;
452    use similar_asserts::assert_eq;
453    use uuid::Uuid;
454
455    use crate::stacktrace::normalize_non_raw_frame;
456    use crate::validation::validate_event;
457    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
458
459    use super::*;
460
461    #[test]
462    fn test_normalize_ai_model_name() {
463        // Test date patterns
464        assert_eq!(
465            normalize_ai_model_name("claude-4-sonnet-20250522"),
466            "claude-4-sonnet"
467        );
468        assert_eq!(normalize_ai_model_name("o3-pro-2025-06-10"), "o3-pro");
469        assert_eq!(
470            normalize_ai_model_name("claude-3-5-haiku@20241022"),
471            "claude-3-5-haiku"
472        );
473
474        // Test date with version patterns
475        assert_eq!(
476            normalize_ai_model_name("claude-opus-4-1-20250805-v1:0"),
477            "claude-opus-4-1"
478        );
479        assert_eq!(
480            normalize_ai_model_name("us.anthropic.claude-sonnet-4-20250514-v1:0"),
481            "us.anthropic.claude-sonnet-4"
482        );
483
484        // Test standalone version patterns with 'v' prefix
485        assert_eq!(normalize_ai_model_name("gpt-4-v1.0"), "gpt-4");
486        assert_eq!(normalize_ai_model_name("gpt-4-v1:0"), "gpt-4");
487        assert_eq!(normalize_ai_model_name("gpt-4-v1"), "gpt-4");
488        assert_eq!(normalize_ai_model_name("model_v2"), "model");
489
490        // Test that version patterns WITHOUT 'v' prefix are NOT stripped
491        assert_eq!(normalize_ai_model_name("gpt-4.5"), "gpt-4.5");
492
493        // Test that if version without 'v' comes after date, nothing is stripped
494        // (date regex requires end of string, so date won't match if followed by version)
495        assert_eq!(
496            normalize_ai_model_name("anthropic.claude-3-5-sonnet-20241022-v2:0"),
497            "anthropic.claude-3-5-sonnet"
498        );
499
500        // Test no pattern (should return original)
501        assert_eq!(normalize_ai_model_name("gpt-4"), "gpt-4");
502        assert_eq!(normalize_ai_model_name("claude-3-opus"), "claude-3-opus");
503    }
504
505    #[test]
506    fn test_merge_builtin_measurement_keys() {
507        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
508        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
509        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
510
511        let proj = MeasurementsConfig {
512            builtin_measurements: vec![foo.clone(), bar.clone()],
513            max_custom_measurements: 4,
514        };
515
516        let glob = MeasurementsConfig {
517            // The 'bar' here will be ignored since it's a duplicate from the project level.
518            builtin_measurements: vec![baz.clone(), bar.clone()],
519            max_custom_measurements: 4,
520        };
521        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
522
523        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
524
525        assert_eq!(keys, vec![&foo, &bar, &baz]);
526    }
527
528    #[test]
529    fn test_max_custom_measurement() {
530        // Empty configs will return a None value for max measurements.
531        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
532        assert!(dynamic_config.max_custom_measurements().is_none());
533
534        let proj = MeasurementsConfig {
535            builtin_measurements: vec![],
536            max_custom_measurements: 3,
537        };
538
539        let glob = MeasurementsConfig {
540            builtin_measurements: vec![],
541            max_custom_measurements: 4,
542        };
543
544        // If only project level measurement config is there, return its max custom measurement variable.
545        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
546        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
547
548        // Same logic for when only global level measurement config exists.
549        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
550        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
551
552        // If both is available, pick the smallest number.
553        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
554        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
555    }
556
557    #[test]
558    fn test_geo_from_ip_address() {
559        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
560
561        let json = r#"{
562            "user": {
563                "ip_address": "2.125.160.216"
564            }
565        }"#;
566        let mut event = Annotated::<Event>::from_json(json).unwrap();
567
568        normalize_event(
569            &mut event,
570            &NormalizationConfig {
571                geoip_lookup: Some(&lookup),
572                ..Default::default()
573            },
574        );
575
576        let expected = Annotated::new(Geo {
577            country_code: Annotated::new("GB".to_owned()),
578            city: Annotated::new("Boxford".to_owned()),
579            subdivision: Annotated::new("England".to_owned()),
580            region: Annotated::new("United Kingdom".to_owned()),
581            ..Geo::default()
582        });
583        assert_eq!(get_value!(event.user!).geo, expected);
584    }
585
586    #[test]
587    fn test_user_ip_from_remote_addr() {
588        let mut event = Annotated::new(Event {
589            request: Annotated::from(Request {
590                env: Annotated::new({
591                    let mut map = Object::new();
592                    map.insert(
593                        "REMOTE_ADDR".to_owned(),
594                        Annotated::new(Value::String("2.125.160.216".to_owned())),
595                    );
596                    map
597                }),
598                ..Request::default()
599            }),
600            platform: Annotated::new("javascript".to_owned()),
601            ..Event::default()
602        });
603
604        normalize_event(&mut event, &NormalizationConfig::default());
605
606        let ip_addr = get_value!(event.user.ip_address!);
607        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
608    }
609
610    #[test]
611    fn test_user_ip_from_invalid_remote_addr() {
612        let mut event = Annotated::new(Event {
613            request: Annotated::from(Request {
614                env: Annotated::new({
615                    let mut map = Object::new();
616                    map.insert(
617                        "REMOTE_ADDR".to_owned(),
618                        Annotated::new(Value::String("whoops".to_owned())),
619                    );
620                    map
621                }),
622                ..Request::default()
623            }),
624            platform: Annotated::new("javascript".to_owned()),
625            ..Event::default()
626        });
627
628        normalize_event(&mut event, &NormalizationConfig::default());
629
630        assert_eq!(Annotated::empty(), event.value().unwrap().user);
631    }
632
633    #[test]
634    fn test_user_ip_from_client_ip_without_auto() {
635        let mut event = Annotated::new(Event {
636            platform: Annotated::new("javascript".to_owned()),
637            ..Default::default()
638        });
639
640        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
641
642        normalize_event(
643            &mut event,
644            &NormalizationConfig {
645                client_ip: Some(&ip_address),
646                infer_ip_address: true,
647                ..Default::default()
648            },
649        );
650
651        let ip_addr = get_value!(event.user.ip_address!);
652        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
653    }
654
655    #[test]
656    fn test_user_ip_from_client_ip_with_auto() {
657        let mut event = Annotated::new(Event {
658            user: Annotated::new(User {
659                ip_address: Annotated::new(IpAddr::auto()),
660                ..Default::default()
661            }),
662            ..Default::default()
663        });
664
665        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
666
667        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
668        normalize_event(
669            &mut event,
670            &NormalizationConfig {
671                client_ip: Some(&ip_address),
672                geoip_lookup: Some(&geo),
673                infer_ip_address: true,
674                ..Default::default()
675            },
676        );
677
678        let user = get_value!(event.user!);
679        let ip_addr = user.ip_address.value().expect("ip address missing");
680
681        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
682        assert!(user.geo.value().is_some());
683    }
684
685    #[test]
686    fn test_user_ip_from_client_ip_without_appropriate_platform() {
687        let mut event = Annotated::new(Event::default());
688
689        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
690        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
691        normalize_event(
692            &mut event,
693            &NormalizationConfig {
694                client_ip: Some(&ip_address),
695                geoip_lookup: Some(&geo),
696                ..Default::default()
697            },
698        );
699
700        let user = get_value!(event.user!);
701        assert!(user.ip_address.value().is_none());
702        assert!(user.geo.value().is_some());
703    }
704
705    #[test]
706    fn test_geo_present_if_ip_inferring_disabled() {
707        let mut event = Annotated::new(Event {
708            user: Annotated::new(User {
709                ip_address: Annotated::new(IpAddr::auto()),
710                ..Default::default()
711            }),
712            ..Default::default()
713        });
714
715        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
716        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
717
718        normalize_event(
719            &mut event,
720            &NormalizationConfig {
721                client_ip: Some(&ip_address),
722                geoip_lookup: Some(&geo),
723                infer_ip_address: false,
724                ..Default::default()
725            },
726        );
727
728        let user = get_value!(event.user!);
729        assert!(user.ip_address.value().unwrap().is_auto());
730        assert!(user.geo.value().is_some());
731    }
732
733    #[test]
734    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
735        let mut event = Annotated::new(Event {
736            user: Annotated::new(User {
737                ip_address: Annotated::new(IpAddr::auto()),
738                ..Default::default()
739            }),
740            ..Default::default()
741        });
742
743        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
744        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
745
746        normalize_event(
747            &mut event,
748            &NormalizationConfig {
749                client_ip: Some(&ip_address),
750                geoip_lookup: Some(&geo),
751                infer_ip_address: true,
752                ..Default::default()
753            },
754        );
755
756        let user = get_value!(event.user!);
757        assert_eq!(
758            user.ip_address.value(),
759            Some(&IpAddr::parse("2.125.160.216").unwrap())
760        );
761        assert!(user.geo.value().is_some());
762    }
763
764    #[test]
765    fn test_event_level_defaulted() {
766        let mut event = Annotated::new(Event::default());
767        normalize_event(&mut event, &NormalizationConfig::default());
768        assert_eq!(get_value!(event.level), Some(&Level::Error));
769    }
770
771    #[test]
772    fn test_transaction_level_untouched() {
773        let mut event = Annotated::new(Event {
774            ty: Annotated::new(EventType::Transaction),
775            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
776            start_timestamp: Annotated::new(
777                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
778            ),
779            contexts: {
780                let mut contexts = Contexts::new();
781                contexts.add(TraceContext {
782                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
783                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
784                    op: Annotated::new("http.server".to_owned()),
785                    ..Default::default()
786                });
787                Annotated::new(contexts)
788            },
789            ..Event::default()
790        });
791        normalize_event(&mut event, &NormalizationConfig::default());
792        assert_eq!(get_value!(event.level), Some(&Level::Info));
793    }
794
795    #[test]
796    fn test_environment_tag_is_moved() {
797        let mut event = Annotated::new(Event {
798            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
799                Annotated::new("environment".to_owned()),
800                Annotated::new("despacito".to_owned()),
801            ))]))),
802            ..Event::default()
803        });
804
805        normalize_event(&mut event, &NormalizationConfig::default());
806
807        let event = event.value().unwrap();
808
809        assert_eq!(event.environment.as_str(), Some("despacito"));
810        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
811    }
812
813    #[test]
814    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
815        let mut event = Annotated::new(Event {
816            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
817                Annotated::new("environment".to_owned()),
818                Annotated::new("despacito".to_owned()),
819            ))]))),
820            environment: Annotated::new("".to_owned()),
821            ..Event::default()
822        });
823
824        normalize_event(&mut event, &NormalizationConfig::default());
825
826        let event = event.value().unwrap();
827
828        assert_eq!(event.environment.as_str(), Some("despacito"));
829        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
830    }
831
832    #[test]
833    fn test_empty_environment_is_removed() {
834        let mut event = Annotated::new(Event {
835            environment: Annotated::new("".to_owned()),
836            ..Event::default()
837        });
838
839        normalize_event(&mut event, &NormalizationConfig::default());
840        assert_eq!(get_value!(event.environment), None);
841    }
842    #[test]
843    fn test_replay_id_added_from_dsc() {
844        let replay_id = Uuid::new_v4();
845        let mut event = Annotated::new(Event {
846            contexts: Annotated::new(Contexts(Object::new())),
847            ..Event::default()
848        });
849        normalize_event(
850            &mut event,
851            &NormalizationConfig {
852                replay_id: Some(replay_id),
853                ..Default::default()
854            },
855        );
856
857        let event = event.value().unwrap();
858
859        assert_eq!(event.contexts, {
860            let mut contexts = Contexts::new();
861            contexts.add(ReplayContext {
862                replay_id: Annotated::new(EventId(replay_id)),
863                other: Object::default(),
864            });
865            Annotated::new(contexts)
866        })
867    }
868
869    #[test]
870    fn test_none_environment_errors() {
871        let mut event = Annotated::new(Event {
872            environment: Annotated::new("none".to_owned()),
873            ..Event::default()
874        });
875
876        normalize_event(&mut event, &NormalizationConfig::default());
877
878        let environment = get_path!(event.environment!);
879        let expected_original = &Value::String("none".to_owned());
880
881        assert_eq!(
882            environment.meta().iter_errors().collect::<Vec<&Error>>(),
883            vec![&Error::new(ErrorKind::InvalidData)],
884        );
885        assert_eq!(
886            environment.meta().original_value().unwrap(),
887            expected_original
888        );
889        assert_eq!(environment.value(), None);
890    }
891
892    #[test]
893    fn test_invalid_release_removed() {
894        let mut event = Annotated::new(Event {
895            release: Annotated::new(LenientString("Latest".to_owned())),
896            ..Event::default()
897        });
898
899        normalize_event(&mut event, &NormalizationConfig::default());
900
901        let release = get_path!(event.release!);
902        let expected_original = &Value::String("Latest".to_owned());
903
904        assert_eq!(
905            release.meta().iter_errors().collect::<Vec<&Error>>(),
906            vec![&Error::new(ErrorKind::InvalidData)],
907        );
908        assert_eq!(release.meta().original_value().unwrap(), expected_original);
909        assert_eq!(release.value(), None);
910    }
911
912    #[test]
913    fn test_top_level_keys_moved_into_tags() {
914        let mut event = Annotated::new(Event {
915            server_name: Annotated::new("foo".to_owned()),
916            site: Annotated::new("foo".to_owned()),
917            tags: Annotated::new(Tags(PairList(vec![
918                Annotated::new(TagEntry(
919                    Annotated::new("site".to_owned()),
920                    Annotated::new("old".to_owned()),
921                )),
922                Annotated::new(TagEntry(
923                    Annotated::new("server_name".to_owned()),
924                    Annotated::new("old".to_owned()),
925                )),
926            ]))),
927            ..Event::default()
928        });
929
930        normalize_event(&mut event, &NormalizationConfig::default());
931
932        assert_eq!(get_value!(event.site), None);
933        assert_eq!(get_value!(event.server_name), None);
934
935        assert_eq!(
936            get_value!(event.tags!),
937            &Tags(PairList(vec![
938                Annotated::new(TagEntry(
939                    Annotated::new("site".to_owned()),
940                    Annotated::new("foo".to_owned()),
941                )),
942                Annotated::new(TagEntry(
943                    Annotated::new("server_name".to_owned()),
944                    Annotated::new("foo".to_owned()),
945                )),
946            ]))
947        );
948    }
949
950    #[test]
951    fn test_internal_tags_removed() {
952        let mut event = Annotated::new(Event {
953            tags: Annotated::new(Tags(PairList(vec![
954                Annotated::new(TagEntry(
955                    Annotated::new("release".to_owned()),
956                    Annotated::new("foo".to_owned()),
957                )),
958                Annotated::new(TagEntry(
959                    Annotated::new("dist".to_owned()),
960                    Annotated::new("foo".to_owned()),
961                )),
962                Annotated::new(TagEntry(
963                    Annotated::new("user".to_owned()),
964                    Annotated::new("foo".to_owned()),
965                )),
966                Annotated::new(TagEntry(
967                    Annotated::new("filename".to_owned()),
968                    Annotated::new("foo".to_owned()),
969                )),
970                Annotated::new(TagEntry(
971                    Annotated::new("function".to_owned()),
972                    Annotated::new("foo".to_owned()),
973                )),
974                Annotated::new(TagEntry(
975                    Annotated::new("something".to_owned()),
976                    Annotated::new("else".to_owned()),
977                )),
978            ]))),
979            ..Event::default()
980        });
981
982        normalize_event(&mut event, &NormalizationConfig::default());
983
984        assert_eq!(get_value!(event.tags!).len(), 1);
985    }
986
987    #[test]
988    fn test_empty_tags_removed() {
989        let mut event = Annotated::new(Event {
990            tags: Annotated::new(Tags(PairList(vec![
991                Annotated::new(TagEntry(
992                    Annotated::new("".to_owned()),
993                    Annotated::new("foo".to_owned()),
994                )),
995                Annotated::new(TagEntry(
996                    Annotated::new("foo".to_owned()),
997                    Annotated::new("".to_owned()),
998                )),
999                Annotated::new(TagEntry(
1000                    Annotated::new("something".to_owned()),
1001                    Annotated::new("else".to_owned()),
1002                )),
1003            ]))),
1004            ..Event::default()
1005        });
1006
1007        normalize_event(&mut event, &NormalizationConfig::default());
1008
1009        assert_eq!(
1010            get_value!(event.tags!),
1011            &Tags(PairList(vec![
1012                Annotated::new(TagEntry(
1013                    Annotated::from_error(Error::nonempty(), None),
1014                    Annotated::new("foo".to_owned()),
1015                )),
1016                Annotated::new(TagEntry(
1017                    Annotated::new("foo".to_owned()),
1018                    Annotated::from_error(Error::nonempty(), None),
1019                )),
1020                Annotated::new(TagEntry(
1021                    Annotated::new("something".to_owned()),
1022                    Annotated::new("else".to_owned()),
1023                )),
1024            ]))
1025        );
1026    }
1027
1028    #[test]
1029    fn test_tags_deduplicated() {
1030        let mut event = Annotated::new(Event {
1031            tags: Annotated::new(Tags(PairList(vec![
1032                Annotated::new(TagEntry(
1033                    Annotated::new("foo".to_owned()),
1034                    Annotated::new("1".to_owned()),
1035                )),
1036                Annotated::new(TagEntry(
1037                    Annotated::new("bar".to_owned()),
1038                    Annotated::new("1".to_owned()),
1039                )),
1040                Annotated::new(TagEntry(
1041                    Annotated::new("foo".to_owned()),
1042                    Annotated::new("2".to_owned()),
1043                )),
1044                Annotated::new(TagEntry(
1045                    Annotated::new("bar".to_owned()),
1046                    Annotated::new("2".to_owned()),
1047                )),
1048                Annotated::new(TagEntry(
1049                    Annotated::new("foo".to_owned()),
1050                    Annotated::new("3".to_owned()),
1051                )),
1052            ]))),
1053            ..Event::default()
1054        });
1055
1056        normalize_event(&mut event, &NormalizationConfig::default());
1057
1058        // should keep the first occurrence of every tag
1059        assert_eq!(
1060            get_value!(event.tags!),
1061            &Tags(PairList(vec![
1062                Annotated::new(TagEntry(
1063                    Annotated::new("foo".to_owned()),
1064                    Annotated::new("1".to_owned()),
1065                )),
1066                Annotated::new(TagEntry(
1067                    Annotated::new("bar".to_owned()),
1068                    Annotated::new("1".to_owned()),
1069                )),
1070            ]))
1071        );
1072    }
1073
1074    #[test]
1075    fn test_transaction_status_defaulted_to_unknown() {
1076        let mut object = Object::new();
1077        let trace_context = TraceContext {
1078            // We assume the status to be null.
1079            status: Annotated::empty(),
1080            ..TraceContext::default()
1081        };
1082        object.insert(
1083            "trace".to_owned(),
1084            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1085        );
1086
1087        let mut event = Annotated::new(Event {
1088            contexts: Annotated::new(Contexts(object)),
1089            ..Event::default()
1090        });
1091        normalize_event(&mut event, &NormalizationConfig::default());
1092
1093        let event = event.value().unwrap();
1094
1095        let event_trace_context = event.context::<TraceContext>().unwrap();
1096        assert_eq!(
1097            event_trace_context.status,
1098            Annotated::new(SpanStatus::Unknown)
1099        )
1100    }
1101
1102    #[test]
1103    fn test_unknown_debug_image() {
1104        let mut event = Annotated::new(Event {
1105            debug_meta: Annotated::new(DebugMeta {
1106                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1107                ..DebugMeta::default()
1108            }),
1109            ..Event::default()
1110        });
1111
1112        normalize_event(&mut event, &NormalizationConfig::default());
1113
1114        assert_eq!(
1115            get_path!(event.debug_meta!),
1116            &Annotated::new(DebugMeta {
1117                images: Annotated::new(vec![Annotated::from_error(
1118                    Error::invalid("unsupported debug image type"),
1119                    Some(Value::Object(Object::default())),
1120                )]),
1121                ..DebugMeta::default()
1122            })
1123        );
1124    }
1125
1126    #[test]
1127    fn test_context_line_default() {
1128        let mut frame = Annotated::new(Frame {
1129            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1130            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1131            ..Frame::default()
1132        });
1133
1134        normalize_non_raw_frame(&mut frame);
1135
1136        let frame = frame.value().unwrap();
1137        assert_eq!(frame.context_line.as_str(), Some(""));
1138    }
1139
1140    #[test]
1141    fn test_context_line_retain() {
1142        let mut frame = Annotated::new(Frame {
1143            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1144            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1145            context_line: Annotated::new("some line".to_owned()),
1146            ..Frame::default()
1147        });
1148
1149        normalize_non_raw_frame(&mut frame);
1150
1151        let frame = frame.value().unwrap();
1152        assert_eq!(frame.context_line.as_str(), Some("some line"));
1153    }
1154
1155    #[test]
1156    fn test_frame_null_context_lines() {
1157        let mut frame = Annotated::new(Frame {
1158            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1159            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1160            ..Frame::default()
1161        });
1162
1163        normalize_non_raw_frame(&mut frame);
1164
1165        assert_eq!(
1166            *get_value!(frame.pre_context!),
1167            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1168        );
1169        assert_eq!(
1170            *get_value!(frame.post_context!),
1171            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1172        );
1173    }
1174
1175    #[test]
1176    fn test_too_long_distribution() {
1177        let json = r#"{
1178  "event_id": "52df9022835246eeb317dbd739ccd059",
1179  "fingerprint": [
1180    "{{ default }}"
1181  ],
1182  "platform": "other",
1183  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1184}"#;
1185
1186        let mut event = Annotated::<Event>::from_json(json).unwrap();
1187
1188        normalize_event(&mut event, &NormalizationConfig::default());
1189
1190        let dist = &event.value().unwrap().dist;
1191        let result = &Annotated::<String>::from_error(
1192            Error::new(ErrorKind::ValueTooLong),
1193            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1194        );
1195        assert_eq!(dist, result);
1196    }
1197
1198    #[test]
1199    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1200        let mut event = Annotated::new(Event {
1201            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1202                ty: Annotated::new("FooDivisionError".to_owned()),
1203                value: Annotated::new("hi".to_owned().into()),
1204                ..Exception::default()
1205            })])),
1206            stacktrace: Annotated::new(
1207                RawStacktrace {
1208                    frames: Annotated::new(vec![Annotated::new(Frame {
1209                        module: Annotated::new("MyModule".to_owned()),
1210                        filename: Annotated::new("MyFilename".into()),
1211                        function: Annotated::new("Void FooBar()".to_owned()),
1212                        ..Frame::default()
1213                    })]),
1214                    ..RawStacktrace::default()
1215                }
1216                .into(),
1217            ),
1218            ..Event::default()
1219        });
1220
1221        normalize_event(&mut event, &NormalizationConfig::default());
1222
1223        assert_eq!(
1224            get_value!(event.exceptions.values[0].stacktrace!),
1225            &Stacktrace(RawStacktrace {
1226                frames: Annotated::new(vec![Annotated::new(Frame {
1227                    module: Annotated::new("MyModule".to_owned()),
1228                    filename: Annotated::new("MyFilename".into()),
1229                    abs_path: Annotated::new("MyFilename".into()),
1230                    function: Annotated::new("Void FooBar()".to_owned()),
1231                    ..Frame::default()
1232                })]),
1233                ..RawStacktrace::default()
1234            })
1235        );
1236    }
1237
1238    #[test]
1239    fn test_parses_sdk_info_from_header() {
1240        let mut event = Annotated::new(Event::default());
1241
1242        normalize_event(
1243            &mut event,
1244            &NormalizationConfig {
1245                client: Some("_fooBar/0.0.0".to_owned()),
1246                ..Default::default()
1247            },
1248        );
1249
1250        assert_eq!(
1251            get_path!(event.client_sdk!),
1252            &Annotated::new(ClientSdkInfo {
1253                name: Annotated::new("_fooBar".to_owned()),
1254                version: Annotated::new("0.0.0".to_owned()),
1255                ..ClientSdkInfo::default()
1256            })
1257        );
1258    }
1259
1260    #[test]
1261    fn test_discards_received() {
1262        let mut event = Annotated::new(Event {
1263            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1264            ..Default::default()
1265        });
1266
1267        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1268        normalize_event(&mut event, &NormalizationConfig::default());
1269
1270        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1271    }
1272
1273    #[test]
1274    fn test_grouping_config() {
1275        let mut event = Annotated::new(Event {
1276            logentry: Annotated::from(LogEntry {
1277                message: Annotated::new("Hello World!".to_owned().into()),
1278                ..Default::default()
1279            }),
1280            ..Default::default()
1281        });
1282
1283        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1284        normalize_event(
1285            &mut event,
1286            &NormalizationConfig {
1287                grouping_config: Some(json!({
1288                    "id": "legacy:1234-12-12".to_owned(),
1289                })),
1290                ..Default::default()
1291            },
1292        );
1293
1294        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1295            ".event_id" => "[event-id]",
1296            ".received" => "[received]",
1297            ".timestamp" => "[timestamp]"
1298        }, @r###"
1299        {
1300          "event_id": "[event-id]",
1301          "level": "error",
1302          "type": "default",
1303          "logentry": {
1304            "formatted": "Hello World!",
1305          },
1306          "logger": "",
1307          "platform": "other",
1308          "timestamp": "[timestamp]",
1309          "received": "[received]",
1310          "grouping_config": {
1311            "id": "legacy:1234-12-12",
1312          },
1313        }
1314        "###);
1315    }
1316
1317    #[test]
1318    fn test_logentry_error() {
1319        let json = r#"
1320{
1321    "event_id": "74ad1301f4df489ead37d757295442b1",
1322    "timestamp": 1668148328.308933,
1323    "received": 1668148328.308933,
1324    "level": "error",
1325    "platform": "python",
1326    "logentry": {
1327        "params": [
1328            "bogus"
1329        ],
1330        "formatted": 42
1331    }
1332}
1333"#;
1334        let mut event = Annotated::<Event>::from_json(json).unwrap();
1335
1336        normalize_event(&mut event, &NormalizationConfig::default());
1337
1338        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1339        {
1340          "event_id": "74ad1301f4df489ead37d757295442b1",
1341          "level": "error",
1342          "type": "default",
1343          "logentry": null,
1344          "logger": "",
1345          "platform": "python",
1346          "timestamp": 1668148328.308933,
1347          "received": "[received]",
1348          "_meta": {
1349            "logentry": {
1350              "": {
1351                "err": [
1352                  [
1353                    "invalid_data",
1354                    {
1355                      "reason": "no message present"
1356                    }
1357                  ]
1358                ],
1359                "val": {
1360                  "formatted": null,
1361                  "message": null,
1362                  "params": [
1363                    "bogus"
1364                  ]
1365                }
1366              }
1367            }
1368          }
1369        }
1370        "###);
1371    }
1372
1373    #[test]
1374    fn test_future_timestamp() {
1375        let mut event = Annotated::new(Event {
1376            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1377            ..Default::default()
1378        });
1379
1380        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1381        let max_secs_in_past = Some(30 * 24 * 3600);
1382        let max_secs_in_future = Some(60);
1383
1384        validate_event(
1385            &mut event,
1386            &EventValidationConfig {
1387                received_at,
1388                max_secs_in_past,
1389                max_secs_in_future,
1390                is_validated: false,
1391                ..Default::default()
1392            },
1393        )
1394        .unwrap();
1395        normalize_event(&mut event, &NormalizationConfig::default());
1396
1397        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1398        ".event_id" => "[event-id]",
1399    }, @r###"
1400        {
1401          "event_id": "[event-id]",
1402          "level": "error",
1403          "type": "default",
1404          "logger": "",
1405          "platform": "other",
1406          "timestamp": 946857600.0,
1407          "received": 946857600.0,
1408          "_meta": {
1409            "timestamp": {
1410              "": Meta(Some(MetaInner(
1411                err: [
1412                  [
1413                    "future_timestamp",
1414                    {
1415                      "sdk_time": "2000-01-03T00:02:00+00:00",
1416                      "server_time": "2000-01-03T00:00:00+00:00",
1417                    },
1418                  ],
1419                ],
1420              ))),
1421            },
1422          },
1423        }
1424        "###);
1425    }
1426
1427    #[test]
1428    fn test_past_timestamp() {
1429        let mut event = Annotated::new(Event {
1430            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1431            ..Default::default()
1432        });
1433
1434        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1435        let max_secs_in_past = Some(30 * 24 * 3600);
1436        let max_secs_in_future = Some(60);
1437
1438        validate_event(
1439            &mut event,
1440            &EventValidationConfig {
1441                received_at,
1442                max_secs_in_past,
1443                max_secs_in_future,
1444                is_validated: false,
1445                ..Default::default()
1446            },
1447        )
1448        .unwrap();
1449        normalize_event(&mut event, &NormalizationConfig::default());
1450
1451        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1452        ".event_id" => "[event-id]",
1453    }, @r###"
1454        {
1455          "event_id": "[event-id]",
1456          "level": "error",
1457          "type": "default",
1458          "logger": "",
1459          "platform": "other",
1460          "timestamp": 952041600.0,
1461          "received": 952041600.0,
1462          "_meta": {
1463            "timestamp": {
1464              "": Meta(Some(MetaInner(
1465                err: [
1466                  [
1467                    "past_timestamp",
1468                    {
1469                      "sdk_time": "2000-01-03T00:00:00+00:00",
1470                      "server_time": "2000-03-03T00:00:00+00:00",
1471                    },
1472                  ],
1473                ],
1474              ))),
1475            },
1476          },
1477        }
1478        "###);
1479    }
1480
1481    #[test]
1482    fn test_normalize_logger_empty() {
1483        let mut event = Event::from_value(
1484            serde_json::json!({
1485                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1486                "platform": "java",
1487                "logger": "",
1488            })
1489            .into(),
1490        );
1491
1492        normalize_event(&mut event, &NormalizationConfig::default());
1493        assert_annotated_snapshot!(event);
1494    }
1495
1496    #[test]
1497    fn test_normalize_logger_trimmed() {
1498        let mut event = Event::from_value(
1499            serde_json::json!({
1500                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1501                "platform": "java",
1502                "logger": " \t  \t   ",
1503            })
1504            .into(),
1505        );
1506
1507        normalize_event(&mut event, &NormalizationConfig::default());
1508        assert_annotated_snapshot!(event);
1509    }
1510
1511    #[test]
1512    fn test_normalize_logger_short_no_trimming() {
1513        let mut event = Event::from_value(
1514            serde_json::json!({
1515                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1516                "platform": "java",
1517                "logger": "my.short-logger.isnt_trimmed",
1518            })
1519            .into(),
1520        );
1521
1522        normalize_event(&mut event, &NormalizationConfig::default());
1523        assert_annotated_snapshot!(event);
1524    }
1525
1526    #[test]
1527    fn test_normalize_logger_exact_length() {
1528        let mut event = Event::from_value(
1529            serde_json::json!({
1530                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1531                "platform": "java",
1532                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1533            })
1534            .into(),
1535        );
1536
1537        normalize_event(&mut event, &NormalizationConfig::default());
1538        assert_annotated_snapshot!(event);
1539    }
1540
1541    #[test]
1542    fn test_normalize_logger_too_long_single_word() {
1543        let mut event = Event::from_value(
1544            serde_json::json!({
1545                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1546                "platform": "java",
1547                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1548            })
1549            .into(),
1550        );
1551
1552        normalize_event(&mut event, &NormalizationConfig::default());
1553        assert_annotated_snapshot!(event);
1554    }
1555
1556    #[test]
1557    fn test_normalize_logger_word_trimmed_at_max() {
1558        let mut event = Event::from_value(
1559            serde_json::json!({
1560                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1561                "platform": "java",
1562                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1563            })
1564            .into(),
1565        );
1566
1567        normalize_event(&mut event, &NormalizationConfig::default());
1568        assert_annotated_snapshot!(event);
1569    }
1570
1571    #[test]
1572    fn test_normalize_logger_word_trimmed_before_max() {
1573        // This test verifies the "smart" trimming on words -- the logger name
1574        // should be cut before the max limit, removing entire words.
1575        let mut event = Event::from_value(
1576            serde_json::json!({
1577                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1578                "platform": "java",
1579                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1580            })
1581            .into(),
1582        );
1583
1584        normalize_event(&mut event, &NormalizationConfig::default());
1585        assert_annotated_snapshot!(event);
1586    }
1587
1588    #[test]
1589    fn test_normalize_logger_word_leading_dots() {
1590        let mut event = Event::from_value(
1591            serde_json::json!({
1592                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1593                "platform": "java",
1594                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1595            })
1596            .into(),
1597        );
1598
1599        normalize_event(&mut event, &NormalizationConfig::default());
1600        assert_annotated_snapshot!(event);
1601    }
1602
1603    #[test]
1604    fn test_normalization_is_idempotent() {
1605        // get an event, normalize it. the result of that must be the same as normalizing it once more
1606        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1607        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1608        let mut event = Annotated::new(Event {
1609            ty: Annotated::new(EventType::Transaction),
1610            transaction: Annotated::new("/".to_owned()),
1611            timestamp: Annotated::new(end.into()),
1612            start_timestamp: Annotated::new(start.into()),
1613            contexts: {
1614                let mut contexts = Contexts::new();
1615                contexts.add(TraceContext {
1616                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1617                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1618                    op: Annotated::new("http.server".to_owned()),
1619                    ..Default::default()
1620                });
1621                Annotated::new(contexts)
1622            },
1623            spans: Annotated::new(vec![Annotated::new(Span {
1624                timestamp: Annotated::new(
1625                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1626                ),
1627                start_timestamp: Annotated::new(
1628                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1629                ),
1630                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1631                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1632
1633                ..Default::default()
1634            })]),
1635            ..Default::default()
1636        });
1637
1638        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1639            relay_event_schema::processor::apply(event, |e, _m| {
1640                e.received = Annotated::empty();
1641                Ok(())
1642            })
1643            .unwrap();
1644            event
1645        }
1646
1647        normalize_event(&mut event, &NormalizationConfig::default());
1648        let first = remove_received_from_event(&mut event.clone())
1649            .to_json()
1650            .unwrap();
1651        // Expected some fields (such as timestamps) exist after first normalization.
1652
1653        normalize_event(&mut event, &NormalizationConfig::default());
1654        let second = remove_received_from_event(&mut event.clone())
1655            .to_json()
1656            .unwrap();
1657        assert_eq!(&first, &second, "idempotency check failed");
1658
1659        normalize_event(&mut event, &NormalizationConfig::default());
1660        let third = remove_received_from_event(&mut event.clone())
1661            .to_json()
1662            .unwrap();
1663        assert_eq!(&second, &third, "idempotency check failed");
1664    }
1665
1666    /// Validate full normalization is idempotent.
1667    ///
1668    /// Both PoPs and processing relays will temporarily run full normalization
1669    /// in events, during the rollout of running normalization once in internal
1670    /// relays.
1671    // TODO(iker): remove this test after the rollout is done.
1672    #[test]
1673    fn test_full_normalization_is_idempotent() {
1674        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1675        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1676        let mut event = Annotated::new(Event {
1677            ty: Annotated::new(EventType::Transaction),
1678            transaction: Annotated::new("/".to_owned()),
1679            timestamp: Annotated::new(end.into()),
1680            start_timestamp: Annotated::new(start.into()),
1681            contexts: {
1682                let mut contexts = Contexts::new();
1683                contexts.add(TraceContext {
1684                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1685                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1686                    op: Annotated::new("http.server".to_owned()),
1687                    ..Default::default()
1688                });
1689                Annotated::new(contexts)
1690            },
1691            spans: Annotated::new(vec![Annotated::new(Span {
1692                timestamp: Annotated::new(
1693                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1694                ),
1695                start_timestamp: Annotated::new(
1696                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1697                ),
1698                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1699                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1700
1701                ..Default::default()
1702            })]),
1703            ..Default::default()
1704        });
1705
1706        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1707            relay_event_schema::processor::apply(event, |e, _m| {
1708                e.received = Annotated::empty();
1709                Ok(())
1710            })
1711            .unwrap();
1712            event
1713        }
1714
1715        let full_normalization_config = NormalizationConfig {
1716            is_renormalize: false,
1717            enable_trimming: true,
1718            remove_other: true,
1719            emit_event_errors: true,
1720            ..Default::default()
1721        };
1722
1723        normalize_event(&mut event, &full_normalization_config);
1724        let first = remove_received_from_event(&mut event.clone())
1725            .to_json()
1726            .unwrap();
1727        // Expected some fields (such as timestamps) exist after first normalization.
1728
1729        normalize_event(&mut event, &full_normalization_config);
1730        let second = remove_received_from_event(&mut event.clone())
1731            .to_json()
1732            .unwrap();
1733        assert_eq!(&first, &second, "idempotency check failed");
1734
1735        normalize_event(&mut event, &full_normalization_config);
1736        let third = remove_received_from_event(&mut event.clone())
1737            .to_json()
1738            .unwrap();
1739        assert_eq!(&second, &third, "idempotency check failed");
1740    }
1741
1742    #[test]
1743    fn test_normalize_validates_spans() {
1744        let event = Annotated::<Event>::from_json(
1745            r#"
1746            {
1747                "type": "transaction",
1748                "transaction": "/",
1749                "timestamp": 946684810.0,
1750                "start_timestamp": 946684800.0,
1751                "contexts": {
1752                    "trace": {
1753                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1754                    "span_id": "fa90fdead5f74053",
1755                    "op": "http.server",
1756                    "type": "trace"
1757                    }
1758                },
1759                "spans": []
1760            }
1761            "#,
1762        )
1763        .unwrap();
1764
1765        // All incomplete spans should be caught by normalization:
1766        for span in [
1767            r#"null"#,
1768            r#"{
1769              "timestamp": 946684810.0,
1770              "start_timestamp": 946684900.0,
1771              "span_id": "fa90fdead5f74053",
1772              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1773            }"#,
1774            r#"{
1775              "timestamp": 946684810.0,
1776              "span_id": "fa90fdead5f74053",
1777              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1778            }"#,
1779            r#"{
1780              "timestamp": 946684810.0,
1781              "start_timestamp": 946684800.0,
1782              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1783            }"#,
1784            r#"{
1785              "timestamp": 946684810.0,
1786              "start_timestamp": 946684800.0,
1787              "span_id": "fa90fdead5f74053"
1788            }"#,
1789        ] {
1790            let mut modified_event = event.clone();
1791            let event_ref = modified_event.value_mut().as_mut().unwrap();
1792            event_ref
1793                .spans
1794                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1795
1796            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1797
1798            assert!(res.is_err(), "{span:?}");
1799        }
1800    }
1801
1802    #[test]
1803    fn test_normalization_respects_is_renormalize() {
1804        let mut event = Annotated::<Event>::from_json(
1805            r#"
1806            {
1807                "type": "default",
1808                "tags": [["environment", "some_environment"]]
1809            }
1810            "#,
1811        )
1812        .unwrap();
1813
1814        normalize_event(
1815            &mut event,
1816            &NormalizationConfig {
1817                is_renormalize: true,
1818                ..Default::default()
1819            },
1820        );
1821
1822        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1823        Tags(
1824            PairList(
1825                [
1826                    TagEntry(
1827                        "environment",
1828                        "some_environment",
1829                    ),
1830                ],
1831            ),
1832        )
1833        "###);
1834    }
1835
1836    #[test]
1837    fn test_geo_in_normalize() {
1838        let mut event = Annotated::<Event>::from_json(
1839            r#"
1840            {
1841                "type": "transaction",
1842                "transaction": "/foo/",
1843                "timestamp": 946684810.0,
1844                "start_timestamp": 946684800.0,
1845                "contexts": {
1846                    "trace": {
1847                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1848                        "span_id": "fa90fdead5f74053",
1849                        "op": "http.server",
1850                        "type": "trace"
1851                    }
1852                },
1853                "transaction_info": {
1854                    "source": "url"
1855                },
1856                "user": {
1857                    "ip_address": "2.125.160.216"
1858                }
1859            }
1860            "#,
1861        )
1862        .unwrap();
1863
1864        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1865
1866        // Extract user's geo information before normalization.
1867        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1868        assert!(user_geo.is_none());
1869
1870        normalize_event(
1871            &mut event,
1872            &NormalizationConfig {
1873                geoip_lookup: Some(&lookup),
1874                ..Default::default()
1875            },
1876        );
1877
1878        // Extract user's geo information after normalization.
1879        let user_geo = event
1880            .value()
1881            .unwrap()
1882            .user
1883            .value()
1884            .unwrap()
1885            .geo
1886            .value()
1887            .unwrap();
1888
1889        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1890        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1891    }
1892
1893    #[test]
1894    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1895        let mut event = Event {
1896            spans: Annotated::new(vec![Annotated::new(Span {
1897                op: Annotated::new("app_start_cold".to_owned()),
1898                ..Default::default()
1899            })]),
1900            client_sdk: Annotated::new(ClientSdkInfo {
1901                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1902                version: Annotated::new("4.5.0".to_owned()),
1903                ..Default::default()
1904            }),
1905            ..Default::default()
1906        };
1907        span::normalize_app_start_spans(&mut event);
1908        assert_debug_snapshot!(event.spans, @r#"
1909        [
1910            Span {
1911                timestamp: ~,
1912                start_timestamp: ~,
1913                exclusive_time: ~,
1914                op: "app_start_cold",
1915                span_id: ~,
1916                parent_span_id: ~,
1917                trace_id: ~,
1918                segment_id: ~,
1919                is_segment: ~,
1920                is_remote: ~,
1921                status: ~,
1922                description: ~,
1923                tags: ~,
1924                origin: ~,
1925                profile_id: ~,
1926                data: ~,
1927                links: ~,
1928                sentry_tags: ~,
1929                received: ~,
1930                measurements: ~,
1931                platform: ~,
1932                was_transaction: ~,
1933                kind: ~,
1934                other: {},
1935            },
1936        ]
1937        "#);
1938    }
1939
1940    #[test]
1941    fn test_normalize_app_start_cold_spans_for_react_native() {
1942        let mut event = Event {
1943            spans: Annotated::new(vec![Annotated::new(Span {
1944                op: Annotated::new("app_start_cold".to_owned()),
1945                ..Default::default()
1946            })]),
1947            client_sdk: Annotated::new(ClientSdkInfo {
1948                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1949                version: Annotated::new("4.4.0".to_owned()),
1950                ..Default::default()
1951            }),
1952            ..Default::default()
1953        };
1954        span::normalize_app_start_spans(&mut event);
1955        assert_debug_snapshot!(event.spans, @r#"
1956        [
1957            Span {
1958                timestamp: ~,
1959                start_timestamp: ~,
1960                exclusive_time: ~,
1961                op: "app.start.cold",
1962                span_id: ~,
1963                parent_span_id: ~,
1964                trace_id: ~,
1965                segment_id: ~,
1966                is_segment: ~,
1967                is_remote: ~,
1968                status: ~,
1969                description: ~,
1970                tags: ~,
1971                origin: ~,
1972                profile_id: ~,
1973                data: ~,
1974                links: ~,
1975                sentry_tags: ~,
1976                received: ~,
1977                measurements: ~,
1978                platform: ~,
1979                was_transaction: ~,
1980                kind: ~,
1981                other: {},
1982            },
1983        ]
1984        "#);
1985    }
1986
1987    #[test]
1988    fn test_normalize_app_start_warm_spans_for_react_native() {
1989        let mut event = Event {
1990            spans: Annotated::new(vec![Annotated::new(Span {
1991                op: Annotated::new("app_start_warm".to_owned()),
1992                ..Default::default()
1993            })]),
1994            client_sdk: Annotated::new(ClientSdkInfo {
1995                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1996                version: Annotated::new("4.4.0".to_owned()),
1997                ..Default::default()
1998            }),
1999            ..Default::default()
2000        };
2001        span::normalize_app_start_spans(&mut event);
2002        assert_debug_snapshot!(event.spans, @r#"
2003        [
2004            Span {
2005                timestamp: ~,
2006                start_timestamp: ~,
2007                exclusive_time: ~,
2008                op: "app.start.warm",
2009                span_id: ~,
2010                parent_span_id: ~,
2011                trace_id: ~,
2012                segment_id: ~,
2013                is_segment: ~,
2014                is_remote: ~,
2015                status: ~,
2016                description: ~,
2017                tags: ~,
2018                origin: ~,
2019                profile_id: ~,
2020                data: ~,
2021                links: ~,
2022                sentry_tags: ~,
2023                received: ~,
2024                measurements: ~,
2025                platform: ~,
2026                was_transaction: ~,
2027                kind: ~,
2028                other: {},
2029            },
2030        ]
2031        "#);
2032    }
2033}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
