relay_event_normalization/normalize/
mod.rs

1use std::hash::Hash;
2use std::{collections::HashMap, sync::LazyLock};
3
4use regex::Regex;
5use relay_base_schema::metrics::MetricUnit;
6use relay_event_schema::protocol::{Event, VALID_PLATFORMS};
7use relay_pattern::Pattern;
8use relay_protocol::{FiniteF64, RuleCondition};
9use serde::{Deserialize, Serialize};
10
11pub mod breakdowns;
12pub mod contexts;
13pub mod nel;
14pub mod request;
15pub mod span;
16pub mod user_agent;
17pub mod utils;
18
19/// Defines a builtin measurement.
20#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
21#[serde(default, rename_all = "camelCase")]
22pub struct BuiltinMeasurementKey {
23    name: String,
24    unit: MetricUnit,
25    #[serde(skip_serializing_if = "is_false")]
26    allow_negative: bool,
27}
28
29fn is_false(b: &bool) -> bool {
30    !b
31}
32
33impl BuiltinMeasurementKey {
34    /// Creates a new [`BuiltinMeasurementKey`].
35    pub fn new(name: impl Into<String>, unit: MetricUnit) -> Self {
36        Self {
37            name: name.into(),
38            unit,
39            allow_negative: false,
40        }
41    }
42
43    /// Returns the name of the built in measurement key.
44    pub fn name(&self) -> &str {
45        &self.name
46    }
47
48    /// Returns the unit of the built in measurement key.
49    pub fn unit(&self) -> &MetricUnit {
50        &self.unit
51    }
52
53    /// Return true if the built in measurement key allows negative values.
54    pub fn allow_negative(&self) -> &bool {
55        &self.allow_negative
56    }
57}
58
59/// Configuration for measurements normalization.
60#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Hash)]
61#[serde(default, rename_all = "camelCase")]
62pub struct MeasurementsConfig {
63    /// A list of measurements that are built-in and are not subject to custom measurement limits.
64    #[serde(default, skip_serializing_if = "Vec::is_empty")]
65    pub builtin_measurements: Vec<BuiltinMeasurementKey>,
66
67    /// The maximum number of measurements allowed per event that are not known measurements.
68    pub max_custom_measurements: usize,
69}
70
71impl MeasurementsConfig {
72    /// The length of a full measurement MRI, minus the name and the unit. This length is the same
73    /// for every measurement-mri.
74    pub const MEASUREMENT_MRI_OVERHEAD: usize = 29;
75}
76
77/// Returns `true` if the given platform string is a known platform identifier.
78///
79/// See [`VALID_PLATFORMS`] for a list of all known platforms.
80pub fn is_valid_platform(platform: &str) -> bool {
81    VALID_PLATFORMS.contains(&platform)
82}
83
84/// Replaces snake_case app start spans op with dot.case op.
85///
86/// This is done for the affected React Native SDK versions (from 3 to 4.4).
87pub fn normalize_app_start_spans(event: &mut Event) {
88    if !event.sdk_name().eq("sentry.javascript.react-native")
89        || !(event.sdk_version().starts_with("4.4")
90            || event.sdk_version().starts_with("4.3")
91            || event.sdk_version().starts_with("4.2")
92            || event.sdk_version().starts_with("4.1")
93            || event.sdk_version().starts_with("4.0")
94            || event.sdk_version().starts_with('3'))
95    {
96        return;
97    }
98
99    if let Some(spans) = event.spans.value_mut() {
100        for span in spans {
101            if let Some(span) = span.value_mut()
102                && let Some(op) = span.op.value()
103            {
104                if op == "app_start_cold" {
105                    span.op.set_value(Some("app.start.cold".to_owned()));
106                    break;
107                } else if op == "app_start_warm" {
108                    span.op.set_value(Some("app.start.warm".to_owned()));
109                    break;
110                }
111            }
112        }
113    }
114}
115
116/// Container for global and project level [`MeasurementsConfig`]. The purpose is to handle
117/// the merging logic.
118#[derive(Clone, Debug)]
119pub struct CombinedMeasurementsConfig<'a> {
120    project: Option<&'a MeasurementsConfig>,
121    global: Option<&'a MeasurementsConfig>,
122}
123
124impl<'a> CombinedMeasurementsConfig<'a> {
125    /// Constructor for [`CombinedMeasurementsConfig`].
126    pub fn new(
127        project: Option<&'a MeasurementsConfig>,
128        global: Option<&'a MeasurementsConfig>,
129    ) -> Self {
130        CombinedMeasurementsConfig { project, global }
131    }
132
133    /// Returns an iterator over the merged builtin measurement keys.
134    ///
135    /// Items from the project config are prioritized over global config, and
136    /// there are no duplicates.
137    pub fn builtin_measurement_keys(
138        &'a self,
139    ) -> impl Iterator<Item = &'a BuiltinMeasurementKey> + 'a {
140        let project = self
141            .project
142            .map(|p| p.builtin_measurements.as_slice())
143            .unwrap_or_default();
144
145        let global = self
146            .global
147            .map(|g| g.builtin_measurements.as_slice())
148            .unwrap_or_default();
149
150        project
151            .iter()
152            .chain(global.iter().filter(|key| !project.contains(key)))
153    }
154
155    /// Gets the max custom measurements value from the [`MeasurementsConfig`] from project level or
156    /// global level. If both of them are available, it will choose the most restrictive.
157    pub fn max_custom_measurements(&'a self) -> Option<usize> {
158        match (&self.project, &self.global) {
159            (None, None) => None,
160            (None, Some(global)) => Some(global.max_custom_measurements),
161            (Some(project), None) => Some(project.max_custom_measurements),
162            (Some(project), Some(global)) => Some(std::cmp::min(
163                project.max_custom_measurements,
164                global.max_custom_measurements,
165            )),
166        }
167    }
168}
169
170/// Defines a weighted component for a performance score.
171///
172/// Weight is the % of score it can take up (eg. LCP is a max of 35% weight for desktops)
173/// Currently also contains (p10, p50) which are used for log CDF normalization of the weight score
174#[derive(Debug, Default, Clone, Serialize, Deserialize)]
175pub struct PerformanceScoreWeightedComponent {
176    /// Measurement (eg. measurements.lcp) to be matched against. If this measurement is missing the entire
177    /// profile will be discarded.
178    pub measurement: String,
179    /// Weight [0,1.0] of this component in the performance score
180    pub weight: FiniteF64,
181    /// p10 used to define the log-normal for calculation
182    pub p10: FiniteF64,
183    /// Median used to define the log-normal for calculation
184    pub p50: FiniteF64,
185    /// Whether the measurement is optional. If the measurement is missing, performance score processing
186    /// may still continue, and the weight will be 0.
187    #[serde(default)]
188    pub optional: bool,
189}
190
191/// Defines a profile for performance score.
192///
193/// A profile contains weights for a score of 100% and match against an event using a condition.
194/// eg. Desktop vs. Mobile(web) profiles for better web vital score calculation.
195#[derive(Debug, Default, Clone, Serialize, Deserialize)]
196#[serde(rename_all = "camelCase")]
197pub struct PerformanceScoreProfile {
198    /// Name of the profile, used for debugging and faceting multiple profiles
199    pub name: Option<String>,
200    /// Score components
201    #[serde(default, skip_serializing_if = "Vec::is_empty")]
202    pub score_components: Vec<PerformanceScoreWeightedComponent>,
203    /// See [`RuleCondition`] for all available options to specify and combine conditions.
204    pub condition: Option<RuleCondition>,
205    /// The version of the profile, used to isolate changes to score calculations.
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub version: Option<String>,
208}
209
210/// Defines the performance configuration for the project.
211///
212/// Includes profiles matching different behaviour (desktop / mobile) and weights matching those
213/// specific conditions.
214#[derive(Debug, Default, Clone, Serialize, Deserialize)]
215pub struct PerformanceScoreConfig {
216    /// List of performance profiles, only the first with matching conditions will be applied.
217    #[serde(default, skip_serializing_if = "Vec::is_empty")]
218    pub profiles: Vec<PerformanceScoreProfile>,
219}
220
221/// Regex that matches version and/or date patterns at the end of a model name.
222///
223/// Examples matched:
224/// - "-20250805-v1:0" in "claude-opus-4-1-20250805-v1:0" (both)
225/// - "-20250514-v1:0" in "claude-sonnet-4-20250514-v1:0" (both)
226/// - "_v2" in "model_v2" (version only)
227/// - "-v1.0" in "gpt-4-v1.0" (version only)
228/// - "-20250522" in "claude-4-sonnet-20250522" (date only)
229/// - "-2025-06-10" in "o3-pro-2025-06-10" (date only)
230/// - "@20241022" in "claude-3-5-haiku@20241022" (date only)
231static VERSION_AND_DATE_REGEX: LazyLock<Regex> = LazyLock::new(|| {
232    Regex::new(
233        r"(?x)
234        (
235            ([-_@])                                    # Required separator before date (-, _, or @)
236            (\d{4}[-/.]\d{2}[-/.]\d{2}|\d{8})          # Date: YYYY-MM-DD/YYYY/MM/DD/YYYY.MM.DD or YYYYMMDD
237        )?                                              # Date is optional
238        (
239            [-_]                                        # Required separator before version (- or _)
240            v\d+[:.]?\d*                                # Version: v1, v1.0, v1:0
241            ([-:].*)?                                   # Optional trailing content
242        )?                                              # Version is optional
243        $  # Must be at the end of the string
244        ",
245    )
246    .unwrap()
247});
248
249/// Normalizes an AI model name by stripping date and version patterns.
250///
251/// This function converts specific model versions into normalized names suitable for matching
252/// against cost configurations, ensuring that different versions of the same model can share
253/// cost information.
254///
255/// The normalization strips version and/or date patterns from the end of the string in a single pass.
256///
257/// Examples:
258/// - "claude-4-sonnet-20250522" -> "claude-4-sonnet"
259/// - "o3-pro-2025-06-10" -> "o3-pro"
260/// - "claude-3-5-haiku@20241022" -> "claude-3-5-haiku"
261/// - "claude-opus-4-1-20250805-v1:0" -> "claude-opus-4-1"
262/// - "us.anthropic.claude-sonnet-4-20250514-v1:0" -> "us.anthropic.claude-sonnet-4"
263/// - "gpt-4-v1.0" -> "gpt-4"
264/// - "gpt-4-v1:0" -> "gpt-4"
265/// - "model_v2" -> "model"
266///
267/// Version patterns:
268/// - Must start with 'v': v1, v1.0, v1:0 (requires -, _ separator)
269///
270/// Date patterns:
271/// - YYYYMMDD: 20250522
272/// - YYYY-MM-DD, YYYY/MM/DD, YYYY.MM.DD: 2025-06-10
273///
274/// Args:
275///     model_id: The original model ID
276///
277/// Returns:
278///     The normalized model name without date and version patterns
279fn normalize_ai_model_name(model_id: &str) -> &str {
280    if let Some(captures) = VERSION_AND_DATE_REGEX.captures(model_id) {
281        let match_idx = captures.get(0).map(|m| m.start()).unwrap_or(model_id.len());
282        &model_id[..match_idx]
283    } else {
284        model_id
285    }
286}
287
288/// Version 2 of a mapping of AI model types (like GPT-4) to their respective costs.
289#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
290#[serde(rename_all = "camelCase")]
291pub struct ModelCostV2 {
292    /// The cost per input token
293    pub input_per_token: f64,
294    /// The cost per output token
295    pub output_per_token: f64,
296    /// The cost per output reasoning token
297    pub output_reasoning_per_token: f64,
298    /// The cost per input cached token
299    pub input_cached_per_token: f64,
300    /// The cost per input cache write token
301    pub input_cache_write_per_token: f64,
302}
303
304/// Metadata for AI models including costs and context size.
305///
306/// Example JSON:
307/// ```json
308/// {
309///   "version": 1,
310///   "models": {
311///     "gpt-4": {
312///       "costs": {
313///         "inputPerToken": 0.0000003,
314///         "outputPerToken": 0.00000165,
315///         "outputReasoningPerToken": 0.0,
316///         "inputCachedPerToken": 0.0000015,
317///         "inputCacheWritePerToken": 0.00001875
318///       },
319///       "contextSize": 1000000
320///     }
321///   }
322/// }
323/// ```
324#[derive(Clone, Default, Debug, Serialize, Deserialize)]
325#[serde(rename_all = "camelCase")]
326pub struct ModelMetadata {
327    /// The version of the model metadata struct.
328    pub version: u16,
329
330    /// The mappings of model ID => metadata as a dictionary.
331    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
332    pub models: HashMap<Pattern, ModelMetadataEntry>,
333}
334
335impl ModelMetadata {
336    const SUPPORTED_VERSION: u16 = 1;
337
338    /// `true` if the model metadata is empty or the version is unsupported.
339    pub fn is_empty(&self) -> bool {
340        self.models.is_empty() || !self.is_enabled()
341    }
342
343    /// `false` if the version is unsupported.
344    pub fn is_enabled(&self) -> bool {
345        self.version == Self::SUPPORTED_VERSION
346    }
347
348    /// Gets the cost per token for a given model, if defined.
349    pub fn cost_per_token(&self, model_id: &str) -> Option<&ModelCostV2> {
350        self.get(model_id).and_then(|entry| entry.costs.as_ref())
351    }
352
353    /// Gets the context window size for a given model, if defined.
354    ///
355    /// Returns `None` for a zero context size, as it is not a meaningful value.
356    pub fn context_size(&self, model_id: &str) -> Option<u64> {
357        self.get(model_id)
358            .and_then(|entry| entry.context_size)
359            .filter(|&size| size > 0)
360    }
361
362    /// Gets the metadata for a given model, if defined.
363    pub fn get(&self, model_id: &str) -> Option<&ModelMetadataEntry> {
364        if !self.is_enabled() {
365            return None;
366        }
367
368        let normalized_model_id = normalize_ai_model_name(model_id);
369
370        // First try exact match.
371        if let Some(value) = self.models.get(normalized_model_id) {
372            return Some(value);
373        }
374
375        // Fall back to glob matching.
376        self.models.iter().find_map(|(key, value)| {
377            if key.is_match(normalized_model_id) {
378                Some(value)
379            } else {
380                None
381            }
382        })
383    }
384}
385
386/// Metadata for a single AI model.
387#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]
388#[serde(rename_all = "camelCase")]
389pub struct ModelMetadataEntry {
390    /// Token costs for this model.
391    #[serde(default, skip_serializing_if = "Option::is_none")]
392    pub costs: Option<ModelCostV2>,
393
394    /// The context window size in tokens.
395    #[serde(default, skip_serializing_if = "Option::is_none")]
396    pub context_size: Option<u64>,
397}
398
399#[cfg(test)]
400mod tests {
401    use chrono::{TimeZone, Utc};
402    use insta::{assert_debug_snapshot, assert_json_snapshot};
403    use itertools::Itertools;
404    use relay_base_schema::events::EventType;
405    use relay_base_schema::metrics::DurationUnit;
406    use relay_base_schema::spans::SpanStatus;
407    use relay_event_schema::protocol::{
408        ClientSdkInfo, Context, ContextInner, Contexts, DebugImage, DebugMeta, EventId, Exception,
409        Frame, Geo, IpAddr, LenientString, Level, LogEntry, PairList, RawStacktrace, ReplayContext,
410        Request, Span, Stacktrace, TagEntry, Tags, TraceContext, User, Values,
411    };
412    use relay_protocol::{
413        Annotated, Error, ErrorKind, FromValue, Object, SerializableAnnotated, Value,
414        assert_annotated_snapshot, get_path, get_value,
415    };
416    use serde_json::json;
417    use similar_asserts::assert_eq;
418    use uuid::Uuid;
419
420    use crate::stacktrace::normalize_non_raw_frame;
421    use crate::validation::validate_event;
422    use crate::{EventValidationConfig, GeoIpLookup, NormalizationConfig, normalize_event};
423
424    use super::*;
425
426    #[test]
427    fn test_normalize_ai_model_name() {
428        // Test date patterns
429        assert_eq!(
430            normalize_ai_model_name("claude-4-sonnet-20250522"),
431            "claude-4-sonnet"
432        );
433        assert_eq!(normalize_ai_model_name("o3-pro-2025-06-10"), "o3-pro");
434        assert_eq!(
435            normalize_ai_model_name("claude-3-5-haiku@20241022"),
436            "claude-3-5-haiku"
437        );
438
439        // Test date with version patterns
440        assert_eq!(
441            normalize_ai_model_name("claude-opus-4-1-20250805-v1:0"),
442            "claude-opus-4-1"
443        );
444        assert_eq!(
445            normalize_ai_model_name("us.anthropic.claude-sonnet-4-20250514-v1:0"),
446            "us.anthropic.claude-sonnet-4"
447        );
448
449        // Test standalone version patterns with 'v' prefix
450        assert_eq!(normalize_ai_model_name("gpt-4-v1.0"), "gpt-4");
451        assert_eq!(normalize_ai_model_name("gpt-4-v1:0"), "gpt-4");
452        assert_eq!(normalize_ai_model_name("gpt-4-v1"), "gpt-4");
453        assert_eq!(normalize_ai_model_name("model_v2"), "model");
454
455        // Test that version patterns WITHOUT 'v' prefix are NOT stripped
456        assert_eq!(normalize_ai_model_name("gpt-4.5"), "gpt-4.5");
457
458        // Test that if version without 'v' comes after date, nothing is stripped
459        // (date regex requires end of string, so date won't match if followed by version)
460        assert_eq!(
461            normalize_ai_model_name("anthropic.claude-3-5-sonnet-20241022-v2:0"),
462            "anthropic.claude-3-5-sonnet"
463        );
464
465        // Test no pattern (should return original)
466        assert_eq!(normalize_ai_model_name("gpt-4"), "gpt-4");
467        assert_eq!(normalize_ai_model_name("claude-3-opus"), "claude-3-opus");
468    }
469
470    #[test]
471    fn test_merge_builtin_measurement_keys() {
472        let foo = BuiltinMeasurementKey::new("foo", MetricUnit::Duration(DurationUnit::Hour));
473        let bar = BuiltinMeasurementKey::new("bar", MetricUnit::Duration(DurationUnit::Day));
474        let baz = BuiltinMeasurementKey::new("baz", MetricUnit::Duration(DurationUnit::Week));
475
476        let proj = MeasurementsConfig {
477            builtin_measurements: vec![foo.clone(), bar.clone()],
478            max_custom_measurements: 4,
479        };
480
481        let glob = MeasurementsConfig {
482            // The 'bar' here will be ignored since it's a duplicate from the project level.
483            builtin_measurements: vec![baz.clone(), bar.clone()],
484            max_custom_measurements: 4,
485        };
486        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
487
488        let keys = dynamic_config.builtin_measurement_keys().collect_vec();
489
490        assert_eq!(keys, vec![&foo, &bar, &baz]);
491    }
492
493    #[test]
494    fn test_max_custom_measurement() {
495        // Empty configs will return a None value for max measurements.
496        let dynamic_config = CombinedMeasurementsConfig::new(None, None);
497        assert!(dynamic_config.max_custom_measurements().is_none());
498
499        let proj = MeasurementsConfig {
500            builtin_measurements: vec![],
501            max_custom_measurements: 3,
502        };
503
504        let glob = MeasurementsConfig {
505            builtin_measurements: vec![],
506            max_custom_measurements: 4,
507        };
508
509        // If only project level measurement config is there, return its max custom measurement variable.
510        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), None);
511        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
512
513        // Same logic for when only global level measurement config exists.
514        let dynamic_config = CombinedMeasurementsConfig::new(None, Some(&glob));
515        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 4);
516
517        // If both is available, pick the smallest number.
518        let dynamic_config = CombinedMeasurementsConfig::new(Some(&proj), Some(&glob));
519        assert_eq!(dynamic_config.max_custom_measurements().unwrap(), 3);
520    }
521
522    #[test]
523    fn test_geo_from_ip_address() {
524        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
525
526        let json = r#"{
527            "user": {
528                "ip_address": "2.125.160.216"
529            }
530        }"#;
531        let mut event = Annotated::<Event>::from_json(json).unwrap();
532
533        normalize_event(
534            &mut event,
535            &NormalizationConfig {
536                geoip_lookup: Some(&lookup),
537                ..Default::default()
538            },
539        );
540
541        let expected = Annotated::new(Geo {
542            country_code: Annotated::new("GB".to_owned()),
543            city: Annotated::new("Boxford".to_owned()),
544            subdivision: Annotated::new("England".to_owned()),
545            region: Annotated::new("United Kingdom".to_owned()),
546            ..Geo::default()
547        });
548        assert_eq!(get_value!(event.user!).geo, expected);
549    }
550
551    #[test]
552    fn test_user_ip_from_remote_addr() {
553        let mut event = Annotated::new(Event {
554            request: Annotated::from(Request {
555                env: Annotated::new({
556                    let mut map = Object::new();
557                    map.insert(
558                        "REMOTE_ADDR".to_owned(),
559                        Annotated::new(Value::String("2.125.160.216".to_owned())),
560                    );
561                    map
562                }),
563                ..Request::default()
564            }),
565            platform: Annotated::new("javascript".to_owned()),
566            ..Event::default()
567        });
568
569        normalize_event(&mut event, &NormalizationConfig::default());
570
571        let ip_addr = get_value!(event.user.ip_address!);
572        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
573    }
574
575    #[test]
576    fn test_user_ip_from_invalid_remote_addr() {
577        let mut event = Annotated::new(Event {
578            request: Annotated::from(Request {
579                env: Annotated::new({
580                    let mut map = Object::new();
581                    map.insert(
582                        "REMOTE_ADDR".to_owned(),
583                        Annotated::new(Value::String("whoops".to_owned())),
584                    );
585                    map
586                }),
587                ..Request::default()
588            }),
589            platform: Annotated::new("javascript".to_owned()),
590            ..Event::default()
591        });
592
593        normalize_event(&mut event, &NormalizationConfig::default());
594
595        assert_eq!(Annotated::empty(), event.value().unwrap().user);
596    }
597
598    #[test]
599    fn test_user_ip_from_client_ip_without_auto() {
600        let mut event = Annotated::new(Event {
601            platform: Annotated::new("javascript".to_owned()),
602            ..Default::default()
603        });
604
605        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
606
607        normalize_event(
608            &mut event,
609            &NormalizationConfig {
610                client_ip: Some(&ip_address),
611                infer_ip_address: true,
612                ..Default::default()
613            },
614        );
615
616        let ip_addr = get_value!(event.user.ip_address!);
617        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
618    }
619
620    #[test]
621    fn test_user_ip_from_client_ip_with_auto() {
622        let mut event = Annotated::new(Event {
623            user: Annotated::new(User {
624                ip_address: Annotated::new(IpAddr::auto()),
625                ..Default::default()
626            }),
627            ..Default::default()
628        });
629
630        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
631
632        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
633        normalize_event(
634            &mut event,
635            &NormalizationConfig {
636                client_ip: Some(&ip_address),
637                geoip_lookup: Some(&geo),
638                infer_ip_address: true,
639                ..Default::default()
640            },
641        );
642
643        let user = get_value!(event.user!);
644        let ip_addr = user.ip_address.value().expect("ip address missing");
645
646        assert_eq!(ip_addr, &IpAddr("2.125.160.216".to_owned()));
647        assert!(user.geo.value().is_some());
648    }
649
650    #[test]
651    fn test_user_ip_from_client_ip_without_appropriate_platform() {
652        let mut event = Annotated::new(Event::default());
653
654        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
655        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
656        normalize_event(
657            &mut event,
658            &NormalizationConfig {
659                client_ip: Some(&ip_address),
660                geoip_lookup: Some(&geo),
661                ..Default::default()
662            },
663        );
664
665        let user = get_value!(event.user!);
666        assert!(user.ip_address.value().is_none());
667        assert!(user.geo.value().is_some());
668    }
669
670    #[test]
671    fn test_geo_present_if_ip_inferring_disabled() {
672        let mut event = Annotated::new(Event {
673            user: Annotated::new(User {
674                ip_address: Annotated::new(IpAddr::auto()),
675                ..Default::default()
676            }),
677            ..Default::default()
678        });
679
680        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
681        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
682
683        normalize_event(
684            &mut event,
685            &NormalizationConfig {
686                client_ip: Some(&ip_address),
687                geoip_lookup: Some(&geo),
688                infer_ip_address: false,
689                ..Default::default()
690            },
691        );
692
693        let user = get_value!(event.user!);
694        assert!(user.ip_address.value().unwrap().is_auto());
695        assert!(user.geo.value().is_some());
696    }
697
698    #[test]
699    fn test_geo_and_ip_present_if_ip_inferring_enabled() {
700        let mut event = Annotated::new(Event {
701            user: Annotated::new(User {
702                ip_address: Annotated::new(IpAddr::auto()),
703                ..Default::default()
704            }),
705            ..Default::default()
706        });
707
708        let ip_address = IpAddr::parse("2.125.160.216").unwrap();
709        let geo = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
710
711        normalize_event(
712            &mut event,
713            &NormalizationConfig {
714                client_ip: Some(&ip_address),
715                geoip_lookup: Some(&geo),
716                infer_ip_address: true,
717                ..Default::default()
718            },
719        );
720
721        let user = get_value!(event.user!);
722        assert_eq!(
723            user.ip_address.value(),
724            Some(&IpAddr::parse("2.125.160.216").unwrap())
725        );
726        assert!(user.geo.value().is_some());
727    }
728
729    #[test]
730    fn test_event_level_defaulted() {
731        let mut event = Annotated::new(Event::default());
732        normalize_event(&mut event, &NormalizationConfig::default());
733        assert_eq!(get_value!(event.level), Some(&Level::Error));
734    }
735
736    #[test]
737    fn test_transaction_level_untouched() {
738        let mut event = Annotated::new(Event {
739            ty: Annotated::new(EventType::Transaction),
740            timestamp: Annotated::new(Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into()),
741            start_timestamp: Annotated::new(
742                Utc.with_ymd_and_hms(1987, 6, 5, 4, 3, 2).unwrap().into(),
743            ),
744            contexts: {
745                let mut contexts = Contexts::new();
746                contexts.add(TraceContext {
747                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
748                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
749                    op: Annotated::new("http.server".to_owned()),
750                    ..Default::default()
751                });
752                Annotated::new(contexts)
753            },
754            ..Event::default()
755        });
756        normalize_event(&mut event, &NormalizationConfig::default());
757        assert_eq!(get_value!(event.level), Some(&Level::Info));
758    }
759
760    #[test]
761    fn test_environment_tag_is_moved() {
762        let mut event = Annotated::new(Event {
763            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
764                Annotated::new("environment".to_owned()),
765                Annotated::new("despacito".to_owned()),
766            ))]))),
767            ..Event::default()
768        });
769
770        normalize_event(&mut event, &NormalizationConfig::default());
771
772        let event = event.value().unwrap();
773
774        assert_eq!(event.environment.as_str(), Some("despacito"));
775        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
776    }
777
778    #[test]
779    fn test_empty_environment_is_removed_and_overwritten_with_tag() {
780        let mut event = Annotated::new(Event {
781            tags: Annotated::new(Tags(PairList(vec![Annotated::new(TagEntry(
782                Annotated::new("environment".to_owned()),
783                Annotated::new("despacito".to_owned()),
784            ))]))),
785            environment: Annotated::new("".to_owned()),
786            ..Event::default()
787        });
788
789        normalize_event(&mut event, &NormalizationConfig::default());
790
791        let event = event.value().unwrap();
792
793        assert_eq!(event.environment.as_str(), Some("despacito"));
794        assert_eq!(event.tags.value(), Some(&Tags(vec![].into())));
795    }
796
797    #[test]
798    fn test_empty_environment_is_removed() {
799        let mut event = Annotated::new(Event {
800            environment: Annotated::new("".to_owned()),
801            ..Event::default()
802        });
803
804        normalize_event(&mut event, &NormalizationConfig::default());
805        assert_eq!(get_value!(event.environment), None);
806    }
807    #[test]
808    fn test_replay_id_added_from_dsc() {
809        let replay_id = Uuid::new_v4();
810        let mut event = Annotated::new(Event {
811            contexts: Annotated::new(Contexts(Object::new())),
812            ..Event::default()
813        });
814        normalize_event(
815            &mut event,
816            &NormalizationConfig {
817                replay_id: Some(replay_id),
818                ..Default::default()
819            },
820        );
821
822        let event = event.value().unwrap();
823
824        assert_eq!(event.contexts, {
825            let mut contexts = Contexts::new();
826            contexts.add(ReplayContext {
827                replay_id: Annotated::new(EventId(replay_id)),
828                other: Object::default(),
829            });
830            Annotated::new(contexts)
831        })
832    }
833
834    #[test]
835    fn test_none_environment_errors() {
836        let mut event = Annotated::new(Event {
837            environment: Annotated::new("none".to_owned()),
838            ..Event::default()
839        });
840
841        normalize_event(&mut event, &NormalizationConfig::default());
842
843        let environment = get_path!(event.environment!);
844        let expected_original = &Value::String("none".to_owned());
845
846        assert_eq!(
847            environment.meta().iter_errors().collect::<Vec<&Error>>(),
848            vec![&Error::new(ErrorKind::InvalidData)],
849        );
850        assert_eq!(
851            environment.meta().original_value().unwrap(),
852            expected_original
853        );
854        assert_eq!(environment.value(), None);
855    }
856
857    #[test]
858    fn test_invalid_release_removed() {
859        let mut event = Annotated::new(Event {
860            release: Annotated::new(LenientString("Latest".to_owned())),
861            ..Event::default()
862        });
863
864        normalize_event(&mut event, &NormalizationConfig::default());
865
866        let release = get_path!(event.release!);
867        let expected_original = &Value::String("Latest".to_owned());
868
869        assert_eq!(
870            release.meta().iter_errors().collect::<Vec<&Error>>(),
871            vec![&Error::new(ErrorKind::InvalidData)],
872        );
873        assert_eq!(release.meta().original_value().unwrap(), expected_original);
874        assert_eq!(release.value(), None);
875    }
876
877    #[test]
878    fn test_top_level_keys_moved_into_tags() {
879        let mut event = Annotated::new(Event {
880            server_name: Annotated::new("foo".to_owned()),
881            site: Annotated::new("foo".to_owned()),
882            tags: Annotated::new(Tags(PairList(vec![
883                Annotated::new(TagEntry(
884                    Annotated::new("site".to_owned()),
885                    Annotated::new("old".to_owned()),
886                )),
887                Annotated::new(TagEntry(
888                    Annotated::new("server_name".to_owned()),
889                    Annotated::new("old".to_owned()),
890                )),
891            ]))),
892            ..Event::default()
893        });
894
895        normalize_event(&mut event, &NormalizationConfig::default());
896
897        assert_eq!(get_value!(event.site), None);
898        assert_eq!(get_value!(event.server_name), None);
899
900        assert_eq!(
901            get_value!(event.tags!),
902            &Tags(PairList(vec![
903                Annotated::new(TagEntry(
904                    Annotated::new("site".to_owned()),
905                    Annotated::new("foo".to_owned()),
906                )),
907                Annotated::new(TagEntry(
908                    Annotated::new("server_name".to_owned()),
909                    Annotated::new("foo".to_owned()),
910                )),
911            ]))
912        );
913    }
914
915    #[test]
916    fn test_internal_tags_removed() {
917        let mut event = Annotated::new(Event {
918            tags: Annotated::new(Tags(PairList(vec![
919                Annotated::new(TagEntry(
920                    Annotated::new("release".to_owned()),
921                    Annotated::new("foo".to_owned()),
922                )),
923                Annotated::new(TagEntry(
924                    Annotated::new("dist".to_owned()),
925                    Annotated::new("foo".to_owned()),
926                )),
927                Annotated::new(TagEntry(
928                    Annotated::new("user".to_owned()),
929                    Annotated::new("foo".to_owned()),
930                )),
931                Annotated::new(TagEntry(
932                    Annotated::new("filename".to_owned()),
933                    Annotated::new("foo".to_owned()),
934                )),
935                Annotated::new(TagEntry(
936                    Annotated::new("function".to_owned()),
937                    Annotated::new("foo".to_owned()),
938                )),
939                Annotated::new(TagEntry(
940                    Annotated::new("something".to_owned()),
941                    Annotated::new("else".to_owned()),
942                )),
943            ]))),
944            ..Event::default()
945        });
946
947        normalize_event(&mut event, &NormalizationConfig::default());
948
949        assert_eq!(get_value!(event.tags!).len(), 1);
950    }
951
952    #[test]
953    fn test_empty_tags_removed() {
954        let mut event = Annotated::new(Event {
955            tags: Annotated::new(Tags(PairList(vec![
956                Annotated::new(TagEntry(
957                    Annotated::new("".to_owned()),
958                    Annotated::new("foo".to_owned()),
959                )),
960                Annotated::new(TagEntry(
961                    Annotated::new("foo".to_owned()),
962                    Annotated::new("".to_owned()),
963                )),
964                Annotated::new(TagEntry(
965                    Annotated::new("something".to_owned()),
966                    Annotated::new("else".to_owned()),
967                )),
968            ]))),
969            ..Event::default()
970        });
971
972        normalize_event(&mut event, &NormalizationConfig::default());
973
974        assert_eq!(
975            get_value!(event.tags!),
976            &Tags(PairList(vec![
977                Annotated::new(TagEntry(
978                    Annotated::from_error(Error::nonempty(), None),
979                    Annotated::new("foo".to_owned()),
980                )),
981                Annotated::new(TagEntry(
982                    Annotated::new("foo".to_owned()),
983                    Annotated::from_error(Error::nonempty(), None),
984                )),
985                Annotated::new(TagEntry(
986                    Annotated::new("something".to_owned()),
987                    Annotated::new("else".to_owned()),
988                )),
989            ]))
990        );
991    }
992
993    #[test]
994    fn test_tags_deduplicated() {
995        let mut event = Annotated::new(Event {
996            tags: Annotated::new(Tags(PairList(vec![
997                Annotated::new(TagEntry(
998                    Annotated::new("foo".to_owned()),
999                    Annotated::new("1".to_owned()),
1000                )),
1001                Annotated::new(TagEntry(
1002                    Annotated::new("bar".to_owned()),
1003                    Annotated::new("1".to_owned()),
1004                )),
1005                Annotated::new(TagEntry(
1006                    Annotated::new("foo".to_owned()),
1007                    Annotated::new("2".to_owned()),
1008                )),
1009                Annotated::new(TagEntry(
1010                    Annotated::new("bar".to_owned()),
1011                    Annotated::new("2".to_owned()),
1012                )),
1013                Annotated::new(TagEntry(
1014                    Annotated::new("foo".to_owned()),
1015                    Annotated::new("3".to_owned()),
1016                )),
1017            ]))),
1018            ..Event::default()
1019        });
1020
1021        normalize_event(&mut event, &NormalizationConfig::default());
1022
1023        // should keep the first occurrence of every tag
1024        assert_eq!(
1025            get_value!(event.tags!),
1026            &Tags(PairList(vec![
1027                Annotated::new(TagEntry(
1028                    Annotated::new("foo".to_owned()),
1029                    Annotated::new("1".to_owned()),
1030                )),
1031                Annotated::new(TagEntry(
1032                    Annotated::new("bar".to_owned()),
1033                    Annotated::new("1".to_owned()),
1034                )),
1035            ]))
1036        );
1037    }
1038
1039    #[test]
1040    fn test_transaction_status_defaulted_to_unknown() {
1041        let mut object = Object::new();
1042        let trace_context = TraceContext {
1043            // We assume the status to be null.
1044            status: Annotated::empty(),
1045            ..TraceContext::default()
1046        };
1047        object.insert(
1048            "trace".to_owned(),
1049            Annotated::new(ContextInner(Context::Trace(Box::new(trace_context)))),
1050        );
1051
1052        let mut event = Annotated::new(Event {
1053            contexts: Annotated::new(Contexts(object)),
1054            ..Event::default()
1055        });
1056        normalize_event(&mut event, &NormalizationConfig::default());
1057
1058        let event = event.value().unwrap();
1059
1060        let event_trace_context = event.context::<TraceContext>().unwrap();
1061        assert_eq!(
1062            event_trace_context.status,
1063            Annotated::new(SpanStatus::Unknown)
1064        )
1065    }
1066
1067    #[test]
1068    fn test_unknown_debug_image() {
1069        let mut event = Annotated::new(Event {
1070            debug_meta: Annotated::new(DebugMeta {
1071                images: Annotated::new(vec![Annotated::new(DebugImage::Other(Object::default()))]),
1072                ..DebugMeta::default()
1073            }),
1074            ..Event::default()
1075        });
1076
1077        normalize_event(&mut event, &NormalizationConfig::default());
1078
1079        assert_eq!(
1080            get_path!(event.debug_meta!),
1081            &Annotated::new(DebugMeta {
1082                images: Annotated::new(vec![Annotated::from_error(
1083                    Error::invalid("unsupported debug image type"),
1084                    Some(Value::Object(Object::default())),
1085                )]),
1086                ..DebugMeta::default()
1087            })
1088        );
1089    }
1090
1091    #[test]
1092    fn test_context_line_default() {
1093        let mut frame = Annotated::new(Frame {
1094            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1095            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1096            ..Frame::default()
1097        });
1098
1099        normalize_non_raw_frame(&mut frame);
1100
1101        let frame = frame.value().unwrap();
1102        assert_eq!(frame.context_line.as_str(), Some(""));
1103    }
1104
1105    #[test]
1106    fn test_context_line_retain() {
1107        let mut frame = Annotated::new(Frame {
1108            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1109            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1110            context_line: Annotated::new("some line".to_owned()),
1111            ..Frame::default()
1112        });
1113
1114        normalize_non_raw_frame(&mut frame);
1115
1116        let frame = frame.value().unwrap();
1117        assert_eq!(frame.context_line.as_str(), Some("some line"));
1118    }
1119
1120    #[test]
1121    fn test_frame_null_context_lines() {
1122        let mut frame = Annotated::new(Frame {
1123            pre_context: Annotated::new(vec![Annotated::default(), Annotated::new("".to_owned())]),
1124            post_context: Annotated::new(vec![Annotated::new("".to_owned()), Annotated::default()]),
1125            ..Frame::default()
1126        });
1127
1128        normalize_non_raw_frame(&mut frame);
1129
1130        assert_eq!(
1131            *get_value!(frame.pre_context!),
1132            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1133        );
1134        assert_eq!(
1135            *get_value!(frame.post_context!),
1136            vec![Annotated::new("".to_owned()), Annotated::new("".to_owned())],
1137        );
1138    }
1139
1140    #[test]
1141    fn test_too_long_distribution() {
1142        let json = r#"{
1143  "event_id": "52df9022835246eeb317dbd739ccd059",
1144  "fingerprint": [
1145    "{{ default }}"
1146  ],
1147  "platform": "other",
1148  "dist": "52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059"
1149}"#;
1150
1151        let mut event = Annotated::<Event>::from_json(json).unwrap();
1152
1153        normalize_event(&mut event, &NormalizationConfig::default());
1154
1155        let dist = &event.value().unwrap().dist;
1156        let result = &Annotated::<String>::from_error(
1157            Error::new(ErrorKind::ValueTooLong),
1158            Some(Value::String("52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059-52df9022835246eeb317dbd739ccd059".to_owned()))
1159        );
1160        assert_eq!(dist, result);
1161    }
1162
1163    #[test]
1164    fn test_regression_backfills_abs_path_even_when_moving_stacktrace() {
1165        let mut event = Annotated::new(Event {
1166            exceptions: Annotated::new(Values::new(vec![Annotated::new(Exception {
1167                ty: Annotated::new("FooDivisionError".to_owned()),
1168                value: Annotated::new("hi".to_owned().into()),
1169                ..Exception::default()
1170            })])),
1171            stacktrace: Annotated::new(
1172                RawStacktrace {
1173                    frames: Annotated::new(vec![Annotated::new(Frame {
1174                        module: Annotated::new("MyModule".to_owned()),
1175                        filename: Annotated::new("MyFilename".into()),
1176                        function: Annotated::new("Void FooBar()".to_owned()),
1177                        ..Frame::default()
1178                    })]),
1179                    ..RawStacktrace::default()
1180                }
1181                .into(),
1182            ),
1183            ..Event::default()
1184        });
1185
1186        normalize_event(&mut event, &NormalizationConfig::default());
1187
1188        assert_eq!(
1189            get_value!(event.exceptions.values[0].stacktrace!),
1190            &Stacktrace(RawStacktrace {
1191                frames: Annotated::new(vec![Annotated::new(Frame {
1192                    module: Annotated::new("MyModule".to_owned()),
1193                    filename: Annotated::new("MyFilename".into()),
1194                    abs_path: Annotated::new("MyFilename".into()),
1195                    function: Annotated::new("Void FooBar()".to_owned()),
1196                    ..Frame::default()
1197                })]),
1198                ..RawStacktrace::default()
1199            })
1200        );
1201    }
1202
1203    #[test]
1204    fn test_parses_sdk_info_from_header() {
1205        let mut event = Annotated::new(Event::default());
1206
1207        normalize_event(
1208            &mut event,
1209            &NormalizationConfig {
1210                client: Some("_fooBar/0.0.0".to_owned()),
1211                ..Default::default()
1212            },
1213        );
1214
1215        assert_eq!(
1216            get_path!(event.client_sdk!),
1217            &Annotated::new(ClientSdkInfo {
1218                name: Annotated::new("_fooBar".to_owned()),
1219                version: Annotated::new("0.0.0".to_owned()),
1220                ..ClientSdkInfo::default()
1221            })
1222        );
1223    }
1224
1225    #[test]
1226    fn test_discards_received() {
1227        let mut event = Annotated::new(Event {
1228            received: FromValue::from_value(Annotated::new(Value::U64(696_969_696_969))),
1229            ..Default::default()
1230        });
1231
1232        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1233        normalize_event(&mut event, &NormalizationConfig::default());
1234
1235        assert_eq!(get_value!(event.received!), get_value!(event.timestamp!));
1236    }
1237
1238    #[test]
1239    fn test_grouping_config() {
1240        let mut event = Annotated::new(Event {
1241            logentry: Annotated::from(LogEntry {
1242                message: Annotated::new("Hello World!".to_owned().into()),
1243                ..Default::default()
1244            }),
1245            ..Default::default()
1246        });
1247
1248        validate_event(&mut event, &EventValidationConfig::default()).unwrap();
1249        normalize_event(
1250            &mut event,
1251            &NormalizationConfig {
1252                grouping_config: Some(json!({
1253                    "id": "legacy:1234-12-12".to_owned(),
1254                })),
1255                ..Default::default()
1256            },
1257        );
1258
1259        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1260            ".event_id" => "[event-id]",
1261            ".received" => "[received]",
1262            ".timestamp" => "[timestamp]"
1263        }, @r###"
1264        {
1265          "event_id": "[event-id]",
1266          "level": "error",
1267          "type": "default",
1268          "logentry": {
1269            "formatted": "Hello World!",
1270          },
1271          "logger": "",
1272          "platform": "other",
1273          "timestamp": "[timestamp]",
1274          "received": "[received]",
1275          "grouping_config": {
1276            "id": "legacy:1234-12-12",
1277          },
1278        }
1279        "###);
1280    }
1281
1282    #[test]
1283    fn test_logentry_error() {
1284        let json = r#"
1285{
1286    "event_id": "74ad1301f4df489ead37d757295442b1",
1287    "timestamp": 1668148328.308933,
1288    "received": 1668148328.308933,
1289    "level": "error",
1290    "platform": "python",
1291    "logentry": {
1292        "params": [
1293            "bogus"
1294        ],
1295        "formatted": 42
1296    }
1297}
1298"#;
1299        let mut event = Annotated::<Event>::from_json(json).unwrap();
1300
1301        normalize_event(&mut event, &NormalizationConfig::default());
1302
1303        assert_json_snapshot!(SerializableAnnotated(&event), {".received" => "[received]"}, @r###"
1304        {
1305          "event_id": "74ad1301f4df489ead37d757295442b1",
1306          "level": "error",
1307          "type": "default",
1308          "logentry": null,
1309          "logger": "",
1310          "platform": "python",
1311          "timestamp": 1668148328.308933,
1312          "received": "[received]",
1313          "_meta": {
1314            "logentry": {
1315              "": {
1316                "err": [
1317                  [
1318                    "invalid_data",
1319                    {
1320                      "reason": "no message present"
1321                    }
1322                  ]
1323                ],
1324                "val": {
1325                  "formatted": null,
1326                  "message": null,
1327                  "params": [
1328                    "bogus"
1329                  ]
1330                }
1331              }
1332            }
1333          }
1334        }
1335        "###)
1336    }
1337
1338    #[test]
1339    fn test_future_timestamp() {
1340        let mut event = Annotated::new(Event {
1341            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 2, 0).unwrap().into()),
1342            ..Default::default()
1343        });
1344
1345        let received_at = Some(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap());
1346        let max_secs_in_past = Some(30 * 24 * 3600);
1347        let max_secs_in_future = Some(60);
1348
1349        validate_event(
1350            &mut event,
1351            &EventValidationConfig {
1352                received_at,
1353                max_secs_in_past,
1354                max_secs_in_future,
1355                is_validated: false,
1356                ..Default::default()
1357            },
1358        )
1359        .unwrap();
1360        normalize_event(&mut event, &NormalizationConfig::default());
1361
1362        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1363        ".event_id" => "[event-id]",
1364    }, @r###"
1365        {
1366          "event_id": "[event-id]",
1367          "level": "error",
1368          "type": "default",
1369          "logger": "",
1370          "platform": "other",
1371          "timestamp": 946857600.0,
1372          "received": 946857600.0,
1373          "_meta": {
1374            "timestamp": {
1375              "": Meta(Some(MetaInner(
1376                err: [
1377                  [
1378                    "future_timestamp",
1379                    {
1380                      "sdk_time": "2000-01-03T00:02:00+00:00",
1381                      "server_time": "2000-01-03T00:00:00+00:00",
1382                    },
1383                  ],
1384                ],
1385              ))),
1386            },
1387          },
1388        }
1389        "###);
1390    }
1391
1392    #[test]
1393    fn test_past_timestamp() {
1394        let mut event = Annotated::new(Event {
1395            timestamp: Annotated::new(Utc.with_ymd_and_hms(2000, 1, 3, 0, 0, 0).unwrap().into()),
1396            ..Default::default()
1397        });
1398
1399        let received_at = Some(Utc.with_ymd_and_hms(2000, 3, 3, 0, 0, 0).unwrap());
1400        let max_secs_in_past = Some(30 * 24 * 3600);
1401        let max_secs_in_future = Some(60);
1402
1403        validate_event(
1404            &mut event,
1405            &EventValidationConfig {
1406                received_at,
1407                max_secs_in_past,
1408                max_secs_in_future,
1409                is_validated: false,
1410                ..Default::default()
1411            },
1412        )
1413        .unwrap();
1414        normalize_event(&mut event, &NormalizationConfig::default());
1415
1416        insta::assert_ron_snapshot!(SerializableAnnotated(&event), {
1417        ".event_id" => "[event-id]",
1418    }, @r###"
1419        {
1420          "event_id": "[event-id]",
1421          "level": "error",
1422          "type": "default",
1423          "logger": "",
1424          "platform": "other",
1425          "timestamp": 952041600.0,
1426          "received": 952041600.0,
1427          "_meta": {
1428            "timestamp": {
1429              "": Meta(Some(MetaInner(
1430                err: [
1431                  [
1432                    "past_timestamp",
1433                    {
1434                      "sdk_time": "2000-01-03T00:00:00+00:00",
1435                      "server_time": "2000-03-03T00:00:00+00:00",
1436                    },
1437                  ],
1438                ],
1439              ))),
1440            },
1441          },
1442        }
1443        "###);
1444    }
1445
1446    #[test]
1447    fn test_normalize_logger_empty() {
1448        let mut event = Event::from_value(
1449            serde_json::json!({
1450                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1451                "platform": "java",
1452                "logger": "",
1453            })
1454            .into(),
1455        );
1456
1457        normalize_event(&mut event, &NormalizationConfig::default());
1458        assert_annotated_snapshot!(event);
1459    }
1460
1461    #[test]
1462    fn test_normalize_logger_trimmed() {
1463        let mut event = Event::from_value(
1464            serde_json::json!({
1465                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1466                "platform": "java",
1467                "logger": " \t  \t   ",
1468            })
1469            .into(),
1470        );
1471
1472        normalize_event(&mut event, &NormalizationConfig::default());
1473        assert_annotated_snapshot!(event);
1474    }
1475
1476    #[test]
1477    fn test_normalize_logger_short_no_trimming() {
1478        let mut event = Event::from_value(
1479            serde_json::json!({
1480                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1481                "platform": "java",
1482                "logger": "my.short-logger.isnt_trimmed",
1483            })
1484            .into(),
1485        );
1486
1487        normalize_event(&mut event, &NormalizationConfig::default());
1488        assert_annotated_snapshot!(event);
1489    }
1490
1491    #[test]
1492    fn test_normalize_logger_exact_length() {
1493        let mut event = Event::from_value(
1494            serde_json::json!({
1495                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1496                "platform": "java",
1497                "logger": "this_is-exactly-the_max_len.012345678901234567890123456789012345",
1498            })
1499            .into(),
1500        );
1501
1502        normalize_event(&mut event, &NormalizationConfig::default());
1503        assert_annotated_snapshot!(event);
1504    }
1505
1506    #[test]
1507    fn test_normalize_logger_too_long_single_word() {
1508        let mut event = Event::from_value(
1509            serde_json::json!({
1510                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1511                "platform": "java",
1512                "logger": "this_is-way_too_long-and_we_only_have_one_word-so_we_cant_smart_trim",
1513            })
1514            .into(),
1515        );
1516
1517        normalize_event(&mut event, &NormalizationConfig::default());
1518        assert_annotated_snapshot!(event);
1519    }
1520
1521    #[test]
1522    fn test_normalize_logger_word_trimmed_at_max() {
1523        let mut event = Event::from_value(
1524            serde_json::json!({
1525                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1526                "platform": "java",
1527                "logger": "already_out.out.in.this_part-is-kept.this_right_here-is_an-extremely_long_word",
1528            })
1529            .into(),
1530        );
1531
1532        normalize_event(&mut event, &NormalizationConfig::default());
1533        assert_annotated_snapshot!(event);
1534    }
1535
1536    #[test]
1537    fn test_normalize_logger_word_trimmed_before_max() {
1538        // This test verifies the "smart" trimming on words -- the logger name
1539        // should be cut before the max limit, removing entire words.
1540        let mut event = Event::from_value(
1541            serde_json::json!({
1542                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1543                "platform": "java",
1544                "logger": "super_out.this_is_already_out_too.this_part-is-kept.this_right_here-is_a-very_long_word",
1545            })
1546            .into(),
1547        );
1548
1549        normalize_event(&mut event, &NormalizationConfig::default());
1550        assert_annotated_snapshot!(event);
1551    }
1552
1553    #[test]
1554    fn test_normalize_logger_word_leading_dots() {
1555        let mut event = Event::from_value(
1556            serde_json::json!({
1557                "event_id": "7637af36578e4e4592692e28a1d6e2ca",
1558                "platform": "java",
1559                "logger": "io.this-tests-the-smart-trimming-and-word-removal-around-dot.words",
1560            })
1561            .into(),
1562        );
1563
1564        normalize_event(&mut event, &NormalizationConfig::default());
1565        assert_annotated_snapshot!(event);
1566    }
1567
1568    #[test]
1569    fn test_normalization_is_idempotent() {
1570        // get an event, normalize it. the result of that must be the same as normalizing it once more
1571        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1572        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1573        let mut event = Annotated::new(Event {
1574            ty: Annotated::new(EventType::Transaction),
1575            transaction: Annotated::new("/".to_owned()),
1576            timestamp: Annotated::new(end.into()),
1577            start_timestamp: Annotated::new(start.into()),
1578            contexts: {
1579                let mut contexts = Contexts::new();
1580                contexts.add(TraceContext {
1581                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1582                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1583                    op: Annotated::new("http.server".to_owned()),
1584                    ..Default::default()
1585                });
1586                Annotated::new(contexts)
1587            },
1588            spans: Annotated::new(vec![Annotated::new(Span {
1589                timestamp: Annotated::new(
1590                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1591                ),
1592                start_timestamp: Annotated::new(
1593                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1594                ),
1595                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1596                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1597
1598                ..Default::default()
1599            })]),
1600            ..Default::default()
1601        });
1602
1603        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1604            relay_event_schema::processor::apply(event, |e, _m| {
1605                e.received = Annotated::empty();
1606                Ok(())
1607            })
1608            .unwrap();
1609            event
1610        }
1611
1612        normalize_event(&mut event, &NormalizationConfig::default());
1613        let first = remove_received_from_event(&mut event.clone())
1614            .to_json()
1615            .unwrap();
1616        // Expected some fields (such as timestamps) exist after first normalization.
1617
1618        normalize_event(&mut event, &NormalizationConfig::default());
1619        let second = remove_received_from_event(&mut event.clone())
1620            .to_json()
1621            .unwrap();
1622        assert_eq!(&first, &second, "idempotency check failed");
1623
1624        normalize_event(&mut event, &NormalizationConfig::default());
1625        let third = remove_received_from_event(&mut event.clone())
1626            .to_json()
1627            .unwrap();
1628        assert_eq!(&second, &third, "idempotency check failed");
1629    }
1630
1631    /// Validate full normalization is idempotent.
1632    ///
1633    /// Both PoPs and processing relays will temporarily run full normalization
1634    /// in events, during the rollout of running normalization once in internal
1635    /// relays.
1636    // TODO(iker): remove this test after the rollout is done.
1637    #[test]
1638    fn test_full_normalization_is_idempotent() {
1639        let start = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap();
1640        let end = Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap();
1641        let mut event = Annotated::new(Event {
1642            ty: Annotated::new(EventType::Transaction),
1643            transaction: Annotated::new("/".to_owned()),
1644            timestamp: Annotated::new(end.into()),
1645            start_timestamp: Annotated::new(start.into()),
1646            contexts: {
1647                let mut contexts = Contexts::new();
1648                contexts.add(TraceContext {
1649                    trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1650                    span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1651                    op: Annotated::new("http.server".to_owned()),
1652                    ..Default::default()
1653                });
1654                Annotated::new(contexts)
1655            },
1656            spans: Annotated::new(vec![Annotated::new(Span {
1657                timestamp: Annotated::new(
1658                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 10).unwrap().into(),
1659                ),
1660                start_timestamp: Annotated::new(
1661                    Utc.with_ymd_and_hms(2000, 1, 1, 0, 0, 0).unwrap().into(),
1662                ),
1663                trace_id: Annotated::new("4c79f60c11214eb38604f4ae0781bfb2".parse().unwrap()),
1664                span_id: Annotated::new("fa90fdead5f74053".parse().unwrap()),
1665
1666                ..Default::default()
1667            })]),
1668            ..Default::default()
1669        });
1670
1671        fn remove_received_from_event(event: &mut Annotated<Event>) -> &mut Annotated<Event> {
1672            relay_event_schema::processor::apply(event, |e, _m| {
1673                e.received = Annotated::empty();
1674                Ok(())
1675            })
1676            .unwrap();
1677            event
1678        }
1679
1680        let full_normalization_config = NormalizationConfig {
1681            is_renormalize: false,
1682            enable_trimming: true,
1683            remove_other: true,
1684            emit_event_errors: true,
1685            ..Default::default()
1686        };
1687
1688        normalize_event(&mut event, &full_normalization_config);
1689        let first = remove_received_from_event(&mut event.clone())
1690            .to_json()
1691            .unwrap();
1692        // Expected some fields (such as timestamps) exist after first normalization.
1693
1694        normalize_event(&mut event, &full_normalization_config);
1695        let second = remove_received_from_event(&mut event.clone())
1696            .to_json()
1697            .unwrap();
1698        assert_eq!(&first, &second, "idempotency check failed");
1699
1700        normalize_event(&mut event, &full_normalization_config);
1701        let third = remove_received_from_event(&mut event.clone())
1702            .to_json()
1703            .unwrap();
1704        assert_eq!(&second, &third, "idempotency check failed");
1705    }
1706
1707    #[test]
1708    fn test_normalize_validates_spans() {
1709        let event = Annotated::<Event>::from_json(
1710            r#"
1711            {
1712                "type": "transaction",
1713                "transaction": "/",
1714                "timestamp": 946684810.0,
1715                "start_timestamp": 946684800.0,
1716                "contexts": {
1717                    "trace": {
1718                    "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1719                    "span_id": "fa90fdead5f74053",
1720                    "op": "http.server",
1721                    "type": "trace"
1722                    }
1723                },
1724                "spans": []
1725            }
1726            "#,
1727        )
1728        .unwrap();
1729
1730        // All incomplete spans should be caught by normalization:
1731        for span in [
1732            r#"null"#,
1733            r#"{
1734              "timestamp": 946684810.0,
1735              "start_timestamp": 946684900.0,
1736              "span_id": "fa90fdead5f74053",
1737              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1738            }"#,
1739            r#"{
1740              "timestamp": 946684810.0,
1741              "span_id": "fa90fdead5f74053",
1742              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1743            }"#,
1744            r#"{
1745              "timestamp": 946684810.0,
1746              "start_timestamp": 946684800.0,
1747              "trace_id": "4c79f60c11214eb38604f4ae0781bfb2"
1748            }"#,
1749            r#"{
1750              "timestamp": 946684810.0,
1751              "start_timestamp": 946684800.0,
1752              "span_id": "fa90fdead5f74053"
1753            }"#,
1754        ] {
1755            let mut modified_event = event.clone();
1756            let event_ref = modified_event.value_mut().as_mut().unwrap();
1757            event_ref
1758                .spans
1759                .set_value(Some(vec![Annotated::<Span>::from_json(span).unwrap()]));
1760
1761            let res = validate_event(&mut modified_event, &EventValidationConfig::default());
1762
1763            assert!(res.is_err(), "{span:?}");
1764        }
1765    }
1766
1767    #[test]
1768    fn test_normalization_respects_is_renormalize() {
1769        let mut event = Annotated::<Event>::from_json(
1770            r#"
1771            {
1772                "type": "default",
1773                "tags": [["environment", "some_environment"]]
1774            }
1775            "#,
1776        )
1777        .unwrap();
1778
1779        normalize_event(
1780            &mut event,
1781            &NormalizationConfig {
1782                is_renormalize: true,
1783                ..Default::default()
1784            },
1785        );
1786
1787        assert_debug_snapshot!(event.value().unwrap().tags, @r###"
1788        Tags(
1789            PairList(
1790                [
1791                    TagEntry(
1792                        "environment",
1793                        "some_environment",
1794                    ),
1795                ],
1796            ),
1797        )
1798        "###);
1799    }
1800
1801    #[test]
1802    fn test_geo_in_normalize() {
1803        let mut event = Annotated::<Event>::from_json(
1804            r#"
1805            {
1806                "type": "transaction",
1807                "transaction": "/foo/",
1808                "timestamp": 946684810.0,
1809                "start_timestamp": 946684800.0,
1810                "contexts": {
1811                    "trace": {
1812                        "trace_id": "4c79f60c11214eb38604f4ae0781bfb2",
1813                        "span_id": "fa90fdead5f74053",
1814                        "op": "http.server",
1815                        "type": "trace"
1816                    }
1817                },
1818                "transaction_info": {
1819                    "source": "url"
1820                },
1821                "user": {
1822                    "ip_address": "2.125.160.216"
1823                }
1824            }
1825            "#,
1826        )
1827        .unwrap();
1828
1829        let lookup = GeoIpLookup::open("tests/fixtures/GeoIP2-Enterprise-Test.mmdb").unwrap();
1830
1831        // Extract user's geo information before normalization.
1832        let user_geo = event.value().unwrap().user.value().unwrap().geo.value();
1833        assert!(user_geo.is_none());
1834
1835        normalize_event(
1836            &mut event,
1837            &NormalizationConfig {
1838                geoip_lookup: Some(&lookup),
1839                ..Default::default()
1840            },
1841        );
1842
1843        // Extract user's geo information after normalization.
1844        let user_geo = event
1845            .value()
1846            .unwrap()
1847            .user
1848            .value()
1849            .unwrap()
1850            .geo
1851            .value()
1852            .unwrap();
1853
1854        assert_eq!(user_geo.country_code.value().unwrap(), "GB");
1855        assert_eq!(user_geo.city.value().unwrap(), "Boxford");
1856    }
1857
1858    #[test]
1859    fn test_normalize_app_start_spans_only_for_react_native_3_to_4_4() {
1860        let mut event = Event {
1861            spans: Annotated::new(vec![Annotated::new(Span {
1862                op: Annotated::new("app_start_cold".to_owned()),
1863                ..Default::default()
1864            })]),
1865            client_sdk: Annotated::new(ClientSdkInfo {
1866                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1867                version: Annotated::new("4.5.0".to_owned()),
1868                ..Default::default()
1869            }),
1870            ..Default::default()
1871        };
1872        normalize_app_start_spans(&mut event);
1873        assert_debug_snapshot!(event.spans, @r###"
1874        [
1875            Span {
1876                timestamp: ~,
1877                start_timestamp: ~,
1878                exclusive_time: ~,
1879                op: "app_start_cold",
1880                span_id: ~,
1881                parent_span_id: ~,
1882                trace_id: ~,
1883                segment_id: ~,
1884                is_segment: ~,
1885                is_remote: ~,
1886                status: ~,
1887                description: ~,
1888                tags: ~,
1889                origin: ~,
1890                profile_id: ~,
1891                data: ~,
1892                links: ~,
1893                sentry_tags: ~,
1894                received: ~,
1895                measurements: ~,
1896                platform: ~,
1897                was_transaction: ~,
1898                kind: ~,
1899                performance_issues_spans: ~,
1900                other: {},
1901            },
1902        ]
1903        "###);
1904    }
1905
1906    #[test]
1907    fn test_normalize_app_start_cold_spans_for_react_native() {
1908        let mut event = Event {
1909            spans: Annotated::new(vec![Annotated::new(Span {
1910                op: Annotated::new("app_start_cold".to_owned()),
1911                ..Default::default()
1912            })]),
1913            client_sdk: Annotated::new(ClientSdkInfo {
1914                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1915                version: Annotated::new("4.4.0".to_owned()),
1916                ..Default::default()
1917            }),
1918            ..Default::default()
1919        };
1920        normalize_app_start_spans(&mut event);
1921        assert_debug_snapshot!(event.spans, @r###"
1922        [
1923            Span {
1924                timestamp: ~,
1925                start_timestamp: ~,
1926                exclusive_time: ~,
1927                op: "app.start.cold",
1928                span_id: ~,
1929                parent_span_id: ~,
1930                trace_id: ~,
1931                segment_id: ~,
1932                is_segment: ~,
1933                is_remote: ~,
1934                status: ~,
1935                description: ~,
1936                tags: ~,
1937                origin: ~,
1938                profile_id: ~,
1939                data: ~,
1940                links: ~,
1941                sentry_tags: ~,
1942                received: ~,
1943                measurements: ~,
1944                platform: ~,
1945                was_transaction: ~,
1946                kind: ~,
1947                performance_issues_spans: ~,
1948                other: {},
1949            },
1950        ]
1951        "###);
1952    }
1953
1954    #[test]
1955    fn test_normalize_app_start_warm_spans_for_react_native() {
1956        let mut event = Event {
1957            spans: Annotated::new(vec![Annotated::new(Span {
1958                op: Annotated::new("app_start_warm".to_owned()),
1959                ..Default::default()
1960            })]),
1961            client_sdk: Annotated::new(ClientSdkInfo {
1962                name: Annotated::new("sentry.javascript.react-native".to_owned()),
1963                version: Annotated::new("4.4.0".to_owned()),
1964                ..Default::default()
1965            }),
1966            ..Default::default()
1967        };
1968        normalize_app_start_spans(&mut event);
1969        assert_debug_snapshot!(event.spans, @r###"
1970        [
1971            Span {
1972                timestamp: ~,
1973                start_timestamp: ~,
1974                exclusive_time: ~,
1975                op: "app.start.warm",
1976                span_id: ~,
1977                parent_span_id: ~,
1978                trace_id: ~,
1979                segment_id: ~,
1980                is_segment: ~,
1981                is_remote: ~,
1982                status: ~,
1983                description: ~,
1984                tags: ~,
1985                origin: ~,
1986                profile_id: ~,
1987                data: ~,
1988                links: ~,
1989                sentry_tags: ~,
1990                received: ~,
1991                measurements: ~,
1992                platform: ~,
1993                was_transaction: ~,
1994                kind: ~,
1995                performance_issues_spans: ~,
1996                other: {},
1997            },
1998        ]
1999        "###);
2000    }
2001}
relay_event_normalization/normalize/mod.rs

relay_event_normalization/normalize/
mod.rs
