relay_server/services/processor/
attachment.rs

Help
1//! Attachments processor code.
2
3use std::error::Error;
4use std::sync::Arc;
5use std::time::Instant;
6
7use relay_pii::{PiiAttachmentsProcessor, SelectorPathItem, SelectorSpec};
8use relay_statsd::metric;
9
10use crate::envelope::{AttachmentType, ContentType, ItemType};
11use crate::statsd::RelayTimers;
12
13use crate::services::projects::project::ProjectInfo;
14use crate::utils::TypedEnvelope;
15use relay_dynamic_config::Feature;
16#[cfg(feature = "processing")]
17use {
18    crate::services::processor::{ErrorGroup, EventFullyNormalized},
19    crate::utils,
20    relay_event_schema::protocol::{Event, Metrics},
21    relay_protocol::Annotated,
22};
23
24/// Adds processing placeholders for special attachments.
25///
26/// If special attachments are present in the envelope, this adds placeholder payloads to the
27/// event. This indicates to the pipeline that the event needs special processing.
28///
29/// If the event payload was empty before, it is created.
30#[cfg(feature = "processing")]
31pub fn create_placeholders(
32    managed_envelope: &mut TypedEnvelope<ErrorGroup>,
33    event: &mut Annotated<Event>,
34    metrics: &mut Metrics,
35) -> Option<EventFullyNormalized> {
36    let envelope = managed_envelope.envelope();
37    let minidump_attachment =
38        envelope.get_item_by(|item| item.attachment_type() == Some(&AttachmentType::Minidump));
39    let apple_crash_report_attachment = envelope
40        .get_item_by(|item| item.attachment_type() == Some(&AttachmentType::AppleCrashReport));
41
42    if let Some(item) = minidump_attachment {
43        let event = event.get_or_insert_with(Event::default);
44        metrics.bytes_ingested_event_minidump = Annotated::new(item.len() as u64);
45        utils::process_minidump(event, &item.payload());
46        return Some(EventFullyNormalized(false));
47    } else if let Some(item) = apple_crash_report_attachment {
48        let event = event.get_or_insert_with(Event::default);
49        metrics.bytes_ingested_event_applecrashreport = Annotated::new(item.len() as u64);
50        utils::process_apple_crash_report(event, &item.payload());
51        return Some(EventFullyNormalized(false));
52    }
53
54    None
55}
56
57/// Apply data privacy rules to attachments in the envelope.
58///
59/// This only applies the new PII rules that explicitly select `ValueType::Binary` or one of the
60/// attachment types. When special attachments are detected, these are scrubbed with custom
61/// logic; otherwise the entire attachment is treated as a single binary blob.
62pub fn scrub<Group>(managed_envelope: &mut TypedEnvelope<Group>, project_info: Arc<ProjectInfo>) {
63    let envelope = managed_envelope.envelope_mut();
64    if let Some(ref config) = project_info.config.pii_config {
65        let view_hierarchy_scrubbing_enabled = project_info
66            .config
67            .features
68            .has(Feature::ViewHierarchyScrubbing);
69        for item in envelope.items_mut() {
70            if view_hierarchy_scrubbing_enabled
71                && item.attachment_type() == Some(&AttachmentType::ViewHierarchy)
72            {
73                scrub_view_hierarchy(item, config)
74            } else if item.attachment_type() == Some(&AttachmentType::Minidump) {
75                scrub_minidump(item, config)
76            } else if item.ty() == &ItemType::Attachment && has_simple_attachment_selector(config) {
77                // We temporarily only scrub attachments to projects that have at least one simple attachment rule,
78                // such as `$attachments.'foo.txt'`.
79                // After we have assessed the impact on performance we can relax this condition.
80                scrub_attachment(item, config)
81            }
82        }
83    }
84}
85
86fn scrub_minidump(item: &mut crate::envelope::Item, config: &relay_pii::PiiConfig) {
87    debug_assert_eq!(item.attachment_type(), Some(&AttachmentType::Minidump));
88    let filename = item.filename().unwrap_or_default();
89    let mut payload = item.payload().to_vec();
90
91    let processor = PiiAttachmentsProcessor::new(config.compiled());
92
93    // Minidump scrubbing can fail if the minidump cannot be parsed. In this case, we
94    // must be conservative and treat it as a plain attachment. Under extreme
95    // conditions, this could destroy stack memory.
96    let start = Instant::now();
97    match processor.scrub_minidump(filename, &mut payload) {
98        Ok(modified) => {
99            metric!(
100                timer(RelayTimers::MinidumpScrubbing) = start.elapsed(),
101                status = if modified { "ok" } else { "n/a" },
102            );
103        }
104        Err(scrub_error) => {
105            metric!(
106                timer(RelayTimers::MinidumpScrubbing) = start.elapsed(),
107                status = "error"
108            );
109            relay_log::warn!(
110                error = &scrub_error as &dyn Error,
111                "failed to scrub minidump",
112            );
113            metric!(
114                timer(RelayTimers::AttachmentScrubbing),
115                attachment_type = "minidump",
116                {
117                    processor.scrub_attachment(filename, &mut payload);
118                }
119            )
120        }
121    }
122
123    let content_type = item
124        .content_type()
125        .unwrap_or(&ContentType::Minidump)
126        .clone();
127
128    item.set_payload(content_type, payload);
129}
130
131fn scrub_view_hierarchy(item: &mut crate::envelope::Item, config: &relay_pii::PiiConfig) {
132    let processor = PiiAttachmentsProcessor::new(config.compiled());
133
134    let payload = item.payload();
135    let start = Instant::now();
136    match processor.scrub_json(&payload) {
137        Ok(output) => {
138            metric!(
139                timer(RelayTimers::ViewHierarchyScrubbing) = start.elapsed(),
140                status = "ok"
141            );
142            let content_type = item.content_type().unwrap_or(&ContentType::Json).clone();
143            item.set_payload(content_type, output);
144        }
145        Err(e) => {
146            relay_log::warn!(error = &e as &dyn Error, "failed to scrub view hierarchy",);
147            metric!(
148                timer(RelayTimers::ViewHierarchyScrubbing) = start.elapsed(),
149                status = "error"
150            )
151        }
152    }
153}
154
155fn has_simple_attachment_selector(config: &relay_pii::PiiConfig) -> bool {
156    for application in &config.applications {
157        if let SelectorSpec::Path(vec) = &application.0 {
158            let Some([a, b]) = vec.get(0..2) else {
159                continue;
160            };
161            if matches!(
162                a,
163                SelectorPathItem::Type(relay_event_schema::processor::ValueType::Attachments)
164            ) && matches!(b, SelectorPathItem::Key(_))
165            {
166                return true;
167            }
168        }
169    }
170    false
171}
172
173fn scrub_attachment(item: &mut crate::envelope::Item, config: &relay_pii::PiiConfig) {
174    let filename = item.filename().unwrap_or_default();
175    let mut payload = item.payload().to_vec();
176
177    let processor = PiiAttachmentsProcessor::new(config.compiled());
178    let attachment_type_tag = match item.attachment_type() {
179        Some(t) => t.to_string(),
180        None => "".to_owned(),
181    };
182    metric!(
183        timer(RelayTimers::AttachmentScrubbing),
184        attachment_type = &attachment_type_tag,
185        {
186            processor.scrub_attachment(filename, &mut payload);
187        }
188    );
189
190    item.set_payload_without_content_type(payload);
191}
192
193#[cfg(test)]
194mod tests {
195    use relay_pii::PiiConfig;
196
197    use super::*;
198
199    #[test]
200    fn matches_attachment_selector() {
201        let config = r#"{
202            "rules": {"0": {"type": "ip", "redaction": {"method": "remove"}}},
203            "applications": {"$attachments.'foo.txt'": ["0"]}
204        }"#;
205        let config: PiiConfig = serde_json::from_str(config).unwrap();
206        assert!(has_simple_attachment_selector(&config));
207    }
208
209    #[test]
210    fn does_not_match_wildcard() {
211        let config = r#"{
212            "rules": {},
213            "applications": {"$attachments.**":["0"]}
214        }"#;
215        let config: PiiConfig = serde_json::from_str(config).unwrap();
216        assert!(!has_simple_attachment_selector(&config));
217    }
218
219    #[test]
220    fn does_not_match_empty() {
221        let config = r#"{
222            "rules": {},
223            "applications": {}
224        }"#;
225        let config: PiiConfig = serde_json::from_str(config).unwrap();
226        assert!(!has_simple_attachment_selector(&config));
227    }
228
229    #[test]
230    fn does_not_match_something_else() {
231        let config = r#"{
232            "rules": {},
233            "applications": {
234                "**": ["0"]
235            }
236        }"#;
237        let config: PiiConfig = serde_json::from_str(config).unwrap();
238        assert!(!has_simple_attachment_selector(&config));
239    }
240}
relay_server/services/processor/attachment.rs

relay_server/services/processor/
attachment.rs
