Skip to main content

relay_dynamic_config/
metrics.rs

1//! Dynamic configuration for metrics extraction from sessions and transactions.
2
3use core::fmt;
4use std::collections::{BTreeMap, BTreeSet};
5use std::convert::Infallible;
6use std::str::FromStr;
7
8use relay_base_schema::data_category::DataCategory;
9use relay_common::glob2::LazyGlob;
10use relay_common::impl_str_serde;
11use relay_pattern::{Patterns, TypedPatterns};
12use relay_protocol::RuleCondition;
13use serde::{Deserialize, Serialize};
14
15use crate::project::ProjectConfig;
16
17/// Configuration for removing tags matching the `tag` pattern on metrics whose name matches the `name` pattern.
18#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq)]
19#[serde(default)]
20pub struct TagBlock {
21    /// Name of metric of which we want to remove certain tags.
22    #[serde(skip_serializing_if = "Patterns::is_empty")]
23    pub name: TypedPatterns,
24    /// Pattern to match keys of tags that we want to remove.
25    #[serde(skip_serializing_if = "Patterns::is_empty")]
26    pub tags: TypedPatterns,
27}
28
29/// Rule defining when a target tag should be set on a metric.
30#[derive(Debug, Clone, Serialize, Deserialize)]
31#[serde(rename_all = "camelCase")]
32pub struct TaggingRule {
33    // note: could add relay_sampling::RuleType here, but right now we only support transaction
34    // events
35    /// Condition that defines when to set the tag.
36    pub condition: RuleCondition,
37    /// Metrics on which the tag is set.
38    pub target_metrics: BTreeSet<String>,
39    /// Name of the tag that is set.
40    pub target_tag: String,
41    /// Value of the tag that is set.
42    pub tag_value: String,
43}
44
45/// Current version of metrics extraction.
46const SESSION_EXTRACT_VERSION: u16 = 3;
47const EXTRACT_ABNORMAL_MECHANISM_VERSION: u16 = 2;
48
49/// Configuration for metric extraction from sessions.
50#[derive(Debug, Clone, Copy, Default, Deserialize, Serialize)]
51#[serde(default, rename_all = "camelCase")]
52pub struct SessionMetricsConfig {
53    /// The revision of the extraction algorithm.
54    ///
55    /// Provided the revision is lower than or equal to the revision supported by this Relay,
56    /// metrics are extracted. If the revision is higher than what this Relay supports, it does not
57    /// extract metrics from sessions, and instead forwards them to the upstream.
58    ///
59    /// Version `0` (default) disables extraction.
60    version: u16,
61}
62
63impl SessionMetricsConfig {
64    /// Returns `true` if session metrics is enabled and compatible.
65    pub fn is_enabled(&self) -> bool {
66        self.version > 0 && self.version <= SESSION_EXTRACT_VERSION
67    }
68
69    /// Returns `true` if Relay should not extract metrics from sessions.
70    pub fn is_disabled(&self) -> bool {
71        !self.is_enabled()
72    }
73
74    /// Whether or not the abnormal mechanism should be extracted as a tag.
75    pub fn should_extract_abnormal_mechanism(&self) -> bool {
76        self.version >= EXTRACT_ABNORMAL_MECHANISM_VERSION
77    }
78}
79
80/// Configuration for extracting custom measurements from transaction payloads.
81#[derive(Default, Debug, Clone, Serialize, Deserialize)]
82#[serde(default, rename_all = "camelCase")]
83pub struct CustomMeasurementConfig {
84    /// The maximum number of custom measurements to extract. Defaults to zero.
85    limit: usize,
86}
87
88/// Combined view of global and project-specific metrics extraction configs.
89#[derive(Debug, Clone, Copy)]
90pub struct CombinedMetricExtractionConfig<'a> {
91    global: &'a MetricExtractionGroups,
92    project: &'a MetricExtractionConfig,
93}
94
95impl<'a> CombinedMetricExtractionConfig<'a> {
96    /// Creates a new combined view from two references.
97    pub fn new(global: &'a MetricExtractionGroups, project: &'a MetricExtractionConfig) -> Self {
98        for key in project.global_groups.keys() {
99            if !global.groups.contains_key(key) {
100                relay_log::error!(
101                    "Metrics group configured for project missing in global config: {key:?}"
102                )
103            }
104        }
105
106        Self { global, project }
107    }
108
109    /// Returns an iterator of metric specs.
110    pub fn metrics(&self) -> impl Iterator<Item = &MetricSpec> {
111        let project = self.project.metrics.iter();
112        let enabled_global = self
113            .enabled_groups()
114            .flat_map(|template| template.metrics.iter());
115
116        project.chain(enabled_global)
117    }
118
119    /// Returns an iterator of tag mappings.
120    pub fn tags(&self) -> impl Iterator<Item = &TagMapping> {
121        let project = self.project.tags.iter();
122        let enabled_global = self
123            .enabled_groups()
124            .flat_map(|template| template.tags.iter());
125
126        project.chain(enabled_global)
127    }
128
129    fn enabled_groups(&self) -> impl Iterator<Item = &MetricExtractionGroup> {
130        self.global.groups.iter().filter_map(|(key, template)| {
131            let is_enabled_by_override = self.project.global_groups.get(key).map(|c| c.is_enabled);
132            let is_enabled = is_enabled_by_override.unwrap_or(template.is_enabled);
133
134            is_enabled.then_some(template)
135        })
136    }
137}
138
139impl<'a> From<&'a MetricExtractionConfig> for CombinedMetricExtractionConfig<'a> {
140    /// Creates a combined config with an empty global component. Used in tests.
141    fn from(value: &'a MetricExtractionConfig) -> Self {
142        Self::new(MetricExtractionGroups::EMPTY, value)
143    }
144}
145
146/// Global groups for metric extraction.
147///
148/// Templates can be enabled or disabled by project configs.
149#[derive(Clone, Default, Debug, Serialize, Deserialize)]
150#[serde(rename_all = "camelCase")]
151pub struct MetricExtractionGroups {
152    /// Mapping from group name to metrics specs & tags.
153    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
154    pub groups: BTreeMap<GroupKey, MetricExtractionGroup>,
155}
156
157impl MetricExtractionGroups {
158    /// Empty config, used in tests and as a fallback.
159    pub const EMPTY: &'static Self = &Self {
160        groups: BTreeMap::new(),
161    };
162
163    /// Returns `true` if the contained groups are empty.
164    pub fn is_empty(&self) -> bool {
165        self.groups.is_empty()
166    }
167}
168
169/// Group of metrics & tags that can be enabled or disabled as a group.
170#[derive(Clone, Debug, Serialize, Deserialize)]
171#[serde(rename_all = "camelCase")]
172pub struct MetricExtractionGroup {
173    /// Whether the set is enabled by default.
174    ///
175    /// Project configs can overwrite this flag to opt-in or out of a set.
176    pub is_enabled: bool,
177
178    /// A list of metric specifications to extract.
179    #[serde(default, skip_serializing_if = "Vec::is_empty")]
180    pub metrics: Vec<MetricSpec>,
181
182    /// A list of tags to add to previously extracted metrics.
183    ///
184    /// These tags add further tags to a range of metrics. If some metrics already have a matching
185    /// tag extracted, the existing tag is left unchanged.
186    #[serde(default, skip_serializing_if = "Vec::is_empty")]
187    pub tags: Vec<TagMapping>,
188}
189
190/// Configuration for generic extraction of metrics from all data categories.
191#[derive(Clone, Default, Debug, Serialize, Deserialize)]
192#[serde(rename_all = "camelCase")]
193pub struct MetricExtractionConfig {
194    /// Versioning of metrics extraction. Relay skips extraction if the version is not supported.
195    pub version: u16,
196
197    /// Configuration of global metric groups.
198    ///
199    /// The groups themselves are configured in [`crate::GlobalConfig`],
200    /// but can be enabled or disabled here.
201    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
202    pub global_groups: BTreeMap<GroupKey, MetricExtractionGroupOverride>,
203
204    /// A list of metric specifications to extract.
205    #[serde(default, skip_serializing_if = "Vec::is_empty")]
206    pub metrics: Vec<MetricSpec>,
207
208    /// A list of tags to add to previously extracted metrics.
209    ///
210    /// These tags add further tags to a range of metrics. If some metrics already have a matching
211    /// tag extracted, the existing tag is left unchanged.
212    #[serde(default, skip_serializing_if = "Vec::is_empty")]
213    pub tags: Vec<TagMapping>,
214
215    /// This config has been extended with fields from `conditional_tagging`.
216    ///
217    /// At the moment, Relay will parse `conditional_tagging` rules and insert them into the `tags`
218    /// mapping in this struct. If the flag is `true`, this has already happened and should not be
219    /// repeated.
220    ///
221    /// This is a temporary flag that will be removed once the transaction metric extraction version
222    /// is bumped to `2`.
223    #[serde(default)]
224    pub _conditional_tags_extended: bool,
225
226    /// This config has been extended with default span metrics.
227    ///
228    /// Relay checks for the span extraction flag and adds built-in metrics and tags to this struct.
229    /// If the flag is `true`, this has already happened and should not be repeated.
230    ///
231    /// This is a temporary flag that will be removed once the transaction metric extraction version
232    /// is bumped to `2`.
233    #[serde(default)]
234    pub _span_metrics_extended: bool,
235}
236
237impl MetricExtractionConfig {
238    /// The latest version for this config struct.
239    ///
240    /// This is the maximum version supported by this Relay instance.
241    pub const MAX_SUPPORTED_VERSION: u16 = 4;
242
243    /// Returns an empty `MetricExtractionConfig` with the latest version.
244    ///
245    /// As opposed to `default()`, this will be enabled once populated with specs.
246    pub fn empty() -> Self {
247        Self {
248            version: Self::MAX_SUPPORTED_VERSION,
249            global_groups: BTreeMap::new(),
250            metrics: Default::default(),
251            tags: Default::default(),
252            _conditional_tags_extended: false,
253            _span_metrics_extended: false,
254        }
255    }
256
257    /// Returns `true` if the version of this metric extraction config is supported.
258    pub fn is_supported(&self) -> bool {
259        self.version <= Self::MAX_SUPPORTED_VERSION
260    }
261
262    /// Returns `true` if metric extraction is configured and compatible with this Relay.
263    pub fn is_enabled(&self) -> bool {
264        self.version > 0
265            && self.is_supported()
266            && !(self.metrics.is_empty() && self.tags.is_empty() && self.global_groups.is_empty())
267    }
268}
269
270/// Configures global metrics extraction groups.
271///
272/// Project configs can enable or disable globally defined groups.
273#[derive(Clone, Default, Debug, Serialize, Deserialize)]
274#[serde(rename_all = "camelCase")]
275pub struct MetricExtractionGroupOverride {
276    /// `true` if a template should be enabled.
277    pub is_enabled: bool,
278}
279
280/// Enumeration of keys in [`MetricExtractionGroups`]. In JSON, this is simply a string.
281#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
282pub enum GroupKey {
283    /// Metric extracted for all plans.
284    SpanMetricsCommon,
285    /// "addon" metrics.
286    SpanMetricsAddons,
287    /// Metrics extracted from spans in the transaction namespace.
288    SpanMetricsTx,
289    /// Any other group defined by the upstream.
290    Other(String),
291}
292
293impl fmt::Display for GroupKey {
294    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
295        write!(
296            f,
297            "{}",
298            match self {
299                GroupKey::SpanMetricsCommon => "span_metrics_common",
300                GroupKey::SpanMetricsAddons => "span_metrics_addons",
301                GroupKey::SpanMetricsTx => "span_metrics_tx",
302                GroupKey::Other(s) => &s,
303            }
304        )
305    }
306}
307
308impl FromStr for GroupKey {
309    type Err = Infallible;
310
311    fn from_str(s: &str) -> Result<Self, Self::Err> {
312        Ok(match s {
313            "span_metrics_common" => GroupKey::SpanMetricsCommon,
314            "span_metrics_addons" => GroupKey::SpanMetricsAddons,
315            "span_metrics_tx" => GroupKey::SpanMetricsTx,
316            s => GroupKey::Other(s.to_owned()),
317        })
318    }
319}
320
321impl_str_serde!(GroupKey, "a metrics extraction group key");
322
323/// Specification for a metric to extract from some data.
324#[derive(Clone, Debug, Serialize, Deserialize)]
325#[serde(rename_all = "camelCase")]
326pub struct MetricSpec {
327    /// Category of data to extract this metric for.
328    pub category: DataCategory,
329
330    /// The Metric Resource Identifier (MRI) of the metric to extract.
331    pub mri: String,
332
333    /// A path to the field to extract the metric from.
334    ///
335    /// This value contains a fully qualified expression pointing at the data field in the payload
336    /// to extract the metric from. It follows the `Getter` syntax that is also used for dynamic
337    /// sampling.
338    ///
339    /// How the value is treated depends on the metric type:
340    ///
341    /// - **Counter** metrics are a special case, since the default product counters do not count
342    ///   any specific field but rather the occurrence of the event. As such, there is no value
343    ///   expression, and the field is set to `None`. Semantics of specifying remain undefined at
344    ///   this point.
345    /// - **Distribution** metrics require a numeric value. If the value at the specified path is
346    ///   not numeric, metric extraction will be skipped.
347    /// - **Set** metrics require a string value, which is then emitted into the set as unique
348    ///   value. Insertion of numbers and other types is undefined.
349    ///
350    /// If the field does not exist, extraction is skipped.
351    #[serde(default, skip_serializing_if = "Option::is_none")]
352    pub field: Option<String>,
353
354    /// An optional condition to meet before extraction.
355    ///
356    /// See [`RuleCondition`] for all available options to specify and combine conditions. If no
357    /// condition is specified, the metric is extracted unconditionally.
358    #[serde(default, skip_serializing_if = "Option::is_none")]
359    pub condition: Option<RuleCondition>,
360
361    /// A list of tags to add to the metric.
362    ///
363    /// Tags can be conditional, see [`TagSpec`] for configuration options. For this reason, it is
364    /// possible to list tag keys multiple times, each with different conditions. The first matching
365    /// condition will be applied.
366    #[serde(default, skip_serializing_if = "Vec::is_empty")]
367    pub tags: Vec<TagSpec>,
368}
369
370/// Mapping between extracted metrics and additional tags to extract.
371#[derive(Clone, Debug, Serialize, Deserialize)]
372#[serde(rename_all = "camelCase")]
373pub struct TagMapping {
374    /// A list of Metric Resource Identifiers (MRI) to apply tags to.
375    ///
376    /// Entries in this list can contain wildcards to match metrics with dynamic MRIs.
377    #[serde(default)]
378    pub metrics: Vec<LazyGlob>,
379
380    /// A list of tags to add to the metric.
381    ///
382    /// Tags can be conditional, see [`TagSpec`] for configuration options. For this reason, it is
383    /// possible to list tag keys multiple times, each with different conditions. The first matching
384    /// condition will be applied.
385    #[serde(default)]
386    pub tags: Vec<TagSpec>,
387}
388
389impl TagMapping {
390    /// Returns `true` if this mapping matches the provided MRI.
391    pub fn matches(&self, mri: &str) -> bool {
392        // TODO: Use a globset, instead.
393        self.metrics
394            .iter()
395            .any(|glob| glob.compiled().is_match(mri))
396    }
397}
398
399/// Configuration for a tag to add to a metric.
400///
401/// Tags values can be static if defined through `value` or dynamically queried from the payload if
402/// defined through `field`. These two options are mutually exclusive, behavior is undefined if both
403/// are specified.
404#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
405#[serde(rename_all = "camelCase")]
406pub struct TagSpec {
407    /// The key of the tag to extract.
408    pub key: String,
409
410    /// Path to a field containing the tag's value.
411    ///
412    /// It follows the `Getter` syntax to read data from the payload.
413    ///
414    /// Mutually exclusive with `value`.
415    #[serde(default, skip_serializing_if = "Option::is_none")]
416    pub field: Option<String>,
417
418    /// Literal value of the tag.
419    ///
420    /// Mutually exclusive with `field`.
421    #[serde(default, skip_serializing_if = "Option::is_none")]
422    pub value: Option<String>,
423
424    /// An optional condition to meet before extraction.
425    ///
426    /// See [`RuleCondition`] for all available options to specify and combine conditions. If no
427    /// condition is specified, the tag is added unconditionally, provided it is not already there.
428    #[serde(default, skip_serializing_if = "Option::is_none")]
429    pub condition: Option<RuleCondition>,
430}
431
432impl TagSpec {
433    /// Returns the source of tag values, either literal or a field.
434    pub fn source(&self) -> TagSource<'_> {
435        if let Some(ref field) = self.field {
436            TagSource::Field(field)
437        } else if let Some(ref value) = self.value {
438            TagSource::Literal(value)
439        } else {
440            TagSource::Unknown
441        }
442    }
443}
444
445/// Builder for [`TagSpec`].
446pub struct Tag {
447    key: String,
448}
449
450impl Tag {
451    /// Prepares a tag with a given tag name.
452    pub fn with_key(key: impl Into<String>) -> Self {
453        Self { key: key.into() }
454    }
455
456    /// Defines the field from which the tag value gets its data.
457    pub fn from_field(self, field_name: impl Into<String>) -> TagWithSource {
458        let Self { key } = self;
459        TagWithSource {
460            key,
461            field: Some(field_name.into()),
462            value: None,
463        }
464    }
465
466    /// Defines what value to set for a tag.
467    pub fn with_value(self, value: impl Into<String>) -> TagWithSource {
468        let Self { key } = self;
469        TagWithSource {
470            key,
471            field: None,
472            value: Some(value.into()),
473        }
474    }
475}
476
477/// Intermediate result of the tag spec builder.
478///
479/// Can be transformed into [`TagSpec`].
480pub struct TagWithSource {
481    key: String,
482    field: Option<String>,
483    value: Option<String>,
484}
485
486impl TagWithSource {
487    /// Defines a tag that is extracted unconditionally.
488    pub fn always(self) -> TagSpec {
489        let Self { key, field, value } = self;
490        TagSpec {
491            key,
492            field,
493            value,
494            condition: None,
495        }
496    }
497
498    /// Defines a tag that is extracted under the given condition.
499    pub fn when(self, condition: RuleCondition) -> TagSpec {
500        let Self { key, field, value } = self;
501        TagSpec {
502            key,
503            field,
504            value,
505            condition: Some(condition),
506        }
507    }
508}
509
510/// Specifies how to obtain the value of a tag in [`TagSpec`].
511#[derive(Clone, Debug, PartialEq)]
512pub enum TagSource<'a> {
513    /// A literal value.
514    Literal(&'a str),
515    /// Path to a field to evaluate.
516    Field(&'a str),
517    /// An unsupported or unknown source.
518    Unknown,
519}
520
521/// Converts the given tagging rules from `conditional_tagging` to the newer metric extraction
522/// config.
523pub fn convert_conditional_tagging(project_config: &mut ProjectConfig) {
524    // NOTE: This clones the rules so that they remain in the project state for old Relays that
525    // do not support generic metrics extraction. Once the migration is complete, this can be
526    // removed with a version bump of the transaction metrics config.
527    let rules = &project_config.metric_conditional_tagging;
528    if rules.is_empty() {
529        return;
530    }
531
532    let config = project_config
533        .metric_extraction
534        .get_or_insert_with(MetricExtractionConfig::empty);
535
536    if !config.is_supported() || config._conditional_tags_extended {
537        return;
538    }
539
540    config.tags.extend(TaggingRuleConverter {
541        rules: rules.iter().cloned().peekable(),
542        tags: Vec::new(),
543    });
544
545    config._conditional_tags_extended = true;
546    if config.version == 0 {
547        config.version = MetricExtractionConfig::MAX_SUPPORTED_VERSION;
548    }
549}
550
551struct TaggingRuleConverter<I: Iterator<Item = TaggingRule>> {
552    rules: std::iter::Peekable<I>,
553    tags: Vec<TagSpec>,
554}
555
556impl<I> Iterator for TaggingRuleConverter<I>
557where
558    I: Iterator<Item = TaggingRule>,
559{
560    type Item = TagMapping;
561
562    fn next(&mut self) -> Option<Self::Item> {
563        loop {
564            let old = self.rules.next()?;
565
566            self.tags.push(TagSpec {
567                key: old.target_tag,
568                field: None,
569                value: Some(old.tag_value),
570                condition: Some(old.condition),
571            });
572
573            // Optimization: Collect tags for consecutive tagging rules for the same set of metrics.
574            // Then, emit a single entry with all tag specs at once.
575            if self.rules.peek().map(|r| &r.target_metrics) == Some(&old.target_metrics) {
576                continue;
577            }
578
579            return Some(TagMapping {
580                metrics: old.target_metrics.into_iter().map(LazyGlob::new).collect(),
581                tags: std::mem::take(&mut self.tags),
582            });
583        }
584    }
585}
586
587#[cfg(test)]
588mod tests {
589    use super::*;
590    use similar_asserts::assert_eq;
591
592    #[test]
593    fn parse_tag_spec_value() {
594        let json = r#"{"key":"foo","value":"bar"}"#;
595        let spec: TagSpec = serde_json::from_str(json).unwrap();
596        assert_eq!(spec.source(), TagSource::Literal("bar"));
597    }
598
599    #[test]
600    fn parse_tag_spec_field() {
601        let json = r#"{"key":"foo","field":"bar"}"#;
602        let spec: TagSpec = serde_json::from_str(json).unwrap();
603        assert_eq!(spec.source(), TagSource::Field("bar"));
604    }
605
606    #[test]
607    fn parse_tag_spec_unsupported() {
608        let json = r#"{"key":"foo","somethingNew":"bar"}"#;
609        let spec: TagSpec = serde_json::from_str(json).unwrap();
610        assert_eq!(spec.source(), TagSource::Unknown);
611    }
612
613    #[test]
614    fn parse_tag_mapping() {
615        let json = r#"{"metrics": ["d:spans/*"], "tags": [{"key":"foo","field":"bar"}]}"#;
616        let mapping: TagMapping = serde_json::from_str(json).unwrap();
617        assert!(mapping.metrics[0].compiled().is_match("d:spans/foo"));
618    }
619
620    fn groups() -> MetricExtractionGroups {
621        serde_json::from_value::<MetricExtractionGroups>(serde_json::json!({
622            "groups": {
623                "group1": {
624                    "isEnabled": false,
625                    "metrics": [{
626                        "category": "transaction",
627                        "mri": "c:metric1/counter@none",
628                    }],
629                    "tags": [
630                        {
631                            "metrics": ["c:metric1/counter@none"],
632                            "tags": [{
633                                "key": "tag1",
634                                "value": "value1"
635                            }]
636                        }
637                    ]
638                },
639                "group2": {
640                    "isEnabled": true,
641                    "metrics": [{
642                        "category": "transaction",
643                        "mri": "c:metric2/counter@none",
644                    }],
645                    "tags": [
646                        {
647                            "metrics": ["c:metric2/counter@none"],
648                            "tags": [{
649                                "key": "tag2",
650                                "value": "value2"
651                            }]
652                        }
653                    ]
654                }
655            }
656        }))
657        .unwrap()
658    }
659
660    #[test]
661    fn metric_extraction_global_defaults() {
662        let global = groups();
663        let project: MetricExtractionConfig = serde_json::from_value(serde_json::json!({
664            "version": 1,
665            "global_templates": {}
666        }))
667        .unwrap();
668        let combined = CombinedMetricExtractionConfig::new(&global, &project);
669
670        assert_eq!(
671            combined
672                .metrics()
673                .map(|m| m.mri.as_str())
674                .collect::<Vec<_>>(),
675            vec!["c:metric2/counter@none"]
676        );
677        assert_eq!(
678            combined
679                .tags()
680                .map(|t| t.tags[0].key.as_str())
681                .collect::<Vec<_>>(),
682            vec!["tag2"]
683        );
684    }
685
686    #[test]
687    fn metric_extraction_override() {
688        let global = groups();
689        let project: MetricExtractionConfig = serde_json::from_value(serde_json::json!({
690            "version": 1,
691            "globalGroups": {
692                "group1": {"isEnabled": true},
693                "group2": {"isEnabled": false}
694            }
695        }))
696        .unwrap();
697        let combined = CombinedMetricExtractionConfig::new(&global, &project);
698
699        assert_eq!(
700            combined
701                .metrics()
702                .map(|m| m.mri.as_str())
703                .collect::<Vec<_>>(),
704            vec!["c:metric1/counter@none"]
705        );
706        assert_eq!(
707            combined
708                .tags()
709                .map(|t| t.tags[0].key.as_str())
710                .collect::<Vec<_>>(),
711            vec!["tag1"]
712        );
713    }
714}