objectstore_types/
lib.rs

1//! This is a collection of types shared among various objectstore crates.
2//!
3//! It primarily includes metadata-related structures being used by both the client and server/service
4//! components.
5
6#![warn(missing_docs)]
7#![warn(missing_debug_implementations)]
8
9use std::borrow::Cow;
10use std::collections::BTreeMap;
11use std::fmt;
12use std::str::FromStr;
13use std::time::{Duration, SystemTime};
14
15use http::header::{self, HeaderMap, HeaderName};
16use humantime::{
17    format_duration, format_rfc3339_micros, format_rfc3339_seconds, parse_duration, parse_rfc3339,
18};
19use serde::{Deserialize, Serialize};
20
21/// The custom HTTP header that contains the serialized [`ExpirationPolicy`].
22pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
23/// The custom HTTP header that contains the serialized redirect tombstone.
24pub const HEADER_REDIRECT_TOMBSTONE: &str = "x-sn-redirect-tombstone";
25/// The custom HTTP header that contains the object creation time.
26pub const HEADER_TIME_CREATED: &str = "x-sn-time-created";
27/// The custom HTTP header that contains the object expiration time.
28pub const HEADER_TIME_EXPIRES: &str = "x-sn-time-expires";
29/// The prefix for custom HTTP headers containing custom per-object metadata.
30pub const HEADER_META_PREFIX: &str = "x-snme-";
31
32/// The default content type for objects without a known content type.
33pub const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
34
35/// Errors that can happen dealing with metadata
36#[derive(Debug, thiserror::Error)]
37pub enum Error {
38    /// Any problems dealing with http headers, essentially converting to/from [`str`].
39    #[error("error dealing with http headers")]
40    Header(#[from] Option<http::Error>),
41    /// The value for the expiration policy is invalid.
42    #[error("invalid expiration policy value")]
43    InvalidExpiration(#[from] Option<humantime::DurationError>),
44    /// The compression algorithm is invalid.
45    #[error("invalid compression value")]
46    InvalidCompression,
47    /// The content type is invalid.
48    #[error("invalid content type")]
49    InvalidContentType(#[from] mediatype::MediaTypeError),
50    /// The creation time is invalid.
51    #[error("invalid creation time")]
52    InvalidCreationTime(#[from] humantime::TimestampError),
53}
54impl From<http::header::InvalidHeaderValue> for Error {
55    fn from(err: http::header::InvalidHeaderValue) -> Self {
56        Self::Header(Some(err.into()))
57    }
58}
59impl From<http::header::InvalidHeaderName> for Error {
60    fn from(err: http::header::InvalidHeaderName) -> Self {
61        Self::Header(Some(err.into()))
62    }
63}
64impl From<http::header::ToStrError> for Error {
65    fn from(_err: http::header::ToStrError) -> Self {
66        // the error happens when converting a header value back to a `str`
67        Self::Header(None)
68    }
69}
70
71/// The per-object expiration policy
72///
73/// We support automatic time-to-live and time-to-idle policies.
74/// Setting this to `Manual` means that the object has no automatic policy, and will not be
75/// garbage-collected automatically. It essentially lives forever until manually deleted.
76#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
77pub enum ExpirationPolicy {
78    /// Manual expiration, meaning no automatic cleanup.
79    // IMPORTANT: Do not change the default, we rely on this for persisted objects.
80    #[default]
81    Manual,
82    /// Time to live, with expiration after the specified duration.
83    TimeToLive(Duration),
84    /// Time to idle, with expiration once the object has not been accessed within the specified duration.
85    TimeToIdle(Duration),
86}
87impl ExpirationPolicy {
88    /// Returns the duration after which the object expires.
89    pub fn expires_in(&self) -> Option<Duration> {
90        match self {
91            ExpirationPolicy::Manual => None,
92            ExpirationPolicy::TimeToLive(duration) => Some(*duration),
93            ExpirationPolicy::TimeToIdle(duration) => Some(*duration),
94        }
95    }
96
97    /// Returns `true` if this policy indicates time-based expiry.
98    pub fn is_timeout(&self) -> bool {
99        match self {
100            ExpirationPolicy::TimeToLive(_) => true,
101            ExpirationPolicy::TimeToIdle(_) => true,
102            ExpirationPolicy::Manual => false,
103        }
104    }
105
106    /// Returns `true` if this policy is `Manual`.
107    pub fn is_manual(&self) -> bool {
108        *self == ExpirationPolicy::Manual
109    }
110}
111impl fmt::Display for ExpirationPolicy {
112    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
113        match self {
114            ExpirationPolicy::TimeToLive(duration) => {
115                write!(f, "ttl:{}", format_duration(*duration))
116            }
117            ExpirationPolicy::TimeToIdle(duration) => {
118                write!(f, "tti:{}", format_duration(*duration))
119            }
120            ExpirationPolicy::Manual => f.write_str("manual"),
121        }
122    }
123}
124impl FromStr for ExpirationPolicy {
125    type Err = Error;
126
127    fn from_str(s: &str) -> Result<Self, Self::Err> {
128        if s == "manual" {
129            return Ok(ExpirationPolicy::Manual);
130        }
131        if let Some(duration) = s.strip_prefix("ttl:") {
132            return Ok(ExpirationPolicy::TimeToLive(parse_duration(duration)?));
133        }
134        if let Some(duration) = s.strip_prefix("tti:") {
135            return Ok(ExpirationPolicy::TimeToIdle(parse_duration(duration)?));
136        }
137        Err(Error::InvalidExpiration(None))
138    }
139}
140
141/// The compression algorithm of an object to upload.
142#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
143pub enum Compression {
144    /// Compressed using `zstd`.
145    Zstd,
146    // /// Compressed using `gzip`.
147    // Gzip,
148    // /// Compressed using `lz4`.
149    // Lz4,
150}
151
152impl Compression {
153    /// Returns a string representation of the compression algorithm.
154    pub fn as_str(&self) -> &str {
155        match self {
156            Compression::Zstd => "zstd",
157            // Compression::Gzip => "gzip",
158            // Compression::Lz4 => "lz4",
159        }
160    }
161}
162
163impl fmt::Display for Compression {
164    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
165        f.write_str(self.as_str())
166    }
167}
168
169impl FromStr for Compression {
170    type Err = Error;
171
172    fn from_str(s: &str) -> Result<Self, Self::Err> {
173        match s {
174            "zstd" => Ok(Compression::Zstd),
175            // "gzip" => Compression::Gzip,
176            // "lz4" => Compression::Lz4,
177            _ => Err(Error::InvalidCompression),
178        }
179    }
180}
181
182/// Per-object Metadata.
183///
184/// This includes special metadata like the expiration policy and compression used,
185/// as well as arbitrary user-provided metadata.
186#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
187#[serde(default)]
188pub struct Metadata {
189    /// The object/metadata denotes a "redirect key".
190    ///
191    /// This means that this particular object is just a tombstone, and the real thing
192    /// is rather found on the other backend.
193    /// In practice this means that the tombstone is stored on the "HighVolume" backend,
194    /// to avoid unnecessarily slow "not found" requests on the "LongTerm" backend.
195    #[serde(skip_serializing_if = "Option::is_none")]
196    pub is_redirect_tombstone: Option<bool>,
197
198    /// The expiration policy of the object.
199    #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")]
200    pub expiration_policy: ExpirationPolicy,
201
202    /// The creation/last replacement time of the object, if known.
203    ///
204    /// This is set by the server every time an object is put, i.e. when objects are first created
205    /// and when existing objects are overwritten.
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub time_created: Option<SystemTime>,
208
209    /// The expiration time of the object, if any, in accordance with its expiration policy.
210    ///
211    /// When using a Time To Idle expiration policy, this value will reflect the expiration
212    /// timestamp present prior to the current access to the object.
213    #[serde(skip_serializing_if = "Option::is_none")]
214    pub time_expires: Option<SystemTime>,
215
216    /// The content type of the object, if known.
217    pub content_type: Cow<'static, str>,
218
219    /// The compression algorithm used for this object, if any.
220    #[serde(skip_serializing_if = "Option::is_none")]
221    pub compression: Option<Compression>,
222
223    /// Size of the data in bytes, if known.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub size: Option<usize>,
226
227    /// Some arbitrary user-provided metadata.
228    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
229    pub custom: BTreeMap<String, String>,
230}
231
232impl Metadata {
233    /// Extracts metadata from the given [`HeaderMap`].
234    ///
235    /// A prefix can be also be provided which is being stripped from custom non-standard headers.
236    pub fn from_headers(headers: &HeaderMap, prefix: &str) -> Result<Self, Error> {
237        let mut metadata = Metadata::default();
238
239        for (name, value) in headers {
240            match *name {
241                // standard HTTP headers
242                header::CONTENT_TYPE => {
243                    let content_type = value.to_str()?;
244                    validate_content_type(content_type)?;
245                    metadata.content_type = content_type.to_owned().into();
246                }
247                header::CONTENT_ENCODING => {
248                    let compression = value.to_str()?;
249                    metadata.compression = Some(Compression::from_str(compression)?);
250                }
251                _ => {
252                    let Some(name) = name.as_str().strip_prefix(prefix) else {
253                        continue;
254                    };
255
256                    match name {
257                        // Objectstore first-class metadata
258                        HEADER_EXPIRATION => {
259                            let expiration_policy = value.to_str()?;
260                            metadata.expiration_policy =
261                                ExpirationPolicy::from_str(expiration_policy)?;
262                        }
263                        HEADER_REDIRECT_TOMBSTONE => {
264                            if value.to_str()? == "true" {
265                                metadata.is_redirect_tombstone = Some(true);
266                            }
267                        }
268                        HEADER_TIME_CREATED => {
269                            let timestamp = value.to_str()?;
270                            let time = parse_rfc3339(timestamp)?;
271                            metadata.time_created = Some(time);
272                        }
273                        HEADER_TIME_EXPIRES => {
274                            let timestamp = value.to_str()?;
275                            let time = parse_rfc3339(timestamp)?;
276                            metadata.time_expires = Some(time);
277                        }
278                        _ => {
279                            // customer-provided metadata
280                            if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
281                                let value = value.to_str()?;
282                                metadata.custom.insert(name.into(), value.into());
283                            }
284                        }
285                    }
286                }
287            }
288        }
289
290        Ok(metadata)
291    }
292
293    /// Turns the metadata into a [`HeaderMap`].
294    ///
295    /// It will prefix any non-standard headers with the given `prefix`.
296    /// If the `with_expiration` parameter is set, it will additionally resolve the expiration policy
297    /// into a specific RFC3339 datetime, and set that as the `Custom-Time` header.
298    pub fn to_headers(&self, prefix: &str, with_expiration: bool) -> Result<HeaderMap, Error> {
299        let Self {
300            is_redirect_tombstone,
301            content_type,
302            compression,
303            expiration_policy,
304            time_created,
305            time_expires,
306            size: _,
307            custom,
308        } = self;
309
310        let mut headers = HeaderMap::new();
311
312        // standard headers
313        headers.append(header::CONTENT_TYPE, content_type.parse()?);
314        if let Some(compression) = compression {
315            headers.append(header::CONTENT_ENCODING, compression.as_str().parse()?);
316        }
317
318        // Objectstore first-class metadata
319        if matches!(is_redirect_tombstone, Some(true)) {
320            let name = HeaderName::try_from(format!("{prefix}{HEADER_REDIRECT_TOMBSTONE}"))?;
321            headers.append(name, "true".parse()?);
322        }
323        if *expiration_policy != ExpirationPolicy::Manual {
324            let name = HeaderName::try_from(format!("{prefix}{HEADER_EXPIRATION}"))?;
325            headers.append(name, expiration_policy.to_string().parse()?);
326            if with_expiration {
327                let expires_in = expiration_policy.expires_in().unwrap_or_default();
328                let expires_at = format_rfc3339_seconds(SystemTime::now() + expires_in);
329                headers.append("x-goog-custom-time", expires_at.to_string().parse()?);
330            }
331        }
332        if let Some(time) = time_created {
333            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_CREATED}"))?;
334            let timestamp = format_rfc3339_micros(*time);
335            headers.append(name, timestamp.to_string().parse()?);
336        }
337        if let Some(time) = time_expires {
338            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_EXPIRES}"))?;
339            let timestamp = format_rfc3339_micros(*time);
340            headers.append(name, timestamp.to_string().parse()?);
341        }
342
343        // customer-provided metadata
344        for (key, value) in custom {
345            let name = HeaderName::try_from(format!("{prefix}{HEADER_META_PREFIX}{key}"))?;
346            headers.append(name, value.parse()?);
347        }
348
349        Ok(headers)
350    }
351}
352
353/// Validates that `content_type` is a valid [IANA Media
354/// Type](https://www.iana.org/assignments/media-types/media-types.xhtml).
355fn validate_content_type(content_type: &str) -> Result<(), Error> {
356    mediatype::MediaType::parse(content_type)?;
357    Ok(())
358}
359
360impl Default for Metadata {
361    fn default() -> Self {
362        Self {
363            is_redirect_tombstone: None,
364            expiration_policy: ExpirationPolicy::Manual,
365            time_created: None,
366            time_expires: None,
367            content_type: DEFAULT_CONTENT_TYPE.into(),
368            compression: None,
369            size: None,
370            custom: BTreeMap::new(),
371        }
372    }
373}
objectstore_types/lib.rs

objectstore_types/
lib.rs
