objectstore_types/
lib.rs

1//! This is a collection of types shared among various objectstore crates.
2//!
3//! It primarily includes metadata-related structures being used by both the client and server/service
4//! components.
5
6#![warn(missing_docs)]
7#![warn(missing_debug_implementations)]
8
9use std::borrow::Cow;
10use std::collections::BTreeMap;
11use std::fmt;
12use std::str::FromStr;
13use std::time::{Duration, SystemTime};
14
15use http::header::{self, HeaderMap, HeaderName};
16use humantime::{format_duration, format_rfc3339_seconds, parse_duration};
17use serde::{Deserialize, Serialize};
18
19/// The custom HTTP header that contains the serialized [`ExpirationPolicy`].
20pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
21/// The custom HTTP header that contains the serialized redirect tombstone.
22pub const HEADER_REDIRECT_TOMBSTONE: &str = "x-sn-redirect-tombstone";
23/// The prefix for custom HTTP headers containing custom per-object metadata.
24pub const HEADER_META_PREFIX: &str = "x-snme-";
25
26/// HTTP request query parameter that contains the request scope.
27pub const PARAM_SCOPE: &str = "scope";
28/// HTTP request query parameter that contains the request usecase.
29pub const PARAM_USECASE: &str = "usecase";
30
31/// The default content type for objects without a known content type.
32pub const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
33
34/// Errors that can happen dealing with metadata
35#[derive(Debug, thiserror::Error)]
36pub enum Error {
37    /// Any problems dealing with http headers, essentially converting to/from [`str`].
38    #[error("error dealing with http headers")]
39    Header(#[from] Option<http::Error>),
40    /// The value for the expiration policy is invalid.
41    #[error("invalid expiration policy value")]
42    InvalidExpiration(#[from] Option<humantime::DurationError>),
43    /// The compression algorithm is invalid.
44    #[error("invalid compression value")]
45    InvalidCompression,
46}
47impl From<http::header::InvalidHeaderValue> for Error {
48    fn from(err: http::header::InvalidHeaderValue) -> Self {
49        Self::Header(Some(err.into()))
50    }
51}
52impl From<http::header::InvalidHeaderName> for Error {
53    fn from(err: http::header::InvalidHeaderName) -> Self {
54        Self::Header(Some(err.into()))
55    }
56}
57impl From<http::header::ToStrError> for Error {
58    fn from(_err: http::header::ToStrError) -> Self {
59        // the error happens when converting a header value back to a `str`
60        Self::Header(None)
61    }
62}
63
64/// The per-object expiration policy
65///
66/// We support automatic time-to-live and time-to-idle policies.
67/// Setting this to `Manual` means that the object has no automatic policy, and will not be
68/// garbage-collected automatically. It essentially lives forever until manually deleted.
69#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
70pub enum ExpirationPolicy {
71    /// Manual expiration, meaning no automatic cleanup.
72    // IMPORTANT: Do not change the default, we rely on this for persisted objects.
73    #[default]
74    Manual,
75    /// Time to live, with expiration after the specified duration.
76    TimeToLive(Duration),
77    /// Time to idle, with expiration once the object has not been accessed within the specified duration.
78    TimeToIdle(Duration),
79}
80impl ExpirationPolicy {
81    /// Returns the duration after which the object expires.
82    pub fn expires_in(&self) -> Option<Duration> {
83        match self {
84            ExpirationPolicy::Manual => None,
85            ExpirationPolicy::TimeToLive(duration) => Some(*duration),
86            ExpirationPolicy::TimeToIdle(duration) => Some(*duration),
87        }
88    }
89
90    /// Returns `true` if this policy indicates time-based expiry.
91    pub fn is_timeout(&self) -> bool {
92        match self {
93            ExpirationPolicy::TimeToLive(_) => true,
94            ExpirationPolicy::TimeToIdle(_) => true,
95            ExpirationPolicy::Manual => false,
96        }
97    }
98
99    /// Returns `true` if this policy is `Manual`.
100    pub fn is_manual(&self) -> bool {
101        *self == ExpirationPolicy::Manual
102    }
103}
104impl fmt::Display for ExpirationPolicy {
105    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
106        match self {
107            ExpirationPolicy::TimeToLive(duration) => {
108                write!(f, "ttl:{}", format_duration(*duration))
109            }
110            ExpirationPolicy::TimeToIdle(duration) => {
111                write!(f, "tti:{}", format_duration(*duration))
112            }
113            ExpirationPolicy::Manual => f.write_str("manual"),
114        }
115    }
116}
117impl FromStr for ExpirationPolicy {
118    type Err = Error;
119
120    fn from_str(s: &str) -> Result<Self, Self::Err> {
121        if s == "manual" {
122            return Ok(ExpirationPolicy::Manual);
123        }
124        if let Some(duration) = s.strip_prefix("ttl:") {
125            return Ok(ExpirationPolicy::TimeToLive(parse_duration(duration)?));
126        }
127        if let Some(duration) = s.strip_prefix("tti:") {
128            return Ok(ExpirationPolicy::TimeToIdle(parse_duration(duration)?));
129        }
130        Err(Error::InvalidExpiration(None))
131    }
132}
133
134/// The compression algorithm of an object to upload.
135#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
136pub enum Compression {
137    /// Compressed using `zstd`.
138    Zstd,
139    // /// Compressed using `gzip`.
140    // Gzip,
141    // /// Compressed using `lz4`.
142    // Lz4,
143}
144
145impl Compression {
146    /// Returns a string representation of the compression algorithm.
147    pub fn as_str(&self) -> &str {
148        match self {
149            Compression::Zstd => "zstd",
150            // Compression::Gzip => "gzip",
151            // Compression::Lz4 => "lz4",
152        }
153    }
154}
155
156impl fmt::Display for Compression {
157    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
158        f.write_str(self.as_str())
159    }
160}
161
162impl FromStr for Compression {
163    type Err = Error;
164
165    fn from_str(s: &str) -> Result<Self, Self::Err> {
166        match s {
167            "zstd" => Ok(Compression::Zstd),
168            // "gzip" => Compression::Gzip,
169            // "lz4" => Compression::Lz4,
170            _ => Err(Error::InvalidCompression),
171        }
172    }
173}
174
175/// Per-object Metadata.
176///
177/// This includes special metadata like the expiration policy and compression used,
178/// as well as arbitrary user-provided metadata.
179#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
180#[serde(default)]
181pub struct Metadata {
182    /// The object/metadata denotes a "redirect key".
183    ///
184    /// This means that this particular object is just a tombstone, and the real thing
185    /// is rather found on the other backend.
186    /// In practice this means that the tombstone is stored on the "HighVolume" backend,
187    /// to avoid unnecessarily slow "not found" requests on the "LongTerm" backend.
188    #[serde(skip_serializing_if = "Option::is_none")]
189    pub is_redirect_tombstone: Option<bool>,
190
191    /// The expiration policy of the object.
192    #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")]
193    pub expiration_policy: ExpirationPolicy,
194
195    /// The content type of the object, if known.
196    pub content_type: Cow<'static, str>,
197
198    /// The compression algorithm used for this object, if any.
199    #[serde(skip_serializing_if = "Option::is_none")]
200    pub compression: Option<Compression>,
201
202    /// Size of the data in bytes, if known.
203    #[serde(skip_serializing_if = "Option::is_none")]
204    pub size: Option<usize>,
205
206    /// Some arbitrary user-provided metadata.
207    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
208    pub custom: BTreeMap<String, String>,
209}
210
211impl Metadata {
212    /// Extracts metadata from the given [`HeaderMap`].
213    ///
214    /// A prefix can be also be provided which is being stripped from custom non-standard headers.
215    pub fn from_headers(headers: &HeaderMap, prefix: &str) -> Result<Self, Error> {
216        let mut metadata = Metadata::default();
217
218        for (name, value) in headers {
219            if name == header::CONTENT_TYPE {
220                let content_type = value.to_str()?;
221                metadata.content_type = content_type.to_owned().into();
222            } else if name == header::CONTENT_ENCODING {
223                let compression = value.to_str()?;
224                metadata.compression = Some(Compression::from_str(compression)?);
225            } else if let Some(name) = name.as_str().strip_prefix(prefix) {
226                if name == HEADER_EXPIRATION {
227                    let expiration_policy = value.to_str()?;
228                    metadata.expiration_policy = ExpirationPolicy::from_str(expiration_policy)?;
229                } else if name == HEADER_REDIRECT_TOMBSTONE {
230                    if value.to_str()? == "true" {
231                        metadata.is_redirect_tombstone = Some(true);
232                    }
233                } else if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
234                    let value = value.to_str()?;
235                    metadata.custom.insert(name.into(), value.into());
236                }
237            }
238        }
239
240        Ok(metadata)
241    }
242
243    /// Turns the metadata into a [`HeaderMap`].
244    ///
245    /// It will prefix any non-standard headers with the given `prefix`.
246    /// If the `with_expiration` parameter is set, it will additionally resolve the expiration policy
247    /// into a specific RFC3339 datetime, and set that as the `Custom-Time` header.
248    pub fn to_headers(&self, prefix: &str, with_expiration: bool) -> Result<HeaderMap, Error> {
249        let Self {
250            is_redirect_tombstone,
251            content_type,
252            compression,
253            expiration_policy,
254            size: _,
255            custom,
256        } = self;
257
258        let mut headers = HeaderMap::new();
259        headers.append(header::CONTENT_TYPE, content_type.parse()?);
260
261        if matches!(is_redirect_tombstone, Some(true)) {
262            let name = HeaderName::try_from(format!("{prefix}{HEADER_REDIRECT_TOMBSTONE}"))?;
263            headers.append(name, "true".parse()?);
264        }
265
266        if let Some(compression) = compression {
267            headers.append(header::CONTENT_ENCODING, compression.as_str().parse()?);
268        }
269
270        if *expiration_policy != ExpirationPolicy::Manual {
271            let name = HeaderName::try_from(format!("{prefix}{HEADER_EXPIRATION}"))?;
272            headers.append(name, expiration_policy.to_string().parse()?);
273            if with_expiration {
274                let expires_in = expiration_policy.expires_in().unwrap_or_default();
275                let expires_at = format_rfc3339_seconds(SystemTime::now() + expires_in);
276                headers.append("x-goog-custom-time", expires_at.to_string().parse()?);
277            }
278        }
279
280        for (key, value) in custom {
281            let name = HeaderName::try_from(format!("{prefix}{HEADER_META_PREFIX}{key}"))?;
282            headers.append(name, value.parse()?);
283        }
284
285        Ok(headers)
286    }
287}
288
289impl Default for Metadata {
290    fn default() -> Self {
291        Self {
292            is_redirect_tombstone: None,
293            expiration_policy: ExpirationPolicy::Manual,
294            content_type: DEFAULT_CONTENT_TYPE.into(),
295            compression: None,
296            size: None,
297            custom: BTreeMap::new(),
298        }
299    }
300}
objectstore_types/lib.rs

objectstore_types/
lib.rs
