objectstore_types/
lib.rs

1//! This is a collection of types shared among various objectstore crates.
2//!
3//! It primarily includes metadata-related structures being used by both the client and server/service
4//! components.
5
6#![warn(missing_docs)]
7#![warn(missing_debug_implementations)]
8
9pub mod scope;
10
11use std::borrow::Cow;
12use std::collections::{BTreeMap, HashSet};
13use std::fmt;
14use std::str::FromStr;
15use std::time::{Duration, SystemTime};
16
17use http::header::{self, HeaderMap, HeaderName};
18use humantime::{
19    format_duration, format_rfc3339_micros, format_rfc3339_seconds, parse_duration, parse_rfc3339,
20};
21use serde::{Deserialize, Serialize};
22
23/// The custom HTTP header that contains the serialized [`ExpirationPolicy`].
24pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
25/// The custom HTTP header that contains the serialized redirect tombstone.
26pub const HEADER_REDIRECT_TOMBSTONE: &str = "x-sn-redirect-tombstone";
27/// The custom HTTP header that contains the object creation time.
28pub const HEADER_TIME_CREATED: &str = "x-sn-time-created";
29/// The custom HTTP header that contains the object expiration time.
30pub const HEADER_TIME_EXPIRES: &str = "x-sn-time-expires";
31/// The prefix for custom HTTP headers containing custom per-object metadata.
32pub const HEADER_META_PREFIX: &str = "x-snme-";
33
34/// The default content type for objects without a known content type.
35pub const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
36
37/// Errors that can happen dealing with metadata
38#[derive(Debug, thiserror::Error)]
39pub enum Error {
40    /// Any problems dealing with http headers, essentially converting to/from [`str`].
41    #[error("error dealing with http headers")]
42    Header(#[from] Option<http::Error>),
43    /// The value for the expiration policy is invalid.
44    #[error("invalid expiration policy value")]
45    InvalidExpiration(#[from] Option<humantime::DurationError>),
46    /// The compression algorithm is invalid.
47    #[error("invalid compression value")]
48    InvalidCompression,
49    /// The content type is invalid.
50    #[error("invalid content type")]
51    InvalidContentType(#[from] mediatype::MediaTypeError),
52    /// The creation time is invalid.
53    #[error("invalid creation time")]
54    InvalidCreationTime(#[from] humantime::TimestampError),
55}
56impl From<http::header::InvalidHeaderValue> for Error {
57    fn from(err: http::header::InvalidHeaderValue) -> Self {
58        Self::Header(Some(err.into()))
59    }
60}
61impl From<http::header::InvalidHeaderName> for Error {
62    fn from(err: http::header::InvalidHeaderName) -> Self {
63        Self::Header(Some(err.into()))
64    }
65}
66impl From<http::header::ToStrError> for Error {
67    fn from(_err: http::header::ToStrError) -> Self {
68        // the error happens when converting a header value back to a `str`
69        Self::Header(None)
70    }
71}
72
73/// The per-object expiration policy
74///
75/// We support automatic time-to-live and time-to-idle policies.
76/// Setting this to `Manual` means that the object has no automatic policy, and will not be
77/// garbage-collected automatically. It essentially lives forever until manually deleted.
78#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
79pub enum ExpirationPolicy {
80    /// Manual expiration, meaning no automatic cleanup.
81    // IMPORTANT: Do not change the default, we rely on this for persisted objects.
82    #[default]
83    Manual,
84    /// Time to live, with expiration after the specified duration.
85    TimeToLive(Duration),
86    /// Time to idle, with expiration once the object has not been accessed within the specified duration.
87    TimeToIdle(Duration),
88}
89impl ExpirationPolicy {
90    /// Returns the duration after which the object expires.
91    pub fn expires_in(&self) -> Option<Duration> {
92        match self {
93            ExpirationPolicy::Manual => None,
94            ExpirationPolicy::TimeToLive(duration) => Some(*duration),
95            ExpirationPolicy::TimeToIdle(duration) => Some(*duration),
96        }
97    }
98
99    /// Returns `true` if this policy indicates time-based expiry.
100    pub fn is_timeout(&self) -> bool {
101        match self {
102            ExpirationPolicy::TimeToLive(_) => true,
103            ExpirationPolicy::TimeToIdle(_) => true,
104            ExpirationPolicy::Manual => false,
105        }
106    }
107
108    /// Returns `true` if this policy is `Manual`.
109    pub fn is_manual(&self) -> bool {
110        *self == ExpirationPolicy::Manual
111    }
112}
113impl fmt::Display for ExpirationPolicy {
114    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
115        match self {
116            ExpirationPolicy::TimeToLive(duration) => {
117                write!(f, "ttl:{}", format_duration(*duration))
118            }
119            ExpirationPolicy::TimeToIdle(duration) => {
120                write!(f, "tti:{}", format_duration(*duration))
121            }
122            ExpirationPolicy::Manual => f.write_str("manual"),
123        }
124    }
125}
126impl FromStr for ExpirationPolicy {
127    type Err = Error;
128
129    fn from_str(s: &str) -> Result<Self, Self::Err> {
130        if s == "manual" {
131            return Ok(ExpirationPolicy::Manual);
132        }
133        if let Some(duration) = s.strip_prefix("ttl:") {
134            return Ok(ExpirationPolicy::TimeToLive(parse_duration(duration)?));
135        }
136        if let Some(duration) = s.strip_prefix("tti:") {
137            return Ok(ExpirationPolicy::TimeToIdle(parse_duration(duration)?));
138        }
139        Err(Error::InvalidExpiration(None))
140    }
141}
142
143/// The compression algorithm of an object to upload.
144#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
145pub enum Compression {
146    /// Compressed using `zstd`.
147    Zstd,
148    // /// Compressed using `gzip`.
149    // Gzip,
150    // /// Compressed using `lz4`.
151    // Lz4,
152}
153
154impl Compression {
155    /// Returns a string representation of the compression algorithm.
156    pub fn as_str(&self) -> &str {
157        match self {
158            Compression::Zstd => "zstd",
159            // Compression::Gzip => "gzip",
160            // Compression::Lz4 => "lz4",
161        }
162    }
163}
164
165impl fmt::Display for Compression {
166    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
167        f.write_str(self.as_str())
168    }
169}
170
171impl FromStr for Compression {
172    type Err = Error;
173
174    fn from_str(s: &str) -> Result<Self, Self::Err> {
175        match s {
176            "zstd" => Ok(Compression::Zstd),
177            // "gzip" => Compression::Gzip,
178            // "lz4" => Compression::Lz4,
179            _ => Err(Error::InvalidCompression),
180        }
181    }
182}
183
184/// Permissions that control whether different operations are authorized.
185#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq, Hash)]
186pub enum Permission {
187    /// The permission required to read objects from objectstore.
188    #[serde(rename = "object.read")]
189    ObjectRead,
190
191    /// The permission required to write/overwrite objects in objectstore.
192    #[serde(rename = "object.write")]
193    ObjectWrite,
194
195    /// The permission required to delete objects from objectstore.
196    #[serde(rename = "object.delete")]
197    ObjectDelete,
198}
199
200impl Permission {
201    /// Convenience function for creating a set with read, write, and delete permissions.
202    pub fn rwd() -> HashSet<Permission> {
203        HashSet::from([
204            Permission::ObjectRead,
205            Permission::ObjectWrite,
206            Permission::ObjectDelete,
207        ])
208    }
209}
210
211/// Per-object Metadata.
212///
213/// This includes special metadata like the expiration policy and compression used,
214/// as well as arbitrary user-provided metadata.
215#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
216#[serde(default)]
217pub struct Metadata {
218    /// The object/metadata denotes a "redirect key".
219    ///
220    /// This means that this particular object is just a tombstone, and the real thing
221    /// is rather found on the other backend.
222    /// In practice this means that the tombstone is stored on the "HighVolume" backend,
223    /// to avoid unnecessarily slow "not found" requests on the "LongTerm" backend.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub is_redirect_tombstone: Option<bool>,
226
227    /// The expiration policy of the object.
228    #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")]
229    pub expiration_policy: ExpirationPolicy,
230
231    /// The creation/last replacement time of the object, if known.
232    ///
233    /// This is set by the server every time an object is put, i.e. when objects are first created
234    /// and when existing objects are overwritten.
235    #[serde(skip_serializing_if = "Option::is_none")]
236    pub time_created: Option<SystemTime>,
237
238    /// The expiration time of the object, if any, in accordance with its expiration policy.
239    ///
240    /// When using a Time To Idle expiration policy, this value will reflect the expiration
241    /// timestamp present prior to the current access to the object.
242    #[serde(skip_serializing_if = "Option::is_none")]
243    pub time_expires: Option<SystemTime>,
244
245    /// The content type of the object, if known.
246    pub content_type: Cow<'static, str>,
247
248    /// The compression algorithm used for this object, if any.
249    #[serde(skip_serializing_if = "Option::is_none")]
250    pub compression: Option<Compression>,
251
252    /// Size of the data in bytes, if known.
253    #[serde(skip_serializing_if = "Option::is_none")]
254    pub size: Option<usize>,
255
256    /// Some arbitrary user-provided metadata.
257    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
258    pub custom: BTreeMap<String, String>,
259}
260
261impl Metadata {
262    /// Extracts metadata from the given [`HeaderMap`].
263    ///
264    /// A prefix can be also be provided which is being stripped from custom non-standard headers.
265    pub fn from_headers(headers: &HeaderMap, prefix: &str) -> Result<Self, Error> {
266        let mut metadata = Metadata::default();
267
268        for (name, value) in headers {
269            match *name {
270                // standard HTTP headers
271                header::CONTENT_TYPE => {
272                    let content_type = value.to_str()?;
273                    validate_content_type(content_type)?;
274                    metadata.content_type = content_type.to_owned().into();
275                }
276                header::CONTENT_ENCODING => {
277                    let compression = value.to_str()?;
278                    metadata.compression = Some(Compression::from_str(compression)?);
279                }
280                _ => {
281                    let Some(name) = name.as_str().strip_prefix(prefix) else {
282                        continue;
283                    };
284
285                    match name {
286                        // Objectstore first-class metadata
287                        HEADER_EXPIRATION => {
288                            let expiration_policy = value.to_str()?;
289                            metadata.expiration_policy =
290                                ExpirationPolicy::from_str(expiration_policy)?;
291                        }
292                        HEADER_REDIRECT_TOMBSTONE => {
293                            if value.to_str()? == "true" {
294                                metadata.is_redirect_tombstone = Some(true);
295                            }
296                        }
297                        HEADER_TIME_CREATED => {
298                            let timestamp = value.to_str()?;
299                            let time = parse_rfc3339(timestamp)?;
300                            metadata.time_created = Some(time);
301                        }
302                        HEADER_TIME_EXPIRES => {
303                            let timestamp = value.to_str()?;
304                            let time = parse_rfc3339(timestamp)?;
305                            metadata.time_expires = Some(time);
306                        }
307                        _ => {
308                            // customer-provided metadata
309                            if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
310                                let value = value.to_str()?;
311                                metadata.custom.insert(name.into(), value.into());
312                            }
313                        }
314                    }
315                }
316            }
317        }
318
319        Ok(metadata)
320    }
321
322    /// Turns the metadata into a [`HeaderMap`].
323    ///
324    /// It will prefix any non-standard headers with the given `prefix`.
325    /// If the `with_expiration` parameter is set, it will additionally resolve the expiration policy
326    /// into a specific RFC3339 datetime, and set that as the `Custom-Time` header.
327    pub fn to_headers(&self, prefix: &str, with_expiration: bool) -> Result<HeaderMap, Error> {
328        let Self {
329            is_redirect_tombstone,
330            content_type,
331            compression,
332            expiration_policy,
333            time_created,
334            time_expires,
335            size: _,
336            custom,
337        } = self;
338
339        let mut headers = HeaderMap::new();
340
341        // standard headers
342        headers.append(header::CONTENT_TYPE, content_type.parse()?);
343        if let Some(compression) = compression {
344            headers.append(header::CONTENT_ENCODING, compression.as_str().parse()?);
345        }
346
347        // Objectstore first-class metadata
348        if matches!(is_redirect_tombstone, Some(true)) {
349            let name = HeaderName::try_from(format!("{prefix}{HEADER_REDIRECT_TOMBSTONE}"))?;
350            headers.append(name, "true".parse()?);
351        }
352        if *expiration_policy != ExpirationPolicy::Manual {
353            let name = HeaderName::try_from(format!("{prefix}{HEADER_EXPIRATION}"))?;
354            headers.append(name, expiration_policy.to_string().parse()?);
355            if with_expiration {
356                let expires_in = expiration_policy.expires_in().unwrap_or_default();
357                let expires_at = format_rfc3339_seconds(SystemTime::now() + expires_in);
358                headers.append("x-goog-custom-time", expires_at.to_string().parse()?);
359            }
360        }
361        if let Some(time) = time_created {
362            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_CREATED}"))?;
363            let timestamp = format_rfc3339_micros(*time);
364            headers.append(name, timestamp.to_string().parse()?);
365        }
366        if let Some(time) = time_expires {
367            let name = HeaderName::try_from(format!("{prefix}{HEADER_TIME_EXPIRES}"))?;
368            let timestamp = format_rfc3339_micros(*time);
369            headers.append(name, timestamp.to_string().parse()?);
370        }
371
372        // customer-provided metadata
373        for (key, value) in custom {
374            let name = HeaderName::try_from(format!("{prefix}{HEADER_META_PREFIX}{key}"))?;
375            headers.append(name, value.parse()?);
376        }
377
378        Ok(headers)
379    }
380}
381
382/// Validates that `content_type` is a valid [IANA Media
383/// Type](https://www.iana.org/assignments/media-types/media-types.xhtml).
384fn validate_content_type(content_type: &str) -> Result<(), Error> {
385    mediatype::MediaType::parse(content_type)?;
386    Ok(())
387}
388
389impl Default for Metadata {
390    fn default() -> Self {
391        Self {
392            is_redirect_tombstone: None,
393            expiration_policy: ExpirationPolicy::Manual,
394            time_created: None,
395            time_expires: None,
396            content_type: DEFAULT_CONTENT_TYPE.into(),
397            compression: None,
398            size: None,
399            custom: BTreeMap::new(),
400        }
401    }
402}
objectstore_types/lib.rs

objectstore_types/
lib.rs
