objectstore_service/backend/
mod.rs

1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14pub mod bigtable;
15pub mod changelog;
16pub mod common;
17pub mod gcs;
18pub mod in_memory;
19pub mod local_fs;
20pub mod s3_compatible;
21pub mod tiered;
22
23#[cfg(test)]
24pub(crate) mod testing;
25
26/// Storage backend configuration.
27///
28/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
29///
30/// Used to configure storage backends via [`from_config`].
31#[derive(Debug, Clone, Deserialize, Serialize)]
32#[serde(tag = "type", rename_all = "lowercase")]
33pub enum StorageConfig {
34    /// Local filesystem storage backend (type `"filesystem"`).
35    FileSystem(local_fs::FileSystemConfig),
36
37    /// S3-compatible storage backend (type `"s3compatible"`).
38    S3Compatible(s3_compatible::S3CompatibleConfig),
39
40    /// [Google Cloud Storage] backend (type `"gcs"`).
41    ///
42    /// [Google Cloud Storage]: https://cloud.google.com/storage
43    Gcs(gcs::GcsConfig),
44
45    /// [Google Bigtable] backend (type `"bigtable"`).
46    ///
47    /// [Google Bigtable]: https://cloud.google.com/bigtable
48    BigTable(bigtable::BigTableConfig),
49
50    /// Tiered storage backend (type `"tiered"`).
51    ///
52    /// Routes objects across two backends based on size: small objects go to
53    /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
54    /// another `Tiered` is not supported and will return an error at startup.
55    Tiered(tiered::TieredStorageConfig),
56}
57
58/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
59pub async fn from_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
60    Ok(match config {
61        StorageConfig::Tiered(c) => {
62            let hv = hv_from_config(c.high_volume).await?;
63            let lt = from_leaf_config(*c.long_term).await?;
64            let log = Box::new(changelog::NoopChangeLog);
65            Box::new(tiered::TieredStorage::new(hv, lt, log))
66        }
67        // All non-Tiered variants are handled by from_leaf_config. A wildcard
68        // is intentional here: any new leaf variant should fall through to
69        // from_leaf_config, which will handle it or produce a compile error.
70        _ => from_leaf_config(config).await?,
71    })
72}
73
74async fn from_leaf_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
75    Ok(match config {
76        StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c)),
77        StorageConfig::S3Compatible(c) => {
78            Box::new(s3_compatible::S3CompatibleBackend::without_token(c))
79        }
80        StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c).await?),
81        StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
82        StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
83    })
84}
85
86/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
87///
88/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
89/// Currently this is limited to BigTable.
90#[derive(Debug, Clone, Deserialize, Serialize)]
91#[serde(tag = "type", rename_all = "lowercase")]
92pub enum HighVolumeStorageConfig {
93    /// [Google Bigtable] backend.
94    ///
95    /// [Google Bigtable]: https://cloud.google.com/bigtable
96    BigTable(bigtable::BigTableConfig),
97}
98
99/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
100async fn hv_from_config(
101    config: HighVolumeStorageConfig,
102) -> anyhow::Result<Box<dyn common::HighVolumeBackend>> {
103    Ok(match config {
104        HighVolumeStorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
105    })
106}
objectstore_service/backend/mod.rs

objectstore_service/backend/
mod.rs
