objectstore_service/backend/
mod.rs

1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14pub mod bigtable;
15pub mod changelog;
16pub mod common;
17pub mod counting;
18mod extensions;
19pub mod gcs;
20pub mod in_memory;
21pub mod local_fs;
22pub mod s3_compatible;
23pub mod tiered;
24
25#[cfg(test)]
26pub(crate) mod testing;
27
28/// Storage backend configuration.
29///
30/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
31///
32/// Used to configure storage backends via [`from_config`].
33#[derive(Debug, Clone, Deserialize, Serialize)]
34#[serde(tag = "type", rename_all = "lowercase")]
35pub enum StorageConfig {
36    /// Local filesystem storage backend (type `"filesystem"`).
37    FileSystem(local_fs::FileSystemConfig),
38
39    /// S3-compatible storage backend (type `"s3compatible"`).
40    S3Compatible(s3_compatible::S3CompatibleConfig),
41
42    /// [Google Cloud Storage] backend (type `"gcs"`).
43    ///
44    /// [Google Cloud Storage]: https://cloud.google.com/storage
45    Gcs(gcs::GcsConfig),
46
47    /// [Google Bigtable] backend (type `"bigtable"`).
48    ///
49    /// [Google Bigtable]: https://cloud.google.com/bigtable
50    BigTable(bigtable::BigTableConfig),
51
52    /// Tiered storage backend (type `"tiered"`).
53    ///
54    /// Routes objects across two backends based on size: small objects go to
55    /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
56    /// another `Tiered` is not supported and will return an error at startup.
57    Tiered(tiered::TieredStorageConfig),
58}
59
60/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
61pub async fn from_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
62    Ok(match config {
63        StorageConfig::Tiered(c) => {
64            let hv = hv_from_config(c.high_volume).await?;
65            let lt = lt_from_config(c.long_term).await?;
66            let log = Box::new(changelog::NoopChangeLog);
67            Box::new(tiered::TieredStorage::new(hv, lt, log))
68        }
69        // All non-Tiered variants are handled by from_leaf_config. A wildcard
70        // is intentional here: any new leaf variant should fall through to
71        // from_leaf_config, which will handle it or produce a compile error.
72        _ => from_leaf_config(config).await?,
73    })
74}
75
76async fn from_leaf_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
77    Ok(match config {
78        StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c)),
79        StorageConfig::S3Compatible(c) => {
80            Box::new(s3_compatible::S3CompatibleBackend::without_token(c))
81        }
82        StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c).await?),
83        StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
84        StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
85    })
86}
87
88/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
89///
90/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
91/// Currently this is limited to BigTable.
92#[derive(Debug, Clone, Deserialize, Serialize)]
93#[serde(tag = "type", rename_all = "lowercase")]
94pub enum HighVolumeStorageConfig {
95    /// [Google Bigtable] backend.
96    ///
97    /// [Google Bigtable]: https://cloud.google.com/bigtable
98    BigTable(bigtable::BigTableConfig),
99}
100
101/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
102async fn hv_from_config(
103    config: HighVolumeStorageConfig,
104) -> Result<Box<dyn common::HighVolumeBackend>> {
105    Ok(match config {
106        HighVolumeStorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
107    })
108}
109
110/// Configuration for the long-term backend in a [`tiered::TieredStorageConfig`].
111///
112/// Only backends that implement [`common::MultipartUploadBackend`] are valid here.
113#[derive(Debug, Clone, Deserialize, Serialize)]
114#[serde(tag = "type", rename_all = "lowercase")]
115pub enum MultipartUploadStorageConfig {
116    /// Local filesystem storage backend (type `"filesystem"`).
117    FileSystem(local_fs::FileSystemConfig),
118
119    /// [Google Cloud Storage] backend (type `"gcs"`).
120    ///
121    /// [Google Cloud Storage]: https://cloud.google.com/storage
122    Gcs(gcs::GcsConfig),
123}
124
125/// Constructs a type-erased [`common::MultipartUploadBackend`] from the given config.
126async fn lt_from_config(
127    config: MultipartUploadStorageConfig,
128) -> Result<Box<dyn common::MultipartUploadBackend>> {
129    Ok(match config {
130        MultipartUploadStorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c)),
131        MultipartUploadStorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c).await?),
132    })
133}
objectstore_service/backend/mod.rs

objectstore_service/backend/
mod.rs
