objectstore_service/backend/
mod.rs

1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14pub mod bigtable;
15pub mod changelog;
16pub mod common;
17pub mod counting;
18pub mod gcs;
19pub mod in_memory;
20pub mod local_fs;
21pub mod s3_compatible;
22pub mod tiered;
23
24#[cfg(test)]
25pub(crate) mod testing;
26
27/// Storage backend configuration.
28///
29/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
30///
31/// Used to configure storage backends via [`from_config`].
32#[derive(Debug, Clone, Deserialize, Serialize)]
33#[serde(tag = "type", rename_all = "lowercase")]
34pub enum StorageConfig {
35    /// Local filesystem storage backend (type `"filesystem"`).
36    FileSystem(local_fs::FileSystemConfig),
37
38    /// S3-compatible storage backend (type `"s3compatible"`).
39    S3Compatible(s3_compatible::S3CompatibleConfig),
40
41    /// [Google Cloud Storage] backend (type `"gcs"`).
42    ///
43    /// [Google Cloud Storage]: https://cloud.google.com/storage
44    Gcs(gcs::GcsConfig),
45
46    /// [Google Bigtable] backend (type `"bigtable"`).
47    ///
48    /// [Google Bigtable]: https://cloud.google.com/bigtable
49    BigTable(bigtable::BigTableConfig),
50
51    /// Tiered storage backend (type `"tiered"`).
52    ///
53    /// Routes objects across two backends based on size: small objects go to
54    /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
55    /// another `Tiered` is not supported and will return an error at startup.
56    Tiered(tiered::TieredStorageConfig),
57}
58
59/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
60pub async fn from_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
61    Ok(match config {
62        StorageConfig::Tiered(c) => {
63            let hv = hv_from_config(c.high_volume).await?;
64            let lt = lt_from_config(c.long_term).await?;
65            let log = Box::new(changelog::NoopChangeLog);
66            Box::new(tiered::TieredStorage::new(hv, lt, log))
67        }
68        // All non-Tiered variants are handled by from_leaf_config. A wildcard
69        // is intentional here: any new leaf variant should fall through to
70        // from_leaf_config, which will handle it or produce a compile error.
71        _ => from_leaf_config(config).await?,
72    })
73}
74
75async fn from_leaf_config(config: StorageConfig) -> Result<Box<dyn common::Backend>> {
76    Ok(match config {
77        StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c)),
78        StorageConfig::S3Compatible(c) => {
79            Box::new(s3_compatible::S3CompatibleBackend::without_token(c))
80        }
81        StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c).await?),
82        StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
83        StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
84    })
85}
86
87/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
88///
89/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
90/// Currently this is limited to BigTable.
91#[derive(Debug, Clone, Deserialize, Serialize)]
92#[serde(tag = "type", rename_all = "lowercase")]
93pub enum HighVolumeStorageConfig {
94    /// [Google Bigtable] backend.
95    ///
96    /// [Google Bigtable]: https://cloud.google.com/bigtable
97    BigTable(bigtable::BigTableConfig),
98}
99
100/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
101async fn hv_from_config(
102    config: HighVolumeStorageConfig,
103) -> Result<Box<dyn common::HighVolumeBackend>> {
104    Ok(match config {
105        HighVolumeStorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c).await?),
106    })
107}
108
109/// Configuration for the long-term backend in a [`tiered::TieredStorageConfig`].
110///
111/// Only backends that implement [`common::MultipartUploadBackend`] are valid here.
112#[derive(Debug, Clone, Deserialize, Serialize)]
113#[serde(tag = "type", rename_all = "lowercase")]
114pub enum MultipartUploadStorageConfig {
115    /// Local filesystem storage backend (type `"filesystem"`).
116    FileSystem(local_fs::FileSystemConfig),
117
118    /// [Google Cloud Storage] backend (type `"gcs"`).
119    ///
120    /// [Google Cloud Storage]: https://cloud.google.com/storage
121    Gcs(gcs::GcsConfig),
122}
123
124/// Constructs a type-erased [`common::MultipartUploadBackend`] from the given config.
125async fn lt_from_config(
126    config: MultipartUploadStorageConfig,
127) -> Result<Box<dyn common::MultipartUploadBackend>> {
128    Ok(match config {
129        MultipartUploadStorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c)),
130        MultipartUploadStorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c).await?),
131    })
132}
objectstore_service/backend/mod.rs

objectstore_service/backend/
mod.rs
