objectstore_service/backend/
mod.rs

1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14use crate::change_stream::ChangeStreamFactory;
15
16pub mod bigtable;
17pub mod changelog;
18pub mod common;
19pub mod counting;
20mod extensions;
21pub mod gcs;
22pub mod in_memory;
23pub mod local_fs;
24pub mod s3_compatible;
25pub mod tiered;
26
27#[cfg(test)]
28pub(crate) mod testing;
29
30/// Storage backend configuration.
31///
32/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
33///
34/// Used to configure storage backends via [`from_config`].
35#[derive(Debug, Clone, Deserialize, Serialize)]
36#[serde(tag = "type", rename_all = "lowercase")]
37pub enum StorageConfig {
38    /// Local filesystem storage backend (type `"filesystem"`).
39    FileSystem(local_fs::FileSystemConfig),
40
41    /// S3-compatible storage backend (type `"s3compatible"`).
42    S3Compatible(s3_compatible::S3CompatibleConfig),
43
44    /// [Google Cloud Storage] backend (type `"gcs"`).
45    ///
46    /// [Google Cloud Storage]: https://cloud.google.com/storage
47    Gcs(gcs::GcsConfig),
48
49    /// [Google Bigtable] backend (type `"bigtable"`).
50    ///
51    /// [Google Bigtable]: https://cloud.google.com/bigtable
52    BigTable(bigtable::BigTableConfig),
53
54    /// Tiered storage backend (type `"tiered"`).
55    ///
56    /// Routes objects across two backends based on size: small objects go to
57    /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
58    /// another `Tiered` is not supported and will return an error at startup.
59    Tiered(tiered::TieredStorageConfig),
60}
61
62/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
63///
64/// Backends that configured a [`ChangeStream`](crate::change_stream::ChangeStream) get one
65/// from `streams`; the rest report nothing.
66pub async fn from_config(
67    config: StorageConfig,
68    streams: &ChangeStreamFactory,
69) -> Result<Box<dyn common::Backend>> {
70    Ok(match config {
71        StorageConfig::Tiered(c) => {
72            let hv = hv_from_config(c.high_volume, streams).await?;
73            let lt = lt_from_config(c.long_term, streams).await?;
74            let log = Box::new(changelog::NoopChangeLog);
75            Box::new(tiered::TieredStorage::new(hv, lt, log))
76        }
77        // All non-Tiered variants are handled by from_leaf_config. A wildcard
78        // is intentional here: any new leaf variant should fall through to
79        // from_leaf_config, which will handle it or produce a compile error.
80        _ => from_leaf_config(config, streams).await?,
81    })
82}
83
84async fn from_leaf_config(
85    config: StorageConfig,
86    streams: &ChangeStreamFactory,
87) -> Result<Box<dyn common::Backend>> {
88    Ok(match config {
89        StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c, streams)),
90        StorageConfig::S3Compatible(c) => Box::new(
91            s3_compatible::S3CompatibleBackend::without_token(c, streams),
92        ),
93        StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c, streams).await?),
94        StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c, streams).await?),
95        StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
96    })
97}
98
99/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
100///
101/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
102/// Currently this is limited to BigTable.
103#[derive(Debug, Clone, Deserialize, Serialize)]
104#[serde(tag = "type", rename_all = "lowercase")]
105pub enum HighVolumeStorageConfig {
106    /// [Google Bigtable] backend.
107    ///
108    /// [Google Bigtable]: https://cloud.google.com/bigtable
109    BigTable(bigtable::BigTableConfig),
110}
111
112/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
113async fn hv_from_config(
114    config: HighVolumeStorageConfig,
115    streams: &ChangeStreamFactory,
116) -> Result<Box<dyn common::HighVolumeBackend>> {
117    Ok(match config {
118        HighVolumeStorageConfig::BigTable(c) => {
119            Box::new(bigtable::BigTableBackend::new(c, streams).await?)
120        }
121    })
122}
123
124/// Configuration for the long-term backend in a [`tiered::TieredStorageConfig`].
125///
126/// Only backends that implement [`common::MultipartUploadBackend`] are valid here.
127#[derive(Debug, Clone, Deserialize, Serialize)]
128#[serde(tag = "type", rename_all = "lowercase")]
129pub enum MultipartUploadStorageConfig {
130    /// Local filesystem storage backend (type `"filesystem"`).
131    FileSystem(local_fs::FileSystemConfig),
132
133    /// [Google Cloud Storage] backend (type `"gcs"`).
134    ///
135    /// [Google Cloud Storage]: https://cloud.google.com/storage
136    Gcs(gcs::GcsConfig),
137}
138
139/// Constructs a type-erased [`common::MultipartUploadBackend`] from the given config.
140async fn lt_from_config(
141    config: MultipartUploadStorageConfig,
142    streams: &ChangeStreamFactory,
143) -> Result<Box<dyn common::MultipartUploadBackend>> {
144    Ok(match config {
145        MultipartUploadStorageConfig::FileSystem(c) => {
146            Box::new(local_fs::LocalFsBackend::new(c, streams))
147        }
148        MultipartUploadStorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c, streams).await?),
149    })
150}
objectstore_service/backend/mod.rs

objectstore_service/backend/
mod.rs
