Skip to main content

objectstore_service/backend/
mod.rs

1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14use crate::change_stream::ChangeStreamFactory;
15
16pub mod bigtable;
17pub mod changelog;
18pub mod common;
19pub mod counting;
20mod extensions;
21pub mod gcs;
22pub mod in_memory;
23pub mod local_fs;
24pub mod s3_compatible;
25pub mod tiered;
26
27#[cfg(test)]
28pub(crate) mod testing;
29
30/// Storage backend configuration.
31///
32/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
33///
34/// Used to configure storage backends via [`from_config`].
35#[derive(Debug, Clone, Deserialize, Serialize)]
36#[serde(tag = "type", rename_all = "lowercase")]
37pub enum StorageConfig {
38    /// Local filesystem storage backend (type `"filesystem"`).
39    FileSystem(local_fs::FileSystemConfig),
40
41    /// S3-compatible storage backend (type `"s3compatible"`).
42    S3Compatible(s3_compatible::S3CompatibleConfig),
43
44    /// [Google Cloud Storage] backend (type `"gcs"`).
45    ///
46    /// [Google Cloud Storage]: https://cloud.google.com/storage
47    Gcs(gcs::GcsConfig),
48
49    /// [Google Bigtable] backend (type `"bigtable"`).
50    ///
51    /// [Google Bigtable]: https://cloud.google.com/bigtable
52    BigTable(bigtable::BigTableConfig),
53
54    /// Tiered storage backend (type `"tiered"`).
55    ///
56    /// Routes objects across two backends based on size: small objects go to
57    /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
58    /// another `Tiered` is not supported and will return an error at startup.
59    Tiered(tiered::TieredStorageConfig),
60}
61
62/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
63///
64/// Backends that configured a [`ChangeStream`](crate::change_stream::ChangeStream) get one
65/// from `streams`; the rest report nothing.
66pub async fn from_config(
67    config: StorageConfig,
68    streams: &ChangeStreamFactory,
69) -> Result<Box<dyn common::Backend>> {
70    Ok(match config {
71        StorageConfig::Tiered(c) => {
72            let hv = hv_from_config(c.high_volume, streams).await?;
73            let lt = from_leaf_config(*c.long_term, streams).await?;
74            let log = Box::new(changelog::NoopChangeLog);
75            let mut backend = tiered::TieredStorage::new(hv, lt, log);
76            if let Some(limits) = c.concurrency {
77                backend = backend.with_concurrency(limits);
78            }
79            Box::new(backend)
80        }
81        // All non-Tiered variants are handled by from_leaf_config. A wildcard
82        // is intentional here: any new leaf variant should fall through to
83        // from_leaf_config, which will handle it or produce a compile error.
84        _ => from_leaf_config(config, streams).await?,
85    })
86}
87
88async fn from_leaf_config(
89    config: StorageConfig,
90    streams: &ChangeStreamFactory,
91) -> Result<Box<dyn common::Backend>> {
92    Ok(match config {
93        StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c, streams)),
94        StorageConfig::S3Compatible(c) => Box::new(
95            s3_compatible::S3CompatibleBackend::without_token(c, streams),
96        ),
97        StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c, streams).await?),
98        StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c, streams).await?),
99        StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
100    })
101}
102
103/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
104///
105/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
106/// Currently this is limited to BigTable.
107#[derive(Debug, Clone, Deserialize, Serialize)]
108#[serde(tag = "type", rename_all = "lowercase")]
109pub enum HighVolumeStorageConfig {
110    /// [Google Bigtable] backend.
111    ///
112    /// [Google Bigtable]: https://cloud.google.com/bigtable
113    BigTable(bigtable::BigTableConfig),
114}
115
116/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
117async fn hv_from_config(
118    config: HighVolumeStorageConfig,
119    streams: &ChangeStreamFactory,
120) -> Result<Box<dyn common::HighVolumeBackend>> {
121    Ok(match config {
122        HighVolumeStorageConfig::BigTable(c) => {
123            Box::new(bigtable::BigTableBackend::new(c, streams).await?)
124        }
125    })
126}