objectstore_service/backend/mod.rs
1//! Storage backend implementations.
2//!
3//! This module contains the [`Backend`](common::Backend) trait and its
4//! implementations. Each backend adapts a specific storage system (BigTable,
5//! GCS, local filesystem, S3-compatible) to a uniform interface that
6//! [`StorageService`](crate::StorageService) consumes.
7//!
8//! Two-tier routing is encapsulated in [`TieredStorage`](tiered::TieredStorage)
9//! and can be configured via [`StorageConfig::Tiered`].
10
11use anyhow::Result;
12use serde::{Deserialize, Serialize};
13
14use crate::change_stream::ChangeStreamFactory;
15
16pub mod bigtable;
17pub mod changelog;
18pub mod common;
19pub mod counting;
20mod extensions;
21pub mod gcs;
22pub mod in_memory;
23pub mod local_fs;
24pub mod s3_compatible;
25pub mod tiered;
26
27#[cfg(test)]
28pub(crate) mod testing;
29
30/// Storage backend configuration.
31///
32/// The `type` field in YAML or `__TYPE` in environment variables determines which variant is used.
33///
34/// Used to configure storage backends via [`from_config`].
35#[derive(Debug, Clone, Deserialize, Serialize)]
36#[serde(tag = "type", rename_all = "lowercase")]
37pub enum StorageConfig {
38 /// Local filesystem storage backend (type `"filesystem"`).
39 FileSystem(local_fs::FileSystemConfig),
40
41 /// S3-compatible storage backend (type `"s3compatible"`).
42 S3Compatible(s3_compatible::S3CompatibleConfig),
43
44 /// [Google Cloud Storage] backend (type `"gcs"`).
45 ///
46 /// [Google Cloud Storage]: https://cloud.google.com/storage
47 Gcs(gcs::GcsConfig),
48
49 /// [Google Bigtable] backend (type `"bigtable"`).
50 ///
51 /// [Google Bigtable]: https://cloud.google.com/bigtable
52 BigTable(bigtable::BigTableConfig),
53
54 /// Tiered storage backend (type `"tiered"`).
55 ///
56 /// Routes objects across two backends based on size: small objects go to
57 /// `high_volume`, large objects go to `long_term`. Nesting `Tiered` inside
58 /// another `Tiered` is not supported and will return an error at startup.
59 Tiered(tiered::TieredStorageConfig),
60}
61
62/// Constructs a type-erased [`Backend`](common::Backend) from the given [`StorageConfig`].
63///
64/// Backends that configured a [`ChangeStream`](crate::change_stream::ChangeStream) get one
65/// from `streams`; the rest report nothing.
66pub async fn from_config(
67 config: StorageConfig,
68 streams: &ChangeStreamFactory,
69) -> Result<Box<dyn common::Backend>> {
70 Ok(match config {
71 StorageConfig::Tiered(c) => {
72 let hv = hv_from_config(c.high_volume, streams).await?;
73 let lt = from_leaf_config(*c.long_term, streams).await?;
74 let log = Box::new(changelog::NoopChangeLog);
75 let mut backend = tiered::TieredStorage::new(hv, lt, log);
76 if let Some(limits) = c.concurrency {
77 backend = backend.with_concurrency(limits);
78 }
79 Box::new(backend)
80 }
81 // All non-Tiered variants are handled by from_leaf_config. A wildcard
82 // is intentional here: any new leaf variant should fall through to
83 // from_leaf_config, which will handle it or produce a compile error.
84 _ => from_leaf_config(config, streams).await?,
85 })
86}
87
88async fn from_leaf_config(
89 config: StorageConfig,
90 streams: &ChangeStreamFactory,
91) -> Result<Box<dyn common::Backend>> {
92 Ok(match config {
93 StorageConfig::FileSystem(c) => Box::new(local_fs::LocalFsBackend::new(c, streams)),
94 StorageConfig::S3Compatible(c) => Box::new(
95 s3_compatible::S3CompatibleBackend::without_token(c, streams),
96 ),
97 StorageConfig::Gcs(c) => Box::new(gcs::GcsBackend::new(c, streams).await?),
98 StorageConfig::BigTable(c) => Box::new(bigtable::BigTableBackend::new(c, streams).await?),
99 StorageConfig::Tiered(_) => anyhow::bail!("nested tiered storage is not supported"),
100 })
101}
102
103/// Configuration for the high-volume backend in a [`tiered::TieredStorageConfig`].
104///
105/// Only backends that implement [`common::HighVolumeBackend`] are valid here.
106/// Currently this is limited to BigTable.
107#[derive(Debug, Clone, Deserialize, Serialize)]
108#[serde(tag = "type", rename_all = "lowercase")]
109pub enum HighVolumeStorageConfig {
110 /// [Google Bigtable] backend.
111 ///
112 /// [Google Bigtable]: https://cloud.google.com/bigtable
113 BigTable(bigtable::BigTableConfig),
114}
115
116/// Constructs a type-erased [`common::HighVolumeBackend`] from the given config.
117async fn hv_from_config(
118 config: HighVolumeStorageConfig,
119 streams: &ChangeStreamFactory,
120) -> Result<Box<dyn common::HighVolumeBackend>> {
121 Ok(match config {
122 HighVolumeStorageConfig::BigTable(c) => {
123 Box::new(bigtable::BigTableBackend::new(c, streams).await?)
124 }
125 })
126}