Skip to main content

objectstore_service/
streaming.rs

1//! Streaming operation types and concurrent executor.
2//!
3//! [`StreamExecutor`] processes a stream of `(idx, Result<`[`Operation`]`, E>)` tuples concurrently
4//! within a bounded window. Errors in the input stream pass through unchanged; successful
5//! operations are executed against the backend directly, with [`tokio::spawn`] for panic isolation
6//! and run-to-completion guarantees.
7//!
8//! ## Permit Acquisition
9//!
10//! Streaming / batch operations are subject to the service's concurrency limiter. They count as
11//! "bulk" operations, which are capped at a lower limit than regular operations. This ensures a
12//! large bulk request doesn't automatically bring the service to its limits and leaves room for
13//! regular requests.
14//!
15//! The regular acquire timeout applies: Operations that cannot acquire a permit within the
16//! configured queue timeout fail with [`AtCapacity`](crate::error::ErrorKind::AtCapacity).
17//!
18//! ## Concurrency Model
19//!
20//! [`StreamExecutor::execute`] uses `buffer_unordered` with the bulk budget as the concurrency
21//! bound. The input stream is pulled lazily and results are yielded in completion order. Each
22//! operation is wrapped in a [`tokio::spawn`] for panic isolation: a panic in one operation
23//! surfaces as a [`Panic`](crate::error::ErrorKind::Panic) for that item and does not affect
24//! the others.
25
26use std::sync::Arc;
27
28use futures_util::{Stream, StreamExt};
29use objectstore_types::batch::OperationKind;
30use objectstore_types::metadata::Metadata;
31use objectstore_types::time::Timestamp;
32
33use crate::backend::common::Backend;
34use crate::background::RenewalScheduler;
35use crate::concurrency::ConcurrencyLimiter;
36use crate::error::{Error, Result};
37use crate::id::{ObjectContext, ObjectId, ObjectKey};
38use crate::service::GetResponse;
39
40/// An insert operation: stores an object at the given key.
41#[derive(Debug)]
42pub struct Insert {
43    /// The key to store the object under. When `None`, the service generates a key.
44    pub key: Option<ObjectKey>,
45    /// Metadata for the object.
46    pub metadata: Metadata,
47    /// The object payload. Batch inserts are fully buffered (≤1 MiB).
48    pub payload: bytes::Bytes,
49}
50
51/// A get operation: retrieves an existing object by key.
52#[derive(Debug)]
53pub struct Get {
54    /// The key of the object to retrieve.
55    pub key: ObjectKey,
56}
57
58/// A delete operation: removes an object by key.
59#[derive(Debug)]
60pub struct Delete {
61    /// The key of the object to delete.
62    pub key: ObjectKey,
63}
64
65/// A head (metadata-only) operation: checks existence and retrieves metadata by key.
66#[derive(Debug)]
67pub struct Head {
68    /// The key of the object to check.
69    pub key: ObjectKey,
70}
71
72/// A single streaming operation.
73#[derive(Debug)]
74pub enum Operation {
75    /// Insert a new object.
76    Insert(Box<Insert>),
77    /// Get an existing object.
78    Get(Get),
79    /// Delete an object.
80    Delete(Delete),
81    /// Head (metadata-only) check for an object.
82    Head(Head),
83}
84
85impl Operation {
86    /// Returns the key for this operation, if one was provided.
87    pub fn key(&self) -> Option<&ObjectKey> {
88        match self {
89            Operation::Insert(op) => op.key.as_ref(),
90            Operation::Get(op) => Some(&op.key),
91            Operation::Delete(op) => Some(&op.key),
92            Operation::Head(op) => Some(&op.key),
93        }
94    }
95
96    /// Returns the permission required to perform this operation.
97    pub fn permission(&self) -> objectstore_types::auth::Permission {
98        match self {
99            Operation::Get(_) | Operation::Head(_) => {
100                objectstore_types::auth::Permission::ObjectRead
101            }
102            Operation::Insert(_) => objectstore_types::auth::Permission::ObjectWrite,
103            Operation::Delete(_) => objectstore_types::auth::Permission::ObjectDelete,
104        }
105    }
106
107    /// Returns the kind of this operation.
108    pub fn kind(&self) -> OperationKind {
109        match self {
110            Operation::Insert(_) => OperationKind::Insert,
111            Operation::Get(_) => OperationKind::Get,
112            Operation::Delete(_) => OperationKind::Delete,
113            Operation::Head(_) => OperationKind::Head,
114        }
115    }
116}
117
118/// The response of a single executed streaming operation.
119///
120/// Each variant carries the fields needed to render a response part.
121/// The kind (`"insert"`, `"get"`, `"delete"`) is derivable via [`OpResponse::kind`].
122pub enum OpResponse {
123    /// An insert completed successfully.
124    Inserted {
125        /// The fully-qualified identifier assigned to the inserted object.
126        id: ObjectId,
127    },
128    /// A get completed.
129    Got {
130        /// The key that was looked up.
131        key: ObjectKey,
132        /// The object content, or `None` if the object was not found.
133        response: GetResponse,
134    },
135    /// A delete completed successfully.
136    Deleted {
137        /// The key that was deleted.
138        key: ObjectKey,
139    },
140    /// A head (metadata-only) check completed.
141    Head {
142        /// The key that was checked.
143        key: ObjectKey,
144        /// The metadata, or `None` if the object was not found.
145        metadata: Option<Metadata>,
146    },
147}
148
149impl OpResponse {
150    /// Returns the kind of the operation.
151    pub fn kind(&self) -> OperationKind {
152        match self {
153            OpResponse::Inserted { .. } => OperationKind::Insert,
154            OpResponse::Got { .. } => OperationKind::Get,
155            OpResponse::Deleted { .. } => OperationKind::Delete,
156            OpResponse::Head { .. } => OperationKind::Head,
157        }
158    }
159
160    /// Returns the object key for this response.
161    pub fn key(&self) -> &ObjectKey {
162        match self {
163            OpResponse::Inserted { id } => &id.key,
164            OpResponse::Got { key, .. } => key,
165            OpResponse::Deleted { key } => key,
166            OpResponse::Head { key, .. } => key,
167        }
168    }
169}
170
171impl std::fmt::Debug for OpResponse {
172    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173        match self {
174            OpResponse::Inserted { id } => f.debug_struct("Inserted").field("id", id).finish(),
175            OpResponse::Got {
176                key,
177                response: Some(_),
178            } => f
179                .debug_struct("Got")
180                .field("key", key)
181                .field("response", &format_args!("Some(<stream>)"))
182                .finish(),
183            OpResponse::Got {
184                key,
185                response: None,
186            } => f
187                .debug_struct("Got")
188                .field("key", key)
189                .field("response", &format_args!("None"))
190                .finish(),
191            OpResponse::Deleted { key } => f.debug_struct("Deleted").field("key", key).finish(),
192            OpResponse::Head { key, metadata } => f
193                .debug_struct("Head")
194                .field("key", key)
195                .field("metadata", &metadata.is_some())
196                .finish(),
197        }
198    }
199}
200
201/// Executes streaming operations with bounded concurrency.
202///
203/// Construct via [`StorageService::stream`](crate::service::StorageService::stream).
204/// Each operation acquires a bulk permit individually; aggregate bulk
205/// concurrency is bounded by the bulk semaphore on the limiter.
206///
207/// See the [module documentation](self) for the concurrency model.
208#[derive(Debug)]
209pub struct StreamExecutor {
210    backend: Arc<dyn Backend>,
211    concurrency: ConcurrencyLimiter,
212    renewals: RenewalScheduler,
213}
214
215impl StreamExecutor {
216    /// Creates a new `StreamExecutor` with the given backend and limiter.
217    pub(crate) fn new(
218        backend: Arc<dyn Backend>,
219        concurrency: ConcurrencyLimiter,
220        renewals: RenewalScheduler,
221    ) -> Self {
222        Self {
223            backend,
224            concurrency,
225            renewals,
226        }
227    }
228
229    /// Executes the operations stream with bounded concurrency.
230    ///
231    /// Each item is a `(index, Result<Operation, E>)` tuple where `index` is the
232    /// 0-based position of the operation in the original request. Error items pass
233    /// through immediately; successful items acquire a bulk permit and execute
234    /// in an isolated [`tokio::spawn`].
235    ///
236    /// Permit acquisition is sequential — only one acquire is in flight at a
237    /// time, ensuring fairness with other streams and preventing a large
238    /// batch from racing for all permits at once. Execution of operations
239    /// that already hold a permit proceeds concurrently.
240    ///
241    /// Operations that cannot acquire a permit within the configured queue
242    /// timeout fail with [`AtCapacity`](crate::error::ErrorKind::AtCapacity).
243    /// Results are yielded in completion order (not submission order).
244    ///
245    /// All operations use the supplied `access_time`, including operations
246    /// parsed or admitted later. Use the same timestamp when resolving insert
247    /// metadata so creation, expiry checks, and TTI calculations share an anchor.
248    pub fn execute<E>(
249        self,
250        context: ObjectContext,
251        operations: impl Stream<Item = (usize, Result<Operation, E>)> + Send + 'static,
252        access_time: Timestamp,
253    ) -> impl Stream<Item = (usize, Result<OpResponse, E>)> + Send + 'static
254    where
255        E: From<Error> + Send + 'static,
256    {
257        let StreamExecutor {
258            backend,
259            concurrency,
260            renewals,
261        } = self;
262
263        let buffer = concurrency.total_bulk().max(1) as usize;
264
265        operations
266            // `then` awaits each closure before pulling the next item, making
267            // permit acquisition sequential. this ensures fairness with other
268            // streams and prevents a large batch from racing for all permits
269            // at once.
270            .then(move |(idx, item)| {
271                let concurrency = concurrency.clone();
272                async move {
273                    let op = match item {
274                        Ok(op) => op,
275                        Err(e) => return (idx, Err(e)),
276                    };
277                    match concurrency.acquire_bulk().await {
278                        Ok(permit) => (idx, Ok((op, permit))),
279                        Err(e) => {
280                            objectstore_metrics::count!(
281                                "service.concurrency.rejected",
282                                class = "bulk"
283                            );
284                            objectstore_log::warn!("Bulk operation rejected: service at capacity");
285                            (idx, Err(E::from(e)))
286                        }
287                    }
288                }
289            })
290            .map(move |(idx, result)| {
291                let backend = Arc::clone(&backend);
292                let context = context.clone();
293                let renewals = renewals.clone();
294                async move {
295                    let (op, permit) = match result {
296                        Ok(pair) => pair,
297                        Err(e) => return (idx, Err(e)),
298                    };
299
300                    let spawn = crate::concurrency::run_metered(op.kind().as_str(), permit, {
301                        execute_operation(backend, renewals, context, op, access_time)
302                    });
303                    (idx, spawn.await.map_err(E::from))
304                }
305            })
306            .buffer_unordered(buffer)
307    }
308}
309
310async fn execute_operation(
311    backend: Arc<dyn Backend>,
312    renewals: RenewalScheduler,
313    context: ObjectContext,
314    op: Operation,
315    access_time: Timestamp,
316) -> Result<OpResponse> {
317    match op {
318        Operation::Get(get) => {
319            let id = ObjectId::new(context, get.key);
320            let response = backend.get_object(&id, access_time, None).await?;
321            if let Some((metadata, _, _)) = &response
322                && let Some(expire_at) = metadata.check_tti_bump(access_time)
323            {
324                renewals.schedule(id.clone(), expire_at);
325            }
326            Ok(OpResponse::Got {
327                key: id.key,
328                response,
329            })
330        }
331        Operation::Insert(insert) => {
332            let id = ObjectId::optional(context, insert.key);
333            let stream = crate::stream::single(insert.payload);
334            backend
335                .put_object(&id, &insert.metadata, stream, access_time)
336                .await?;
337            Ok(OpResponse::Inserted { id })
338        }
339        Operation::Delete(delete) => {
340            let id = ObjectId::new(context, delete.key);
341            backend.delete_object(&id, access_time).await?;
342            Ok(OpResponse::Deleted { key: id.key })
343        }
344        Operation::Head(head) => {
345            let id = ObjectId::new(context, head.key);
346            let metadata = backend.get_metadata(&id, access_time).await?;
347            if let Some(metadata) = &metadata
348                && let Some(expire_at) = metadata.check_tti_bump(access_time)
349            {
350                renewals.schedule(id.clone(), expire_at);
351            }
352            Ok(OpResponse::Head {
353                key: id.key,
354                metadata,
355            })
356        }
357    }
358}
359
360#[cfg(test)]
361mod tests {
362    use std::sync::Arc;
363    use std::sync::atomic::{AtomicUsize, Ordering};
364    use std::time::Duration;
365
366    use bytes::Bytes;
367    use futures_util::StreamExt;
368    use objectstore_types::metadata::{ExpirationPolicy, Metadata};
369    use objectstore_types::scope::{Scope, Scopes};
370    use objectstore_types::time::Timestamp;
371
372    use super::*;
373    use crate::backend::common::{ExpiryUpdate, PutResponse, SetExpiryResponse};
374    use crate::backend::in_memory::InMemoryBackend;
375    use crate::backend::testing::{Hooks, TestBackend};
376    use crate::concurrency::ConcurrencyLimiter;
377    use crate::encryption::Cipher;
378    use crate::error::{Error, ErrorKind};
379    use crate::service::StorageService;
380    use crate::stream::{self, ClientStream};
381
382    fn make_context() -> ObjectContext {
383        ObjectContext {
384            usecase: "testing".parse().unwrap(),
385            scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
386        }
387    }
388
389    fn make_service_with_limit(limit: u32) -> StorageService {
390        StorageService::new(
391            Box::new(InMemoryBackend::new("in-memory")),
392            Cipher::ephemeral().unwrap(),
393        )
394        .with_concurrency(ConcurrencyLimiter::new(limit))
395    }
396
397    fn make_service() -> StorageService {
398        make_service_with_limit(500)
399    }
400
401    // Wraps a plain `Vec<Operation>` as an indexed `Ok`-stream for `execute`.
402    fn indexed_ok(ops: Vec<Operation>) -> impl Stream<Item = (usize, Result<Operation, Error>)> {
403        futures_util::stream::iter(ops.into_iter().enumerate().map(|(i, op)| (i, Ok(op))))
404    }
405
406    #[derive(Clone, Debug, Default)]
407    struct GateOnExpiry {
408        calls: Arc<AtomicUsize>,
409        started: Arc<tokio::sync::Notify>,
410        resume: Arc<tokio::sync::Notify>,
411    }
412
413    #[async_trait::async_trait]
414    impl Hooks for GateOnExpiry {
415        async fn set_expiry(
416            &self,
417            inner: &InMemoryBackend,
418            id: &ObjectId,
419            target: ExpiryUpdate,
420            access_time: Timestamp,
421        ) -> Result<SetExpiryResponse> {
422            self.calls.fetch_add(1, Ordering::SeqCst);
423            self.started.notify_one();
424            self.resume.notified().await;
425            inner.set_expiry(id, target, access_time).await
426        }
427    }
428
429    #[tokio::test]
430    async fn batch_renewal() {
431        let backend = TestBackend::new(GateOnExpiry::default());
432        let context = make_context();
433        let metadata = Metadata {
434            expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_hours(1)),
435            time_expires: Some(Timestamp::now() + Duration::from_mins(1)),
436            ..Default::default()
437        };
438        for key in ["get", "head"] {
439            let id = ObjectId::new(context.clone(), key.into());
440            backend
441                .inner
442                .put_object(&id, &metadata, stream::single("payload"), Timestamp::now())
443                .await
444                .unwrap();
445        }
446        let mut service =
447            StorageService::new(Box::new(backend.clone()), Cipher::ephemeral().unwrap());
448        service.start();
449        let outcomes = tokio::time::timeout(
450            Duration::from_secs(1),
451            service
452                .stream()
453                .execute(
454                    context,
455                    indexed_ok(vec![
456                        Operation::Get(Get { key: "get".into() }),
457                        Operation::Head(Head { key: "head".into() }),
458                    ]),
459                    Timestamp::now(),
460                )
461                .collect::<Vec<_>>(),
462        )
463        .await
464        .expect("batch reads waited for background renewal");
465        assert_eq!(outcomes.len(), 2);
466        tokio::time::timeout(Duration::from_secs(1), async {
467            while backend.hooks.calls.load(Ordering::SeqCst) < 2 {
468                tokio::task::yield_now().await;
469            }
470        })
471        .await
472        .expect("renewals did not start");
473        assert_eq!(backend.hooks.calls.load(Ordering::SeqCst), 2);
474
475        backend.hooks.resume.notify_waiters();
476        service.join().await;
477    }
478
479    // --- StreamExecutor correctness tests ---
480
481    #[tokio::test]
482    async fn execute_empty_stream() {
483        let service = make_service();
484        let executor = service.stream();
485        let outcomes: Vec<_> = executor
486            .execute(
487                make_context(),
488                futures_util::stream::empty::<(usize, Result<Operation, Error>)>(),
489                Timestamp::now(),
490            )
491            .collect()
492            .await;
493        assert!(outcomes.is_empty());
494    }
495
496    #[tokio::test]
497    async fn execute_runs_all_operations() {
498        let service = make_service();
499        let context = make_context();
500
501        // Seed an object to retrieve and delete.
502        service
503            .insert_object(
504                context.clone(),
505                Some("key1".into()),
506                Metadata::default(),
507                stream::single("hello"),
508                Timestamp::now(),
509            )
510            .await
511            .unwrap();
512
513        let ops = vec![
514            Operation::Get(Get { key: "key1".into() }),
515            Operation::Get(Get {
516                key: "nonexistent".into(),
517            }),
518            Operation::Insert(Box::new(Insert {
519                key: Some("key2".into()),
520                metadata: Metadata::default(),
521                payload: Bytes::from("world"),
522            })),
523            Operation::Delete(Delete { key: "key1".into() }),
524        ];
525
526        let executor = service.stream();
527        let outcomes: Vec<_> = executor
528            .execute(context, indexed_ok(ops), Timestamp::now())
529            .collect()
530            .await;
531
532        assert_eq!(outcomes.len(), 4);
533
534        for (_, result) in &outcomes {
535            let response = result
536                .as_ref()
537                .unwrap_or_else(|e| panic!("unexpected error: {e:?}"));
538            assert!(
539                !response.key().as_str().is_empty(),
540                "response must have a non-empty key"
541            );
542        }
543    }
544
545    #[tokio::test]
546    async fn execute_head_operation() {
547        let service = make_service();
548        let context = make_context();
549
550        service
551            .insert_object(
552                context.clone(),
553                Some("exists".into()),
554                Metadata::default(),
555                stream::single("data"),
556                Timestamp::now(),
557            )
558            .await
559            .unwrap();
560
561        let ops = vec![
562            Operation::Head(Head {
563                key: "exists".into(),
564            }),
565            Operation::Head(Head {
566                key: "missing".into(),
567            }),
568        ];
569
570        let executor = service.stream();
571        let mut outcomes: Vec<_> = executor
572            .execute(context, indexed_ok(ops), Timestamp::now())
573            .collect()
574            .await;
575        outcomes.sort_by_key(|(idx, _)| *idx);
576
577        assert_eq!(outcomes.len(), 2);
578
579        match &outcomes[0].1 {
580            Ok(OpResponse::Head {
581                key,
582                metadata: Some(_),
583            }) => assert_eq!(key.as_str(), "exists"),
584            other => panic!("expected Head with metadata, got: {other:?}"),
585        }
586
587        match &outcomes[1].1 {
588            Ok(OpResponse::Head {
589                key,
590                metadata: None,
591            }) => assert_eq!(key.as_str(), "missing"),
592            other => panic!("expected Head with None, got: {other:?}"),
593        }
594    }
595
596    // --- Service-level concurrent execution and capacity tests ---
597
598    struct GateOnPut {
599        paused_tx: tokio::sync::mpsc::Sender<()>,
600        resume: Arc<tokio::sync::Notify>,
601        in_flight: Arc<AtomicUsize>,
602    }
603
604    impl std::fmt::Debug for GateOnPut {
605        fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
606            f.debug_struct("GateOnPut").finish()
607        }
608    }
609
610    #[async_trait::async_trait]
611    impl Hooks for GateOnPut {
612        async fn put_object(
613            &self,
614            inner: &InMemoryBackend,
615            id: &ObjectId,
616            metadata: &Metadata,
617            stream: ClientStream,
618            access_time: Timestamp,
619        ) -> Result<PutResponse> {
620            self.in_flight.fetch_add(1, Ordering::SeqCst);
621            let _ = self.paused_tx.send(()).await;
622            self.resume.notified().await;
623            let result = inner.put_object(id, metadata, stream, access_time).await;
624            self.in_flight.fetch_sub(1, Ordering::SeqCst);
625            result
626        }
627    }
628
629    #[tokio::test]
630    async fn concurrent_execution() {
631        let (paused_tx, mut paused_rx) = tokio::sync::mpsc::channel::<()>(20);
632        let resume = Arc::new(tokio::sync::Notify::new());
633        let in_flight = Arc::new(AtomicUsize::new(0));
634
635        let gated = TestBackend::new(GateOnPut {
636            paused_tx,
637            resume: Arc::clone(&resume),
638            in_flight: Arc::clone(&in_flight),
639        });
640        let service = StorageService::new(Box::new(gated), Cipher::ephemeral().unwrap())
641            .with_concurrency(ConcurrencyLimiter::new(100));
642
643        let ops: Vec<Operation> = (0..10)
644            .map(|i| {
645                Operation::Insert(Box::new(Insert {
646                    key: Some(format!("key{i}")),
647                    metadata: Metadata::default(),
648                    payload: Bytes::from(format!("data{i}")),
649                }))
650            })
651            .collect();
652
653        let executor = service.stream();
654        let exec_handle = tokio::spawn(async move {
655            executor
656                .execute(make_context(), indexed_ok(ops), Timestamp::now())
657                .collect::<Vec<_>>()
658                .await
659        });
660
661        // Wait for all 10 operations to pause inside the backend.
662        for _ in 0..10 {
663            paused_rx.recv().await.unwrap();
664        }
665        assert_eq!(in_flight.load(Ordering::SeqCst), 10);
666
667        // Release all.
668        resume.notify_waiters();
669
670        let outcomes = exec_handle.await.unwrap();
671        assert_eq!(outcomes.len(), 10);
672        for (_, result) in &outcomes {
673            assert!(
674                matches!(result, Ok(OpResponse::Inserted { .. })),
675                "unexpected result: {result:?}",
676            );
677        }
678    }
679
680    #[tokio::test]
681    async fn bulk_respects_budget() {
682        // Bulk budget = 1 (100% of max=1). Hold the permit via a normal
683        // acquire; the bulk op should wait and eventually time out.
684        let service = StorageService::new(
685            Box::new(InMemoryBackend::new("in-memory")),
686            Cipher::ephemeral().unwrap(),
687        )
688        .with_concurrency(
689            ConcurrencyLimiter::new(1)
690                .with_queue(0)
691                .with_timeout(Duration::from_millis(1))
692                .with_bulk(100),
693        );
694
695        let _held = service.concurrency_limiter().acquire().await.unwrap();
696
697        let ops = vec![Operation::Insert(Box::new(Insert {
698            key: Some("blocked".into()),
699            metadata: Metadata::default(),
700            payload: Bytes::from("data"),
701        }))];
702
703        let executor = service.stream();
704        let outcomes: Vec<_> = executor
705            .execute(make_context(), indexed_ok(ops), Timestamp::now())
706            .collect()
707            .await;
708
709        assert_eq!(outcomes.len(), 1);
710        assert!(
711            outcomes[0]
712                .1
713                .as_ref()
714                .is_err_and(|error| error.kind() == ErrorKind::AtCapacity),
715            "expected AtCapacity, got {:?}",
716            outcomes[0].1,
717        );
718    }
719}