1use std::sync::Arc;
27
28use futures_util::{Stream, StreamExt};
29use objectstore_types::batch::OperationKind;
30use objectstore_types::metadata::Metadata;
31use objectstore_types::time::Timestamp;
32
33use crate::backend::common::Backend;
34use crate::background::RenewalScheduler;
35use crate::concurrency::ConcurrencyLimiter;
36use crate::error::{Error, Result};
37use crate::id::{ObjectContext, ObjectId, ObjectKey};
38use crate::service::GetResponse;
39
40#[derive(Debug)]
42pub struct Insert {
43 pub key: Option<ObjectKey>,
45 pub metadata: Metadata,
47 pub payload: bytes::Bytes,
49}
50
51#[derive(Debug)]
53pub struct Get {
54 pub key: ObjectKey,
56}
57
58#[derive(Debug)]
60pub struct Delete {
61 pub key: ObjectKey,
63}
64
65#[derive(Debug)]
67pub struct Head {
68 pub key: ObjectKey,
70}
71
72#[derive(Debug)]
74pub enum Operation {
75 Insert(Box<Insert>),
77 Get(Get),
79 Delete(Delete),
81 Head(Head),
83}
84
85impl Operation {
86 pub fn key(&self) -> Option<&ObjectKey> {
88 match self {
89 Operation::Insert(op) => op.key.as_ref(),
90 Operation::Get(op) => Some(&op.key),
91 Operation::Delete(op) => Some(&op.key),
92 Operation::Head(op) => Some(&op.key),
93 }
94 }
95
96 pub fn permission(&self) -> objectstore_types::auth::Permission {
98 match self {
99 Operation::Get(_) | Operation::Head(_) => {
100 objectstore_types::auth::Permission::ObjectRead
101 }
102 Operation::Insert(_) => objectstore_types::auth::Permission::ObjectWrite,
103 Operation::Delete(_) => objectstore_types::auth::Permission::ObjectDelete,
104 }
105 }
106
107 pub fn kind(&self) -> OperationKind {
109 match self {
110 Operation::Insert(_) => OperationKind::Insert,
111 Operation::Get(_) => OperationKind::Get,
112 Operation::Delete(_) => OperationKind::Delete,
113 Operation::Head(_) => OperationKind::Head,
114 }
115 }
116}
117
118pub enum OpResponse {
123 Inserted {
125 id: ObjectId,
127 },
128 Got {
130 key: ObjectKey,
132 response: GetResponse,
134 },
135 Deleted {
137 key: ObjectKey,
139 },
140 Head {
142 key: ObjectKey,
144 metadata: Option<Metadata>,
146 },
147}
148
149impl OpResponse {
150 pub fn kind(&self) -> OperationKind {
152 match self {
153 OpResponse::Inserted { .. } => OperationKind::Insert,
154 OpResponse::Got { .. } => OperationKind::Get,
155 OpResponse::Deleted { .. } => OperationKind::Delete,
156 OpResponse::Head { .. } => OperationKind::Head,
157 }
158 }
159
160 pub fn key(&self) -> &ObjectKey {
162 match self {
163 OpResponse::Inserted { id } => &id.key,
164 OpResponse::Got { key, .. } => key,
165 OpResponse::Deleted { key } => key,
166 OpResponse::Head { key, .. } => key,
167 }
168 }
169}
170
171impl std::fmt::Debug for OpResponse {
172 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
173 match self {
174 OpResponse::Inserted { id } => f.debug_struct("Inserted").field("id", id).finish(),
175 OpResponse::Got {
176 key,
177 response: Some(_),
178 } => f
179 .debug_struct("Got")
180 .field("key", key)
181 .field("response", &format_args!("Some(<stream>)"))
182 .finish(),
183 OpResponse::Got {
184 key,
185 response: None,
186 } => f
187 .debug_struct("Got")
188 .field("key", key)
189 .field("response", &format_args!("None"))
190 .finish(),
191 OpResponse::Deleted { key } => f.debug_struct("Deleted").field("key", key).finish(),
192 OpResponse::Head { key, metadata } => f
193 .debug_struct("Head")
194 .field("key", key)
195 .field("metadata", &metadata.is_some())
196 .finish(),
197 }
198 }
199}
200
201#[derive(Debug)]
209pub struct StreamExecutor {
210 backend: Arc<dyn Backend>,
211 concurrency: ConcurrencyLimiter,
212 renewals: RenewalScheduler,
213}
214
215impl StreamExecutor {
216 pub(crate) fn new(
218 backend: Arc<dyn Backend>,
219 concurrency: ConcurrencyLimiter,
220 renewals: RenewalScheduler,
221 ) -> Self {
222 Self {
223 backend,
224 concurrency,
225 renewals,
226 }
227 }
228
229 pub fn execute<E>(
249 self,
250 context: ObjectContext,
251 operations: impl Stream<Item = (usize, Result<Operation, E>)> + Send + 'static,
252 access_time: Timestamp,
253 ) -> impl Stream<Item = (usize, Result<OpResponse, E>)> + Send + 'static
254 where
255 E: From<Error> + Send + 'static,
256 {
257 let StreamExecutor {
258 backend,
259 concurrency,
260 renewals,
261 } = self;
262
263 let buffer = concurrency.total_bulk().max(1) as usize;
264
265 operations
266 .then(move |(idx, item)| {
271 let concurrency = concurrency.clone();
272 async move {
273 let op = match item {
274 Ok(op) => op,
275 Err(e) => return (idx, Err(e)),
276 };
277 match concurrency.acquire_bulk().await {
278 Ok(permit) => (idx, Ok((op, permit))),
279 Err(e) => {
280 objectstore_metrics::count!(
281 "service.concurrency.rejected",
282 class = "bulk"
283 );
284 objectstore_log::warn!("Bulk operation rejected: service at capacity");
285 (idx, Err(E::from(e)))
286 }
287 }
288 }
289 })
290 .map(move |(idx, result)| {
291 let backend = Arc::clone(&backend);
292 let context = context.clone();
293 let renewals = renewals.clone();
294 async move {
295 let (op, permit) = match result {
296 Ok(pair) => pair,
297 Err(e) => return (idx, Err(e)),
298 };
299
300 let spawn = crate::concurrency::run_metered(op.kind().as_str(), permit, {
301 execute_operation(backend, renewals, context, op, access_time)
302 });
303 (idx, spawn.await.map_err(E::from))
304 }
305 })
306 .buffer_unordered(buffer)
307 }
308}
309
310async fn execute_operation(
311 backend: Arc<dyn Backend>,
312 renewals: RenewalScheduler,
313 context: ObjectContext,
314 op: Operation,
315 access_time: Timestamp,
316) -> Result<OpResponse> {
317 match op {
318 Operation::Get(get) => {
319 let id = ObjectId::new(context, get.key);
320 let response = backend.get_object(&id, access_time, None).await?;
321 if let Some((metadata, _, _)) = &response
322 && let Some(expire_at) = metadata.check_tti_bump(access_time)
323 {
324 renewals.schedule(id.clone(), expire_at);
325 }
326 Ok(OpResponse::Got {
327 key: id.key,
328 response,
329 })
330 }
331 Operation::Insert(insert) => {
332 let id = ObjectId::optional(context, insert.key);
333 let stream = crate::stream::single(insert.payload);
334 backend
335 .put_object(&id, &insert.metadata, stream, access_time)
336 .await?;
337 Ok(OpResponse::Inserted { id })
338 }
339 Operation::Delete(delete) => {
340 let id = ObjectId::new(context, delete.key);
341 backend.delete_object(&id, access_time).await?;
342 Ok(OpResponse::Deleted { key: id.key })
343 }
344 Operation::Head(head) => {
345 let id = ObjectId::new(context, head.key);
346 let metadata = backend.get_metadata(&id, access_time).await?;
347 if let Some(metadata) = &metadata
348 && let Some(expire_at) = metadata.check_tti_bump(access_time)
349 {
350 renewals.schedule(id.clone(), expire_at);
351 }
352 Ok(OpResponse::Head {
353 key: id.key,
354 metadata,
355 })
356 }
357 }
358}
359
360#[cfg(test)]
361mod tests {
362 use std::sync::Arc;
363 use std::sync::atomic::{AtomicUsize, Ordering};
364 use std::time::Duration;
365
366 use bytes::Bytes;
367 use futures_util::StreamExt;
368 use objectstore_types::metadata::{ExpirationPolicy, Metadata};
369 use objectstore_types::scope::{Scope, Scopes};
370 use objectstore_types::time::Timestamp;
371
372 use super::*;
373 use crate::backend::common::{ExpiryUpdate, PutResponse, SetExpiryResponse};
374 use crate::backend::in_memory::InMemoryBackend;
375 use crate::backend::testing::{Hooks, TestBackend};
376 use crate::concurrency::ConcurrencyLimiter;
377 use crate::encryption::Cipher;
378 use crate::error::{Error, ErrorKind};
379 use crate::service::StorageService;
380 use crate::stream::{self, ClientStream};
381
382 fn make_context() -> ObjectContext {
383 ObjectContext {
384 usecase: "testing".parse().unwrap(),
385 scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
386 }
387 }
388
389 fn make_service_with_limit(limit: u32) -> StorageService {
390 StorageService::new(
391 Box::new(InMemoryBackend::new("in-memory")),
392 Cipher::ephemeral().unwrap(),
393 )
394 .with_concurrency(ConcurrencyLimiter::new(limit))
395 }
396
397 fn make_service() -> StorageService {
398 make_service_with_limit(500)
399 }
400
401 fn indexed_ok(ops: Vec<Operation>) -> impl Stream<Item = (usize, Result<Operation, Error>)> {
403 futures_util::stream::iter(ops.into_iter().enumerate().map(|(i, op)| (i, Ok(op))))
404 }
405
406 #[derive(Clone, Debug, Default)]
407 struct GateOnExpiry {
408 calls: Arc<AtomicUsize>,
409 started: Arc<tokio::sync::Notify>,
410 resume: Arc<tokio::sync::Notify>,
411 }
412
413 #[async_trait::async_trait]
414 impl Hooks for GateOnExpiry {
415 async fn set_expiry(
416 &self,
417 inner: &InMemoryBackend,
418 id: &ObjectId,
419 target: ExpiryUpdate,
420 access_time: Timestamp,
421 ) -> Result<SetExpiryResponse> {
422 self.calls.fetch_add(1, Ordering::SeqCst);
423 self.started.notify_one();
424 self.resume.notified().await;
425 inner.set_expiry(id, target, access_time).await
426 }
427 }
428
429 #[tokio::test]
430 async fn batch_renewal() {
431 let backend = TestBackend::new(GateOnExpiry::default());
432 let context = make_context();
433 let metadata = Metadata {
434 expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_hours(1)),
435 time_expires: Some(Timestamp::now() + Duration::from_mins(1)),
436 ..Default::default()
437 };
438 for key in ["get", "head"] {
439 let id = ObjectId::new(context.clone(), key.into());
440 backend
441 .inner
442 .put_object(&id, &metadata, stream::single("payload"), Timestamp::now())
443 .await
444 .unwrap();
445 }
446 let mut service =
447 StorageService::new(Box::new(backend.clone()), Cipher::ephemeral().unwrap());
448 service.start();
449 let outcomes = tokio::time::timeout(
450 Duration::from_secs(1),
451 service
452 .stream()
453 .execute(
454 context,
455 indexed_ok(vec![
456 Operation::Get(Get { key: "get".into() }),
457 Operation::Head(Head { key: "head".into() }),
458 ]),
459 Timestamp::now(),
460 )
461 .collect::<Vec<_>>(),
462 )
463 .await
464 .expect("batch reads waited for background renewal");
465 assert_eq!(outcomes.len(), 2);
466 tokio::time::timeout(Duration::from_secs(1), async {
467 while backend.hooks.calls.load(Ordering::SeqCst) < 2 {
468 tokio::task::yield_now().await;
469 }
470 })
471 .await
472 .expect("renewals did not start");
473 assert_eq!(backend.hooks.calls.load(Ordering::SeqCst), 2);
474
475 backend.hooks.resume.notify_waiters();
476 service.join().await;
477 }
478
479 #[tokio::test]
482 async fn execute_empty_stream() {
483 let service = make_service();
484 let executor = service.stream();
485 let outcomes: Vec<_> = executor
486 .execute(
487 make_context(),
488 futures_util::stream::empty::<(usize, Result<Operation, Error>)>(),
489 Timestamp::now(),
490 )
491 .collect()
492 .await;
493 assert!(outcomes.is_empty());
494 }
495
496 #[tokio::test]
497 async fn execute_runs_all_operations() {
498 let service = make_service();
499 let context = make_context();
500
501 service
503 .insert_object(
504 context.clone(),
505 Some("key1".into()),
506 Metadata::default(),
507 stream::single("hello"),
508 Timestamp::now(),
509 )
510 .await
511 .unwrap();
512
513 let ops = vec![
514 Operation::Get(Get { key: "key1".into() }),
515 Operation::Get(Get {
516 key: "nonexistent".into(),
517 }),
518 Operation::Insert(Box::new(Insert {
519 key: Some("key2".into()),
520 metadata: Metadata::default(),
521 payload: Bytes::from("world"),
522 })),
523 Operation::Delete(Delete { key: "key1".into() }),
524 ];
525
526 let executor = service.stream();
527 let outcomes: Vec<_> = executor
528 .execute(context, indexed_ok(ops), Timestamp::now())
529 .collect()
530 .await;
531
532 assert_eq!(outcomes.len(), 4);
533
534 for (_, result) in &outcomes {
535 let response = result
536 .as_ref()
537 .unwrap_or_else(|e| panic!("unexpected error: {e:?}"));
538 assert!(
539 !response.key().as_str().is_empty(),
540 "response must have a non-empty key"
541 );
542 }
543 }
544
545 #[tokio::test]
546 async fn execute_head_operation() {
547 let service = make_service();
548 let context = make_context();
549
550 service
551 .insert_object(
552 context.clone(),
553 Some("exists".into()),
554 Metadata::default(),
555 stream::single("data"),
556 Timestamp::now(),
557 )
558 .await
559 .unwrap();
560
561 let ops = vec![
562 Operation::Head(Head {
563 key: "exists".into(),
564 }),
565 Operation::Head(Head {
566 key: "missing".into(),
567 }),
568 ];
569
570 let executor = service.stream();
571 let mut outcomes: Vec<_> = executor
572 .execute(context, indexed_ok(ops), Timestamp::now())
573 .collect()
574 .await;
575 outcomes.sort_by_key(|(idx, _)| *idx);
576
577 assert_eq!(outcomes.len(), 2);
578
579 match &outcomes[0].1 {
580 Ok(OpResponse::Head {
581 key,
582 metadata: Some(_),
583 }) => assert_eq!(key.as_str(), "exists"),
584 other => panic!("expected Head with metadata, got: {other:?}"),
585 }
586
587 match &outcomes[1].1 {
588 Ok(OpResponse::Head {
589 key,
590 metadata: None,
591 }) => assert_eq!(key.as_str(), "missing"),
592 other => panic!("expected Head with None, got: {other:?}"),
593 }
594 }
595
596 struct GateOnPut {
599 paused_tx: tokio::sync::mpsc::Sender<()>,
600 resume: Arc<tokio::sync::Notify>,
601 in_flight: Arc<AtomicUsize>,
602 }
603
604 impl std::fmt::Debug for GateOnPut {
605 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
606 f.debug_struct("GateOnPut").finish()
607 }
608 }
609
610 #[async_trait::async_trait]
611 impl Hooks for GateOnPut {
612 async fn put_object(
613 &self,
614 inner: &InMemoryBackend,
615 id: &ObjectId,
616 metadata: &Metadata,
617 stream: ClientStream,
618 access_time: Timestamp,
619 ) -> Result<PutResponse> {
620 self.in_flight.fetch_add(1, Ordering::SeqCst);
621 let _ = self.paused_tx.send(()).await;
622 self.resume.notified().await;
623 let result = inner.put_object(id, metadata, stream, access_time).await;
624 self.in_flight.fetch_sub(1, Ordering::SeqCst);
625 result
626 }
627 }
628
629 #[tokio::test]
630 async fn concurrent_execution() {
631 let (paused_tx, mut paused_rx) = tokio::sync::mpsc::channel::<()>(20);
632 let resume = Arc::new(tokio::sync::Notify::new());
633 let in_flight = Arc::new(AtomicUsize::new(0));
634
635 let gated = TestBackend::new(GateOnPut {
636 paused_tx,
637 resume: Arc::clone(&resume),
638 in_flight: Arc::clone(&in_flight),
639 });
640 let service = StorageService::new(Box::new(gated), Cipher::ephemeral().unwrap())
641 .with_concurrency(ConcurrencyLimiter::new(100));
642
643 let ops: Vec<Operation> = (0..10)
644 .map(|i| {
645 Operation::Insert(Box::new(Insert {
646 key: Some(format!("key{i}")),
647 metadata: Metadata::default(),
648 payload: Bytes::from(format!("data{i}")),
649 }))
650 })
651 .collect();
652
653 let executor = service.stream();
654 let exec_handle = tokio::spawn(async move {
655 executor
656 .execute(make_context(), indexed_ok(ops), Timestamp::now())
657 .collect::<Vec<_>>()
658 .await
659 });
660
661 for _ in 0..10 {
663 paused_rx.recv().await.unwrap();
664 }
665 assert_eq!(in_flight.load(Ordering::SeqCst), 10);
666
667 resume.notify_waiters();
669
670 let outcomes = exec_handle.await.unwrap();
671 assert_eq!(outcomes.len(), 10);
672 for (_, result) in &outcomes {
673 assert!(
674 matches!(result, Ok(OpResponse::Inserted { .. })),
675 "unexpected result: {result:?}",
676 );
677 }
678 }
679
680 #[tokio::test]
681 async fn bulk_respects_budget() {
682 let service = StorageService::new(
685 Box::new(InMemoryBackend::new("in-memory")),
686 Cipher::ephemeral().unwrap(),
687 )
688 .with_concurrency(
689 ConcurrencyLimiter::new(1)
690 .with_queue(0)
691 .with_timeout(Duration::from_millis(1))
692 .with_bulk(100),
693 );
694
695 let _held = service.concurrency_limiter().acquire().await.unwrap();
696
697 let ops = vec![Operation::Insert(Box::new(Insert {
698 key: Some("blocked".into()),
699 metadata: Metadata::default(),
700 payload: Bytes::from("data"),
701 }))];
702
703 let executor = service.stream();
704 let outcomes: Vec<_> = executor
705 .execute(make_context(), indexed_ok(ops), Timestamp::now())
706 .collect()
707 .await;
708
709 assert_eq!(outcomes.len(), 1);
710 assert!(
711 outcomes[0]
712 .1
713 .as_ref()
714 .is_err_and(|error| error.kind() == ErrorKind::AtCapacity),
715 "expected AtCapacity, got {:?}",
716 outcomes[0].1,
717 );
718 }
719}