1use std::error::Error as StdError;
4use std::sync::Arc;
5use std::sync::atomic::Ordering;
6use std::time::Duration;
7use std::{fmt, io};
8
9use futures_util::{StreamExt, TryStreamExt};
10use objectstore_types::metadata::{HEADER_SIZE, Metadata};
11use objectstore_types::range::{ByteRange, ContentRange};
12use objectstore_types::time::Timestamp;
13use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
14use reqwest::{Body, IntoUrl, Method, RequestBuilder, Response, StatusCode};
15
16use super::extensions::{ResponseExt, SendTraced};
17use crate::backend::common::{
18 self, Backend, DeleteResponse, ExpiryUpdate, GetResponse, MetadataResponse, PutResponse,
19 SetExpiryResponse,
20};
21use crate::backend::extensions::ReqwestResultExt;
22use crate::change_stream::{
23 ChangeStream, ChangeStreamFactory, CostTrackerStreamConfig, flush_change_stream,
24};
25use crate::error::{Error, ErrorKind, Result, ResultExt as _};
26use crate::id::ObjectId;
27use crate::stream::{ClientStream, counting_stream};
28
29#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
45pub struct S3CompatibleConfig {
46 pub endpoint: String,
55
56 pub bucket: String,
64
65 #[serde(default = "common::default_tcp_user_timeout", with = "humantime_serde")]
79 pub tcp_user_timeout: Duration,
80
81 #[serde(default, skip_serializing_if = "Option::is_none")]
92 pub cogs: Option<CostTrackerStreamConfig>,
93}
94
95const GCS_CUSTOM_PREFIX: &str = "x-goog-meta-";
99const GCS_CUSTOM_TIME: &str = "x-goog-custom-time";
104
105pub trait Token: Send + Sync {
107 fn as_str(&self) -> &str;
109}
110
111pub trait TokenProvider: Send + Sync + 'static {
113 type Error: StdError + Send + Sync + 'static;
115
116 fn get_token(
118 &self,
119 ) -> impl Future<Output = std::result::Result<impl Token, Self::Error>> + Send;
120}
121
122impl TokenProvider for ! {
124 type Error = !;
125
126 #[allow(refining_impl_trait)]
127 async fn get_token(&self) -> std::result::Result<!, !> {
128 match *self {}
129 }
130}
131
132impl Token for ! {
133 fn as_str(&self) -> &str {
134 match *self {}
135 }
136}
137
138pub struct S3CompatibleBackend<T> {
140 client: reqwest::Client,
141
142 endpoint: String,
143 bucket: String,
144
145 token_provider: Option<T>,
146
147 change_stream: Arc<dyn ChangeStream>,
148}
149
150impl<T> S3CompatibleBackend<T> {
151 pub fn new(
153 config: S3CompatibleConfig,
154 token_provider: T,
155 streams: &ChangeStreamFactory,
156 ) -> Self {
157 Self::build(config, Some(token_provider), streams)
158 }
159
160 fn build(
161 config: S3CompatibleConfig,
162 token_provider: Option<T>,
163 streams: &ChangeStreamFactory,
164 ) -> Self {
165 let S3CompatibleConfig {
166 endpoint,
167 bucket,
168 tcp_user_timeout,
169 cogs,
170 } = config;
171 Self {
172 client: common::reqwest_client(tcp_user_timeout),
173 endpoint,
174 bucket,
175 token_provider,
176 change_stream: streams.build(cogs.as_ref()),
177 }
178 }
179
180 fn object_url(&self, id: &ObjectId) -> String {
182 format!("{}/{}/{}", self.endpoint, self.bucket, id.as_storage_path())
183 }
184}
185
186fn headers_size(headers: &HeaderMap) -> u64 {
188 headers
189 .iter()
190 .map(|(name, value)| name.as_str().len() as u64 + value.len() as u64)
191 .sum()
192}
193
194fn metadata_to_gcs_headers(
196 metadata: &Metadata,
197 prefix: &str,
198) -> Result<HeaderMap, objectstore_types::metadata::Error> {
199 let mut headers = metadata.to_headers(prefix)?;
200
201 let size = HeaderName::try_from(format!("{prefix}{HEADER_SIZE}"))?;
205 headers.remove(&size);
206
207 if let Some(expires_at) = metadata.time_expires {
209 let expires_at = expires_at.as_rfc3339();
210 headers.append(GCS_CUSTOM_TIME, expires_at.to_string().parse()?);
211 }
212 Ok(headers)
213}
214
215impl<T> S3CompatibleBackend<T>
216where
217 T: TokenProvider,
218{
219 async fn request(&self, method: Method, url: impl IntoUrl) -> Result<RequestBuilder> {
221 let mut builder = self.client.request(method, url);
222 if let Some(provider) = &self.token_provider {
223 builder = builder.bearer_auth(
224 provider
225 .get_token()
226 .await
227 .context(ErrorKind::BackendFailure, "getting S3 authentication token")?
228 .as_str(),
229 );
230 }
231 Ok(builder)
232 }
233
234 async fn request_object(
237 &self,
238 method: Method,
239 id: &ObjectId,
240 access_time: Timestamp,
241 range: Option<ByteRange>,
242 ) -> Result<Option<(Metadata, Option<ContentRange>, Response)>> {
243 let object_url = self.object_url(id);
244
245 let mut builder = self.request(method, &object_url).await?;
246 if let Some(r) = range {
247 builder = builder.header(reqwest::header::RANGE, r.to_header_value());
248 }
249 let response = builder
250 .send_traced()
251 .await
252 .reqwest_context("sending an S3 object request")?;
253
254 if response.status() == StatusCode::NOT_FOUND {
255 objectstore_log::debug!("Object not found");
256 response.drain_body().await;
257 return Ok(None);
258 }
259
260 if response.status() == StatusCode::RANGE_NOT_SATISFIABLE {
261 let raw = response
262 .headers()
263 .get(reqwest::header::CONTENT_RANGE)
264 .and_then(|v| v.to_str().ok());
265 let total = raw.and_then(ContentRange::parse_unsatisfiable_total);
266 let err = match total {
267 Some(total) => ErrorKind::RangeNotSatisfiable { total }.into(),
268 None => Error::new(ErrorKind::BackendFailure, "invalid S3 416 Content-Range"),
269 };
270 response.drain_body().await;
271 return Err(err);
272 }
273
274 let response = response.check_error("getting an S3 object").await?;
275
276 let headers = response.headers();
277 let mut metadata = Metadata::from_headers(headers, GCS_CUSTOM_PREFIX)
278 .context(ErrorKind::CorruptData, "decoding S3 object metadata")?;
279
280 let content_range = if response.status() == StatusCode::PARTIAL_CONTENT {
281 let range = headers
282 .get(reqwest::header::CONTENT_RANGE)
283 .and_then(|v| v.to_str().ok())
284 .and_then(|s| s.parse::<ContentRange>().ok())
285 .ok_or_else(|| {
286 Error::new(ErrorKind::BackendFailure, "missing S3 206 Content-Range")
287 })?;
288 metadata.size = Some(range.total as usize);
289 Some(range)
290 } else {
291 let size = headers
294 .get(reqwest::header::CONTENT_LENGTH)
295 .and_then(|value| value.to_str().ok())
296 .map(|value| value.parse::<usize>())
297 .transpose()
298 .context(ErrorKind::CorruptData, "decoding S3 Content-Length")?;
299
300 if let Some(size) = size {
301 metadata.size = Some(size);
302 } else {
303 objectstore_log::warn!("S3: 200 response missing Content-Length header");
304 }
305 None
306 };
307
308 if metadata.is_expired(access_time) {
310 objectstore_log::debug!("Object found but past expiry");
311 response.drain_body().await;
312 return Ok(None);
313 }
314
315 Ok(Some((metadata, content_range, response)))
316 }
317
318 async fn update_metadata(
320 &self,
321 id: &ObjectId,
322 metadata: &Metadata,
323 deadline: Timestamp,
324 etag: &HeaderValue,
325 ) -> Result<SetExpiryResponse> {
326 let request = self
329 .request(Method::PUT, self.object_url(id))
330 .await?
331 .header(
332 "x-amz-copy-source",
333 format!("/{}/{}", self.bucket, id.as_storage_path()),
334 )
335 .header("x-amz-metadata-directive", "REPLACE")
336 .header("x-amz-copy-source-if-match", etag.clone())
337 .headers(
338 metadata_to_gcs_headers(metadata, GCS_CUSTOM_PREFIX)
339 .context(ErrorKind::InvalidMetadata, "encoding S3 object metadata")?,
340 );
341
342 let response = request.send_traced().await;
343 let response = response.reqwest_context("updating S3 expiration")?;
344 let outcome = match response.status() {
345 StatusCode::NOT_FOUND => Some(SetExpiryResponse::NotFound),
346 StatusCode::CONFLICT | StatusCode::PRECONDITION_FAILED => {
347 Some(SetExpiryResponse::Rejected)
348 }
349 _ => None,
350 };
351 if let Some(outcome) = outcome {
352 response.drain_body().await;
353 return Ok(outcome);
354 }
355 response
356 .check_error("updating S3 expiration")
357 .await?
358 .drain_body()
359 .await;
360
361 Ok(SetExpiryResponse::Satisfied(deadline))
362 }
363}
364
365impl<T> fmt::Debug for S3CompatibleBackend<T> {
366 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
367 f.debug_struct("S3Compatible")
368 .field("client", &self.client)
369 .field("endpoint", &self.endpoint)
370 .field("bucket", &self.bucket)
371 .finish_non_exhaustive()
372 }
373}
374
375impl S3CompatibleBackend<!> {
376 pub fn without_token(config: S3CompatibleConfig, streams: &ChangeStreamFactory) -> Self {
378 Self::build(config, None, streams)
379 }
380}
381
382#[async_trait::async_trait]
383impl<T: TokenProvider> Backend for S3CompatibleBackend<T> {
384 fn name(&self) -> &'static str {
385 "s3-compatible"
386 }
387
388 #[tracing::instrument(level = "debug", fields(?id), skip_all)]
389 async fn put_object(
390 &self,
391 id: &ObjectId,
392 metadata: &Metadata,
393 stream: ClientStream,
394 _access_time: Timestamp,
395 ) -> Result<PutResponse> {
396 objectstore_log::debug!("Writing to s3_compatible backend");
397 self.change_stream
398 .begin_write(id, metadata.time_expires)
399 .await?;
400 let headers = metadata_to_gcs_headers(metadata, GCS_CUSTOM_PREFIX)
401 .context(ErrorKind::InvalidMetadata, "encoding S3 object metadata")?;
402 let metadata_size = headers_size(&headers);
403
404 let (payload_size, counted) = counting_stream(stream);
406
407 self.request(Method::PUT, self.object_url(id))
408 .await?
409 .headers(headers)
410 .body(Body::wrap_stream(counted))
411 .send_traced()
412 .await
413 .check_error("uploading an S3 object")
414 .await?
415 .drain_body()
416 .await;
417
418 self.change_stream
419 .commit_write(
420 id,
421 metadata_size + payload_size.load(Ordering::Relaxed),
422 metadata.time_expires,
423 )
424 .await;
425
426 Ok(())
427 }
428
429 #[tracing::instrument(level = "debug", skip(self))]
430 async fn get_object(
431 &self,
432 id: &ObjectId,
433 access_time: Timestamp,
434 range: Option<ByteRange>,
435 ) -> Result<GetResponse> {
436 objectstore_log::debug!("Reading from s3_compatible backend");
437
438 let Some((metadata, content_range, response)) = self
439 .request_object(Method::GET, id, access_time, range)
440 .await?
441 else {
442 return Ok(None);
443 };
444
445 let stream = response.bytes_stream().map_err(io::Error::other);
446 Ok(Some((metadata, content_range, stream.boxed())))
447 }
448
449 #[tracing::instrument(level = "debug", skip(self))]
450 async fn get_metadata(
451 &self,
452 id: &ObjectId,
453 access_time: Timestamp,
454 ) -> Result<MetadataResponse> {
455 objectstore_log::debug!("Reading metadata from s3_compatible backend");
456 let response = self
457 .request_object(Method::HEAD, id, access_time, None)
458 .await?;
459 Ok(response.map(|(metadata, _, _)| metadata))
460 }
461
462 #[tracing::instrument(level = "debug", skip(self))]
463 async fn set_expiry(
464 &self,
465 id: &ObjectId,
466 target: ExpiryUpdate,
467 access_time: Timestamp,
468 ) -> Result<SetExpiryResponse> {
469 let Some((mut metadata, _, response)) = self
470 .request_object(Method::HEAD, id, access_time, None)
471 .await?
472 else {
473 return Ok(SetExpiryResponse::NotFound);
474 };
475
476 let etag = response.headers().get(reqwest::header::ETAG).cloned();
477 response.drain_body().await;
478
479 let Some(current_expiry) = metadata.time_expires else {
480 return Ok(SetExpiryResponse::Rejected);
481 };
482 let Some(expire_at) = target.resolve(metadata.time_created, access_time)? else {
483 return Ok(SetExpiryResponse::Rejected);
484 };
485 if current_expiry >= expire_at {
486 return Ok(SetExpiryResponse::Satisfied(expire_at)); }
488
489 let etag = etag.ok_or_else(|| {
490 Error::new(ErrorKind::BackendFailure, "S3 HEAD response missing ETag")
491 })?;
492
493 metadata.expiration_policy = common::extended_expiration_policy(
494 metadata.expiration_policy,
495 metadata.time_created,
496 current_expiry,
497 expire_at,
498 )?;
499 metadata.time_expires = Some(expire_at);
500 self.change_stream.begin_update(id, Some(expire_at)).await?;
501 let outcome = self
502 .update_metadata(id, &metadata, expire_at, &etag)
503 .await?;
504 if matches!(outcome, SetExpiryResponse::Satisfied(_)) {
505 self.change_stream.commit_update(id, Some(expire_at)).await;
506 }
507
508 Ok(outcome)
509 }
510
511 #[tracing::instrument(level = "debug", skip(self))]
512 async fn delete_object(
513 &self,
514 id: &ObjectId,
515 _access_time: Timestamp,
516 ) -> Result<DeleteResponse> {
517 objectstore_log::debug!("Deleting from s3_compatible backend");
518 let response = self
519 .request(Method::DELETE, self.object_url(id))
520 .await?
521 .send_traced()
522 .await
523 .reqwest_context("sending an S3 delete request")?;
524
525 if response.status() == StatusCode::NOT_FOUND {
528 response.drain_body().await;
529 return Ok(());
530 }
531
532 response
533 .check_error("deleting an S3 object")
534 .await?
535 .drain_body()
536 .await;
537
538 self.change_stream.commit_delete(id).await;
541
542 Ok(())
543 }
544
545 async fn join(&self) {
546 flush_change_stream(&self.change_stream).await;
547 }
548}
549
550#[cfg(test)]
551mod tests {
552 use std::collections::BTreeMap;
553 use std::io::{Read, Write};
554 use std::net::{TcpListener, TcpStream};
555 use std::sync::mpsc;
556 use std::thread;
557 use std::time::Duration;
558
559 use anyhow::Result;
560 use objectstore_types::metadata::ExpirationPolicy;
561 use objectstore_types::scope::{Scope, Scopes};
562
563 use super::*;
564 use crate::backend::common::Backend;
565 use crate::id::ObjectContext;
566 use crate::stream;
567
568 fn create_test_backend() -> S3CompatibleBackend<!> {
574 S3CompatibleBackend::without_token(
575 S3CompatibleConfig {
576 endpoint: "http://localhost:8089".into(),
577 bucket: "test-bucket".into(),
578 tcp_user_timeout: Duration::from_secs(30),
579 cogs: None,
580 },
581 &ChangeStreamFactory::default(),
582 )
583 }
584
585 fn make_id() -> ObjectId {
586 ObjectId::random(ObjectContext {
587 usecase: "testing".parse().unwrap(),
588 scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
589 })
590 }
591
592 fn read_http_request(connection: &mut TcpStream) -> String {
593 let mut bytes = Vec::new();
594 let mut byte = [0];
595 while !bytes.ends_with(b"\r\n\r\n") {
596 connection.read_exact(&mut byte).unwrap();
597 bytes.push(byte[0]);
598 }
599 String::from_utf8(bytes).unwrap()
600 }
601
602 fn start_copy_server(
603 copy_status: &'static str,
604 metadata: Metadata,
605 ) -> (String, mpsc::Receiver<String>, thread::JoinHandle<()>) {
606 let listener = TcpListener::bind(("127.0.0.1", 0)).unwrap();
607 let endpoint = format!("http://{}", listener.local_addr().unwrap());
608 let (request_tx, request_rx) = mpsc::channel();
609 let server = thread::spawn(move || {
610 let (mut head, _) = listener.accept().unwrap();
611 assert!(read_http_request(&mut head).starts_with("HEAD "));
612 write!(
613 head,
614 "HTTP/1.1 200 OK\r\nContent-Length: 0\r\nETag: \"etag\"\r\nConnection: close\r\n"
615 )
616 .unwrap();
617 for (name, value) in metadata.to_headers(GCS_CUSTOM_PREFIX).unwrap().iter() {
618 write!(head, "{name}: {}\r\n", value.to_str().unwrap()).unwrap();
619 }
620 write!(head, "\r\n").unwrap();
621 let (mut copy, _) = listener.accept().unwrap();
622 request_tx.send(read_http_request(&mut copy)).unwrap();
623 write!(
624 copy,
625 "HTTP/1.1 {copy_status}\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
626 )
627 .unwrap();
628 });
629 (endpoint, request_rx, server)
630 }
631
632 #[tokio::test]
633 async fn update_metadata_uses_conditional_s3_copy() {
634 let created = Timestamp::now();
635 let deadline = created + Duration::from_hours(2);
636 for (status, expected) in [
637 ("200 OK", SetExpiryResponse::Satisfied(deadline)),
638 ("404 Not Found", SetExpiryResponse::NotFound),
639 ("409 Conflict", SetExpiryResponse::Rejected),
640 ("412 Precondition Failed", SetExpiryResponse::Rejected),
641 ] {
642 let (endpoint, request_rx, server) = start_copy_server(
643 status,
644 Metadata {
645 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
646 time_created: Some(created),
647 time_expires: Some(created + Duration::from_hours(1)),
648 custom: [("preserved".into(), "yes".into())].into(),
649 ..Default::default()
650 },
651 );
652 let backend = S3CompatibleBackend::without_token(
653 S3CompatibleConfig {
654 endpoint,
655 bucket: "bucket".into(),
656 tcp_user_timeout: Duration::from_secs(30),
657 cogs: None,
658 },
659 &ChangeStreamFactory::default(),
660 );
661
662 assert_eq!(
663 backend
664 .set_expiry(
665 &make_id(),
666 common::ExpiryTarget::At(deadline).into(),
667 created
668 )
669 .await
670 .unwrap(),
671 expected
672 );
673 let request = request_rx.recv().unwrap().to_ascii_lowercase();
674 assert!(request.contains("x-amz-copy-source: /bucket/"));
675 assert!(request.contains("x-amz-metadata-directive: replace"));
676 assert!(request.contains("x-amz-copy-source-if-match: \"etag\""));
677 let expected_headers = Metadata {
678 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(2)),
679 time_created: Some(created),
680 time_expires: Some(deadline),
681 custom: [("preserved".into(), "yes".into())].into(),
682 ..Default::default()
683 }
684 .to_headers(GCS_CUSTOM_PREFIX)
685 .unwrap();
686 for (name, value) in &expected_headers {
687 assert!(request.contains(
688 &format!("{name}: {}", value.to_str().unwrap()).to_ascii_lowercase()
689 ));
690 }
691 server.join().unwrap();
692 }
693 }
694
695 #[test]
696 fn metadata_to_gcs_headers_omits_size() {
697 let metadata = Metadata {
698 size: Some(4096),
699 ..Default::default()
700 };
701
702 let headers = metadata_to_gcs_headers(&metadata, GCS_CUSTOM_PREFIX).unwrap();
703
704 assert!(headers.get("x-goog-meta-x-sn-size").is_none());
706 }
707
708 #[test]
709 fn metadata_to_gcs_headers_uses_time_expires() {
710 let expires = Timestamp::now() + Duration::from_hours(1);
711 let metadata = Metadata {
712 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
713 time_expires: Some(expires),
714 ..Default::default()
715 };
716
717 let headers = metadata_to_gcs_headers(&metadata, GCS_CUSTOM_PREFIX).unwrap();
718 let custom_time = headers.get(GCS_CUSTOM_TIME).unwrap().to_str().unwrap();
719 let expected = expires.as_rfc3339().to_string();
720 assert_eq!(custom_time, expected);
721 }
722
723 #[test]
724 fn metadata_to_gcs_headers_escapes_unicode() {
725 let metadata = Metadata {
726 filename: Some("réport-📄.pdf".into()),
727 custom: BTreeMap::from_iter([("release".into(), "vérsion-1.0-🚀".into())]),
728 ..Default::default()
729 };
730
731 let headers = metadata_to_gcs_headers(&metadata, GCS_CUSTOM_PREFIX).unwrap();
732 assert_eq!(
733 headers.get("x-goog-meta-x-sn-filename").unwrap(),
734 "r%C3%A9port-%F0%9F%93%84.pdf",
735 );
736 assert_eq!(
737 headers.get("x-goog-meta-x-snme-release").unwrap(),
738 "v%C3%A9rsion-1.0-%F0%9F%9A%80",
739 );
740
741 let roundtripped = Metadata::from_headers(&headers, GCS_CUSTOM_PREFIX).unwrap();
743 assert_eq!(roundtripped.filename, metadata.filename);
744 assert_eq!(roundtripped.custom, metadata.custom);
745 }
746
747 #[test]
748 fn headers_size_counts_names_and_values() {
749 let mut headers = HeaderMap::new();
750 headers.insert("x-goog-meta-a", "1".parse().unwrap());
751 headers.insert("x-goog-meta-bb", "22".parse().unwrap());
752
753 assert_eq!(
754 headers_size(&headers),
755 ("x-goog-meta-a".len() + 1 + "x-goog-meta-bb".len() + 2) as u64
756 );
757 }
758
759 #[tokio::test]
760 async fn test_get_metadata_nonexistent() -> Result<()> {
761 let backend = create_test_backend();
762 let id = make_id();
763 let result = backend.get_metadata(&id, Timestamp::now()).await?;
764 assert!(result.is_none());
765 Ok(())
766 }
767
768 #[tokio::test]
769 async fn test_get_metadata_reports_size() -> Result<()> {
770 let backend = create_test_backend();
771 let id = make_id();
772 let payload = "hello, world";
773
774 backend
775 .put_object(
776 &id,
777 &Metadata::default(),
778 stream::single(payload),
779 Timestamp::now(),
780 )
781 .await?;
782
783 let metadata = backend
786 .get_metadata(&id, Timestamp::now())
787 .await?
788 .expect("object exists");
789 assert_eq!(metadata.size, Some(payload.len()));
790
791 Ok(())
792 }
793
794 #[tokio::test]
795 #[ignore = "SeaweedFS does not enforce expiration from GCS custom-time metadata"]
796 async fn test_ttl_immediate() -> Result<()> {
797 let backend = create_test_backend();
798 let id = make_id();
799 let metadata = Metadata {
800 expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_secs(0)),
801 time_expires: Some(Timestamp::now() - Duration::from_secs(1)),
802 ..Default::default()
803 };
804
805 backend
806 .put_object(
807 &id,
808 &metadata,
809 stream::single("hello, world"),
810 Timestamp::now(),
811 )
812 .await?;
813
814 let get_result = backend.get_object(&id, Timestamp::now(), None).await?;
815 assert!(get_result.is_none());
816
817 let head_result = backend.get_metadata(&id, Timestamp::now()).await?;
818 assert!(head_result.is_none());
819
820 Ok(())
821 }
822
823 #[tokio::test]
824 #[ignore = "SeaweedFS does not enforce expiration from GCS custom-time metadata"]
825 async fn test_tti_immediate() -> Result<()> {
826 let backend = create_test_backend();
827 let id = make_id();
828 let metadata = Metadata {
829 expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_secs(0)),
830 time_expires: Some(Timestamp::now() - Duration::from_secs(1)),
831 ..Default::default()
832 };
833
834 backend
835 .put_object(
836 &id,
837 &metadata,
838 stream::single("hello, world"),
839 Timestamp::now(),
840 )
841 .await?;
842
843 let get_result = backend.get_object(&id, Timestamp::now(), None).await?;
844 assert!(get_result.is_none());
845
846 let head_result = backend.get_metadata(&id, Timestamp::now()).await?;
847 assert!(head_result.is_none());
848
849 Ok(())
850 }
851}