Skip to main content

objectstore_service/backend/
local_fs.rs

1//! Local filesystem backend for development and testing.
2//!
3//! Stores each object as one file below the configured root directory. The file holds the
4//! object's metadata as a single JSON line, followed by the raw payload.
5//!
6//! ## Layout
7//!
8//! ```text
9//! {root}/
10//! ├── {usecase}/{scopes}/objects/{key}.obj   object files
11//! ├── .uploads/{upload_id}                   in-progress resumable uploads
12//! └── .locks/{xx}/{yy}                       lock slots
13//! ```
14//!
15//! Internal directories start with `.`, which usecase names cannot contain, so they never collide
16//! with objects.
17//!
18//! Each scope becomes one directory level named `{name}.{value}`, in scope order. Objects without
19//! scopes are stored directly under `{usecase}/objects/`.
20//!
21//! Each `/`-separated segment of the key becomes one directory level, and the last segment gets a
22//! `.obj` suffix. Segments are [encoded](#key-encoding) first. For example, the key `docs/report`
23//! in usecase `attachments` with scopes `org=17;project=42` is stored at
24//! `attachments/org.17/project.42/objects/docs/report.obj`. Because of the suffix, keys like `a`
25//! and `a/b` can coexist.
26//!
27//! ### Key encoding
28//!
29//! Key segments are stored mostly verbatim, so that paths stay readable. To keep every key inside
30//! the root and every key mapped to a distinct path, these characters are percent-encoded as
31//! `%XX`:
32//!
33//! - `%` itself and ASCII control characters, including NUL.
34//! - Characters reserved on Windows: `\ < > : " | ? *`.
35//! - A `.` at the start of a segment, so no segment is `.` or `..`, or a hidden file.
36//! - A `.` or space at the end of a segment, which Windows would strip.
37//! - The `.` of a trailing `.obj` (in any case), so no directory looks like an object file.
38//!
39//! Empty segments, such as in `a//b`, are stored as `%`.
40//!
41//! ## Concurrency
42//!
43//! Complete object files are published by atomically renaming same-directory drafts, so readers
44//! observe either the previous or next complete file. Unpublished drafts are removed automatically
45//! when dropped.
46//!
47//! Deleting an object also removes its directories while they are empty. This is not synchronized
48//! with writers: directory removal only succeeds on empty directories, and writers recreate a
49//! directory removed before they placed their file in it.
50//!
51//! To avoid races on metadata, expiry, and upload updates, this backend uses locks placed under
52//! `.locks/` to synchronize mutations across backend instances and cooperating processes. The first
53//! two bytes of the BLAKE3 hash of an object's storage path select one of 65,536 permanent lock
54//! slots under `.locks/<first byte>/<second byte>` (lowercase hexadecimal). An object and all of its
55//! resumable uploads share the same lock.
56//!
57//! Shared filesystems are supported only when locks propagate across the cluster, pathname
58//! visibility is coherent, and rename is atomic.
59//!
60//! ## Limitations
61//!
62//! - Keys that differ only in case or Unicode normalization collide on filesystems that ignore
63//!   those differences, such as the macOS default.
64//! - Each encoded key segment must fit the filesystem's file name limit, typically 255 bytes.
65//! - Windows additionally rejects reserved device names such as `CON` or `NUL`. These are stored
66//!   unescaped.
67//!
68//! ## Permissions
69//!
70//! Newly created files and directories are owner-only on Unix (0600 and 0700 respectively,
71//! further restricted by umask). Existing paths retain their permissions.
72
73use std::fs::File;
74use std::io;
75use std::mem::DropGuard;
76use std::num::NonZeroU64;
77use std::path::{Path, PathBuf};
78use std::pin::pin;
79use std::sync::Arc;
80
81use futures_util::StreamExt;
82use objectstore_types::metadata::Metadata;
83use objectstore_types::range::ByteRange;
84use objectstore_types::resumable::UploadProgress;
85use objectstore_types::time::Timestamp;
86use tokio::fs::OpenOptions;
87use tokio::io::{
88    AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt, BufReader, BufWriter,
89};
90use tokio::sync::Semaphore;
91use tokio_util::io::{ReaderStream, StreamReader};
92use uuid::Uuid;
93
94use crate::backend::common::{
95    self, Backend, DeleteResponse, ExpiryUpdate, GetResponse, PutResponse, SetExpiryResponse,
96};
97use crate::change_stream::{
98    ChangeStream, ChangeStreamFactory, CostTrackerStreamConfig, flush_change_stream,
99};
100use crate::error::{Error, ErrorKind, Result, ResultExt as _};
101use crate::id::ObjectId;
102use crate::resumable::{BackendToken, Session};
103use crate::stream::{self, ClientStream};
104
105/// Number of times [`DraftDirectory::run`] creates the directories before giving up.
106const MAX_DIRECTORY_ATTEMPTS: usize = 3;
107
108/// Options for owner-only files on Unix, further restricted by umask.
109fn file_options() -> std::fs::OpenOptions {
110    let mut options = std::fs::OpenOptions::new();
111    #[cfg(unix)]
112    {
113        use std::os::unix::fs::OpenOptionsExt as _;
114        options.mode(0o600);
115    }
116    options
117}
118
119/// Directory below the usecase and scopes that holds the object files.
120const OBJECTS_DIR: &str = "objects";
121
122/// Encoded form of an empty key segment, which escaping can never produce otherwise.
123const EMPTY_KEY_SEGMENT: &str = "%";
124
125/// Suffix appended to the last key segment to form an object's file name.
126///
127/// [`encode_key`] escapes the `.` of a trailing `.obj` in any case, so encoded segments never end
128/// in this suffix. Object files therefore never collide with directories of keys nested below them.
129const OBJECT_FILE_SUFFIX: &str = ".obj";
130
131/// Encodes an object key into a relative path of safe directory and file names.
132///
133/// Each `/`-separated segment becomes one path component, and the last one gets
134/// [`OBJECT_FILE_SUFFIX`]. See the [module documentation](self#key-encoding) for the escaping rules.
135fn encode_key(key: &str) -> String {
136    let mut encoded = String::with_capacity(key.len() + OBJECT_FILE_SUFFIX.len());
137    for (segment_index, segment) in key.split('/').enumerate() {
138        if segment_index > 0 {
139            encoded.push('/');
140        }
141        if segment.is_empty() {
142            encoded.push_str(EMPTY_KEY_SEGMENT);
143            continue;
144        }
145
146        // Matched case-insensitively, since case-insensitive filesystems would treat them the same.
147        let suffix_start = segment
148            .len()
149            .checked_sub(OBJECT_FILE_SUFFIX.len())
150            .filter(|&start| {
151                segment.as_bytes()[start..].eq_ignore_ascii_case(OBJECT_FILE_SUFFIX.as_bytes())
152            });
153
154        let last_index = segment.len() - 1;
155        for (index, c) in segment.char_indices() {
156            let escape = match c {
157                '%' | '\\' | '<' | '>' | ':' | '"' | '|' | '?' | '*' => true,
158                // Windows strips trailing dots and spaces, which would merge `a.` into `a`.
159                '.' => index == 0 || index == last_index || Some(index) == suffix_start,
160                ' ' => index == last_index,
161                c => c.is_ascii_control(),
162            };
163            if escape {
164                // `escape` is only true for ASCII characters, so `c as u8` is lossless.
165                encoded.push_str(percent_encoding::percent_encode_byte(c as u8));
166            } else {
167                encoded.push(c);
168            }
169        }
170    }
171    encoded.push_str(OBJECT_FILE_SUFFIX);
172    encoded
173}
174
175/// Builder for owner-only directories on Unix that leaves existing permissions unchanged.
176fn directory_builder() -> std::fs::DirBuilder {
177    let mut builder = std::fs::DirBuilder::new();
178    builder.recursive(true);
179    #[cfg(unix)]
180    {
181        use std::os::unix::fs::DirBuilderExt as _;
182        builder.mode(0o700);
183    }
184    builder
185}
186
187/// Creates owner-only directories on Unix without changing existing permissions.
188async fn create_directories(path: &Path) -> io::Result<()> {
189    let path = path.to_path_buf();
190    tokio::task::spawn_blocking(move || directory_builder().create(path))
191        .await
192        .map_err(io::Error::other)?
193}
194
195/// Removes the directories of a deleted object below `root` as long as they are empty.
196///
197/// Runs [`remove_empty_directories_sync`] on the blocking thread pool.
198async fn remove_empty_directories(root: &Path, object_path: &Path) {
199    let root = root.to_path_buf();
200    let object_path = object_path.to_path_buf();
201    let task = tokio::task::spawn_blocking(move || {
202        remove_empty_directories_sync(&root, &object_path);
203    });
204
205    // A leftover empty directory is harmless, so a failed cleanup must not fail the delete.
206    let _ = task.await;
207}
208
209/// Removes the directories of an object below `root` as long as they are empty.
210///
211/// Stops at the first directory that cannot be removed, typically because it still holds other
212/// objects. Removal is atomic and only succeeds on empty directories, so this never races with
213/// stored objects. Writers handle directories removed under them with [`DraftDirectory::run`].
214fn remove_empty_directories_sync(root: &Path, object_path: &Path) {
215    let mut object_dirs = object_path
216        .ancestors()
217        .skip(1)
218        .take_while(|dir| *dir != root && dir.starts_with(root));
219
220    // A leftover empty directory is harmless, so cleanup ignores failures.
221    let _ = object_dirs.try_for_each(std::fs::remove_dir);
222}
223
224/// Configuration for [`LocalFsBackend`].
225///
226/// Stores objects as files on the local filesystem. Suitable for development, testing,
227/// and single-server deployments.
228///
229/// # Example
230///
231/// ```yaml
232/// storage:
233///   type: filesystem
234///   path: /data
235/// ```
236#[derive(Debug, Clone, serde::Deserialize, serde::Serialize)]
237pub struct FileSystemConfig {
238    /// Directory path for storing objects.
239    ///
240    /// The directory will be created if it doesn't exist. Relative paths are resolved from
241    /// the server's working directory.
242    ///
243    /// # Default
244    ///
245    /// `"data"` (relative to the server's working directory)
246    ///
247    /// # Environment Variables
248    ///
249    /// - `OS__STORAGE__TYPE=filesystem`
250    /// - `OS__STORAGE__PATH=/path/to/storage`
251    pub path: PathBuf,
252
253    /// Reports what this backend stores, for per-usecase cost attribution.
254    ///
255    /// # Default
256    ///
257    /// `None`, which disables reporting for this backend.
258    ///
259    /// # Environment Variables
260    ///
261    /// - `OS__STORAGE__COGS__SHARED_RESOURCE_ID=filesystem_objectstore`
262    /// - `OS__STORAGE__COGS__SAMPLE_RATE=1.0` (optional)
263    #[serde(default, skip_serializing_if = "Option::is_none")]
264    pub cogs: Option<CostTrackerStreamConfig>,
265}
266
267/// Local filesystem backend for development and testing.
268#[derive(Debug)]
269pub struct LocalFsBackend {
270    path: PathBuf,
271    locks: ObjectLocks,
272
273    change_stream: Arc<dyn ChangeStream>,
274}
275
276impl LocalFsBackend {
277    /// Creates a new [`LocalFsBackend`] rooted at the directory in `config`.
278    pub fn new(config: FileSystemConfig, streams: &ChangeStreamFactory) -> Self {
279        let FileSystemConfig { path, cogs } = config;
280        let locks = ObjectLocks::new(&path);
281        Self {
282            path,
283            locks,
284            change_stream: streams.build(cogs.as_ref()),
285        }
286    }
287
288    /// Returns the filesystem path for the given object ID.
289    ///
290    /// See the [module documentation](self) for the layout.
291    fn path(&self, id: &ObjectId) -> PathBuf {
292        let mut path = self.path.join(id.usecase().as_str());
293        if !id.scopes().is_empty() {
294            // Scopes are `/`-separated, so each one becomes its own directory.
295            path.push(id.scopes().as_storage_path().to_string());
296        }
297        path.push(OBJECTS_DIR);
298        path.push(encode_key(id.key()));
299        path
300    }
301
302    fn upload_path(&self, upload_id: Uuid) -> PathBuf {
303        self.path.join(".uploads").join(upload_id.to_string())
304    }
305}
306
307#[async_trait::async_trait]
308impl Backend for LocalFsBackend {
309    fn name(&self) -> &'static str {
310        "local-fs"
311    }
312
313    #[tracing::instrument(level = "debug", fields(?id), skip_all)]
314    async fn put_object(
315        &self,
316        id: &ObjectId,
317        metadata: &Metadata,
318        stream: ClientStream,
319        _access_time: Timestamp,
320    ) -> Result<PutResponse> {
321        let path = self.path(id);
322        objectstore_log::debug!(path=%path.display(), "Writing to local_fs backend");
323        self.change_stream
324            .begin_write(id, metadata.time_expires)
325            .await?;
326
327        let mut draft = Draft::create(&self.path, &path, metadata).await?;
328        let mut reader = pin!(StreamReader::new(stream));
329        let payload_size = tokio::io::copy(&mut reader, draft.writer())
330            .await
331            .map_err(|e| match stream::unpack_client_error(&e) {
332                Some(ce) => Error::from(ce),
333                None => Error::with_context(
334                    ErrorKind::BackendFailure,
335                    "writing local-fs object payload",
336                    e,
337                ),
338            })?;
339
340        let stored_size = draft.preamble_len() + payload_size;
341
342        draft.prepare().await?;
343        let _guard = self.locks.acquire(id).await?;
344        draft.publish().await?;
345
346        self.change_stream
347            .commit_write(id, stored_size, metadata.time_expires)
348            .await;
349
350        Ok(())
351    }
352
353    #[tracing::instrument(level = "debug", skip(self))]
354    async fn get_object(
355        &self,
356        id: &ObjectId,
357        access_time: Timestamp,
358        range: Option<ByteRange>,
359    ) -> Result<GetResponse> {
360        objectstore_log::debug!("Reading from local_fs backend");
361        let path = self.path(id);
362        let Some(object) = ObjectFile::try_open(&path, access_time).await? else {
363            objectstore_log::debug!("Object not found");
364            return Ok(None);
365        };
366        let ObjectFile {
367            metadata,
368            preamble_len,
369            payload_size,
370            mut reader,
371        } = object;
372
373        let (content_range, stream) = match range {
374            Some(byte_range) => {
375                let content_range =
376                    byte_range
377                        .resolve(payload_size)
378                        .ok_or(ErrorKind::RangeNotSatisfiable {
379                            total: payload_size,
380                        })?;
381                let payload_start = preamble_len + content_range.start;
382                reader
383                    .seek(std::io::SeekFrom::Start(payload_start))
384                    .await
385                    .context(ErrorKind::BackendFailure, "seeking local-fs object payload")?;
386                let limited = reader.take(content_range.len());
387                (Some(content_range), ReaderStream::new(limited).boxed())
388            }
389            None => (None, ReaderStream::new(reader).boxed()),
390        };
391        Ok(Some((metadata, content_range, stream)))
392    }
393
394    #[tracing::instrument(level = "debug", skip(self))]
395    async fn set_expiry(
396        &self,
397        id: &ObjectId,
398        target: ExpiryUpdate,
399        access_time: Timestamp,
400    ) -> Result<SetExpiryResponse> {
401        let _guard = self.locks.acquire(id).await?;
402
403        let path = self.path(id);
404        let Some(object) = ObjectFile::try_open(&path, access_time).await? else {
405            return Ok(SetExpiryResponse::NotFound);
406        };
407        let ObjectFile {
408            mut metadata,
409            mut reader,
410            ..
411        } = object;
412
413        let Some(current_expiry) = metadata.time_expires else {
414            return Ok(SetExpiryResponse::Rejected);
415        };
416        let Some(expire_at) = target.resolve(metadata.time_created, access_time)? else {
417            return Ok(SetExpiryResponse::Rejected);
418        };
419        if current_expiry >= expire_at {
420            return Ok(SetExpiryResponse::Satisfied(expire_at)); // already satisfied
421        }
422        metadata.expiration_policy = common::extended_expiration_policy(
423            metadata.expiration_policy,
424            metadata.time_created,
425            current_expiry,
426            expire_at,
427        )?;
428        metadata.time_expires = Some(expire_at);
429        self.change_stream.begin_update(id, Some(expire_at)).await?;
430
431        let mut draft = Draft::create(&self.path, &path, &metadata).await?;
432        tokio::io::copy(&mut reader, draft.writer()).await.context(
433            ErrorKind::BackendFailure,
434            "copying local-fs object payload for expiry extension",
435        )?;
436
437        draft.prepare().await?;
438        draft.publish().await?;
439
440        self.change_stream.commit_update(id, Some(expire_at)).await;
441
442        Ok(SetExpiryResponse::Satisfied(expire_at))
443    }
444
445    #[tracing::instrument(level = "debug", skip(self))]
446    async fn delete_object(
447        &self,
448        id: &ObjectId,
449        _access_time: Timestamp,
450    ) -> Result<DeleteResponse> {
451        let guard = self.locks.acquire(id).await?;
452
453        objectstore_log::debug!("Deleting from local_fs backend");
454        let path = self.path(id);
455        match tokio::fs::remove_file(&path).await {
456            Ok(()) => {
457                self.change_stream.commit_delete(id).await;
458                drop(guard);
459                remove_empty_directories(&self.path, &path).await;
460            }
461            Err(error) if error.kind() == io::ErrorKind::NotFound => {
462                objectstore_log::debug!("Object not found");
463            }
464            result => {
465                result.context(ErrorKind::BackendFailure, "deleting local-fs object")?;
466            }
467        }
468
469        Ok(())
470    }
471
472    #[tracing::instrument(level = "debug", fields(?id), skip_all)]
473    async fn create_upload_session(
474        &self,
475        id: &ObjectId,
476        metadata: &Metadata,
477        _upload_length: NonZeroU64,
478    ) -> Result<Option<BackendToken>> {
479        self.change_stream
480            .begin_write(id, metadata.time_expires)
481            .await?;
482        let upload_id = uuid::Uuid::now_v7();
483        let path = self.upload_path(upload_id);
484        create_directories(path.parent().unwrap()).await.context(
485            ErrorKind::BackendFailure,
486            "creating local-fs object directory",
487        )?;
488        UploadFile::create(&path, metadata).await?;
489        Ok(Some(upload_id.to_string()))
490    }
491
492    // In this backend, if all the bytes of a resumable upload have been written but publication failed,
493    // an empty chunk or offset query will retry publication.
494    #[tracing::instrument(level = "debug", fields(?session, offset, content_length), skip_all)]
495    async fn put_chunk(
496        &self,
497        session: &Session,
498        offset: u64,
499        content_length: u64,
500        stream: ClientStream,
501    ) -> Result<UploadProgress> {
502        let upload_id =
503            Uuid::parse_str(&session.backend_token).map_err(|_| ErrorKind::UnknownUploadSession)?;
504        offset
505            .checked_add(content_length)
506            .filter(|end| *end <= session.upload_length.get())
507            .ok_or(ErrorKind::ChunkExceedsUploadLength {
508                offset,
509                content_length,
510                upload_length: session.upload_length.get(),
511            })?;
512        let _guard = self.locks.acquire(&session.object_id).await?;
513
514        let upload_path = self.upload_path(upload_id);
515        let mut upload = UploadFile::open(&upload_path).await?;
516        if content_length != 0 && offset != upload.offset() {
517            return Err(ErrorKind::UploadOffsetMismatch {
518                offset: upload.offset(),
519            }
520            .into());
521        }
522
523        let reader = StreamReader::new(stream).take(content_length);
524        let persisted_offset = upload.append(reader).await?;
525
526        if persisted_offset != session.upload_length.get() {
527            return Ok(UploadProgress::Incomplete {
528                offset: persisted_offset,
529            });
530        }
531
532        let directory = DraftDirectory::new(&self.path, &self.path(&session.object_id));
533        let (stored_size, expires_at) = upload.publish(directory).await?;
534        self.change_stream
535            .commit_write(&session.object_id, stored_size, expires_at)
536            .await;
537        Ok(UploadProgress::Complete)
538    }
539
540    #[tracing::instrument(level = "debug", fields(?session), skip_all)]
541    async fn upload_offset(&self, session: &Session) -> Result<UploadProgress> {
542        self.put_chunk(session, 0, 0, futures_util::stream::empty().boxed())
543            .await
544    }
545
546    #[tracing::instrument(level = "debug", fields(?session), skip_all)]
547    async fn cancel_upload(&self, session: &Session) -> Result<()> {
548        let upload_id =
549            Uuid::parse_str(&session.backend_token).map_err(|_| ErrorKind::UnknownUploadSession)?;
550        let _guard = self.locks.acquire(&session.object_id).await?;
551        let path = self.upload_path(upload_id);
552        match tokio::fs::remove_file(&path).await {
553            Ok(()) => Ok(()),
554            Err(error) if error.kind() == io::ErrorKind::NotFound => {
555                Err(ErrorKind::UnknownUploadSession.into())
556            }
557            result => result.context(
558                ErrorKind::BackendFailure,
559                "canceling local-fs resumable upload",
560            ),
561        }
562    }
563
564    async fn join(&self) {
565        flush_change_stream(&self.change_stream).await;
566    }
567}
568
569// Must be lower than the tokio runtime `max_blocking_threads` setting.
570const MAX_BLOCKING_LOCK_WAITERS: usize = 256;
571
572#[derive(Debug)]
573struct ObjectLocks {
574    root: PathBuf,
575    blocking_waiters: Arc<Semaphore>,
576}
577
578impl ObjectLocks {
579    pub fn new(storage_root: &Path) -> Self {
580        Self {
581            root: storage_root.join(".locks"),
582            blocking_waiters: Arc::new(Semaphore::new(MAX_BLOCKING_LOCK_WAITERS)),
583        }
584    }
585
586    fn path(&self, id: &ObjectId) -> PathBuf {
587        let hash = blake3::hash(id.as_storage_path().to_string().as_bytes());
588        let bytes = hash.as_bytes();
589        self.root
590            .join(format!("{:02x}", bytes[0]))
591            .join(format!("{:02x}", bytes[1]))
592    }
593
594    /// Acquires a slot lock, released when the returned file is dropped.
595    pub async fn acquire(&self, id: &ObjectId) -> Result<File> {
596        let path = self.path(id);
597        create_directories(path.parent().unwrap()).await.context(
598            ErrorKind::BackendFailure,
599            "creating local-fs object lock directory",
600        )?;
601
602        // Leave blocking-pool capacity available for the current lock holder's filesystem work.
603        let permit = Arc::clone(&self.blocking_waiters)
604            .acquire_owned()
605            .await
606            .expect("local-fs lock semaphore is never closed");
607
608        tokio::task::spawn_blocking(move || -> io::Result<File> {
609            let _permit = permit;
610            let file = file_options()
611                .create(true)
612                .truncate(false)
613                .read(true)
614                .write(true)
615                .open(&path)?;
616            file.lock()?;
617            Ok(file)
618        })
619        .await
620        .context(ErrorKind::Internal, "waiting for local-fs object lock")?
621        .context(ErrorKind::BackendFailure, "acquiring local-fs object lock")
622    }
623}
624
625/// An open resumable upload containing a metadata preamble followed by payload bytes.
626struct UploadFile {
627    file: tokio::fs::File,
628    path: PathBuf,
629    /// Number of payload bytes stored after the metadata preamble.
630    payload_size: u64,
631    /// Total size and expiration, for reporting to the `ChangeStream`.
632    stored_size: u64,
633    expires_at: Option<Timestamp>,
634}
635
636impl UploadFile {
637    async fn create(path: &Path, metadata: &Metadata) -> Result<()> {
638        let mut options = OpenOptions::from(file_options());
639        options.create_new(true).read(true).write(true);
640
641        let mut file = options.open(path).await.context(
642            ErrorKind::BackendFailure,
643            "creating local-fs resumable upload",
644        )?;
645        write_metadata_preamble(&mut file, metadata).await?;
646        file.sync_data().await.context(
647            ErrorKind::BackendFailure,
648            "syncing local-fs resumable upload",
649        )
650    }
651
652    async fn open(path: &Path) -> Result<Self> {
653        let file = match OpenOptions::new().read(true).write(true).open(path).await {
654            Ok(file) => file,
655            Err(error) if error.kind() == io::ErrorKind::NotFound => {
656                return Err(ErrorKind::UnknownUploadSession.into());
657            }
658            result => result.context(
659                ErrorKind::BackendFailure,
660                "opening local-fs resumable upload",
661            )?,
662        };
663        let mut reader = BufReader::new(file);
664        let (metadata, preamble_len) = read_metadata_preamble(&mut reader).await?;
665        let file = reader.into_inner();
666        let stored_size = file
667            .metadata()
668            .await
669            .context(
670                ErrorKind::BackendFailure,
671                "reading local-fs resumable upload size",
672            )?
673            .len();
674        let preamble_len = preamble_len as u64;
675        let payload_size = stored_size.checked_sub(preamble_len).ok_or_else(|| {
676            Error::new(
677                ErrorKind::CorruptData,
678                "reading truncated local-fs resumable upload",
679            )
680        })?;
681        Ok(Self {
682            file,
683            path: path.to_path_buf(),
684            stored_size,
685            expires_at: metadata.time_expires,
686            payload_size,
687        })
688    }
689
690    fn offset(&self) -> u64 {
691        self.payload_size
692    }
693
694    async fn append(&mut self, mut reader: impl AsyncRead + Unpin) -> Result<u64> {
695        self.file.seek(std::io::SeekFrom::End(0)).await.context(
696            ErrorKind::BackendFailure,
697            "seeking local-fs resumable upload",
698        )?;
699
700        let copied = match tokio::io::copy(&mut reader, &mut self.file).await {
701            Ok(copied) => copied,
702            Err(error) => {
703                let client_error = stream::unpack_client_error(&error);
704                self.file.sync_data().await.context(
705                    ErrorKind::BackendFailure,
706                    "syncing partial local-fs resumable chunk",
707                )?;
708                return Err(match client_error {
709                    Some(client_error) => Error::from(client_error),
710                    None => Error::with_context(
711                        ErrorKind::BackendFailure,
712                        "writing local-fs resumable chunk",
713                        error,
714                    ),
715                });
716            }
717        };
718
719        let payload_size = self.payload_size.checked_add(copied).ok_or_else(|| {
720            Error::new(
721                ErrorKind::BackendFailure,
722                "local-fs resumable payload size overflow",
723            )
724        })?;
725        let stored_size = self.stored_size.checked_add(copied).ok_or_else(|| {
726            Error::new(
727                ErrorKind::BackendFailure,
728                "local-fs resumable stored size overflow",
729            )
730        })?;
731        self.file.sync_data().await.context(
732            ErrorKind::BackendFailure,
733            "syncing local-fs resumable chunk",
734        )?;
735        self.payload_size = payload_size;
736        self.stored_size = stored_size;
737        Ok(payload_size)
738    }
739
740    /// Publishes the upload; requires a successful `append()` with no subsequent writes.
741    async fn publish(self, directory: DraftGuard) -> Result<(u64, Option<Timestamp>)> {
742        let Self {
743            file,
744            path,
745            stored_size,
746            expires_at,
747            ..
748        } = self;
749        drop(file);
750
751        let target = directory.object_path().to_path_buf();
752        directory
753            .run(move || std::fs::rename(&path, &target))
754            .await
755            .context(
756                ErrorKind::BackendFailure,
757                "publishing local-fs resumable upload",
758            )?;
759
760        DropGuard::dismiss(directory);
761        Ok((stored_size, expires_at))
762    }
763}
764
765struct ObjectFile {
766    metadata: Metadata,
767    preamble_len: u64,
768    payload_size: u64,
769    reader: BufReader<tokio::fs::File>,
770}
771
772async fn write_metadata_preamble<W>(writer: &mut W, metadata: &Metadata) -> Result<u64>
773where
774    W: tokio::io::AsyncWrite + Unpin,
775{
776    let metadata_json = serde_json::to_string(metadata)
777        .context(ErrorKind::Internal, "encoding local-fs object metadata")?;
778    writer.write_all(metadata_json.as_bytes()).await.context(
779        ErrorKind::BackendFailure,
780        "writing local-fs object metadata",
781    )?;
782    writer.write_all(b"\n").await.context(
783        ErrorKind::BackendFailure,
784        "writing local-fs object metadata",
785    )?;
786
787    Ok(metadata_json.len() as u64 + 1)
788}
789
790async fn read_metadata_preamble<R>(reader: &mut R) -> Result<(Metadata, usize)>
791where
792    R: tokio::io::AsyncBufRead + Unpin,
793{
794    let mut metadata_line = String::new();
795    let preamble_len = reader.read_line(&mut metadata_line).await.context(
796        ErrorKind::BackendFailure,
797        "reading local-fs object metadata",
798    )?;
799    let metadata = serde_json::from_str(metadata_line.trim_end())
800        .context(ErrorKind::CorruptData, "decoding local-fs object metadata")?;
801    Ok((metadata, preamble_len))
802}
803
804impl ObjectFile {
805    /// Opens an object file, returning `None` when it does not exist.
806    async fn try_open(path: &Path, access_time: Timestamp) -> Result<Option<Self>> {
807        let file = match OpenOptions::new().read(true).open(path).await {
808            Ok(file) => file,
809            Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(None),
810            result => result.context(ErrorKind::BackendFailure, "opening local-fs object")?,
811        };
812
813        let mut reader = BufReader::new(file);
814        let (mut metadata, preamble_len) = read_metadata_preamble(&mut reader).await?;
815
816        if metadata.is_expired(access_time) {
817            objectstore_log::debug!("Object found but past expiry");
818            return Ok(None);
819        }
820
821        let preamble_len = preamble_len as u64;
822        let file_len = reader
823            .get_ref()
824            .metadata()
825            .await
826            .context(ErrorKind::BackendFailure, "reading local-fs object size")?
827            .len();
828
829        let payload_size = file_len.checked_sub(preamble_len).ok_or_else(|| {
830            Error::new(ErrorKind::CorruptData, "reading truncated local-fs object")
831        })?;
832
833        metadata.size = Some(payload_size as usize);
834
835        Ok(Some(Self {
836            metadata,
837            preamble_len,
838            payload_size,
839            reader,
840        }))
841    }
842}
843
844/// A [`DraftDirectory`] that is removed again on drop.
845///
846/// Dismiss the guard with [`DropGuard::dismiss`] once the published object lives in it.
847type DraftGuard = DropGuard<DraftDirectory, fn(DraftDirectory)>;
848
849/// The directories of an object that is not published yet.
850///
851/// Writers create the directories and place their entry with [`run`](Self::run). When the
852/// [`DraftGuard`] is dropped without being dismissed, for example after a failed or cancelled
853/// write, this removes the directories again as long as they are empty.
854struct DraftDirectory {
855    root: PathBuf,
856    object_path: PathBuf,
857}
858
859impl DraftDirectory {
860    fn new(root: &Path, object_path: &Path) -> DraftGuard {
861        let directory = Self {
862            root: root.to_path_buf(),
863            object_path: object_path.to_path_buf(),
864        };
865        DropGuard::new(directory, Self::cleanup)
866    }
867
868    /// Returns the path the object is published at.
869    fn object_path(&self) -> &Path {
870        &self.object_path
871    }
872
873    /// Creates the directories and runs `f` on the blocking thread pool to place a new entry in them.
874    ///
875    /// [`remove_empty_directories`] can remove the directories while they are being created, or
876    /// before `f` places its entry. When either step fails with [`io::ErrorKind::NotFound`], both
877    /// are retried, up to [`MAX_DIRECTORY_ATTEMPTS`] times in total.
878    async fn run<T, F>(&self, mut f: F) -> io::Result<T>
879    where
880        T: Send + 'static,
881        F: FnMut() -> io::Result<T> + Send + 'static,
882    {
883        let dir = self.object_path.parent().map(Path::to_path_buf);
884        let task = tokio::task::spawn_blocking(move || {
885            let mut attempt = || {
886                if let Some(dir) = &dir {
887                    directory_builder().create(dir)?;
888                }
889                f()
890            };
891
892            // A concurrent delete can remove the empty directory before `f` places its entry in it.
893            for _ in 1..MAX_DIRECTORY_ATTEMPTS {
894                match attempt() {
895                    Err(error) if error.kind() == io::ErrorKind::NotFound => continue,
896                    result => return result,
897                }
898            }
899
900            attempt()
901        });
902
903        task.await.map_err(io::Error::other)?
904    }
905
906    /// Removes the directories if they are empty.
907    fn cleanup(self) {
908        let cleanup = move || remove_empty_directories_sync(&self.root, &self.object_path);
909
910        // Drafts are usually dropped on an async worker, which must not block on filesystem calls.
911        // Without a runtime, there is no blocking pool, so clean up in place.
912        match tokio::runtime::Handle::try_current() {
913            Ok(runtime) => drop(runtime.spawn_blocking(cleanup)),
914            Err(_) => cleanup(),
915        }
916    }
917}
918
919struct Draft {
920    writer: BufWriter<tokio::fs::File>,
921    path: tempfile::TempPath,
922    target: PathBuf,
923    /// Bytes the metadata preamble occupies, counted as it is written.
924    preamble_len: u64,
925    /// Declared after `path`, so an unpublished draft's file is removed before its directories.
926    directory: DraftGuard,
927}
928
929impl Draft {
930    async fn create(root: &Path, target: &Path, metadata: &Metadata) -> Result<Self> {
931        let directory = DraftDirectory::new(root, target);
932        let parent = target.parent().unwrap().to_path_buf();
933        let tempfile = directory
934            .run(move || create_tempfile(&parent))
935            .await
936            .context(ErrorKind::BackendFailure, "creating local-fs object draft")?;
937        let (file, path) = tempfile.into_parts();
938
939        let mut writer = BufWriter::new(tokio::fs::File::from_std(file));
940        let preamble_len = write_metadata_preamble(&mut writer, metadata).await?;
941
942        Ok(Self {
943            writer,
944            path,
945            target: target.to_path_buf(),
946            preamble_len,
947            directory,
948        })
949    }
950
951    fn writer(&mut self) -> &mut BufWriter<tokio::fs::File> {
952        &mut self.writer
953    }
954
955    /// Bytes the metadata preamble occupies, for sizing the stored object.
956    fn preamble_len(&self) -> u64 {
957        self.preamble_len
958    }
959
960    async fn prepare(&mut self) -> Result<()> {
961        self.writer
962            .flush()
963            .await
964            .context(ErrorKind::BackendFailure, "flushing local-fs object draft")?;
965        self.writer
966            .get_ref()
967            .sync_data()
968            .await
969            .context(ErrorKind::BackendFailure, "syncing local-fs object draft")
970    }
971
972    /// Publishes the draft; requires a successful `prepare()` with no subsequent writes.
973    async fn publish(self) -> Result<()> {
974        let Self {
975            writer,
976            path,
977            target,
978            directory,
979            ..
980        } = self;
981
982        drop(writer);
983        let task = tokio::task::spawn_blocking(move || {
984            // The error contains the draft file. Drop it first so the directory gets cleaned up.
985            path.persist(target).map_err(|error| error.error)
986        });
987
988        task.await
989            .context(
990                ErrorKind::Internal,
991                "waiting to publish local-fs object draft",
992            )?
993            .context(
994                ErrorKind::BackendFailure,
995                "publishing local-fs object draft",
996            )?;
997
998        DropGuard::dismiss(directory);
999        Ok(())
1000    }
1001}
1002
1003fn create_tempfile(parent: &Path) -> io::Result<tempfile::NamedTempFile> {
1004    let mut builder = tempfile::Builder::new();
1005    builder.suffix(".draft");
1006
1007    builder.tempfile_in(parent)
1008}
1009
1010#[cfg(test)]
1011mod tests {
1012    use std::collections::HashMap;
1013    use std::sync::Arc;
1014    use std::time::Duration;
1015
1016    use bytes::{Bytes, BytesMut};
1017    use futures_util::{TryStreamExt, stream as futures_stream};
1018    use objectstore_types::metadata::{Compression, ExpirationPolicy};
1019    use objectstore_types::scope::{Scope, Scopes};
1020    use objectstore_types::time::Timestamp;
1021
1022    #[cfg(feature = "storage-cogs")]
1023    use objectstore_inventory_tracker::OpType;
1024    #[cfg(feature = "storage-cogs")]
1025    use objectstore_inventory_tracker::test_utils::DummyProducer;
1026
1027    use super::*;
1028    use crate::backend::common::ExpiryTarget;
1029    use crate::id::ObjectContext;
1030    use crate::stream;
1031
1032    async fn upload_token(backend: &LocalFsBackend, id: &ObjectId, length: u64) -> Session {
1033        let backend_token = backend
1034            .create_upload_session(id, &Metadata::default(), NonZeroU64::new(length).unwrap())
1035            .await
1036            .unwrap()
1037            .unwrap();
1038        Session {
1039            object_id: id.clone(),
1040            upload_length: NonZeroU64::new(length).unwrap(),
1041            backend_token,
1042        }
1043    }
1044
1045    #[tokio::test]
1046    async fn resumable_upload() {
1047        let (_tempdir, backend) = make_backend();
1048        let id = make_id();
1049        let metadata = Metadata {
1050            content_type: "text/resumable".into(),
1051            ..Default::default()
1052        };
1053
1054        // Create a session and verify its initial state on disk.
1055        let upload_length = NonZeroU64::new(6).unwrap();
1056        let token = backend
1057            .create_upload_session(&id, &metadata, upload_length)
1058            .await
1059            .unwrap()
1060            .unwrap();
1061        let token = Session {
1062            object_id: id.clone(),
1063            upload_length,
1064            backend_token: token,
1065        };
1066        let upload_id = Uuid::parse_str(&token.backend_token).unwrap();
1067        let upload_path = backend.upload_path(upload_id);
1068        assert_eq!(
1069            upload_path.parent().unwrap().file_name().unwrap(),
1070            ".uploads"
1071        );
1072        assert!(!backend.path(&id).exists());
1073
1074        // Upload the first chunk and verify that only the session advances.
1075        assert_eq!(
1076            backend.upload_offset(&token).await.unwrap(),
1077            UploadProgress::Incomplete { offset: 0 }
1078        );
1079        assert_eq!(
1080            backend
1081                .put_chunk(&token, 2, 0, stream::single(""))
1082                .await
1083                .unwrap(),
1084            UploadProgress::Incomplete { offset: 0 }
1085        );
1086        assert_eq!(
1087            backend
1088                .put_chunk(&token, 0, 3, stream::single("abc"))
1089                .await
1090                .unwrap(),
1091            UploadProgress::Incomplete { offset: 3 }
1092        );
1093        assert!(!backend.path(&id).exists());
1094        assert_eq!(
1095            backend.upload_offset(&token).await.unwrap(),
1096            UploadProgress::Incomplete { offset: 3 }
1097        );
1098
1099        // Zero-length chunks report current progress, while stale non-empty offsets fail.
1100        assert_eq!(
1101            backend
1102                .put_chunk(&token, 0, 0, stream::single(""))
1103                .await
1104                .unwrap(),
1105            UploadProgress::Incomplete { offset: 3 }
1106        );
1107        let error = backend
1108            .put_chunk(&token, 0, 3, stream::single("abc"))
1109            .await
1110            .unwrap_err();
1111        assert_eq!(error.kind(), ErrorKind::UploadOffsetMismatch { offset: 3 });
1112
1113        // Upload the final chunk and verify atomic publication removes the session.
1114        assert_eq!(
1115            backend
1116                .put_chunk(&token, 3, 3, stream::single("def"))
1117                .await
1118                .unwrap(),
1119            UploadProgress::Complete
1120        );
1121        assert!(!upload_path.exists());
1122        let (stored_metadata, _, payload) = backend
1123            .get_object(&id, Timestamp::now(), None)
1124            .await
1125            .unwrap()
1126            .unwrap();
1127        assert_eq!(stored_metadata.content_type, metadata.content_type);
1128        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"abcdef");
1129        assert_eq!(
1130            backend.upload_offset(&token).await.unwrap_err().kind(),
1131            ErrorKind::UnknownUploadSession
1132        );
1133    }
1134
1135    #[tokio::test]
1136    async fn resumable_publication_can_be_retried() {
1137        for query_offset in [false, true] {
1138            let (_tempdir, backend) = make_backend();
1139            let id = make_id();
1140            let token = upload_token(&backend, &id, 4).await;
1141            let object_path = backend.path(&id);
1142
1143            // A directory at the destination prevents rename after all bytes are persisted.
1144            tokio::fs::create_dir_all(&object_path).await.unwrap();
1145            let error = backend
1146                .put_chunk(&token, 0, 4, stream::single("data"))
1147                .await
1148                .unwrap_err();
1149            assert_eq!(error.kind(), ErrorKind::BackendFailure);
1150            tokio::fs::remove_dir(&object_path).await.unwrap();
1151
1152            let progress = if query_offset {
1153                backend.upload_offset(&token).await
1154            } else {
1155                backend.put_chunk(&token, 4, 0, stream::single("")).await
1156            };
1157            assert_eq!(progress.unwrap(), UploadProgress::Complete);
1158            let (_, _, payload) = backend
1159                .get_object(&id, Timestamp::now(), None)
1160                .await
1161                .unwrap()
1162                .unwrap();
1163            assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"data");
1164            assert_eq!(
1165                backend.upload_offset(&token).await.unwrap_err().kind(),
1166                ErrorKind::UnknownUploadSession
1167            );
1168        }
1169    }
1170
1171    #[tokio::test]
1172    async fn failed_resumable_chunk_preserves_partial_progress() {
1173        let (_tempdir, backend) = make_backend();
1174        let id = make_id();
1175        let token = upload_token(&backend, &id, 4).await;
1176
1177        // Persist a prefix, then disconnect after writing one byte of the next chunk.
1178        backend
1179            .put_chunk(&token, 0, 2, stream::single("ab"))
1180            .await
1181            .unwrap();
1182        let broken = futures_stream::iter([
1183            Ok(Bytes::from_static(b"c")),
1184            Err(stream::ClientError::new(io::Error::other(
1185                "client disconnected",
1186            ))),
1187        ])
1188        .boxed();
1189        let error = backend.put_chunk(&token, 2, 2, broken).await.unwrap_err();
1190        assert_eq!(error.kind(), ErrorKind::ClientStream);
1191
1192        // Resume from the partial byte and verify the complete object.
1193        assert_eq!(
1194            backend.upload_offset(&token).await.unwrap(),
1195            UploadProgress::Incomplete { offset: 3 }
1196        );
1197        assert_eq!(
1198            backend
1199                .put_chunk(&token, 3, 1, stream::single("d"))
1200                .await
1201                .unwrap(),
1202            UploadProgress::Complete
1203        );
1204        let (_, _, payload) = backend
1205            .get_object(&id, Timestamp::now(), None)
1206            .await
1207            .unwrap()
1208            .unwrap();
1209        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"abcd");
1210    }
1211
1212    #[tokio::test]
1213    async fn concurrent_uploads_for_same_object_do_not_deadlock() {
1214        let (_tempdir, backend) = make_backend();
1215        let id = make_id();
1216        let first = upload_token(&backend, &id, 3).await;
1217        let second = upload_token(&backend, &id, 3).await;
1218
1219        // Complete both sessions concurrently through their shared object lock.
1220        let writes = async {
1221            tokio::join!(
1222                backend.put_chunk(&first, 0, 3, stream::single("one")),
1223                backend.put_chunk(&second, 0, 3, stream::single("two")),
1224            )
1225        };
1226        let (first_result, second_result) = tokio::time::timeout(Duration::from_secs(2), writes)
1227            .await
1228            .unwrap();
1229
1230        // Both requests finish without deadlock; the last publication wins.
1231        assert_eq!(first_result.unwrap(), UploadProgress::Complete);
1232        assert_eq!(second_result.unwrap(), UploadProgress::Complete);
1233        let (_, _, payload) = backend
1234            .get_object(&id, Timestamp::now(), None)
1235            .await
1236            .unwrap()
1237            .unwrap();
1238        let payload = stream::read_to_vec(payload).await.unwrap();
1239        assert!(payload == b"one" || payload == b"two");
1240    }
1241
1242    #[tokio::test]
1243    async fn stores_metadata() {
1244        let tempdir = tempfile::tempdir().unwrap();
1245        let backend = LocalFsBackend::new(
1246            FileSystemConfig {
1247                path: tempdir.path().to_path_buf(),
1248                cogs: None,
1249            },
1250            &ChangeStreamFactory::default(),
1251        );
1252
1253        let id = ObjectId::random(ObjectContext {
1254            usecase: "testing".parse().unwrap(),
1255            scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
1256        });
1257
1258        let metadata = Metadata {
1259            content_type: "text/plain".into(),
1260            expiration_policy: ExpirationPolicy::TimeToIdle(Duration::from_hours(1)),
1261            time_created: Some(Timestamp::now()),
1262            time_expires: Some(Timestamp::now() + Duration::from_hours(1)),
1263            compression: Some(Compression::Zstd),
1264            origin: Some("203.0.113.42".into()),
1265            filename: Some("hello.txt".into()),
1266            custom: [("foo".into(), "bar".into())].into(),
1267            size: None,
1268        };
1269        backend
1270            .put_object(&id, &metadata, stream::single("oh hai!"), Timestamp::now())
1271            .await
1272            .unwrap();
1273
1274        let (read_metadata, _, stream) = backend
1275            .get_object(&id, Timestamp::now(), None)
1276            .await
1277            .unwrap()
1278            .unwrap();
1279        let file_contents: BytesMut = stream.try_collect().await.unwrap();
1280
1281        assert_eq!(
1282            read_metadata,
1283            Metadata {
1284                size: Some(file_contents.len()),
1285                ..metadata
1286            }
1287        );
1288        assert_eq!(file_contents.as_ref(), b"oh hai!");
1289
1290        let lock_path = backend.locks.path(&id);
1291        assert!(lock_path.exists());
1292        backend.delete_object(&id, Timestamp::now()).await.unwrap();
1293        assert!(lock_path.exists());
1294    }
1295
1296    #[tokio::test]
1297    async fn object_locks_coordinate_by_slot() {
1298        let tempdir = tempfile::tempdir().unwrap();
1299        let first_locks = ObjectLocks::new(tempdir.path());
1300        let second_locks = ObjectLocks::new(tempdir.path());
1301        let mut slots = HashMap::new();
1302        let (first_id, colliding_id) = (0..=65_536)
1303            .find_map(|key| {
1304                let id = ObjectId::from_parts(
1305                    "testing".parse().unwrap(),
1306                    Scopes::empty(),
1307                    key.to_string(),
1308                );
1309                let path = first_locks.path(&id);
1310                slots.insert(path, id.clone()).map(|first| (first, id))
1311            })
1312            .expect("65,537 keys must collide in 65,536 slots");
1313        let lock_path = first_locks.path(&first_id);
1314        let other_id = slots
1315            .values()
1316            .find(|id| first_locks.path(id) != lock_path)
1317            .unwrap();
1318        assert_eq!(second_locks.path(&colliding_id), lock_path);
1319
1320        let first_guard = first_locks.acquire(&first_id).await.unwrap();
1321        let mut waiter =
1322            tokio::spawn(async move { second_locks.acquire(&colliding_id).await.unwrap() });
1323        assert!(
1324            tokio::time::timeout(Duration::from_millis(50), &mut waiter)
1325                .await
1326                .is_err()
1327        );
1328
1329        let other_guard =
1330            tokio::time::timeout(Duration::from_secs(1), first_locks.acquire(other_id))
1331                .await
1332                .unwrap()
1333                .unwrap();
1334        drop(other_guard);
1335        drop(first_guard);
1336        let guard = tokio::time::timeout(Duration::from_secs(1), waiter)
1337            .await
1338            .unwrap()
1339            .unwrap();
1340        drop(guard);
1341        assert!(lock_path.exists());
1342    }
1343
1344    #[tokio::test]
1345    async fn missing_object_expiry_does_not_block_descendant() {
1346        let (_tempdir, backend) = make_backend();
1347        let id = ObjectId::from_parts("testing".parse().unwrap(), Scopes::empty(), "foo".into());
1348        assert_eq!(
1349            backend
1350                .set_expiry(
1351                    &id,
1352                    ExpiryTarget::At(Timestamp::now() + Duration::from_hours(1)).into(),
1353                    Timestamp::now()
1354                )
1355                .await
1356                .unwrap(),
1357            SetExpiryResponse::NotFound
1358        );
1359        let descendant = ObjectId::new(id.context.clone(), "foo/bar".into());
1360        backend
1361            .put_object(
1362                &descendant,
1363                &Metadata::default(),
1364                stream::single("payload"),
1365                Timestamp::now(),
1366            )
1367            .await
1368            .unwrap();
1369        let (_, _, payload) = backend
1370            .get_object(&descendant, Timestamp::now(), None)
1371            .await
1372            .unwrap()
1373            .unwrap();
1374        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"payload");
1375    }
1376
1377    #[tokio::test]
1378    async fn traversal_keys_stay_inside_root() {
1379        let tempdir = tempfile::tempdir().unwrap();
1380        let root = tempdir.path().join("root");
1381        let backend = LocalFsBackend::new(
1382            FileSystemConfig {
1383                path: root.clone(),
1384                cogs: None,
1385            },
1386            &ChangeStreamFactory::default(),
1387        );
1388
1389        for key in [
1390            "..",
1391            "../escape",
1392            "../../../../escape",
1393            "a/../../..",
1394            "/abs",
1395            "./x",
1396        ] {
1397            let id = ObjectId::from_parts("testing".parse().unwrap(), Scopes::empty(), key.into());
1398            let path = backend.path(&id);
1399            assert!(path.starts_with(&root), "{key:?} maps to {path:?}");
1400            assert!(
1401                path.components()
1402                    .all(|c| !matches!(c, std::path::Component::ParentDir)),
1403                "{key:?} maps to {path:?}"
1404            );
1405
1406            backend
1407                .put_object(
1408                    &id,
1409                    &Metadata::default(),
1410                    stream::single(key),
1411                    Timestamp::now(),
1412                )
1413                .await
1414                .unwrap();
1415            let (_, _, payload) = backend
1416                .get_object(&id, Timestamp::now(), None)
1417                .await
1418                .unwrap()
1419                .unwrap();
1420            assert_eq!(stream::read_to_vec(payload).await.unwrap(), key.as_bytes());
1421        }
1422
1423        let outside: Vec<_> = std::fs::read_dir(tempdir.path())
1424            .unwrap()
1425            .map(|entry| entry.unwrap().file_name())
1426            .collect();
1427        assert_eq!(outside, ["root"]);
1428    }
1429
1430    #[tokio::test]
1431    async fn parent_and_child_keys_coexist() {
1432        let (_tempdir, backend) = make_backend();
1433        let parent = ObjectId::from_parts("testing".parse().unwrap(), Scopes::empty(), "a".into());
1434        let child = ObjectId::new(parent.context.clone(), "a/b".into());
1435        // Directories named like the parent's object file must not collide with it.
1436        let suffixed = ObjectId::new(parent.context.clone(), "a.obj/b".into());
1437
1438        for id in [&parent, &child, &suffixed] {
1439            backend
1440                .put_object(
1441                    id,
1442                    &Metadata::default(),
1443                    stream::single(id.key().to_owned()),
1444                    Timestamp::now(),
1445                )
1446                .await
1447                .unwrap();
1448        }
1449        backend
1450            .delete_object(&parent, Timestamp::now())
1451            .await
1452            .unwrap();
1453
1454        assert!(
1455            backend
1456                .get_object(&parent, Timestamp::now(), None)
1457                .await
1458                .unwrap()
1459                .is_none()
1460        );
1461        let (_, _, payload) = backend
1462            .get_object(&child, Timestamp::now(), None)
1463            .await
1464            .unwrap()
1465            .unwrap();
1466        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"a/b");
1467
1468        // Deleting the remaining objects removes their empty directories up to the root.
1469        for id in [&child, &suffixed] {
1470            backend.delete_object(id, Timestamp::now()).await.unwrap();
1471        }
1472        assert!(!backend.path.join("testing").exists());
1473        assert!(backend.path.exists());
1474    }
1475
1476    #[test]
1477    fn distinct_keys_map_to_distinct_paths() {
1478        let (_tempdir, backend) = make_backend();
1479        let path = |key: &str| {
1480            let id = ObjectId::from_parts("testing".parse().unwrap(), Scopes::empty(), key.into());
1481            backend.path(&id)
1482        };
1483
1484        // Ordinary keys stay readable.
1485        assert_eq!(
1486            path("docs/report.pdf"),
1487            backend.path.join("testing/objects/docs/report.pdf.obj")
1488        );
1489
1490        // Escaped forms must not collide with the literal keys that produce them.
1491        for (key, escaped) in [
1492            ("a//b", "a/%/b"),    // empty segment
1493            ("a:b", "a%3Ab"),     // reserved character
1494            ("..", "%2E%2E"),     // dot segment
1495            ("a.obj", "a%2Eobj"), // object file suffix
1496        ] {
1497            assert_ne!(path(key), path(escaped), "{key:?} vs {escaped:?}");
1498        }
1499    }
1500
1501    #[test]
1502    fn trailing_dots_and_spaces_are_escaped() {
1503        let (_tempdir, backend) = make_backend();
1504        let path = |key: &str| {
1505            let id = ObjectId::from_parts("testing".parse().unwrap(), Scopes::empty(), key.into());
1506            backend.path(&id)
1507        };
1508
1509        // Windows strips trailing dots and spaces, which would turn the directory `x.obj ` into
1510        // `x.obj` and collide with the object file of key `x`.
1511        assert_eq!(
1512            path("x.obj /y"),
1513            backend.path.join("testing/objects/x.obj%20/y.obj")
1514        );
1515        assert_eq!(
1516            path("a./b"),
1517            backend.path.join("testing/objects/a%2E/b.obj")
1518        );
1519        // Only the last character needs escaping, inner spaces stay readable.
1520        assert_eq!(
1521            path("a b . /c"),
1522            backend.path.join("testing/objects/a b .%20/c.obj")
1523        );
1524    }
1525
1526    #[tokio::test]
1527    async fn failed_put_preserves_published_object() {
1528        let (_tempdir, backend) = make_backend();
1529        let id = make_id();
1530        let original_metadata = Metadata {
1531            content_type: "text/original".into(),
1532            ..Default::default()
1533        };
1534        backend
1535            .put_object(
1536                &id,
1537                &original_metadata,
1538                stream::single("original"),
1539                Timestamp::now(),
1540            )
1541            .await
1542            .unwrap();
1543
1544        let replacement_metadata = Metadata {
1545            content_type: "text/replacement".into(),
1546            ..Default::default()
1547        };
1548        let replacement = futures_stream::iter([
1549            Ok(Bytes::from_static(b"partial")),
1550            Err(stream::ClientError::new(io::Error::other(
1551                "replacement stream failed",
1552            ))),
1553        ])
1554        .boxed();
1555        let error = backend
1556            .put_object(&id, &replacement_metadata, replacement, Timestamp::now())
1557            .await
1558            .unwrap_err();
1559        assert_eq!(error.kind(), ErrorKind::ClientStream);
1560
1561        let (metadata, _, payload) = backend
1562            .get_object(&id, Timestamp::now(), None)
1563            .await
1564            .unwrap()
1565            .unwrap();
1566        assert_eq!(metadata.content_type, original_metadata.content_type);
1567        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"original");
1568
1569        assert_eq!(draft_count(&backend, &id), 0);
1570    }
1571
1572    #[tokio::test]
1573    async fn cancelled_put_removes_draft() {
1574        let (_tempdir, backend) = make_backend();
1575        let backend = Arc::new(backend);
1576        let id = make_id();
1577        backend
1578            .put_object(
1579                &id,
1580                &Metadata::default(),
1581                stream::single("original"),
1582                Timestamp::now(),
1583            )
1584            .await
1585            .unwrap();
1586
1587        let (writing, writing_started) = tokio::sync::oneshot::channel();
1588        let replacement = futures_stream::once(async move {
1589            let _ = writing.send(());
1590            Ok(Bytes::from_static(b"partial"))
1591        })
1592        .chain(futures_stream::pending())
1593        .boxed();
1594
1595        let task = tokio::spawn({
1596            let backend = Arc::clone(&backend);
1597            let id = id.clone();
1598            async move {
1599                backend
1600                    .put_object(&id, &Metadata::default(), replacement, Timestamp::now())
1601                    .await
1602            }
1603        });
1604
1605        writing_started.await.unwrap();
1606        assert_eq!(draft_count(&backend, &id), 1);
1607        task.abort();
1608        assert!(task.await.unwrap_err().is_cancelled());
1609
1610        let (_, _, payload) = backend
1611            .get_object(&id, Timestamp::now(), None)
1612            .await
1613            .unwrap()
1614            .unwrap();
1615        assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"original");
1616
1617        assert_eq!(draft_count(&backend, &id), 0);
1618    }
1619
1620    #[tokio::test]
1621    async fn set_expiry() {
1622        for is_ttl in [false, true] {
1623            let (_tempdir, backend) = make_backend();
1624            let id = make_id();
1625            let created = Timestamp::now();
1626            let old_expiry = created + Duration::from_hours(1);
1627            let metadata = Metadata {
1628                expiration_policy: if is_ttl {
1629                    ExpirationPolicy::TimeToLive(Duration::from_hours(1))
1630                } else {
1631                    ExpirationPolicy::TimeToIdle(Duration::from_hours(1))
1632                },
1633                time_created: Some(created),
1634                time_expires: Some(old_expiry),
1635                custom: [("preserved".into(), "yes".into())].into(),
1636                ..Default::default()
1637            };
1638            backend
1639                .put_object(&id, &metadata, stream::single("payload"), Timestamp::now())
1640                .await
1641                .unwrap();
1642
1643            let duration = Duration::from_hours(2) + Duration::from_nanos(999);
1644            let requested = created + duration;
1645            for target in [
1646                ExpiryTarget::At(requested),
1647                ExpiryTarget::FromCreation(duration),
1648            ] {
1649                assert_eq!(
1650                    backend
1651                        .set_expiry(&id, target.into(), Timestamp::now())
1652                        .await
1653                        .unwrap(),
1654                    SetExpiryResponse::Satisfied(requested)
1655                );
1656            }
1657            let (updated, _, payload) = backend
1658                .get_object(&id, Timestamp::now(), None)
1659                .await
1660                .unwrap()
1661                .unwrap();
1662            assert_eq!(
1663                updated.expiration_policy,
1664                if is_ttl {
1665                    ExpirationPolicy::TimeToLive(Duration::from_hours(2))
1666                } else {
1667                    metadata.expiration_policy
1668                }
1669            );
1670            assert_eq!(updated.time_created, metadata.time_created);
1671            assert_eq!(updated.custom, metadata.custom);
1672            assert_eq!(updated.time_expires, Some(requested));
1673            assert_eq!(stream::read_to_vec(payload).await.unwrap(), b"payload");
1674
1675            assert_eq!(draft_count(&backend, &id), 0);
1676        }
1677    }
1678
1679    #[tokio::test]
1680    async fn expired_object() {
1681        let (_tempdir, backend) = make_backend();
1682        let id = make_id();
1683        let metadata = Metadata {
1684            expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1685            time_expires: Some(Timestamp::now() - Duration::from_secs(1)),
1686            ..Default::default()
1687        };
1688        backend
1689            .put_object(&id, &metadata, stream::single("expired"), Timestamp::now())
1690            .await
1691            .unwrap();
1692
1693        assert!(
1694            backend
1695                .get_object(&id, Timestamp::now(), None)
1696                .await
1697                .unwrap()
1698                .is_none()
1699        );
1700        assert_eq!(
1701            backend
1702                .set_expiry(
1703                    &id,
1704                    ExpiryTarget::At(Timestamp::now() + Duration::from_hours(1)).into(),
1705                    Timestamp::now()
1706                )
1707                .await
1708                .unwrap(),
1709            SetExpiryResponse::NotFound
1710        );
1711    }
1712
1713    #[tokio::test]
1714    async fn get_metadata_returns_metadata() {
1715        let tempdir = tempfile::tempdir().unwrap();
1716        let backend = LocalFsBackend::new(
1717            FileSystemConfig {
1718                path: tempdir.path().to_path_buf(),
1719                cogs: None,
1720            },
1721            &ChangeStreamFactory::default(),
1722        );
1723
1724        let id = ObjectId::random(ObjectContext {
1725            usecase: "testing".parse().unwrap(),
1726            scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
1727        });
1728
1729        let metadata = Metadata {
1730            content_type: "text/plain".into(),
1731            compression: Some(Compression::Zstd),
1732            origin: Some("203.0.113.42".into()),
1733            custom: [("foo".into(), "bar".into())].into(),
1734            ..Default::default()
1735        };
1736        backend
1737            .put_object(&id, &metadata, stream::single("oh hai!"), Timestamp::now())
1738            .await
1739            .unwrap();
1740
1741        let read_metadata = backend
1742            .get_metadata(&id, Timestamp::now())
1743            .await
1744            .unwrap()
1745            .unwrap();
1746        assert_eq!(
1747            read_metadata,
1748            Metadata {
1749                size: Some(7),
1750                ..metadata
1751            }
1752        );
1753    }
1754
1755    #[tokio::test]
1756    async fn get_metadata_nonexistent() {
1757        let tempdir = tempfile::tempdir().unwrap();
1758        let backend = LocalFsBackend::new(
1759            FileSystemConfig {
1760                path: tempdir.path().to_path_buf(),
1761                cogs: None,
1762            },
1763            &ChangeStreamFactory::default(),
1764        );
1765
1766        let id = ObjectId::random(ObjectContext {
1767            usecase: "testing".parse().unwrap(),
1768            scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
1769        });
1770
1771        let result = backend.get_metadata(&id, Timestamp::now()).await.unwrap();
1772        assert!(result.is_none());
1773    }
1774
1775    fn make_id() -> ObjectId {
1776        ObjectId::random(ObjectContext {
1777            usecase: "testing".parse().unwrap(),
1778            scopes: Scopes::from_iter([Scope::create("testing", "value").unwrap()]),
1779        })
1780    }
1781
1782    #[cfg(feature = "storage-cogs")]
1783    fn make_backend_with_change_stream() -> (tempfile::TempDir, LocalFsBackend, DummyProducer) {
1784        let tempdir = tempfile::tempdir().unwrap();
1785        let (streams, producer) = crate::change_stream::dummy_factory();
1786        let backend = LocalFsBackend::new(
1787            FileSystemConfig {
1788                path: tempdir.path().to_path_buf(),
1789                cogs: Some(CostTrackerStreamConfig {
1790                    shared_resource_id: "filesystem_objectstore".into(),
1791                    sample_rate: 1.0,
1792                }),
1793            },
1794            &streams,
1795        );
1796        (tempdir, backend, producer)
1797    }
1798
1799    fn make_backend() -> (tempfile::TempDir, LocalFsBackend) {
1800        let tempdir = tempfile::tempdir().unwrap();
1801        let backend = LocalFsBackend::new(
1802            FileSystemConfig {
1803                path: tempdir.path().to_path_buf(),
1804                cogs: None,
1805            },
1806            &ChangeStreamFactory::default(),
1807        );
1808        (tempdir, backend)
1809    }
1810
1811    fn draft_count(backend: &LocalFsBackend, id: &ObjectId) -> usize {
1812        let object_path = backend.path(id);
1813        std::fs::read_dir(object_path.parent().unwrap())
1814            .unwrap()
1815            .flat_map(|entry| entry.map(|entry| entry.path()))
1816            .filter(|path| path.to_string_lossy().ends_with(".draft"))
1817            .count()
1818    }
1819
1820    #[tokio::test]
1821    async fn get_object_range_bounded() {
1822        let (_tempdir, backend) = make_backend();
1823        let id = make_id();
1824        let metadata = Metadata::default();
1825
1826        let payload = b"Hello, range requests!";
1827        backend
1828            .put_object(
1829                &id,
1830                &metadata,
1831                stream::single(payload.to_vec()),
1832                Timestamp::now(),
1833            )
1834            .await
1835            .unwrap();
1836
1837        // Request bytes 7-11 → "range"
1838        let (_, content_range, body) = backend
1839            .get_object(&id, Timestamp::now(), Some(ByteRange::Bounded(7, 11)))
1840            .await
1841            .unwrap()
1842            .unwrap();
1843        let data: BytesMut = body.try_collect().await.unwrap();
1844
1845        assert_eq!(data.as_ref(), b"range");
1846        let content_range = content_range.unwrap();
1847        assert_eq!(content_range.start, 7);
1848        assert_eq!(content_range.end, 11);
1849        assert_eq!(content_range.total, payload.len() as u64);
1850    }
1851
1852    #[tokio::test]
1853    async fn get_object_range_from() {
1854        let (_tempdir, backend) = make_backend();
1855        let id = make_id();
1856        let metadata = Metadata::default();
1857
1858        let payload = b"Hello, range requests!";
1859        backend
1860            .put_object(
1861                &id,
1862                &metadata,
1863                stream::single(payload.to_vec()),
1864                Timestamp::now(),
1865            )
1866            .await
1867            .unwrap();
1868
1869        // Request bytes 7- → "range requests!"
1870        let (_, content_range, body) = backend
1871            .get_object(&id, Timestamp::now(), Some(ByteRange::From(7)))
1872            .await
1873            .unwrap()
1874            .unwrap();
1875        let data: BytesMut = body.try_collect().await.unwrap();
1876
1877        assert_eq!(data.as_ref(), b"range requests!");
1878        let content_range = content_range.unwrap();
1879        assert_eq!(content_range.start, 7);
1880        assert_eq!(content_range.end, 21);
1881        assert_eq!(content_range.total, payload.len() as u64);
1882    }
1883
1884    #[tokio::test]
1885    async fn get_object_range_last() {
1886        let (_tempdir, backend) = make_backend();
1887        let id = make_id();
1888        let metadata = Metadata::default();
1889
1890        let payload = b"Hello, range requests!";
1891        backend
1892            .put_object(
1893                &id,
1894                &metadata,
1895                stream::single(payload.to_vec()),
1896                Timestamp::now(),
1897            )
1898            .await
1899            .unwrap();
1900
1901        // Request last 9 bytes → "requests!"
1902        let (_, content_range, body) = backend
1903            .get_object(&id, Timestamp::now(), Some(ByteRange::Last(9)))
1904            .await
1905            .unwrap()
1906            .unwrap();
1907        let data: BytesMut = body.try_collect().await.unwrap();
1908
1909        assert_eq!(data.as_ref(), b"requests!");
1910        let content_range = content_range.unwrap();
1911        assert_eq!(content_range.start, 13);
1912        assert_eq!(content_range.end, 21);
1913        assert_eq!(content_range.total, payload.len() as u64);
1914    }
1915
1916    #[tokio::test]
1917    async fn get_object_range_unsatisfiable() {
1918        let (_tempdir, backend) = make_backend();
1919        let id = make_id();
1920        let metadata = Metadata::default();
1921
1922        backend
1923            .put_object(
1924                &id,
1925                &metadata,
1926                stream::single(b"short".to_vec()),
1927                Timestamp::now(),
1928            )
1929            .await
1930            .unwrap();
1931
1932        match backend
1933            .get_object(&id, Timestamp::now(), Some(ByteRange::From(100)))
1934            .await
1935        {
1936            Err(error) if matches!(error.kind(), ErrorKind::RangeNotSatisfiable { total: 5 }) => {}
1937            Err(other) => panic!("expected RangeNotSatisfiable, got: {other:?}"),
1938            Ok(_) => panic!("expected RangeNotSatisfiable, got Ok"),
1939        }
1940    }
1941
1942    #[cfg(feature = "storage-cogs")]
1943    #[tokio::test]
1944    async fn change_stream_reports_size_written_to_disk() {
1945        let (_tempdir, backend, producer) = make_backend_with_change_stream();
1946        let id = make_id();
1947        let metadata = Metadata {
1948            expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1949            time_expires: Some(Timestamp::now() + Duration::from_hours(1)),
1950            ..Default::default()
1951        };
1952        let payload = b"oh hai!";
1953
1954        backend
1955            .put_object(
1956                &id,
1957                &metadata,
1958                stream::single(payload.to_vec()),
1959                Timestamp::now(),
1960            )
1961            .await
1962            .unwrap();
1963
1964        let file = tokio::fs::read(backend.path(&id)).await.unwrap();
1965
1966        let records = producer.records();
1967        assert_eq!(records.len(), 1);
1968        assert_eq!(records[0].op_type, OpType::Write);
1969        assert_eq!(records[0].shared_resource_id, "filesystem_objectstore");
1970        assert_eq!(records[0].app_feature, "testing");
1971        assert_eq!(
1972            records[0].size,
1973            Some(file.len() as u64),
1974            "the metadata header line counts towards the reported size"
1975        );
1976        assert!(records[0].expiration_time.is_some());
1977    }
1978
1979    #[cfg(feature = "storage-cogs")]
1980    #[tokio::test]
1981    async fn change_stream_reports_resumable_upload_size() {
1982        let (_tempdir, backend, producer) = make_backend_with_change_stream();
1983        let id = make_id();
1984        let metadata = Metadata {
1985            expiration_policy: ExpirationPolicy::TimeToLive(Duration::from_hours(1)),
1986            time_expires: Some(Timestamp::now() + Duration::from_hours(1)),
1987            ..Default::default()
1988        };
1989        let payload = b"oh hai!";
1990        let upload_length = NonZeroU64::new(payload.len() as u64).unwrap();
1991        let token = backend
1992            .create_upload_session(&id, &metadata, upload_length)
1993            .await
1994            .unwrap()
1995            .unwrap();
1996        let token = Session {
1997            object_id: id.clone(),
1998            upload_length,
1999            backend_token: token,
2000        };
2001
2002        // Session creation alone does not report a stored object.
2003        assert!(producer.records().is_empty());
2004
2005        // Completing the upload reports the published file's full stored size.
2006        assert_eq!(
2007            backend
2008                .put_chunk(
2009                    &token,
2010                    0,
2011                    payload.len() as u64,
2012                    stream::single(payload.to_vec()),
2013                )
2014                .await
2015                .unwrap(),
2016            UploadProgress::Complete
2017        );
2018
2019        let file = tokio::fs::read(backend.path(&id)).await.unwrap();
2020        let records = producer.records();
2021        assert_eq!(records.len(), 1);
2022        assert_eq!(records[0].op_type, OpType::Write);
2023        assert_eq!(records[0].size, Some(file.len() as u64));
2024        assert!(records[0].expiration_time.is_some());
2025    }
2026
2027    #[cfg(feature = "storage-cogs")]
2028    #[tokio::test]
2029    async fn change_stream_reports_deletes_on_success() {
2030        let (_tempdir, backend, producer) = make_backend_with_change_stream();
2031        let id = make_id();
2032
2033        // Try to delete a non-existent object. Don't emit a message.
2034        backend
2035            .delete_object(&id, Timestamp::now())
2036            .await
2037            .expect("deleting a non-existent object returns Ok(())");
2038        assert!(producer.records().is_empty());
2039
2040        backend
2041            .put_object(
2042                &id,
2043                &Metadata::default(),
2044                stream::single(b"hi".to_vec()),
2045                Timestamp::now(),
2046            )
2047            .await
2048            .unwrap();
2049        producer.clear();
2050
2051        backend.delete_object(&id, Timestamp::now()).await.unwrap();
2052
2053        let records = producer.records();
2054        assert_eq!(records.len(), 1);
2055        assert_eq!(records[0].op_type, OpType::Delete);
2056    }
2057}