fix: content store background task shouldn't hard fail

packages/app-lib/src/state/content_store/adapters/filesystem.rs

@@ -270,9 +270,10 @@ pub(crate) async fn remove_abandoned_staging(
     while let Some(entry) = entries.next_entry().await? {
         let name = entry.file_name();
         if name.to_string_lossy().starts_with(".tmp")
-            && entry.file_type().await?.is_file()
+            && entry.file_type().await.is_ok_and(|kind| kind.is_file())
+            && let Err(error) = fs::remove_file(entry.path()).await
         {
-            fs::remove_file(entry.path()).await?;
+            tracing::warn!(path = %entry.path().display(), "Could not remove abandoned staging file: {error}");
         }
     }
     Ok(())

packages/app-lib/src/state/content_store/adapters/stored_objects.rs

@@ -9,7 +9,7 @@ use crate::util::content_hash::{FileHashes, hash_file};
 use anyhow::Context;
 use std::collections::HashSet;
 use std::path::Path;
-use tokio::fs::{self, ReadDir};
+use tokio::fs::{self, DirEntry, ReadDir};
 
 pub(in crate::state::content_store) struct UnregisteredFiles {
     prefixes: ReadDir,
@@ -40,51 +40,30 @@ impl UnregisteredFiles {
         }))
     }
 
+    /// Unreadable entries are skipped so one damaged file cannot block the
+    /// rest of the scan.
     pub(in crate::state::content_store) async fn next_file(
         &mut self,
     ) -> crate::Result<Option<StoredFileMetadata>> {
         loop {
             if let Some(entries) = &mut self.entries {
-                while let Some(entry) = entries.next_entry().await? {
-                    let hash = entry.file_name().to_string_lossy().into_owned();
-                    if validate_digest(&hash, 128).is_err()
-                        || !hash.starts_with(&self.prefix)
-                        || self.known.contains(&hash)
-                        || !entry.file_type().await?.is_file()
-                    {
-                        continue;
-                    }
-                    let path = entry.path();
-                    let Some(metadata) =
-                        symlink_metadata_if_exists(&path).await?
-                    else {
-                        continue;
+                loop {
+                    let entry = match entries.next_entry().await {
+                        Ok(Some(entry)) => entry,
+                        Ok(None) => break,
+                        Err(error) => {
+                            tracing::warn!(prefix = %self.prefix, "Skipping unreadable store directory: {error}");
+                            break;
+                        }
                     };
-                    if !metadata.is_file() {
-                        continue;
-                    }
-                    let FileHashes { sha512, size } = hash_file(&path).await?;
-                    if sha512 != hash {
-                        tracing::warn!(path = %path.display(), "Preserving an unregistered store object with an unexpected hash");
-                        continue;
+                    let path = entry.path();
+                    match adopt(&entry, &self.prefix, &self.known).await {
+                        Ok(Some(file)) => return Ok(Some(file)),
+                        Ok(None) => {}
+                        Err(error) => {
+                            tracing::warn!(path = %path.display(), "Skipping unreadable store object: {error}");
+                        }
                     }
-                    let mut permissions = metadata.permissions();
-                    permissions.set_readonly(true);
-                    fs::set_permissions(&path, permissions).await?;
-                    return Ok(Some(StoredFileMetadata {
-                        sha512,
-                        size: size.try_into().context(
-                            ErrorKind::ContentStoreError(
-                                "content file is too large".into(),
-                            ),
-                        )?,
-                        status: StoredFileStatus::Ready,
-                        modified_as: crate::state::file_modified_at_ns(
-                            &metadata,
-                        )? as i64,
-                        last_used_at: chrono::Utc::now().timestamp(),
-                        sources: "[]".to_string(),
-                    }));
                 }
                 self.entries = None;
             }
@@ -93,12 +72,59 @@ impl UnregisteredFiles {
             };
             let name = prefix.file_name().to_string_lossy().into_owned();
             if validate_digest(&name, 2).is_err()
-                || !prefix.file_type().await?.is_dir()
+                || !prefix.file_type().await.is_ok_and(|kind| kind.is_dir())
             {
                 continue;
             }
-            self.entries = Some(fs::read_dir(prefix.path()).await?);
-            self.prefix = name;
+            match fs::read_dir(prefix.path()).await {
+                Ok(entries) => {
+                    self.entries = Some(entries);
+                    self.prefix = name;
+                }
+                Err(error) => {
+                    tracing::warn!(path = %prefix.path().display(), "Skipping unreadable store directory: {error}");
+                }
+            }
         }
     }
 }
+
+async fn adopt(
+    entry: &DirEntry,
+    prefix: &str,
+    known: &HashSet<String>,
+) -> crate::Result<Option<StoredFileMetadata>> {
+    let hash = entry.file_name().to_string_lossy().into_owned();
+    if validate_digest(&hash, 128).is_err()
+        || !hash.starts_with(prefix)
+        || known.contains(&hash)
+        || !entry.file_type().await?.is_file()
+    {
+        return Ok(None);
+    }
+    let path = entry.path();
+    let Some(metadata) = symlink_metadata_if_exists(&path).await? else {
+        return Ok(None);
+    };
+    if !metadata.is_file() {
+        return Ok(None);
+    }
+    let FileHashes { sha512, size } = hash_file(&path).await?;
+    if sha512 != hash {
+        tracing::warn!(path = %path.display(), "Preserving an unregistered store object with an unexpected hash");
+        return Ok(None);
+    }
+    let mut permissions = metadata.permissions();
+    permissions.set_readonly(true);
+    fs::set_permissions(&path, permissions).await?;
+    Ok(Some(StoredFileMetadata {
+        sha512,
+        size: size.try_into().context(ErrorKind::ContentStoreError(
+            "content file is too large".into(),
+        ))?,
+        status: StoredFileStatus::Ready,
+        modified_as: crate::state::file_modified_at_ns(&metadata)? as i64,
+        last_used_at: chrono::Utc::now().timestamp(),
+        sources: "[]".to_string(),
+    }))
+}

packages/app-lib/src/state/content_store/commands/migration.rs

@@ -116,8 +116,12 @@ pub(crate) async fn migrate(state: &State) -> crate::Result<()> {
             store.release("rollback", &owner).await?;
         }
     }
-    store.remove_abandoned_staging().await?;
-    store.recover_unregistered_files().await?;
+    if let Err(error) = store.remove_abandoned_staging().await {
+        tracing::warn!("Abandoned staging cleanup deferred: {error}");
+    }
+    if let Err(error) = store.recover_unregistered_files().await {
+        tracing::warn!("Unregistered content recovery deferred: {error}");
+    }
     Ok(())
 }