Milestone 2: persistent blob store, add/materialize/gc
iroh-blobs 0.35 fs store under <store_dir>/blobs. Add imports files or whole directory trees (as Collections, deterministic order), Materialize exports them with a real FICLONE reflink attempt and streaming-copy fallback, Gc is an explicit mark-and-sweep rooted at the pins. Pins, trusted peers and hash formats persist in meta.json (atomic writes). Store reads run on a LocalPool because iroh-blobs entry readers are not Send. Integration tests drive the real daemon binary: directory round trip byte-identical, gc keeps pinned/drops unpinned, reflink verified on the repo's own filesystem (XFS), plus a 32-case proptest round trip. Dependencies: iroh-blobs =0.35.0 (the store itself; pinned per spec), iroh-io (AsyncSliceReader traits to read store entries), reflink-copy (FICLONE with copy fallback, per spec), proptest (dev-only, round-trip property test). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+142
-11
@@ -1,29 +1,58 @@
|
||||
//! Request dispatch and daemon state.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
|
||||
use anyhow::Context;
|
||||
use iroh_blobs::{BlobFormat, Hash, HashAndFormat};
|
||||
use tokio::sync::broadcast;
|
||||
use varde_proto::{Envelope, ErrorCode, Event, GlobalStatus, Request, ResponseData};
|
||||
use varde_proto::{
|
||||
Envelope, ErrorCode, Event, GlobalStatus, HashStatus, PinInfo, Request, ResponseData,
|
||||
};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::meta::{Meta, StoredFormat};
|
||||
use crate::store::{BlobStore, OpError};
|
||||
|
||||
/// Shared daemon state. Handlers grow with the milestones; anything not
|
||||
/// yet wired returns [`ErrorCode::Unimplemented`] rather than pretending.
|
||||
pub struct Daemon {
|
||||
config: Config,
|
||||
store: BlobStore,
|
||||
meta: Meta,
|
||||
events: broadcast::Sender<Event>,
|
||||
}
|
||||
|
||||
impl Daemon {
|
||||
pub fn new(config: Config) -> Self {
|
||||
/// Open the blob store and metadata under the configured store dir.
|
||||
pub async fn open(config: Config) -> anyhow::Result<Daemon> {
|
||||
std::fs::create_dir_all(&config.store_dir)
|
||||
.with_context(|| format!("creating store directory {}", config.store_dir.display()))?;
|
||||
let store = BlobStore::open(&config.store_dir).await?;
|
||||
let meta = Meta::load(&config.store_dir)?;
|
||||
// Capacity bounds memory if a subscriber stalls; laggards get a
|
||||
// Lagged error, not unbounded buffering.
|
||||
let (events, _) = broadcast::channel(1024);
|
||||
Daemon { config, events }
|
||||
Ok(Daemon {
|
||||
config,
|
||||
store,
|
||||
meta,
|
||||
events,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &Config {
|
||||
&self.config
|
||||
}
|
||||
|
||||
pub fn store(&self) -> &BlobStore {
|
||||
&self.store
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &Meta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// New receiver for the daemon event stream.
|
||||
pub fn subscribe_events(&self) -> broadcast::Receiver<Event> {
|
||||
self.events.subscribe()
|
||||
@@ -32,21 +61,123 @@ impl Daemon {
|
||||
/// Handle one API request. Infallible at this level: all failures are
|
||||
/// mapped to structured error envelopes.
|
||||
pub async fn handle(&self, request: Request) -> Envelope {
|
||||
match self.dispatch(request).await {
|
||||
Ok(data) => Envelope::ok(data),
|
||||
Err(e) => Envelope::err(error_code(&e), e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(&self, request: Request) -> Result<ResponseData, OpError> {
|
||||
match request {
|
||||
Request::Status { hash: None } => Envelope::ok(ResponseData::Status(GlobalStatus {
|
||||
Request::Add { path, recursive } => {
|
||||
let (hash, bytes, format) =
|
||||
self.store.add_path(&PathBuf::from(path), recursive).await?;
|
||||
self.meta.record_format(&hash.to_hex(), format)?;
|
||||
Ok(ResponseData::Added {
|
||||
hash: hash.to_hex().to_string(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
Request::Pin { hash, policy } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
self.meta.pin(&parsed.to_hex(), policy)?;
|
||||
// Fetching absent pinned content arrives with the
|
||||
// transfer milestone.
|
||||
Ok(ResponseData::Pinned {
|
||||
hash: parsed.to_hex().to_string(),
|
||||
})
|
||||
}
|
||||
Request::Unpin { hash } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
if !self.meta.unpin(&parsed.to_hex())? {
|
||||
return Err(OpError::NotFound(format!("no pin for {}", parsed.to_hex())));
|
||||
}
|
||||
Ok(ResponseData::Done {})
|
||||
}
|
||||
Request::Materialize { hash, dest, mode } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let format = self.meta.format_of(&parsed.to_hex());
|
||||
let (bytes, reflinked) = self
|
||||
.store
|
||||
.materialize(parsed, format, &PathBuf::from(dest), mode)
|
||||
.await?;
|
||||
Ok(ResponseData::Materialized { bytes, reflinked })
|
||||
}
|
||||
Request::Status { hash: None } => Ok(ResponseData::Status(GlobalStatus {
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
protocol: varde_proto::PROTOCOL_VERSION,
|
||||
node_id: None,
|
||||
store_path: self.config.store_dir.display().to_string(),
|
||||
pins: 0,
|
||||
pins: self.meta.pins().len() as u64,
|
||||
connected_peers: 0,
|
||||
})),
|
||||
Request::List {} => Envelope::ok(ResponseData::Pins { pins: Vec::new() }),
|
||||
Request::Subscribe {} => Envelope::ok(ResponseData::Done {}),
|
||||
other => Envelope::err(
|
||||
ErrorCode::Unimplemented,
|
||||
format!("not implemented yet: {other:?}"),
|
||||
),
|
||||
Request::Status { hash: Some(hash) } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let (have_bytes, total_bytes, complete) = self.store.presence(parsed).await?;
|
||||
Ok(ResponseData::HashStatus(HashStatus {
|
||||
hash: parsed.to_hex().to_string(),
|
||||
have_bytes,
|
||||
total_bytes,
|
||||
complete,
|
||||
pinned: self.meta.is_pinned(&parsed.to_hex()),
|
||||
active_peers: Vec::new(),
|
||||
}))
|
||||
}
|
||||
Request::List {} => {
|
||||
let mut pins = Vec::new();
|
||||
for (hash, record) in self.meta.pins() {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let (have_bytes, total_bytes, complete) = self.store.presence(parsed).await?;
|
||||
pins.push(PinInfo {
|
||||
hash,
|
||||
policy: record.policy,
|
||||
complete,
|
||||
have_bytes,
|
||||
total_bytes,
|
||||
});
|
||||
}
|
||||
Ok(ResponseData::Pins { pins })
|
||||
}
|
||||
Request::Gc {} => {
|
||||
let roots = self.pin_roots()?;
|
||||
let blobs_removed = self.store.gc(roots).await?;
|
||||
Ok(ResponseData::GcDone { blobs_removed })
|
||||
}
|
||||
Request::Subscribe {} => Ok(ResponseData::Done {}),
|
||||
other => Err(OpError::Unimplemented(format!(
|
||||
"not implemented yet: {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The GC roots: every pinned hash with its recorded format.
|
||||
fn pin_roots(&self) -> Result<Vec<HashAndFormat>, OpError> {
|
||||
let mut roots = Vec::new();
|
||||
for (hash, _record) in self.meta.pins() {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let format = match self.meta.format_of(&hash) {
|
||||
StoredFormat::Raw => BlobFormat::Raw,
|
||||
StoredFormat::HashSeq => BlobFormat::HashSeq,
|
||||
};
|
||||
roots.push(HashAndFormat {
|
||||
hash: parsed,
|
||||
format,
|
||||
});
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hash(s: &str) -> Result<Hash, OpError> {
|
||||
Hash::from_str(s).map_err(|e| OpError::InvalidArgument(format!("invalid hash {s:?}: {e}")))
|
||||
}
|
||||
|
||||
fn error_code(e: &OpError) -> ErrorCode {
|
||||
match e {
|
||||
OpError::NotFound(_) => ErrorCode::NotFound,
|
||||
OpError::InvalidArgument(_) => ErrorCode::InvalidArgument,
|
||||
OpError::Io(_) => ErrorCode::Io,
|
||||
OpError::Internal(_) => ErrorCode::Internal,
|
||||
OpError::Unimplemented(_) => ErrorCode::Unimplemented,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,19 +5,20 @@
|
||||
|
||||
pub mod config;
|
||||
pub mod daemon;
|
||||
pub mod meta;
|
||||
pub mod server;
|
||||
pub mod store;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use anyhow::Result;
|
||||
|
||||
/// Run a daemon with the given config until the future is dropped or the
|
||||
/// listener fails. Binds the socket before returning control to the
|
||||
/// runtime so callers can rely on it existing once this future is polled.
|
||||
/// listener fails. The socket is bound before the accept loop starts, so
|
||||
/// once this future has been polled the socket path exists.
|
||||
pub async fn run(config: config::Config) -> Result<()> {
|
||||
std::fs::create_dir_all(&config.store_dir)
|
||||
.with_context(|| format!("creating store directory {}", config.store_dir.display()))?;
|
||||
let listener = server::bind_socket(&config.socket_path)?;
|
||||
let daemon = Arc::new(daemon::Daemon::new(config));
|
||||
let socket_path = config.socket_path.clone();
|
||||
let daemon = Arc::new(daemon::Daemon::open(config).await?);
|
||||
let listener = server::bind_socket(&socket_path)?;
|
||||
server::serve(listener, daemon).await
|
||||
}
|
||||
|
||||
@@ -82,15 +82,16 @@ fn main() -> Result<()> {
|
||||
|
||||
async fn run(config: Config) -> Result<()> {
|
||||
let socket_path = config.socket_path.clone();
|
||||
let daemon = Arc::new(daemon::Daemon::open(config).await?);
|
||||
let listener = server::bind_socket(&socket_path)?;
|
||||
let daemon = Arc::new(daemon::Daemon::new(config));
|
||||
|
||||
let result = tokio::select! {
|
||||
r = server::serve(listener, daemon) => r,
|
||||
r = server::serve(listener, daemon.clone()) => r,
|
||||
r = shutdown_signal() => r.map(|signal| info!(signal, "shutting down")),
|
||||
};
|
||||
|
||||
// Best-effort cleanup so the next start doesn't find a stale socket.
|
||||
// Flush the store and remove the socket so the next start is clean.
|
||||
daemon.store().shutdown().await;
|
||||
let _ = std::fs::remove_file(&socket_path);
|
||||
result
|
||||
}
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
//! Persistent daemon metadata: pins, known formats, trusted peers.
|
||||
//!
|
||||
//! Stored as a single JSON file (`meta.json`) in the store directory —
|
||||
//! the spec allows JSON for the MVP and the data is tiny and human
|
||||
//! auditable. Writes go through a temp file + rename so a crash never
|
||||
//! leaves a torn file.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use varde_proto::PinPolicy;
|
||||
|
||||
/// Whether a hash names a single blob or a HashSeq (directory root).
|
||||
///
|
||||
/// Recorded at import/pin time because the bytes alone don't say; a
|
||||
/// materialize of an unknown hash defaults to `Raw`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StoredFormat {
|
||||
Raw,
|
||||
HashSeq,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PinRecord {
|
||||
pub policy: PinPolicy,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
struct MetaState {
|
||||
/// Pinned root hashes (hex) and their policies.
|
||||
#[serde(default)]
|
||||
pins: BTreeMap<String, PinRecord>,
|
||||
/// Known formats of hashes we imported or fetched (hex).
|
||||
#[serde(default)]
|
||||
formats: BTreeMap<String, StoredFormat>,
|
||||
/// Trusted peer NodeIds (z-base-32).
|
||||
#[serde(default)]
|
||||
trusted_peers: BTreeSet<String>,
|
||||
}
|
||||
|
||||
/// Handle to the metadata file. Cheap to share behind an `Arc`.
|
||||
#[derive(Debug)]
|
||||
pub struct Meta {
|
||||
path: PathBuf,
|
||||
state: Mutex<MetaState>,
|
||||
}
|
||||
|
||||
impl Meta {
|
||||
/// Load metadata from `store_dir/meta.json`, starting empty if absent.
|
||||
pub fn load(store_dir: &Path) -> Result<Meta> {
|
||||
let path = store_dir.join("meta.json");
|
||||
let state = match std::fs::read_to_string(&path) {
|
||||
Ok(text) => serde_json::from_str(&text)
|
||||
.with_context(|| format!("parsing {}", path.display()))?,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => MetaState::default(),
|
||||
Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
|
||||
};
|
||||
Ok(Meta {
|
||||
path,
|
||||
state: Mutex::new(state),
|
||||
})
|
||||
}
|
||||
|
||||
fn mutate<R>(&self, f: impl FnOnce(&mut MetaState) -> R) -> Result<R> {
|
||||
let mut state = self.state.lock().expect("meta lock poisoned");
|
||||
let result = f(&mut state);
|
||||
// Persist inside the lock so concurrent mutations can't write
|
||||
// stale snapshots over each other. The file is tiny.
|
||||
let json = serde_json::to_string_pretty(&*state)?;
|
||||
let tmp = self.path.with_extension("json.tmp");
|
||||
std::fs::write(&tmp, json).with_context(|| format!("writing {}", tmp.display()))?;
|
||||
std::fs::rename(&tmp, &self.path)
|
||||
.with_context(|| format!("renaming into {}", self.path.display()))?;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
/// Create or update a pin.
|
||||
pub fn pin(&self, hash: &str, policy: PinPolicy) -> Result<()> {
|
||||
self.mutate(|s| {
|
||||
s.pins.insert(hash.to_string(), PinRecord { policy });
|
||||
})
|
||||
}
|
||||
|
||||
/// Remove a pin. Returns false if there was none.
|
||||
pub fn unpin(&self, hash: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.pins.remove(hash).is_some())
|
||||
}
|
||||
|
||||
pub fn is_pinned(&self, hash: &str) -> bool {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.pins
|
||||
.contains_key(hash)
|
||||
}
|
||||
|
||||
/// Snapshot of all pins as (hash, record).
|
||||
pub fn pins(&self) -> Vec<(String, PinRecord)> {
|
||||
let state = self.state.lock().expect("meta lock poisoned");
|
||||
state
|
||||
.pins
|
||||
.iter()
|
||||
.map(|(h, r)| (h.clone(), r.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Record the format of a hash we learned about.
|
||||
pub fn record_format(&self, hash: &str, format: StoredFormat) -> Result<()> {
|
||||
self.mutate(|s| {
|
||||
s.formats.insert(hash.to_string(), format);
|
||||
})
|
||||
}
|
||||
|
||||
/// Best knowledge of the format of `hash`; defaults to `Raw`.
|
||||
pub fn format_of(&self, hash: &str) -> StoredFormat {
|
||||
let state = self.state.lock().expect("meta lock poisoned");
|
||||
state
|
||||
.formats
|
||||
.get(hash)
|
||||
.copied()
|
||||
.unwrap_or(StoredFormat::Raw)
|
||||
}
|
||||
|
||||
/// Add a trusted peer. Returns false if it was already trusted.
|
||||
pub fn trust_peer(&self, node_id: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.trusted_peers.insert(node_id.to_string()))
|
||||
}
|
||||
|
||||
/// Remove a trusted peer. Returns false if it was not trusted.
|
||||
pub fn untrust_peer(&self, node_id: &str) -> Result<bool> {
|
||||
self.mutate(|s| s.trusted_peers.remove(node_id))
|
||||
}
|
||||
|
||||
pub fn trusted_peers(&self) -> BTreeSet<String> {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.trusted_peers
|
||||
.clone()
|
||||
}
|
||||
|
||||
pub fn is_trusted(&self, node_id: &str) -> bool {
|
||||
self.state
|
||||
.lock()
|
||||
.expect("meta lock poisoned")
|
||||
.trusted_peers
|
||||
.contains(node_id)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn survives_reload() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
meta.pin("abc", PinPolicy { open_lan: true }).unwrap();
|
||||
meta.record_format("abc", StoredFormat::HashSeq).unwrap();
|
||||
meta.trust_peer("node1").unwrap();
|
||||
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
assert!(meta.is_pinned("abc"));
|
||||
assert_eq!(meta.format_of("abc"), StoredFormat::HashSeq);
|
||||
assert!(meta.is_trusted("node1"));
|
||||
assert_eq!(meta.format_of("unknown"), StoredFormat::Raw);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpin_reports_absence() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let meta = Meta::load(dir.path()).unwrap();
|
||||
assert!(!meta.unpin("nope").unwrap());
|
||||
meta.pin("h", PinPolicy::default()).unwrap();
|
||||
assert!(meta.unpin("h").unwrap());
|
||||
assert!(!meta.is_pinned("h"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,432 @@
|
||||
//! Blob store operations on top of the iroh-blobs persistent fs store.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use anyhow::Context;
|
||||
use iroh_blobs::format::collection::Collection;
|
||||
use iroh_blobs::hashseq::HashSeq;
|
||||
use iroh_blobs::store::{
|
||||
EntryStatus, ImportMode, Map, MapEntry, MapMut, ReadableStore, Store as _,
|
||||
};
|
||||
use iroh_blobs::util::local_pool::{LocalPool, LocalPoolHandle};
|
||||
use iroh_blobs::util::progress::IgnoreProgressSender;
|
||||
use iroh_blobs::{BlobFormat, Hash, HashAndFormat};
|
||||
use iroh_io::{AsyncSliceReader, AsyncSliceReaderExt};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tracing::{debug, info};
|
||||
use varde_proto::MaterializeMode;
|
||||
|
||||
use crate::meta::StoredFormat;
|
||||
|
||||
/// Operation errors that map onto protocol error codes.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum OpError {
|
||||
#[error("{0}")]
|
||||
NotFound(String),
|
||||
#[error("{0}")]
|
||||
InvalidArgument(String),
|
||||
#[error(transparent)]
|
||||
Io(#[from] std::io::Error),
|
||||
#[error(transparent)]
|
||||
Internal(#[from] anyhow::Error),
|
||||
#[error("{0}")]
|
||||
Unimplemented(String),
|
||||
}
|
||||
|
||||
/// The daemon's blob store: an iroh-blobs fs store rooted at
|
||||
/// `<store_dir>/blobs`.
|
||||
///
|
||||
/// Reads from store entries yield non-`Send` futures, so those operations
|
||||
/// run on a dedicated [`LocalPool`] — the same pattern iroh-blobs itself
|
||||
/// uses for its provider and GC tasks.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct BlobStore {
|
||||
store: iroh_blobs::store::fs::Store,
|
||||
data_dir: PathBuf,
|
||||
pool: LocalPoolHandle,
|
||||
// Owns the pool threads; dropped when the last clone goes away.
|
||||
_pool: std::sync::Arc<LocalPool>,
|
||||
}
|
||||
|
||||
/// Reading a blob entry for a streaming copy happens in chunks this size.
|
||||
const COPY_CHUNK: u64 = 1024 * 1024;
|
||||
|
||||
impl BlobStore {
|
||||
pub async fn open(store_dir: &Path) -> anyhow::Result<BlobStore> {
|
||||
let blobs_dir = store_dir.join("blobs");
|
||||
let store = iroh_blobs::store::fs::Store::load(&blobs_dir)
|
||||
.await
|
||||
.with_context(|| format!("opening blob store at {}", blobs_dir.display()))?;
|
||||
let pool = LocalPool::default();
|
||||
Ok(BlobStore {
|
||||
store,
|
||||
data_dir: blobs_dir.join("data"),
|
||||
pool: pool.handle().clone(),
|
||||
_pool: std::sync::Arc::new(pool),
|
||||
})
|
||||
}
|
||||
|
||||
/// Run a non-`Send` store operation on the local pool.
|
||||
async fn on_pool<T, F, Fut>(&self, f: F) -> Result<T, OpError>
|
||||
where
|
||||
F: FnOnce(BlobStore) -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = Result<T, OpError>> + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let this = self.clone();
|
||||
self.pool
|
||||
.spawn(move || f(this))
|
||||
.await
|
||||
.map_err(|e| OpError::Internal(anyhow::anyhow!("local pool: {e}")))?
|
||||
}
|
||||
|
||||
/// Access to the underlying iroh-blobs store (used by the transfer
|
||||
/// layer).
|
||||
pub fn inner(&self) -> &iroh_blobs::store::fs::Store {
|
||||
&self.store
|
||||
}
|
||||
|
||||
/// Import a file or directory. Returns the root hash, total imported
|
||||
/// bytes, and the resulting format (`Raw` for files, `HashSeq` for
|
||||
/// directories).
|
||||
pub async fn add_path(
|
||||
&self,
|
||||
path: &Path,
|
||||
recursive: bool,
|
||||
) -> Result<(Hash, u64, StoredFormat), OpError> {
|
||||
if !path.is_absolute() {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"path must be absolute: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let meta = std::fs::metadata(path)?;
|
||||
if meta.is_file() {
|
||||
let (tag, size) = self.import_one(path.to_owned()).await?;
|
||||
info!(hash = %tag.hash().to_hex(), size, "imported file");
|
||||
return Ok((*tag.hash(), size, StoredFormat::Raw));
|
||||
}
|
||||
if !meta.is_dir() {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"not a file or directory: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
if !recursive {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"{} is a directory; pass recursive=true",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
|
||||
let mut files = Vec::new();
|
||||
collect_files(path, path, &mut files)?;
|
||||
if files.is_empty() {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"directory {} contains no files",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
|
||||
// Temp tags keep the children alive until the collection root is
|
||||
// stored (and this method's caller records/pins the root).
|
||||
let mut children = Vec::new();
|
||||
let mut total = 0u64;
|
||||
for (name, file_path) in files {
|
||||
let (tag, size) = self.import_one(file_path).await?;
|
||||
total += size;
|
||||
children.push((name, tag));
|
||||
}
|
||||
let collection: Collection = children
|
||||
.iter()
|
||||
.map(|(name, tag)| (name.clone(), *tag.hash()))
|
||||
.collect();
|
||||
let root_tag = collection
|
||||
.store(&self.store)
|
||||
.await
|
||||
.context("storing collection")?;
|
||||
let root = *root_tag.hash();
|
||||
info!(hash = %root.to_hex(), files = children.len(), total, "imported directory");
|
||||
Ok((root, total, StoredFormat::HashSeq))
|
||||
}
|
||||
|
||||
async fn import_one(&self, path: PathBuf) -> Result<(iroh_blobs::TempTag, u64), OpError> {
|
||||
let (tag, size) = self
|
||||
.store
|
||||
.import_file(
|
||||
path,
|
||||
ImportMode::Copy,
|
||||
BlobFormat::Raw,
|
||||
IgnoreProgressSender::default(),
|
||||
)
|
||||
.await?;
|
||||
Ok((tag, size))
|
||||
}
|
||||
|
||||
/// Export `hash` to `dest`. Returns bytes written and whether at least
|
||||
/// one file was reflinked.
|
||||
pub async fn materialize(
|
||||
&self,
|
||||
hash: Hash,
|
||||
format: StoredFormat,
|
||||
dest: &Path,
|
||||
mode: MaterializeMode,
|
||||
) -> Result<(u64, bool), OpError> {
|
||||
if !dest.is_absolute() {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"dest must be absolute: {}",
|
||||
dest.display()
|
||||
)));
|
||||
}
|
||||
let dest = dest.to_owned();
|
||||
self.on_pool(
|
||||
move |this| async move { this.materialize_local(hash, format, &dest, mode).await },
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn materialize_local(
|
||||
&self,
|
||||
hash: Hash,
|
||||
format: StoredFormat,
|
||||
dest: &Path,
|
||||
mode: MaterializeMode,
|
||||
) -> Result<(u64, bool), OpError> {
|
||||
let allow_reflink = matches!(mode, MaterializeMode::ReflinkOrCopy);
|
||||
match format {
|
||||
StoredFormat::Raw => self.export_blob(hash, dest, allow_reflink).await,
|
||||
StoredFormat::HashSeq => {
|
||||
let collection = Collection::load_db(&self.store, &hash)
|
||||
.await
|
||||
.map_err(|e| OpError::NotFound(format!("loading collection: {e}")))?;
|
||||
let mut total = 0u64;
|
||||
let mut any_reflink = false;
|
||||
tokio::fs::create_dir_all(dest).await?;
|
||||
for (name, child) in collection.iter() {
|
||||
let rel = sanitize_collection_name(name)?;
|
||||
let (bytes, reflinked) = self
|
||||
.export_blob(*child, &dest.join(rel), allow_reflink)
|
||||
.await?;
|
||||
total += bytes;
|
||||
any_reflink |= reflinked;
|
||||
}
|
||||
Ok((total, any_reflink))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Export one blob. Tries `FICLONE` via reflink first (zero-copy on
|
||||
/// btrfs/XFS when store and dest share a filesystem), falls back to a
|
||||
/// streaming copy. Never hardlinks: store files must stay immutable.
|
||||
async fn export_blob(
|
||||
&self,
|
||||
hash: Hash,
|
||||
dest: &Path,
|
||||
allow_reflink: bool,
|
||||
) -> Result<(u64, bool), OpError> {
|
||||
let entry =
|
||||
self.store.get(&hash).await?.ok_or_else(|| {
|
||||
OpError::NotFound(format!("{} is not in the store", hash.to_hex()))
|
||||
})?;
|
||||
if !entry.is_complete() {
|
||||
return Err(OpError::NotFound(format!(
|
||||
"{} is only partially present",
|
||||
hash.to_hex()
|
||||
)));
|
||||
}
|
||||
let size = entry.size().value();
|
||||
if let Some(parent) = dest.parent() {
|
||||
tokio::fs::create_dir_all(parent).await?;
|
||||
}
|
||||
// Materialize replaces the destination, it never appends.
|
||||
match tokio::fs::remove_file(dest).await {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
|
||||
if allow_reflink {
|
||||
// Complete blobs above the inline threshold live as plain
|
||||
// files at data/<hex>.data; those we can clone directly.
|
||||
// Inline (small) blobs have no backing file and are copied.
|
||||
let src = self.data_dir.join(format!("{}.data", hash.to_hex()));
|
||||
let src_ok = std::fs::metadata(&src)
|
||||
.map(|m| m.len() == size)
|
||||
.unwrap_or(false);
|
||||
if src_ok {
|
||||
match reflink_copy::reflink(&src, dest) {
|
||||
Ok(()) => {
|
||||
debug!(hash = %hash.to_hex(), dest = %dest.display(), "reflinked");
|
||||
return Ok((size, true));
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(error = %e, "reflink failed, falling back to copy");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut reader = entry.data_reader();
|
||||
let mut file = tokio::fs::File::create(dest).await?;
|
||||
let mut offset = 0u64;
|
||||
while offset < size {
|
||||
let len = (size - offset).min(COPY_CHUNK) as usize;
|
||||
let chunk = reader.read_at(offset, len).await?;
|
||||
if chunk.is_empty() {
|
||||
return Err(OpError::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
format!("blob {} truncated at {offset}", hash.to_hex()),
|
||||
)));
|
||||
}
|
||||
file.write_all(&chunk).await?;
|
||||
offset += chunk.len() as u64;
|
||||
}
|
||||
file.flush().await?;
|
||||
Ok((size, false))
|
||||
}
|
||||
|
||||
/// Presence of `hash`: (have_bytes, total_bytes, complete).
|
||||
pub async fn presence(&self, hash: Hash) -> Result<(u64, Option<u64>, bool), OpError> {
|
||||
match self.store.entry_status(&hash).await? {
|
||||
EntryStatus::NotFound => Ok((0, None, false)),
|
||||
EntryStatus::Partial => {
|
||||
let total = self.store.get(&hash).await?.map(|e| e.size().value());
|
||||
// Valid-range accounting for partials arrives with the
|
||||
// transfer milestone; absence of data is the safe report.
|
||||
Ok((0, total, false))
|
||||
}
|
||||
EntryStatus::Complete => {
|
||||
let entry = self
|
||||
.store
|
||||
.get(&hash)
|
||||
.await?
|
||||
.ok_or_else(|| OpError::NotFound(hash.to_hex().to_string()))?;
|
||||
let size = entry.size().value();
|
||||
Ok((size, Some(size), true))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop every blob not reachable from `roots` (mark and sweep).
|
||||
/// In-flight imports are protected by their temp tags; tags stored in
|
||||
/// the blob database are honored too.
|
||||
pub async fn gc(&self, roots: Vec<HashAndFormat>) -> Result<u64, OpError> {
|
||||
self.on_pool(move |this| async move { this.gc_local(roots).await })
|
||||
.await
|
||||
}
|
||||
|
||||
async fn gc_local(&self, roots: Vec<HashAndFormat>) -> Result<u64, OpError> {
|
||||
let mut live: BTreeSet<Hash> = BTreeSet::new();
|
||||
let mut all_roots = roots;
|
||||
all_roots.extend(self.store.temp_tags());
|
||||
for item in self.store.tags(None, None).await.context(TAGS_CONTEXT)? {
|
||||
let (_name, haf) = item.context(TAGS_CONTEXT)?;
|
||||
all_roots.push(haf);
|
||||
}
|
||||
|
||||
for HashAndFormat { hash, format } in all_roots {
|
||||
if !live.insert(hash) || format.is_raw() {
|
||||
continue;
|
||||
}
|
||||
// HashSeq root: its children are live too. A partial root
|
||||
// can't be expanded; its bytes are still protected.
|
||||
let Some(entry) = self.store.get(&hash).await? else {
|
||||
continue;
|
||||
};
|
||||
if !entry.is_complete() {
|
||||
continue;
|
||||
}
|
||||
let mut reader = entry.data_reader();
|
||||
let bytes = reader.read_to_end().await?;
|
||||
let seq = HashSeq::try_from(bytes)
|
||||
.map_err(|e| OpError::Internal(anyhow::anyhow!("invalid hashseq: {e}")))?;
|
||||
live.extend(seq.iter());
|
||||
}
|
||||
|
||||
let mut doomed = Vec::new();
|
||||
for hash in self
|
||||
.store
|
||||
.blobs()
|
||||
.await?
|
||||
.chain(self.store.partial_blobs().await?)
|
||||
{
|
||||
let hash = hash?;
|
||||
if !live.contains(&hash) {
|
||||
doomed.push(hash);
|
||||
}
|
||||
}
|
||||
let removed = doomed.len() as u64;
|
||||
if !doomed.is_empty() {
|
||||
self.store.delete(doomed).await?;
|
||||
}
|
||||
info!(removed, live = live.len(), "gc done");
|
||||
Ok(removed)
|
||||
}
|
||||
|
||||
/// Flush and shut down the store actor.
|
||||
pub async fn shutdown(&self) {
|
||||
self.store.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
const TAGS_CONTEXT: &str = "listing tags";
|
||||
|
||||
/// Recursively collect regular files under `dir` as (relative-name, path),
|
||||
/// sorted for deterministic collection hashes. Symlinks are followed for
|
||||
/// files; symlinked directories are rejected to avoid cycles.
|
||||
fn collect_files(root: &Path, dir: &Path, out: &mut Vec<(String, PathBuf)>) -> Result<(), OpError> {
|
||||
let mut entries: Vec<_> =
|
||||
std::fs::read_dir(dir)?.collect::<Result<Vec<_>, std::io::Error>>()?;
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
for entry in entries {
|
||||
let path = entry.path();
|
||||
let file_type = entry.file_type()?;
|
||||
if file_type.is_dir() {
|
||||
collect_files(root, &path, out)?;
|
||||
} else if file_type.is_symlink() && std::fs::metadata(&path)?.is_dir() {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"symlinked directory not supported: {}",
|
||||
path.display()
|
||||
)));
|
||||
} else {
|
||||
let rel = path
|
||||
.strip_prefix(root)
|
||||
.expect("walked path is under root")
|
||||
.components()
|
||||
.map(|c| c.as_os_str().to_string_lossy())
|
||||
.collect::<Vec<_>>()
|
||||
.join("/");
|
||||
out.push((rel, path));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Turn a collection entry name into a safe relative path.
|
||||
fn sanitize_collection_name(name: &str) -> Result<PathBuf, OpError> {
|
||||
let mut path = PathBuf::new();
|
||||
for comp in name.split('/') {
|
||||
if comp.is_empty() || comp == "." || comp == ".." {
|
||||
return Err(OpError::InvalidArgument(format!(
|
||||
"unsafe name in collection: {name:?}"
|
||||
)));
|
||||
}
|
||||
path.push(comp);
|
||||
}
|
||||
Ok(path)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn collection_names_are_sanitized() {
|
||||
assert!(sanitize_collection_name("a/b.txt").is_ok());
|
||||
assert!(sanitize_collection_name("../etc/passwd").is_err());
|
||||
assert!(sanitize_collection_name("/abs").is_err());
|
||||
assert!(sanitize_collection_name("a//b").is_err());
|
||||
assert!(sanitize_collection_name("a/./b").is_err());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user