Milestone 2: persistent blob store, add/materialize/gc
iroh-blobs 0.35 fs store under <store_dir>/blobs. Add imports files or whole directory trees (as Collections, deterministic order), Materialize exports them with a real FICLONE reflink attempt and streaming-copy fallback, Gc is an explicit mark-and-sweep rooted at the pins. Pins, trusted peers and hash formats persist in meta.json (atomic writes). Store reads run on a LocalPool because iroh-blobs entry readers are not Send. Integration tests drive the real daemon binary: directory round trip byte-identical, gc keeps pinned/drops unpinned, reflink verified on the repo's own filesystem (XFS), plus a 32-case proptest round trip. Dependencies: iroh-blobs =0.35.0 (the store itself; pinned per spec), iroh-io (AsyncSliceReader traits to read store entries), reflink-copy (FICLONE with copy fallback, per spec), proptest (dev-only, round-trip property test). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+142
-11
@@ -1,29 +1,58 @@
|
||||
//! Request dispatch and daemon state.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::str::FromStr;
|
||||
|
||||
use anyhow::Context;
|
||||
use iroh_blobs::{BlobFormat, Hash, HashAndFormat};
|
||||
use tokio::sync::broadcast;
|
||||
use varde_proto::{Envelope, ErrorCode, Event, GlobalStatus, Request, ResponseData};
|
||||
use varde_proto::{
|
||||
Envelope, ErrorCode, Event, GlobalStatus, HashStatus, PinInfo, Request, ResponseData,
|
||||
};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::meta::{Meta, StoredFormat};
|
||||
use crate::store::{BlobStore, OpError};
|
||||
|
||||
/// Shared daemon state. Handlers grow with the milestones; anything not
|
||||
/// yet wired returns [`ErrorCode::Unimplemented`] rather than pretending.
|
||||
pub struct Daemon {
|
||||
config: Config,
|
||||
store: BlobStore,
|
||||
meta: Meta,
|
||||
events: broadcast::Sender<Event>,
|
||||
}
|
||||
|
||||
impl Daemon {
|
||||
pub fn new(config: Config) -> Self {
|
||||
/// Open the blob store and metadata under the configured store dir.
|
||||
pub async fn open(config: Config) -> anyhow::Result<Daemon> {
|
||||
std::fs::create_dir_all(&config.store_dir)
|
||||
.with_context(|| format!("creating store directory {}", config.store_dir.display()))?;
|
||||
let store = BlobStore::open(&config.store_dir).await?;
|
||||
let meta = Meta::load(&config.store_dir)?;
|
||||
// Capacity bounds memory if a subscriber stalls; laggards get a
|
||||
// Lagged error, not unbounded buffering.
|
||||
let (events, _) = broadcast::channel(1024);
|
||||
Daemon { config, events }
|
||||
Ok(Daemon {
|
||||
config,
|
||||
store,
|
||||
meta,
|
||||
events,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn config(&self) -> &Config {
|
||||
&self.config
|
||||
}
|
||||
|
||||
pub fn store(&self) -> &BlobStore {
|
||||
&self.store
|
||||
}
|
||||
|
||||
pub fn meta(&self) -> &Meta {
|
||||
&self.meta
|
||||
}
|
||||
|
||||
/// New receiver for the daemon event stream.
|
||||
pub fn subscribe_events(&self) -> broadcast::Receiver<Event> {
|
||||
self.events.subscribe()
|
||||
@@ -32,21 +61,123 @@ impl Daemon {
|
||||
/// Handle one API request. Infallible at this level: all failures are
|
||||
/// mapped to structured error envelopes.
|
||||
pub async fn handle(&self, request: Request) -> Envelope {
|
||||
match self.dispatch(request).await {
|
||||
Ok(data) => Envelope::ok(data),
|
||||
Err(e) => Envelope::err(error_code(&e), e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn dispatch(&self, request: Request) -> Result<ResponseData, OpError> {
|
||||
match request {
|
||||
Request::Status { hash: None } => Envelope::ok(ResponseData::Status(GlobalStatus {
|
||||
Request::Add { path, recursive } => {
|
||||
let (hash, bytes, format) =
|
||||
self.store.add_path(&PathBuf::from(path), recursive).await?;
|
||||
self.meta.record_format(&hash.to_hex(), format)?;
|
||||
Ok(ResponseData::Added {
|
||||
hash: hash.to_hex().to_string(),
|
||||
bytes,
|
||||
})
|
||||
}
|
||||
Request::Pin { hash, policy } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
self.meta.pin(&parsed.to_hex(), policy)?;
|
||||
// Fetching absent pinned content arrives with the
|
||||
// transfer milestone.
|
||||
Ok(ResponseData::Pinned {
|
||||
hash: parsed.to_hex().to_string(),
|
||||
})
|
||||
}
|
||||
Request::Unpin { hash } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
if !self.meta.unpin(&parsed.to_hex())? {
|
||||
return Err(OpError::NotFound(format!("no pin for {}", parsed.to_hex())));
|
||||
}
|
||||
Ok(ResponseData::Done {})
|
||||
}
|
||||
Request::Materialize { hash, dest, mode } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let format = self.meta.format_of(&parsed.to_hex());
|
||||
let (bytes, reflinked) = self
|
||||
.store
|
||||
.materialize(parsed, format, &PathBuf::from(dest), mode)
|
||||
.await?;
|
||||
Ok(ResponseData::Materialized { bytes, reflinked })
|
||||
}
|
||||
Request::Status { hash: None } => Ok(ResponseData::Status(GlobalStatus {
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
protocol: varde_proto::PROTOCOL_VERSION,
|
||||
node_id: None,
|
||||
store_path: self.config.store_dir.display().to_string(),
|
||||
pins: 0,
|
||||
pins: self.meta.pins().len() as u64,
|
||||
connected_peers: 0,
|
||||
})),
|
||||
Request::List {} => Envelope::ok(ResponseData::Pins { pins: Vec::new() }),
|
||||
Request::Subscribe {} => Envelope::ok(ResponseData::Done {}),
|
||||
other => Envelope::err(
|
||||
ErrorCode::Unimplemented,
|
||||
format!("not implemented yet: {other:?}"),
|
||||
),
|
||||
Request::Status { hash: Some(hash) } => {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let (have_bytes, total_bytes, complete) = self.store.presence(parsed).await?;
|
||||
Ok(ResponseData::HashStatus(HashStatus {
|
||||
hash: parsed.to_hex().to_string(),
|
||||
have_bytes,
|
||||
total_bytes,
|
||||
complete,
|
||||
pinned: self.meta.is_pinned(&parsed.to_hex()),
|
||||
active_peers: Vec::new(),
|
||||
}))
|
||||
}
|
||||
Request::List {} => {
|
||||
let mut pins = Vec::new();
|
||||
for (hash, record) in self.meta.pins() {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let (have_bytes, total_bytes, complete) = self.store.presence(parsed).await?;
|
||||
pins.push(PinInfo {
|
||||
hash,
|
||||
policy: record.policy,
|
||||
complete,
|
||||
have_bytes,
|
||||
total_bytes,
|
||||
});
|
||||
}
|
||||
Ok(ResponseData::Pins { pins })
|
||||
}
|
||||
Request::Gc {} => {
|
||||
let roots = self.pin_roots()?;
|
||||
let blobs_removed = self.store.gc(roots).await?;
|
||||
Ok(ResponseData::GcDone { blobs_removed })
|
||||
}
|
||||
Request::Subscribe {} => Ok(ResponseData::Done {}),
|
||||
other => Err(OpError::Unimplemented(format!(
|
||||
"not implemented yet: {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
/// The GC roots: every pinned hash with its recorded format.
|
||||
fn pin_roots(&self) -> Result<Vec<HashAndFormat>, OpError> {
|
||||
let mut roots = Vec::new();
|
||||
for (hash, _record) in self.meta.pins() {
|
||||
let parsed = parse_hash(&hash)?;
|
||||
let format = match self.meta.format_of(&hash) {
|
||||
StoredFormat::Raw => BlobFormat::Raw,
|
||||
StoredFormat::HashSeq => BlobFormat::HashSeq,
|
||||
};
|
||||
roots.push(HashAndFormat {
|
||||
hash: parsed,
|
||||
format,
|
||||
});
|
||||
}
|
||||
Ok(roots)
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hash(s: &str) -> Result<Hash, OpError> {
|
||||
Hash::from_str(s).map_err(|e| OpError::InvalidArgument(format!("invalid hash {s:?}: {e}")))
|
||||
}
|
||||
|
||||
fn error_code(e: &OpError) -> ErrorCode {
|
||||
match e {
|
||||
OpError::NotFound(_) => ErrorCode::NotFound,
|
||||
OpError::InvalidArgument(_) => ErrorCode::InvalidArgument,
|
||||
OpError::Io(_) => ErrorCode::Io,
|
||||
OpError::Internal(_) => ErrorCode::Internal,
|
||||
OpError::Unimplemented(_) => ErrorCode::Unimplemented,
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user