use std::collections::HashMap;
use std::path::PathBuf;
use walkdir::WalkDir;
pub fn group_by_size(root: &str) -> HashMap<u64, Vec<PathBuf>> {
let mut by_size: HashMap<u64, Vec<PathBuf>> = HashMap::new();
for entry in WalkDir::new(root).follow_links(false) {
let entry = match entry {
Ok(e) => e,
Err(err) => {
eprintln!("skip: {}", err);
continue;
}
};
if !entry.file_type().is_file() {
continue;
}
let len = match entry.metadata() {
Ok(meta) => meta.len(),
Err(_) => continue,
};
by_size.entry(len).or_default().push(entry.into_path());
}
by_size
}
use std::fs::File;
use std::io::{self, Read};
use std::path::Path;
const CHUNK: usize = 64 * 1024;
pub fn hash_file(path: &Path) -> io::Result<String> {
let mut file = File::open(path)?;
let mut hasher = blake3::Hasher::new();
let mut buf = [0u8; CHUNK];
loop {
let read = file.read(&mut buf)?;
if read == 0 {
break;
}
hasher.update(&buf[..read]);
}
Ok(hasher.finalize().to_hex().to_string())
}
use std::collections::HashMap;
use std::path::PathBuf;
use rayon::prelude::*;
use crate::hasher::hash_file;
use crate::scanner::group_by_size;
pub fn find_duplicates(root: &str) -> HashMap<String, Vec<PathBuf>> {
let by_size = group_by_size(root);
let candidates: Vec<PathBuf> = by_size
.into_iter()
.filter(|(_, paths)| paths.len() > 1)
.flat_map(|(_, paths)| paths)
.collect();
let hashed: Vec<(String, PathBuf)> = candidates
.par_iter()
.filter_map(|path| match hash_file(path) {
Ok(digest) => Some((digest, path.clone())),
Err(err) => {
eprintln!("cannot hash {}: {}", path.display(), err);
None
}
})
.collect();
let mut groups: HashMap<String, Vec<PathBuf>> = HashMap::new();
for (digest, path) in hashed {
groups.entry(digest).or_default().push(path);
}
groups.retain(|_, paths| paths.len() > 1);
groups
}
mod dedup;
mod hasher;
mod scanner;
use std::env;
use std::process;
fn main() {
let root = env::args().nth(1).unwrap_or_else(|| {
eprintln!("usage: dupfinder <directory>");
process::exit(1);
});
let duplicates = dedup::find_duplicates(&root);
if duplicates.is_empty() {
println!("no duplicates found under {}", root);
return;
}
for (digest, paths) in &duplicates {
println!("\n{} ({} copies)", &digest[..12], paths.len());
for path in paths {
println!(" {}", path.display());
}
}
}
This snippet implements a two-phase duplicate-file detector, a pattern used by backup tools and disk cleaners to find byte-identical files across a directory tree. Hashing every file is expensive, so the design leans on a cheap pre-filter: files can only be duplicates if they share the same size. scanner.rs walks the tree and groups paths by length, and only groups with more than one member are ever hashed.
In scanner.rs, group_by_size uses walkdir::WalkDir to iterate the tree, skips anything that is not a regular file, and folds paths into a HashMap<u64, Vec<PathBuf>> keyed on metadata.len(). Symlinks are ignored via follow_links(false) so the same inode is not counted twice and cycles cannot trap the walk. The returned map still contains singleton size-buckets; those are cheap to discard later and keeping them keeps this function pure and testable.
hasher.rs holds the actual content hashing. hash_file streams the file through a fixed 64 KiB buffer into a blake3::Hasher, which avoids loading large files fully into memory. Blake3 is chosen over MD5/SHA for speed and because it is cryptographically strong enough that a hash collision is not a practical concern, so matching digests are treated as identical content without a fallback byte comparison. The digest is returned as a hex String for easy use as a map key.
dedup.rs ties the phases together and adds parallelism. find_duplicates first calls group_by_size, then uses rayon's par_iter to hash candidate files concurrently — I/O and CPU-bound hashing scale well across cores here. Files are collected into a HashMap<String, Vec<PathBuf>> keyed by digest, and retain drops any digest seen only once, leaving true duplicate sets. Errors from unreadable files are logged and skipped rather than aborting the whole run, which matters when scanning system directories with permission gaps.
The main trade-off is that the size pre-filter assumes duplicates share exact byte length, which is always true for identical files, making it a safe optimization. The approach reaches for parallel hashing only where it pays off, keeping the common case of unique-sized files nearly free.
Related snips
struct Config<'a> {
name: &'a str,
value: &'a str,
}
fn parse_config(line: &str) -> Config {Lifetime annotations for flexible borrowing in structs
use crossbeam::channel::unbounded;
use std::thread;
fn main() {
let (tx, rx) = unbounded();Crossbeam for advanced concurrent data structures
use tracing::{info, instrument};
#[instrument]
fn process_request(user_id: u64) {
info!(user_id, "Processing request");
// Work happens heretracing for structured logging and distributed tracing
use std::sync::mpsc;
use std::thread;
fn main() {
let (tx, rx) = mpsc::channel();Channels (mpsc) for message passing between threads
use clap::Parser;
#[derive(Parser, Debug)]
#[command(author, version, about)]
struct Args {
#[arg(short, long)]clap for CLI argument parsing with derive macros
macro_rules! create_function {
($func_name:ident) => {
fn $func_name() {
println!("Called {}", stringify!($func_name));
}
};Declarative macros (macro_rules!) for code generation
Share this code
Here's the card — post it anywhere.
