use std::collections::HashMap;
#[derive(Debug, PartialEq)]
pub enum LogLevel {
Info,
Warn,
Error,
}
impl LogLevel {
fn from_str(s: &str) -> Option<LogLevel> {
match s {
"INFO" => Some(LogLevel::Info),
"WARN" => Some(LogLevel::Warn),
"ERROR" => Some(LogLevel::Error),
_ => None,
}
}
}
pub struct ParsedLine {
pub level: LogLevel,
pub endpoint: String,
pub latency_ms: u64,
}
impl ParsedLine {
// Format: "<ts> <LEVEL> <endpoint> <latency_ms>"
pub fn parse(line: &str) -> Option<ParsedLine> {
let mut parts = line.splitn(4, ' ');
let _ts = parts.next()?;
let level = LogLevel::from_str(parts.next()?)?;
let endpoint = parts.next()?;
let latency = parts.next()?.trim().parse::<u64>().ok()?;
Some(ParsedLine {
level,
endpoint: endpoint.to_string(),
latency_ms: latency,
})
}
}
#[derive(Default)]
pub struct Stats {
pub total_lines: u64,
pub parse_failures: u64,
pub error_count: u64,
pub latency_by_endpoint: HashMap<String, (u64, u64)>,
}
use crate::log_line::{LogLevel, ParsedLine, Stats};
use std::fs::File;
use std::io::{self, BufRead, BufReader};
use std::path::Path;
pub fn aggregate_file<P: AsRef<Path>>(path: P) -> io::Result<Stats> {
let file = File::open(path)?;
let reader = BufReader::new(file);
let mut stats = Stats::default();
for line_result in reader.lines() {
let line = line_result?; // real I/O errors abort the run
stats.total_lines += 1;
let parsed = match ParsedLine::parse(&line) {
Some(p) => p,
None => {
stats.parse_failures += 1;
continue;
}
};
if parsed.level == LogLevel::Error {
stats.error_count += 1;
}
let entry = stats
.latency_by_endpoint
.entry(parsed.endpoint)
.or_insert((0, 0));
entry.0 += parsed.latency_ms;
entry.1 += 1;
}
Ok(stats)
}
mod aggregator;
mod log_line;
use std::env;
use std::process;
fn main() {
let path = match env::args().nth(1) {
Some(p) => p,
None => {
eprintln!("usage: logagg <path-to-log>");
process::exit(2);
}
};
let stats = match aggregator::aggregate_file(&path) {
Ok(s) => s,
Err(e) => {
eprintln!("failed to process {}: {}", path, e);
process::exit(1);
}
};
println!("lines: {}", stats.total_lines);
println!("skipped: {}", stats.parse_failures);
println!("errors: {}", stats.error_count);
let mut endpoints: Vec<_> = stats.latency_by_endpoint.iter().collect();
endpoints.sort_by(|a, b| b.1 .1.cmp(&a.1 .1));
for (endpoint, (total, count)) in endpoints.iter().take(10) {
let avg = if *count > 0 { total / count } else { 0 };
println!("{:<24} n={:<8} avg_ms={}", endpoint, count, avg);
}
}
This snippet shows how to process arbitrarily large log files in constant memory by streaming them line by line rather than reading the whole file into a String. The core insight, visible across all three tabs, is that BufReader::lines() yields one owned String at a time, so the peak memory footprint is bounded by the longest line plus a small buffer — even a multi-gigabyte file never fully lands in RAM.
In the log_line parser tab, LogLevel and ParsedLine model just the fields the aggregator cares about. ParsedLine::parse does a cheap splitn-based split instead of a regex, returning Option so malformed lines are skipped rather than aborting the run. Keeping the parser allocation-light matters because it runs once per line; borrowing with &str and only materializing an owned endpoint String avoids copying the entire input.
The stream aggregator tab holds the actual streaming loop. aggregate_file opens the file, wraps it in a BufReader, and drives reader.lines() as an iterator. Each item is a Result<String, io::Error> because I/O can fail mid-stream (a truncated network mount, a bad sector), so the ? on line_result propagates real read errors while ParsedLine::parse(...) quietly drops unparseable rows. The running Stats struct accumulates counts, error tallies, and a per-endpoint HashMap of latencies — state that grows with the number of distinct endpoints, not the number of lines, which is the property that keeps memory flat. A key pitfall this avoids is fs::read_to_string, which would defeat the entire purpose by buffering the whole file.
The main entrypoint tab wires it into a CLI: it reads the path from env::args, calls aggregate_file, and prints a summary, converting any error into a non-zero exit via process::exit. The trade-off of the line-by-line approach is that it assumes newline-delimited records; multi-line stack traces would need a stateful reader. For the common case of one event per line, this pattern gives predictable memory, early failure on I/O errors, and clean separation between parsing and aggregation.
Related snips
struct Config<'a> {
name: &'a str,
value: &'a str,
}
fn parse_config(line: &str) -> Config {Lifetime annotations for flexible borrowing in structs
require "csv"
class PeopleCsvStream
include Enumerable
HEADERS = %w[id full_name email signed_up_at plan].freezeResilient CSV Export as a Streamed Response
use crossbeam::channel::unbounded;
use std::thread;
fn main() {
let (tx, rx) = unbounded();Crossbeam for advanced concurrent data structures
// Creating a Promise
const myPromise = new Promise((resolve, reject) => {
const success = true;
setTimeout(() => {
if (success) {Promises and async/await patterns for asynchronous JavaScript
use tracing::{info, instrument};
#[instrument]
fn process_request(user_id: u64) {
info!(user_id, "Processing request");
// Work happens heretracing for structured logging and distributed tracing
use std::sync::mpsc;
use std::thread;
fn main() {
let (tx, rx) = mpsc::channel();Channels (mpsc) for message passing between threads
Share this code
Here's the card — post it anywhere.
