'use strict';
const fs = require('fs');
const path = require('path');
const zlib = require('zlib');
const crypto = require('crypto');
const { pipeline } = require('stream');
const { promisify } = require('util');
const pipelineAsync = promisify(pipeline);
function safeName(original) {
const base = path.basename(original || 'upload').replace(/[^a-zA-Z0-9._-]/g, '_');
const token = crypto.randomBytes(6).toString('hex');
return `${token}-${base}.gz`;
}
async function storeCompressed(source, { dir, filename }) {
const name = safeName(filename);
const finalPath = path.join(dir, name);
const tmpPath = `${finalPath}.part`;
const gzip = zlib.createGzip({ level: zlib.constants.Z_BEST_SPEED });
const out = fs.createWriteStream(tmpPath);
try {
await pipelineAsync(source, gzip, out);
await fs.promises.rename(tmpPath, finalPath);
} catch (err) {
await fs.promises.unlink(tmpPath).catch(() => {});
throw err;
}
const { size } = await fs.promises.stat(finalPath);
return { name, path: finalPath, compressedBytes: size };
}
module.exports = { storeCompressed };
'use strict';
const express = require('express');
const Busboy = require('busboy');
const { storeCompressed } = require('./uploadStore');
const UPLOAD_DIR = process.env.UPLOAD_DIR || '/var/data/uploads';
const router = express.Router();
router.post('/uploads', (req, res, next) => {
const busboy = Busboy({
headers: req.headers,
limits: { files: 1, fileSize: 200 * 1024 * 1024 },
});
let pending = null;
busboy.on('file', (fieldname, file, info) => {
if (fieldname !== 'artifact') {
file.resume(); // drain unwanted stream so parsing continues
return;
}
file.on('limit', () => {
file.destroy(new Error('file exceeds maximum allowed size'));
});
pending = storeCompressed(file, {
dir: UPLOAD_DIR,
filename: info.filename,
});
});
busboy.on('error', next);
busboy.on('finish', async () => {
if (!pending) {
return res.status(400).json({ error: 'missing "artifact" file field' });
}
try {
const stored = await pending;
res.status(201).json(stored);
} catch (err) {
next(err);
}
});
req.pipe(busboy);
});
module.exports = router;
This snippet shows the correct way to accept a large file upload and write it to disk as a gzip-compressed artifact without ever buffering the whole payload in memory. The core idea is a stream pipeline: the incoming request body is a Readable, gzip is a Transform, and the destination file is a Writable, and stream.pipeline wires them together while propagating backpressure and errors across every stage.
In uploadStore.js, the storeCompressed helper wraps pipeline in its promisified form via util.promisify. Using pipeline rather than a chain of .pipe() calls matters because .pipe() does not forward errors or destroy upstream sources when a downstream stage fails, which classically leaks file descriptors and leaves half-written files. pipeline guarantees that if any stage errors — the socket aborts, the disk fills, gzip throws — every stream in the chain is destroyed. The helper writes to a temporary .part path first and only renames into place on success, so consumers never observe a truncated file; on failure it unlinks the partial and rethrows. The rename is atomic on the same filesystem, which is what makes this crash-safe.
uploadStore.js also derives a safe on-disk name and returns the compressed byte count read from fs.stat, giving the caller something to persist.
In uploadRoutes.js, busboy parses the multipart stream and emits a file event whose value is itself a Readable. That readable is handed straight to storeCompressed, so bytes flow request → busboy → gzip → disk with backpressure intact and nothing accumulating in RAM. A crucial detail is draining unused file streams with file.resume() when the field name is unexpected; an unconsumed busboy stream stalls the whole parse. The route awaits the returned promise, responds with the stored metadata, and lets a rejected pipeline propagate to Express error handling.
The trade-off is that streaming precludes reading the full body before deciding to reject it, so size limits are enforced incrementally via busboy limits rather than after the fact. This pattern is the right reach whenever uploads may be large, memory is constrained, or partial writes must never survive a crash.
Related snips
export interface RetryOptions {
retries: number;
baseMs: number;
maxMs: number;
signal?: AbortSignal;
onRetry?: (attempt: number, delay: number, err: unknown) => void;Exponential backoff with jitter for retries
package dbutil
import (
"context"
"github.com/jackc/pgconn"Retry Postgres serialization failures with bounded attempts
// Creating a Promise
const myPromise = new Promise((resolve, reject) => {
const success = true;
setTimeout(() => {
if (success) {Promises and async/await patterns for asynchronous JavaScript
export type Settled<R> =
| { status: 'fulfilled'; value: R }
| { status: 'rejected'; reason: unknown };
export interface ConcurrencyOptions {
limit: number;Simple concurrency limiter for batch operations
import { randomBytes, createHash } from "crypto";
import jwt from "jsonwebtoken";
import { RefreshTokenStore } from "./store";
const ACCESS_SECRET = process.env.ACCESS_SECRET!;
const ACCESS_TTL = "15m";JWT access + refresh token rotation (conceptual)
# Installation
# rails active_storage:install
# rails db:migrate
# config/storage.yml
local:ActiveStorage for file uploads and attachments
Share this code
Here's the card — post it anywhere.
