javascript 96 lines · 3 tabs

Stream and Import a Large CSV File Line-by-Line with Node.js readline

Shared by codesnips Jul 2026

3 tabs

const fs = require('fs');
const readline = require('readline');

async function* streamCsvRows(filePath) {
  const stream = fs.createReadStream(filePath, { encoding: 'utf8' });
  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });

  let header = null;

  for await (const rawLine of rl) {
    const line = rawLine.trim();
    if (line === '') continue;

    const cells = line.split(',');

    if (header === null) {
      header = cells.map((c) => c.trim());
      continue;
    }

    const row = {};
    for (let i = 0; i < header.length; i++) {
      row[header[i]] = cells[i] !== undefined ? cells[i].trim() : null;
    }
    yield row;
  }
}

module.exports = { streamCsvRows };

const { streamCsvRows } = require('./csvLineStream');

const BATCH_SIZE = 500;

async function flush(pool, rows) {
  if (rows.length === 0) return;

  const values = [];
  const placeholders = rows.map((row, r) => {
    const base = r * 3;
    values.push(row.email, row.name, row.country);
    return `($${base + 1}, $${base + 2}, $${base + 3})`;
  });

  await pool.query(
    `INSERT INTO users (email, name, country)
     VALUES ${placeholders.join(', ')}
     ON CONFLICT (email) DO NOTHING`,
    values
  );
}

async function importCsv(pool, filePath) {
  let batch = [];
  let imported = 0;

  for await (const row of streamCsvRows(filePath)) {
    batch.push(row);
    if (batch.length >= BATCH_SIZE) {
      await flush(pool, batch);
      imported += batch.length;
      batch = [];
    }
  }

  await flush(pool, batch);
  imported += batch.length;
  return imported;
}

module.exports = { importCsv };

const { Pool } = require('pg');
const { importCsv } = require('./batchImport');

async function main() {
  const filePath = process.argv[2];
  if (!filePath) {
    console.error('Usage: node import-users.js <path-to-csv>');
    process.exit(1);
  }

  const pool = new Pool({ connectionString: process.env.DATABASE_URL });

  try {
    const started = Date.now();
    const count = await importCsv(pool, filePath);
    const seconds = ((Date.now() - started) / 1000).toFixed(1);
    console.log(`Imported ${count} rows in ${seconds}s`);
  } catch (err) {
    console.error('Import failed:', err.message);
    process.exitCode = 1;
  } finally {
    await pool.end();
  }
}

main();

3 files · javascript Explain with highlit

Parsing a multi-gigabyte CSV by reading it fully into memory with fs.readFileSync is a common way to crash a Node process. The idiomatic alternative is to treat the file as a stream and consume it one line at a time, which keeps memory bounded regardless of file size. This snippet builds a small CSV import pipeline around Node's built-in readline module, batching rows into database inserts.

In csvLineStream.js, fs.createReadStream opens the file as a readable byte stream and readline.createInterface wraps it, emitting one logical line at a time. The crlfDelay: Infinity option makes readline treat \r\n as a single break so Windows-authored files parse correctly. Because a readline.Interface is an async iterable, the code exposes a clean async function* generator: the first non-empty line is parsed as the header, and every subsequent line is split on commas and zipped into an object keyed by column name. Yielding from a generator gives the consumer natural backpressure — the file is only read as fast as the consumer pulls rows, so a slow database will not let the buffer grow unbounded.

batchImport.js drives that generator with for await...of, which is the modern, readable way to consume async iterators. Rather than inserting one row per statement, rows accumulate into a batch array and are flushed with flush once they reach BATCH_SIZE, dramatically cutting round-trips. A trailing flush after the loop handles the final partial batch. Errors from either the stream or a failed insert propagate out of the await, so a single try/catch at the call site covers the whole pipeline.

import-users.js is the thin entry point wiring a pg Pool to the importer and closing it in a finally. The key trade-off of this design is throughput versus simplicity: it does no CSV-quote handling, so embedded commas or quoted newlines would need a real parser like csv-parse. For clean, well-formed exports it is fast, dependency-light, and constant-memory — exactly what an ETL job streaming from disk needs.

Related snips

package dbutil

import (
  "context"

  "github.com/jackc/pgconn"

Retry Postgres serialization failures with bounded attempts

go postgres transactions

by Leah Thompson 1 tab

ruby

require "csv"

class PeopleCsvStream
  include Enumerable

  HEADERS = %w[id full_name email signed_up_at plan].freeze

Resilient CSV Export as a Streamed Response

rails performance streaming

by codesnips 3 tabs

typescript

export type Settled<R> =
  | { status: 'fulfilled'; value: R }
  | { status: 'rejected'; reason: unknown };

export interface ConcurrencyOptions {
  limit: number;

Simple concurrency limiter for batch operations

node concurrency async

by codesnips 2 tabs

ruby

class AddSettingsToAccounts < ActiveRecord::Migration[7.1]
  disable_ddl_transaction!

  def change
    add_column :accounts, :settings, :jsonb, null: false, default: {}

Postgres JSONB Partial Index for Feature Flags

rails postgres jsonb

by codesnips 3 tabs

ruby

module EmailNormalization
  extend ActiveSupport::Concern

  included do
    attr_accessor :soft_warnings

Soft Validation: Normalize + Validate Email

rails activerecord validations

by codesnips 4 tabs

package pools

import (
  "bytes"
  "sync"
)

sync.Pool for bytes.Buffer to reduce allocations in hot paths

go performance memory

by Leah Thompson 1 tab

Share this code

Here's the card — post it anywhere.

Stream and Import a Large CSV File Line-by-Line with Node.js readline — share card

Link copied