package fetch
import (
"context"
"net/http"
"golang.org/x/sync/errgroup"
)
type Result struct {
URL string
Status int
Body []byte
}
func FetchAll(ctx context.Context, client *http.Client, urls []string, workers int) ([]Result, error) {
g, ctx := errgroup.WithContext(ctx)
jobs := make(chan string, workers)
results := make(chan Result, len(urls))
g.Go(func() error {
defer close(jobs)
for _, u := range urls {
select {
case jobs <- u:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
})
for i := 0; i < workers; i++ {
g.Go(func() error {
return worker(ctx, client, jobs, results)
})
}
go func() {
_ = g.Wait()
close(results)
}()
collected := make([]Result, 0, len(urls))
for r := range results {
collected = append(collected, r)
}
if err := g.Wait(); err != nil {
return nil, err
}
return collected, nil
}package fetch
import (
"context"
"fmt"
"io"
"net/http"
)
func worker(ctx context.Context, client *http.Client, jobs <-chan string, results chan<- Result) error {
for url := range jobs {
res, err := fetch(ctx, client, url)
if err != nil {
return fmt.Errorf("fetch %s: %w", url, err)
}
select {
case results <- res:
case <-ctx.Done():
return ctx.Err()
}
}
return nil
}
func fetch(ctx context.Context, client *http.Client, url string) (Result, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return Result{}, err
}
resp, err := client.Do(req)
if err != nil {
return Result{}, err
}
defer resp.Body.Close()
body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
if err != nil {
return Result{}, err
}
return Result{URL: url, Status: resp.StatusCode, Body: body}, nil
}package main
import (
"context"
"fmt"
"log"
"net/http"
"time"
"example.com/app/fetch"
)
func main() {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
urls := []string{
"https://example.com",
"https://example.org",
"https://example.net",
}
client := &http.Client{Timeout: 5 * time.Second}
results, err := fetch.FetchAll(ctx, client, urls, 4)
if err != nil {
log.Fatalf("fetch failed: %v", err)
}
for _, r := range results {
fmt.Printf("%d\t%d bytes\t%s\n", r.Status, len(r.Body), r.URL)
}
}This snippet shows the idiomatic Go pattern for fanning work out to a fixed pool of goroutines and collecting typed results, without leaking goroutines or letting a slow producer overwhelm the system. The core idea is that unbounded fan-out (spawning one goroutine per input) is easy but dangerous: with large inputs it can exhaust memory or hammer a downstream service. A bounded worker pool caps concurrency to a known number of workers while still letting context cancellation short-circuit the whole batch on the first failure.
In pool.go, FetchAll is the public entry point. It creates a buffered jobs channel and a buffered results channel, then uses golang.org/x/sync/errgroup with errgroup.WithContext so that the first worker to return an error cancels every other worker through the derived ctx. A dedicated feeder goroutine pushes each URL onto jobs and closes the channel when done, but it uses a select on ctx.Done() so it stops early rather than blocking forever if the group has already been cancelled. Exactly workers goroutines are launched with g.Go, and each runs worker, which ranges over jobs until the channel is drained.
The subtle part is result collection. Because results is written by many goroutines, the code must not close it while writers are still active. A separate goroutine waits on g.Wait() and only then closes results, so the for r := range results loop in FetchAll terminates cleanly. g.Wait() also surfaces the first error, which is checked after the drain. Note that results is buffered to len(urls) so workers never block on a full channel even if the collector is momentarily behind.
In worker.go, worker calls fetch per job and forwards a Result onto results, again guarding the send with a select on ctx.Done() to avoid a goroutine leak on cancellation. Returning a non-nil error from any fetch triggers the group-wide cancel.
The main trade-off is throughput versus resource safety: a larger workers count increases parallelism but also load on the target. This pattern is the right reach whenever bounded concurrency, fail-fast semantics, and ordered-independent result gathering are all needed at once.
Related snips
package api
import (
"net/http"
"runtime/debug"
)Expose build metadata for debugging deploys
package dbutil
import (
"context"
"github.com/jackc/pgconn"Retry Postgres serialization failures with bounded attempts
use crossbeam::channel::unbounded;
use std::thread;
fn main() {
let (tx, rx) = unbounded();Crossbeam for advanced concurrent data structures
// Creating a Promise
const myPromise = new Promise((resolve, reject) => {
const success = true;
setTimeout(() => {
if (success) {Promises and async/await patterns for asynchronous JavaScript
use std::sync::mpsc;
use std::thread;
fn main() {
let (tx, rx) = mpsc::channel();Channels (mpsc) for message passing between threads
export type Settled<R> =
| { status: 'fulfilled'; value: R }
| { status: 'rejected'; reason: unknown };
export interface ConcurrencyOptions {
limit: number;Simple concurrency limiter for batch operations
Share this code
Here's the card — post it anywhere.
