Deduplicating Paginated API Results With a Normalized Entity Store and Reducer

import { useCallback, useMemo, useReducer, useRef, useState } from 'react';
import { entityReducer, initialState, Post, State } from './entityReducer';

interface PageResponse {
  posts: Post[];
  totalPages: number;
}

type Fetcher = (page: number) => Promise<PageResponse>;

export function useNormalizedPages(fetchPage: Fetcher) {
  const [state, dispatch] = useReducer(entityReducer, initialState);
  const [loading, setLoading] = useState(false);
  const inFlight = useRef<Set<number>>(new Set());

  const loadPage = useCallback(
    async (page: number) => {
      if (inFlight.current.has(page) || state.loadedPages.has(page)) return;
      inFlight.current.add(page);
      setLoading(true);
      try {
        const res = await fetchPage(page);
        dispatch({ type: 'PAGE_LOADED', page, posts: res.posts, totalPages: res.totalPages });
      } finally {
        inFlight.current.delete(page);
        setLoading(false);
      }
    },
    [fetchPage, state.loadedPages]
  );

  const items = useMemo(() => resolveItems(state), [state]);

  return { items, loading, loadPage, loadedPages: state.loadedPages, totalPages: state.totalPages };
}

function resolveItems(state: State): Post[] {
  const seen = new Set<string>();
  const out: Post[] = [];
  const pages = Object.keys(state.pageOrder).map(Number).sort((a, b) => a - b);
  for (const page of pages) {
    for (const id of state.pageOrder[page]) {
      if (seen.has(id)) continue; // dedupe across pages
      seen.add(id);
      const post = state.byId[id];
      if (post) out.push(post);
    }
  }
  return out;
}

import React, { useEffect, useState } from 'react';
import { useNormalizedPages } from './useNormalizedPages';
import { Post } from './entityReducer';

async function fetchPostsPage(page: number) {
  const res = await fetch(`/api/posts?page=${page}&per=20`);
  if (!res.ok) throw new Error(`Failed to load page ${page}`);
  return (await res.json()) as { posts: Post[]; totalPages: number };
}

export function PostFeed() {
  const { items, loading, loadPage, loadedPages, totalPages } = useNormalizedPages(fetchPostsPage);
  const [page, setPage] = useState(1);

  useEffect(() => {
    loadPage(page);
  }, [page, loadPage]);

  const hasMore = loadedPages.size < totalPages;

  return (
    <div className="post-feed">
      <ul>
        {items.map((post) => (
          <li key={post.id}>{post.title}</li>
        ))}
      </ul>
      <button
        disabled={loading || !hasMore}
        onClick={() => setPage((p) => p + 1)}
      >
        {loading ? 'Loading…' : hasMore ? 'Load more' : 'No more posts'}
      </button>
    </div>
  );
}