Web scraping pipelines with requests and BeautifulSoup

                  import time
import requests
from bs4 import BeautifulSoup

session = requests.Session()
session.headers.update({'User-Agent': 'research-bot/1.0'})

def fetch_article(url: str) -> dict:
    response = session.get(url, timeout=10)
    response.raise_for_status()
    soup = BeautifulSoup(response.text, 'html.parser')
    title = soup.select_one('h1').get_text(strip=True)
    body = ' '.join(node.get_text(' ', strip=True) for node in soup.select('article p'))
    return {'url': url, 'title': title, 'body': body}

records = []
for url in urls:
    records.append(fetch_article(url))
    time.sleep(1)