Puppeteer

Puppeteer for Web Scraping and Automation: Practical Guide

HelpMeTest

HelpMeTest

— 5 min read
Puppeteer for Web Scraping and Automation: Practical Guide

Puppeteer handles scraping scenarios that simple HTTP clients can't: JavaScript-rendered pages, infinite scroll, login-protected content, and multi-step form interactions. This guide covers production-ready patterns — not just the basics.

When to Use Puppeteer for Scraping

Puppeteer makes sense when:

  • The page renders content via JavaScript (React, Vue, Angular apps)
  • The site requires login or session cookies
  • You need to interact with the page (scroll to load content, click pagination)
  • The site uses Cloudflare or bot detection that blocks plain HTTP requests

For static sites with no JavaScript rendering, a simpler HTTP client (axios + cheerio) is faster and cheaper to run.

Basic Setup

const puppeteer = require('puppeteer');

async function scrape(url) {
  const browser = await puppeteer.launch({
    headless: true,
    args: ['--no-sandbox', '--disable-setuid-sandbox'],
  });
  
  const page = await browser.newPage();
  await page.goto(url, { waitUntil: 'networkidle2' });
  
  // Extract data
  const data = await page.evaluate(() => {
    return Array.from(document.querySelectorAll('.product-card')).map(card => ({
      title: card.querySelector('h3')?.textContent?.trim(),
      price: card.querySelector('.price')?.textContent?.trim(),
      url: card.querySelector('a')?.href,
    }));
  });
  
  await browser.close();
  return data;
}

waitUntil Options

// Wait until no network requests for 500ms
await page.goto(url, { waitUntil: 'networkidle0' });

// Wait until no more than 2 requests for 500ms (faster)
await page.goto(url, { waitUntil: 'networkidle2' });

// Wait for DOMContentLoaded only (fastest)
await page.goto(url, { waitUntil: 'domcontentloaded' });

// Wait for initial HTML load
await page.goto(url, { waitUntil: 'load' });

For scraping, networkidle2 is usually the right choice — it waits for the main content to load without waiting for analytics/tracking requests.

Handling Dynamic Content

Waiting for Specific Elements

// Wait for a specific element before extracting
await page.waitForSelector('.product-list .item', { timeout: 10000 });

const items = await page.$$eval('.product-list .item', elements => 
  elements.map(el => el.textContent.trim())
);

Infinite Scroll

async function scrapeAllItems(page) {
  const items = new Set();
  let previousCount = 0;
  
  while (true) {
    // Get current items
    const currentItems = await page.$$eval('.item', els => 
      els.map(el => el.dataset.id)
    );
    currentItems.forEach(id => items.add(id));
    
    // Stop if no new items loaded
    if (items.size === previousCount) break;
    previousCount = items.size;
    
    // Scroll to bottom
    await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));
    
    // Wait for new content to load
    await page.waitForTimeout(1500);
  }
  
  return Array.from(items);
}

Waiting for Network Requests

// Wait for a specific API call to complete
const [response] = await Promise.all([
  page.waitForResponse(res => res.url().includes('/api/products') && res.status() === 200),
  page.click('#load-more'),
]);

const data = await response.json();

This is more reliable than waitForTimeout because you're waiting for the actual data, not guessing how long it takes.

Pagination

Next Button Pagination

async function scrapeAllPages(startUrl) {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  const allData = [];
  
  await page.goto(startUrl);
  
  while (true) {
    await page.waitForSelector('.product-card');
    
    const pageData = await page.$$eval('.product-card', cards => 
      cards.map(card => ({
        title: card.querySelector('h3').textContent,
        price: card.querySelector('.price').textContent,
      }))
    );
    
    allData.push(...pageData);
    
    // Check for next page button
    const nextBtn = await page.$('.pagination .next:not(.disabled)');
    if (!nextBtn) break;
    
    await Promise.all([
      page.waitForNavigation({ waitUntil: 'networkidle2' }),
      nextBtn.click(),
    ]);
  }
  
  await browser.close();
  return allData;
}

URL-based Pagination

async function scrapePages(baseUrl, totalPages) {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  const allData = [];
  
  for (let i = 1; i <= totalPages; i++) {
    await page.goto(`${baseUrl}?page=${i}`, { waitUntil: 'networkidle2' });
    
    const data = await page.$$eval('.item', items => 
      items.map(item => ({ text: item.textContent }))
    );
    
    allData.push(...data);
    
    // Polite delay between requests
    await new Promise(r => setTimeout(r, 1000 + Math.random() * 1000));
  }
  
  await browser.close();
  return allData;
}

Authenticated Scraping

Login Flow

async function loginAndScrape(loginUrl, credentials, targetUrl) {
  const browser = await puppeteer.launch({ headless: true });
  const page = await browser.newPage();
  
  // Login
  await page.goto(loginUrl);
  await page.type('#email', credentials.email);
  await page.type('#password', credentials.password);
  
  await Promise.all([
    page.waitForNavigation(),
    page.click('#login-btn'),
  ]);
  
  // Verify login succeeded
  const loggedIn = await page.$('.user-dashboard');
  if (!loggedIn) throw new Error('Login failed');
  
  // Save session cookies
  const cookies = await page.cookies();
  
  // Navigate to target
  await page.goto(targetUrl, { waitUntil: 'networkidle2' });
  const data = await extractData(page);
  
  await browser.close();
  return data;
}

Reusing Sessions

For multi-page scraping, save and restore cookies to avoid re-logging in:

const fs = require('fs');

async function saveCookies(page, filepath) {
  const cookies = await page.cookies();
  fs.writeFileSync(filepath, JSON.stringify(cookies));
}

async function loadCookies(page, filepath) {
  if (!fs.existsSync(filepath)) return false;
  const cookies = JSON.parse(fs.readFileSync(filepath));
  await page.setCookie(...cookies);
  return true;
}

// Usage
const cookiesFile = './session.json';
const loaded = await loadCookies(page, cookiesFile);

if (!loaded) {
  await performLogin(page);
  await saveCookies(page, cookiesFile);
}

Stealth Mode

Sites with bot detection (Cloudflare, Akamai) detect headless Chrome via various fingerprints. The puppeteer-extra-plugin-stealth package patches these:

npm install puppeteer-extra puppeteer-extra-plugin-stealth
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

puppeteer.use(StealthPlugin());

const browser = await puppeteer.launch({ headless: true });

Stealth patches: navigator.webdriver, Chrome runtime objects, permission handling, plugin enumeration, and several other headless detection vectors.

For pages that specifically check navigator.webdriver:

// Manual patch if not using stealth plugin
await page.evaluateOnNewDocument(() => {
  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
});

Request Interception

Intercept and block unnecessary requests to speed up scraping:

await page.setRequestInterception(true);

page.on('request', request => {
  const blockedTypes = ['image', 'stylesheet', 'font', 'media'];
  
  if (blockedTypes.includes(request.resourceType())) {
    request.abort();
  } else {
    request.continue();
  }
});

Blocking images and CSS typically reduces page load time by 40-60% for content scraping.

Intercepting API Responses

Capture API responses directly instead of scraping HTML:

const apiData = [];

page.on('response', async response => {
  if (response.url().includes('/api/items')) {
    try {
      const json = await response.json();
      apiData.push(...json.items);
    } catch (e) {
      // Not JSON or parse error — skip
    }
  }
});

await page.goto(targetUrl, { waitUntil: 'networkidle2' });
// apiData now contains all items loaded by the page's API calls

Rate Limiting and Politeness

Aggressive scraping can get your IP banned and causes real load on servers:

const sleep = (ms) => new Promise(r => setTimeout(r, ms));

async function scrapeWithDelay(urls, delayMs = 1000) {
  const results = [];
  
  for (const url of urls) {
    const data = await scrapePage(url);
    results.push(data);
    
    // Random delay to appear more human
    const jitter = Math.random() * delayMs;
    await sleep(delayMs + jitter);
  }
  
  return results;
}

For high-volume scraping, use a rotating proxy:

const browser = await puppeteer.launch({
  args: [`--proxy-server=http://proxy.example.com:8080`],
});

Error Handling and Retries

Network errors, timeout errors, and page crashes happen. Build in retries:

async function scrapeWithRetry(url, maxRetries = 3) {
  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      return await scrapePage(url);
    } catch (err) {
      console.error(`Attempt ${attempt} failed for ${url}: ${err.message}`);
      
      if (attempt === maxRetries) throw err;
      
      // Exponential backoff
      await sleep(1000 * Math.pow(2, attempt));
    }
  }
}

Handle page crashes:

page.on('error', err => {
  console.error('Page crash:', err.message);
});

page.on('pageerror', err => {
  console.error('Page JS error:', err.message);
  // Continue scraping — JS errors on the page usually don't affect data extraction
});

Memory Management for Long-Running Scrapers

Browser instances accumulate memory. Recycle them periodically:

async function scrapeInBatches(urls, batchSize = 50) {
  const results = [];
  
  for (let i = 0; i < urls.length; i += batchSize) {
    const batch = urls.slice(i, i + batchSize);
    
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    
    for (const url of batch) {
      await page.goto(url, { waitUntil: 'networkidle2' });
      results.push(await extractData(page));
    }
    
    await browser.close(); // Frees all memory for this batch
  }
  
  return results;
}

Scheduled Scraping vs. Monitored Testing

Scraping and testing overlap: both interact with real pages and extract data. The difference is intent — scraping extracts data, testing verifies behavior.

HelpMeTest fills the gap between one-time scraping scripts and production monitoring: run your Puppeteer scripts on a schedule, get alerts when pages change in ways that break your extraction, and track data over time without managing the infrastructure.

Summary

Puppeteer is production-ready for scraping with the right patterns: proper wait strategies over fixed timeouts, session reuse for authenticated scraping, request interception for speed, stealth mode for bot-detection bypass, and batch processing for memory management. The combination of waitForSelector, waitForResponse, and response interception covers almost all dynamic content scenarios.

Read more

Start now free