Stack Overflow Scraper

Extract questions, answers, vote counts, and tag data from Stack Overflow. Built on spider-browser .

target
stackoverflow.com: success rate
99.9%: latency
~4ms

Quick start

Extract data in minutes.

stackoverflow-scraper.ts

import { SpiderBrowser } from "spider-browser";

const spider = new SpiderBrowser({
  apiKey: process.env.SPIDER_API_KEY!,
});

await spider.connect();
const page = spider.page!;
await page.goto("https://stackoverflow.com/questions/tagged/web-scraping?sort=votes");
await page.content();

const data = await page.evaluate(`(() => {
  const questions = [];
  document.querySelectorAll(".s-post-summary").forEach(el => {
    const title = el.querySelector(".s-link")?.textContent?.trim();
    const votes = el.querySelector("[class*='vote-count'], [class*='score']")?.textContent?.trim();
    const answers = el.querySelector("[class*='answer-count'], [class*='answers'], [class*='accepted'], [class*='answer'] [class*='count']")?.textContent?.trim();
    const views = el.querySelector("[title*='views'] .s-post-summary--stats-item-number")?.textContent?.trim();
    const tags = [...el.querySelectorAll(".post-tag")].map(t => t.textContent?.trim());
    if (title) questions.push({ title, votes, answers, views, tags });
  });
  return JSON.stringify({ total: questions.length, questions: questions.slice(0, 15) });
})()`);

console.log(JSON.parse(data));
await spider.close();