Twitch Scraper

import { SpiderBrowser } from "spider-browser";

const spider = new SpiderBrowser({
  apiKey: process.env.SPIDER_API_KEY!,
});

await spider.connect();
const page = spider.page!;
await page.goto("https://www.twitch.tv/directory");
await page.content(10000);

const data = await page.evaluate(`(() => {
  const categories = [];
  document.querySelectorAll("[data-a-target*='card'], a[href*='/directory/category/'], article").forEach(el => {
    const name = el.querySelector("h2, h3, [data-a-target*='Name'], p")?.textContent?.trim();
    const viewerEl = el.querySelector("[aria-label*='viewer' i]");
    let viewers = viewerEl?.textContent?.trim();
    if (!viewers) {
      el.querySelectorAll("p, span").forEach(s => {
        const t = s.textContent?.trim();
        if (t && /viewer/i.test(t)) viewers = t;
      });
    }
    if (name) categories.push({ name, viewers });
  });
  return JSON.stringify({ total: categories.length, categories: categories.slice(0, 10) });
})()`);

console.log(JSON.parse(data));
await spider.close();