Sitemaps.org Documentation Scraper

import { SpiderBrowser } from "spider-browser";

const spider = new SpiderBrowser({
  apiKey: process.env.SPIDER_API_KEY!,
});

await spider.connect();
const page = spider.page!;
await page.goto("https://www.sitemaps.org");

// No selectors, no schema. Spider reads the page and names the fields.
const data = await page.scrape();

console.log(data);
await spider.close();

import { SpiderBrowser } from "spider-browser";

const spider = new SpiderBrowser({
  apiKey: process.env.SPIDER_API_KEY!,
});

await spider.connect();
const page = spider.page!;
await page.goto("https://www.sitemaps.org");

const data = await page.extractFields({
  faq_link: "a[href="faq.php"]",
  last_updated: "#mainContent .date",
  license: "a[href^="http://creativecommons.org/licenses/by-sa/2.5/"]",
  main_content_text: "#mainContent p",
  protocol_link: "a[href="protocol.php"]",
  terms_link: "a[href="terms.php"]",
});

console.log(data);
await spider.close();