Crawl and scrape any website

curl -X POST https://api.spider.cloud/crawl \
  -H "Authorization: Bearer $SPIDER_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
    "url": "https://example.com",
    "limit": 50,
    "return_format": "markdown"
  }'

import requests

response = requests.post(
    "https://api.spider.cloud/crawl",
    headers={
        "Authorization": f"Bearer {SPIDER_API_KEY}",
    },
    json={
        "url": "https://example.com",
        "limit": 50,
        "return_format": "markdown",
    },
)

pages = response.json()
print(f"Crawled {len(pages)} pages")

const response = await fetch("https://api.spider.cloud/crawl", {
  method: "POST",
  headers: {
    "Authorization": `Bearer ${process.env.SPIDER_API_KEY}`,
    "Content-Type": "application/json",
  },
  body: JSON.stringify({
    url: "https://example.com",
    limit: 50,
    return_format: "markdown",
  }),
});

const pages = await response.json();
console.log(`Crawled ${pages.length} pages`);