Discover every URL on a site.

from spider import Spider

client = Spider()

# Get all URLs from a website
links = client.links(
    "https://example.com",
    params={
        "limit": 0,  # No limit - discover everything
        "subdomains": True,
    },
)

for link in links:
    print(link["url"])
print(f"Total: {len(links)} URLs found")

curl -X POST https://api.spider.cloud/links \
  -H "Authorization: Bearer $SPIDER_API_KEY" \
  -H "Content-Type: application/jsonl" \
  -d '{
    "url": "https://example.com",
    "limit": 1000,
    "depth": 5,
    "subdomains": true
  }'

import Spider from "@spider-cloud/spider-client";

const client = new Spider();

// Step 1: Discover all URLs
const links = await client.links("https://example.com", {
  limit: 500,
});

// Step 2: Filter for blog posts
const blogUrls = links
  .map(l => l.url)
  .filter(url => url.includes("/blog/"));

// Step 3: Scrape only the blog posts
const content = await client.scrape(blogUrls.join(","), {
  return_format: "markdown",
});