Overview

import requests, json

headers = {
    'Authorization': 'Bearer $SPIDER_API_KEY',
    'Content-Type': 'application/json',
}

json_data = {"limit":5,"url":"https://example.com"}

response = requests.post('https://api.spider.cloud/crawl', 
  headers=headers, json=json_data, stream=True)

with response as r:
    r.raise_for_status()
    
    buffer = b""

    for chunk in response.iter_content(chunk_size=8192):
        if chunk:
            buffer += chunk
            try:
                data = json.loads(buffer.decode('utf-8'))
                print(data)
                buffer = b""
            except json.JSONDecodeError:
                continue