Spider as a LlamaIndex data connector.

from llama_index.readers.web import SpiderWebReader

# Initialize the reader
reader = SpiderWebReader()

# Crawl a docs site and get LlamaIndex Documents
documents = reader.load_data(
    url="https://docs.example.com",
    mode="crawl",
    params={
        "return_format": "markdown",
        "limit": 100,
    }
)

# Build a vector index from the crawl results
from llama_index.core import VectorStoreIndex

index = VectorStoreIndex.from_documents(documents)
query_engine = index.as_query_engine()

response = query_engine.query("How do I configure authentication?")
print(response)

from llama_index.core.tools import FunctionTool
from llama_index.llms.openai import OpenAI
from llama_index.core.agent import ReActAgent
from spider import Spider

spider = Spider()

def web_search(query: str) -> str:
    """Search the web and return page contents."""
    results = spider.search(query, params={
        "search_limit": 5,
        "fetch_page_content": True,
        "return_format": "markdown",
    })
    return "\n---\n".join(
        f"Source: {r['url']}\n{r['content'][:2000]}"
        for r in results if r.get("content")
    )

search_tool = FunctionTool.from_defaults(fn=web_search)
agent = ReActAgent.from_tools([search_tool], llm=OpenAI(model="gpt-4o"))

response = agent.chat("What are the latest OWASP Top 10 changes?")
print(response)