Spider as a CrewAI agent tool.

from crewai import Agent, Task, Crew
from crewai_tools import SpiderTool

# Create the Spider tool
spider_tool = SpiderTool()

# Define agents with web access
researcher = Agent(
    role="Senior Researcher",
    goal="Find accurate, up-to-date information from the web",
    tools=[spider_tool],
    verbose=True,
)

analyst = Agent(
    role="Data Analyst",
    goal="Extract structured insights from raw web content",
    tools=[spider_tool],
)

# Define tasks
research_task = Task(
    description="Research the top 5 competitors in the AI search space. Crawl their websites and summarize their offerings.",
    agent=researcher,
    expected_output="A list of competitors with key features and pricing",
)

analysis_task = Task(
    description="Analyze the research and identify market gaps.",
    agent=analyst,
    expected_output="A strategic analysis with actionable recommendations",
)

# Run the crew
crew = Crew(agents=[researcher, analyst], tasks=[research_task, analysis_task])
result = crew.kickoff()
print(result)

from crewai import Agent, Task, Crew
from spider import Spider

spider = Spider()

def search_web(query: str) -> str:
    """Search the web and return relevant content."""
    results = spider.search(query, params={
        "search_limit": 5,
        "fetch_page_content": True,
        "return_format": "markdown",
    })
    return "\n\n".join(
        f"## {r['url']}\n{r['content'][:2000]}"
        for r in results if r.get("content")
    )

def crawl_site(url: str) -> str:
    """Crawl a website and return its contents."""
    pages = spider.crawl_url(url, params={
        "return_format": "markdown",
        "limit": 20,
    })
    return "\n\n".join(
        p["content"][:3000] for p in pages if p.get("content")
    )

# Agents can call search_web() and crawl_site()
# as custom tools alongside SpiderTool for full flexibility