Agno

pip install -U "agno @ git+https://github.com/agno-agi/agno.git#subdirectory=libs/agno" openai scrapegraph-py

export SGAI_API_KEY="your-scrapegraph-key"
export OPENAI_API_KEY="your-openai-key"

from agno.agent import Agent
from agno.models.openai import OpenAIChat
from agno.tools.scrapegraph import ScrapeGraphTools

agent = Agent(
    model=OpenAIChat(id="gpt-4o"),
    tools=[ScrapeGraphTools(all=True)],
    markdown=True,
)

agent.print_response(
    "Use smartscraper on https://example.com to extract the page title and main heading. Return them as JSON.",
    stream=True,
)

tools = ScrapeGraphTools(
    enable_smartscraper=True,
    enable_markdownify=True,
    enable_scrape=True,
    render_heavy_js=True,
    headers={"User-Agent": "MyBot/1.0"},
)

agent = Agent(
    model=OpenAIChat(id="gpt-4o"),
    tools=[ScrapeGraphTools(enable_smartscraper=True)],
    markdown=True,
)

agent.print_response(
    "Extract the product name and price from "
    "https://books.toscrape.com/catalogue/a-light-in-the-attic_1000/ as JSON.",
    stream=True,
)

agent = Agent(
    model=OpenAIChat(id="gpt-4o"),
    tools=[ScrapeGraphTools(enable_markdownify=True)],
    markdown=True,
)

agent.print_response(
    "Fetch https://scrapegraphai.com and summarize the top three product features from the markdown.",
    stream=True,
)

agent = Agent(
    model=OpenAIChat(id="gpt-4o"),
    tools=[ScrapeGraphTools(enable_crawl=True, crawl_max_wait=600)],
    markdown=True,
)

agent.print_response(
    """Crawl https://books.toscrape.com with max_depth=2, max_pages=5.
    Use this schema: {"type": "object", "properties": {"books": {"type": "array", "items": {"type": "object", "properties": {"title": {"type": "string"}, "price": {"type": "string"}}}}}}.
    Prompt: 'Extract every book title and price on the page'. Return the merged JSON.""",
    stream=True,
)