Enterprise web scraping and data collection with Bright Data

pip install langchain-brightdata langchain-google-genai langgraph python-dotenv

# Write to .env (replace with your actual keys)
# BRIGHT_DATA_API_TOKEN=<your-brightdata-api-key>
# GOOGLE_API_KEY=<your-google-api-key>

from langchain_brightdata import BrightDataSERP
from langchain_google_genai import ChatGoogleGenerativeAI
from langgraph.prebuilt import create_react_agent
from dotenv import load_dotenv
import os

load_dotenv()

print(f"Bright Data API Key loaded: {'Yes' if os.getenv('BRIGHT_DATA_API_TOKEN') else 'No'}")
print(f"Google API Key loaded: {'Yes' if os.getenv('GOOGLE_API_KEY') else 'No'}")

llm = ChatGoogleGenerativeAI(
    model="gemini-2.0-flash",
    temperature=0.1,  # low temperature for consistent agent decisions
)

serp_tool = BrightDataSERP(
    search_engine="google",
    country="us",
    language="en",
    results_count=10,
    parse_results=True,  # convert raw HTML to structured data
)

print(f"Search Engine: {serp_tool.search_engine}")
print(f"Country: {serp_tool.country}")
print(f"Language: {serp_tool.language}")
print(f"Results Count: {serp_tool.results_count}")

agent = create_react_agent(
    model=llm,
    tools=[serp_tool],
    prompt=(
        "You are a web researcher agent with access to a SERP tool. "
        "You MUST use the tool to answer user queries. If no specific country, "
        "language, search engine, or vertical is specified, choose what best fits "
        "the user's question."
    ),
)

user_query = "What are the latest developments and news in AI technology in the US?"

for step in agent.stream(
    {"messages": [("human", user_query)]},
    stream_mode="values",
):
    step["messages"][-1].pretty_print()

def create_research_assistant(
    search_engine: str = "google",
    country: str = "us",
    language: str = "en",
):
    """
    Create a research agent configured for a specific locale.

    Args:
        search_engine: "google" or "bing"
        country: ISO country code for localized results
        language: ISO language code
    """
    llm = ChatGoogleGenerativeAI(model="gemini-2.0-flash", temperature=0.1)

    serp_tool = BrightDataSERP(
        search_engine=search_engine,
        country=country,
        language=language,
        results_count=15,
        parse_results=True,
    )

    agent = create_react_agent(llm, [serp_tool])

    print(f"Research Assistant created!")
    print(f"  Engine: {search_engine.title()}")
    print(f"  Location: {country.upper()}")
    print(f"  Language: {language.upper()}")

    return agent

research_assistant = create_research_assistant()

research_query = """
Please research the renewable energy market trends for 2024-2025.
I need information about:
1. Market growth predictions
2. Leading companies and their strategies
3. Recent technological breakthroughs
4. Government policies affecting the sector
"""

for step in research_assistant.stream(
    {"messages": [("human", research_query)]},
    stream_mode="values",
):
    step["messages"][-1].pretty_print()

pip install langgraph langchain-openai mcp-use python-dotenv

# .env
# BRIGHT_DATA_API_TOKEN=<your-brightdata-api-key>
# OPENROUTER_API_KEY=<your-openrouter-api-key>

import asyncio
from langchain_openai import ChatOpenAI
from langgraph.prebuilt import create_react_agent
from mcp_use.client import MCPClient
from mcp_use.adapters.langchain_adapter import LangChainAdapter
from dotenv import load_dotenv
import os

load_dotenv()

async def setup_bright_data_tools():
    """Configure the Bright Data MCP client and convert tools to LangChain format."""
    bright_data_config = {
        "mcpServers": {
            "Bright Data": {
                "command": "npx",
                "args": ["@brightdata/mcp"],
                "env": {
                    "API_TOKEN": os.getenv("BRIGHT_DATA_API_TOKEN"),
                },
            }
        }
    }

    client = MCPClient.from_dict(bright_data_config)
    adapter = LangChainAdapter()

    tools = await adapter.create_tools(client)

    print(f"Connected to Bright Data MCP server")
    print(f"Available tools: {len(tools)}")

    return tools

import datetime

async def create_web_scraper_agent():
    """Create a ReAct agent with full Bright Data MCP tool access."""
    tools = await setup_bright_data_tools()

    current_date = datetime.datetime.now().strftime("%B %d, %Y")

    llm = ChatOpenAI(
        openai_api_key=os.getenv("OPENROUTER_API_KEY"),
        openai_api_base="https://openrouter.ai/api/v1",
        model_name="google/gemini-2.5-flash-lite-preview-06-17",
        temperature=0.1,
    )

    agent = create_react_agent(
        model=llm,
        tools=tools,
        prompt=(
            f"You are a web data extraction specialist. Today is {current_date}. "
            f"You have access to {len(tools)} Bright Data tools including search engines, "
            "platform-specific extractors, and a universal web unlocker. "
            "Always use a tool to answer user requests—do not rely on training data. "
            "Follow this process: 1) Understand the request. 2) Select the best tool. "
            "3) Execute and review results. 4) Return a structured response with sources."
        ),
    )

    return agent

async def test_basic_search(agent):
    print("Testing basic search...")
    print("=" * 50)

    result = await agent.ainvoke({
        "messages": [("human", "Give me the latest AI news from this week. Include full URLs to sources.")],
    })

    print("\nSearch results:")
    print(result["messages"][-1].content)
    return result

agent = await create_web_scraper_agent()
basic_result = await test_basic_search(agent)