web_search

langroid/parsing/web_search.py

Utilities for web search.

NOTE: Using Google Search requires setting the GOOGLE_API_KEY and GOOGLE_CSE_ID environment variables in your .env file, as explained in the README.

WebSearchResult(title, link, max_content_length=3500, max_summary_length=300)

Class representing a Web Search result, containing the title, link, summary and full content of the result.

link (str): The link to the search result.
max_content_length (int): The maximum length of the full content.
max_summary_length (int): The maximum length of the summary.

Source code in langroid/parsing/web_search.py

def __init__(
    self,
    title: str,
    link: str | None,
    max_content_length: int = 3500,
    max_summary_length: int = 300,
):
    """
    Args:
        title (str): The title of the search result.
        link (str): The link to the search result.
        max_content_length (int): The maximum length of the full content.
        max_summary_length (int): The maximum length of the summary.
    """
    self.title = title
    self.link = link
    self.max_content_length = max_content_length
    self.max_summary_length = max_summary_length
    self.full_content = self.get_full_content()
    self.summary = self.get_summary()

google_search(query, num_results=5)

Search via Google's Custom Search JSON API.

.. deprecated:: The underlying API is closed to new customers and is discontinued on 2027-01-01. Existing credentials keep working until then; use :func:tavily_search, :func:exa_search, or :func:duckduckgo_search.

Source code in langroid/parsing/web_search.py

def google_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """Search via Google's Custom Search JSON API.

    .. deprecated::
        The underlying API is closed to new customers and is discontinued on
        2027-01-01. Existing credentials keep working until then; use
        :func:`tavily_search`, :func:`exa_search`, or :func:`duckduckgo_search`.
    """
    warnings.warn(GOOGLE_SEARCH_DEPRECATION, DeprecationWarning, stacklevel=2)
    load_dotenv()
    api_key = os.getenv("GOOGLE_API_KEY")
    cse_id = os.getenv("GOOGLE_CSE_ID")
    service: Resource = build("customsearch", "v1", developerKey=api_key)
    raw_results = (
        service.cse().list(q=query, cx=cse_id, num=num_results).execute()["items"]
    )

    return [
        WebSearchResult(result["title"], result["link"], 3500, 300)
        for result in raw_results
    ]

serpapi_search(query, num_results=5)

Method that makes an API call to SerpApi's Google Search endpoint, which queries the top num_results organic results matching the query. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Notes

Results come from a single SerpApi Google results page: num_results is passed through as the num parameter, and no further pages are fetched via start, so a very large num_results may yield fewer results than requested. Organic results without a link are skipped, since there would be nothing to fetch content from.

SerpApi authenticates via the api_key query parameter, so the request URL that requests embeds in its error messages contains the key. Request failures are re-raised with the key redacted, and without the request / response objects that carry the same URL, to keep the key out of logs and tracebacks.

Source code in langroid/parsing/web_search.py

def serpapi_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call to SerpApi's Google Search endpoint, which
    queries the top `num_results` organic results matching the query.
    Returns a list of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab

    Notes:
        Results come from a single SerpApi Google results page: `num_results`
        is passed through as the `num` parameter, and no further pages are
        fetched via `start`, so a very large `num_results` may yield fewer
        results than requested. Organic results without a `link` are skipped,
        since there would be nothing to fetch content from.

        SerpApi authenticates via the `api_key` query parameter, so the request
        URL that `requests` embeds in its error messages contains the key.
        Request failures are re-raised with the key redacted, and without the
        `request` / `response` objects that carry the same URL, to keep the key
        out of logs and tracebacks.
    """

    load_dotenv()

    api_key = os.getenv("SERPAPI_API_KEY")
    if not api_key:
        raise ValueError(
            "SERPAPI_API_KEY environment variable is not set. "
            "Please set it to your API key and try again."
        )

    params: Dict[str, str | int] = {
        "engine": "google",
        "q": query,
        "num": num_results,
        "api_key": api_key,
    }
    error: requests.RequestException | None = None
    try:
        response = requests.get(
            "https://serpapi.com/search.json",
            params=params,
            timeout=30,
        )
        response.raise_for_status()
    except requests.RequestException as e:
        error = _redacted_request_error(e, api_key)
    if error is not None:
        # raised outside the `except` block so that the original, key-bearing
        # error is not attached to it as __context__
        raise error

    raw_results = response.json().get("organic_results", [])
    # SerpApi extracts organic-result fields opportunistically, so an entry
    # may be missing `title` or `link`; only `link` is essential.
    linked_results = [result for result in raw_results if result.get("link")]

    return [
        WebSearchResult(
            title=result.get("title", ""),
            link=result["link"],
            max_content_length=3500,
            max_summary_length=300,
        )
        for result in linked_results[:num_results]
    ]

metaphor_search(query, num_results=5)

Method that makes an API call by Metaphor client that queries the top num_results links that matches the query. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Source code in langroid/parsing/web_search.py

def metaphor_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call by Metaphor client that queries
    the top num_results links that matches the query. Returns a list
    of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab
    """

    load_dotenv()

    api_key = os.getenv("METAPHOR_API_KEY") or os.getenv("EXA_API_KEY")
    if not api_key:
        raise ValueError(
            """
            Neither METAPHOR_API_KEY nor EXA_API_KEY environment variables are set. 
            Please set one of them to your API key, and try again.
            """
        )

    try:
        from metaphor_python import Metaphor
    except ImportError:
        raise LangroidImportError("metaphor-python", "metaphor")

    client = Metaphor(api_key=api_key)

    response = client.search(
        query=query,
        num_results=num_results,
    )
    raw_results = response.results

    return [
        WebSearchResult(result.title, result.url, 3500, 300) for result in raw_results
    ]

exa_search(query, num_results=5)

Method that makes an API call by Exa client that queries the top num_results links that matches the query. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Source code in langroid/parsing/web_search.py

def exa_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call by Exa client that queries
    the top num_results links that matches the query. Returns a list
    of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab
    """

    load_dotenv()

    api_key = os.getenv("EXA_API_KEY")
    if not api_key:
        raise ValueError(
            """
            EXA_API_KEY environment variables are not set. 
            Please set one of them to your API key, and try again.
            """
        )

    try:
        from exa_py import Exa
    except ImportError:
        raise LangroidImportError("exa-py", "exa")

    client = Exa(api_key=api_key)

    try:
        response = client.search(
            query=query,
            num_results=num_results,
        )
        raw_results = response.results

        return [
            WebSearchResult(
                title=result.title or "",
                link=result.url,
                max_content_length=3500,
                max_summary_length=300,
            )
            for result in raw_results
            if result.url is not None
        ]
    except Exception:
        return [
            WebSearchResult(
                title="Error",
                link=None,
                max_content_length=3500,
                max_summary_length=300,
            )
        ]

duckduckgo_search(query, num_results=5)

Method that makes an API call by DuckDuckGo client that queries the top num_results links that matche the query. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Source code in langroid/parsing/web_search.py

def duckduckgo_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call by DuckDuckGo client that queries
    the top `num_results` links that matche the query. Returns a list
    of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab
    """

    with DDGS() as ddgs:
        search_results = [r for r in ddgs.text(query, max_results=num_results)]

    return [
        WebSearchResult(
            title=result["title"],
            link=result["href"],
            max_content_length=3500,
            max_summary_length=300,
        )
        for result in search_results
    ]

tavily_search(query, num_results=5)

Method that makes an API call to Tavily API that queries the top num_results links that match the query. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Source code in langroid/parsing/web_search.py

def tavily_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call to Tavily API that queries
    the top `num_results` links that match the query. Returns a list
    of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab
    """

    load_dotenv()

    api_key = os.getenv("TAVILY_API_KEY")
    if not api_key:
        raise ValueError(
            "TAVILY_API_KEY environment variable is not set. "
            "Please set it to your API key and try again."
        )

    try:
        from tavily import TavilyClient
    except ImportError:
        raise LangroidImportError("tavily-python", "tavily")

    client = TavilyClient(api_key=api_key)
    response = client.search(query=query, max_results=num_results)
    search_results = response["results"]

    return [
        WebSearchResult(
            title=result["title"],
            link=result["url"],
            max_content_length=3500,
            max_summary_length=300,
        )
        for result in search_results
    ]

seltz_search(query, num_results=5)

Method that makes an API call to Seltz API that queries the top num_results results. Returns a list of WebSearchResult objects.

Parameters:

Name	Type	Description	Default
query	str	The query body that users wants to make.	required
num_results	int	Number of top matching results that we want to grab	5

Source code in langroid/parsing/web_search.py

def seltz_search(query: str, num_results: int = 5) -> List[WebSearchResult]:
    """
    Method that makes an API call to Seltz API that queries
    the top `num_results` results. Returns a list of WebSearchResult objects.

    Args:
        query (str): The query body that users wants to make.
        num_results (int): Number of top matching results that we want
            to grab
    """

    load_dotenv()

    api_key = os.getenv("SELTZ_API_KEY")
    if not api_key:
        raise ValueError(
            "SELTZ_API_KEY environment variable is not set. "
            "Please set it to your API key and try again."
        )

    try:
        from seltz import Includes, Seltz
    except ImportError:
        raise LangroidImportError("seltz", "seltz")

    client = Seltz(api_key=api_key)
    response = client.search(
        query=query,
        includes=Includes(max_documents=num_results),
    )

    results = []
    for doc in response.documents:
        result = WebSearchResult(
            title=doc.url,
            link=None,  # skip HTTP fetch; Seltz already provides content
            max_content_length=3500,
            max_summary_length=300,
        )
        result.link = doc.url
        result.full_content = doc.content[:3500]
        result.summary = doc.content[:300]
        results.append(result)

    return results