xml_tool_message

langroid/agent/xml_tool_message.py

XMLToolMessage

Bases: ToolMessage

Abstract class for tools formatted using XML instead of JSON.

When a subclass defines a field with the attribute verbatim=True, instructions are sent to the LLM to ensure the field's content is: - preserved as is, including whitespace, indents, quotes, newlines, etc with no escaping, and - enclosed in a CDATA section in the XML output. This is useful for LLMs sending code as part of a tool; results can be far superior compared to sending code in JSON-formatted tools, where code needs to confirm to JSON's strict rules and escaping requirements. (see test_xml_tool_message.py for an example).

extract_field_values(formatted_string) classmethod

Extracts field values from an XML-formatted string.

Parameters:

Name	Type	Description	Default
formatted_string	str	The XML-formatted string to parse.	required

Returns:

Type	Description
Optional[Dict[str, Any]]	Optional[Dict[str, Any]]: A dictionary containing the extracted field values, where keys are the XML element names and values are their corresponding contents.
Optional[Dict[str, Any]]	Returns None if parsing fails or the root element is not a dictionary.

Raises:

Type	Description
XMLSyntaxError	If the input string is not valid XML.

Source code in langroid/agent/xml_tool_message.py

@classmethod
def extract_field_values(cls, formatted_string: str) -> Optional[Dict[str, Any]]:
    """
    Extracts field values from an XML-formatted string.

    Args:
        formatted_string (str): The XML-formatted string to parse.

    Returns:
        Optional[Dict[str, Any]]: A dictionary containing the extracted field
            values, where keys are the XML element names and values are their
            corresponding contents.
        Returns None if parsing fails or the root element is not a dictionary.

    Raises:
        etree.XMLSyntaxError: If the input string is not valid XML.
    """
    # SECURITY: Initialize XMLParser with flags to prevent
    # XML External Entity (XXE), billion laughs, and external DTD attacks by
    # disabling entity resolution, DTD loading, and network access;
    # `strip_cdata=False` is needed to preserve
    # content within CDATA sections (e.g., for code).
    parser = etree.XMLParser(
        strip_cdata=False,
        resolve_entities=False,
        load_dtd=False,
        no_network=True,
    )
    root = etree.fromstring(formatted_string.encode("utf-8"), parser=parser)

    def parse_element(
        element: etree._Element, expected_field: Any = None, expected: Any = None
    ) -> Any:
        # Skip elements starting with underscore
        if element.tag.startswith("_"):
            return {}

        # Resolve what this element IS by position, not by tag name. Only a
        # direct child of the root is a top-level field, so only it may be
        # matched to one by name; a deeper element is described solely by
        # what its parent passes down (`expected_field` when the parent is
        # a model and so has field metadata, `expected` for a list's items
        # or a mapping's values). Looking deeper elements up by tag would
        # let a nested tag -- or a dict key -- that happens to collide with
        # a top-level field name borrow that field's type or its
        # `verbatim` flag.
        is_top_level = element is not root and element.getparent() is root
        annotation: Any
        if element is root:
            annotation = None
        elif is_top_level:
            top_field = cls.model_fields.get(element.tag)
            annotation = _unwrap_optional(
                top_field.annotation if top_field is not None else None
            )
        else:
            annotation = _unwrap_optional(
                expected_field.annotation
                if expected_field is not None
                else expected
            )
        origin = get_origin(annotation)
        is_list = origin is list or annotation is list
        is_dict = origin is dict or (
            isinstance(annotation, type) and issubclass(annotation, Mapping)
        )
        # `verbatim` stays a top-level-only flag, as before: nested
        # elements were never verbatim unless their tag happened to match a
        # top-level verbatim field, which is exactly the collision above.
        verbatim_field = cls.model_fields.get(element.tag) if is_top_level else None
        is_verbatim = (
            verbatim_field
            and hasattr(verbatim_field, "json_schema_extra")
            and verbatim_field.json_schema_extra is not None
            and isinstance(verbatim_field.json_schema_extra, dict)
            and verbatim_field.json_schema_extra.get("verbatim", False)
        )

        if is_verbatim:
            # For code elements, preserve the content as is, including whitespace
            content = element.text if element.text else ""
            # Strip leading and trailing triple backticks if present,
            # accounting for whitespace
            return (
                content.strip().removeprefix("```").removesuffix("```").strip()
                if content.strip().startswith("```")
                and content.strip().endswith("```")
                else content
            )
        elif len(element) == 0:
            # An empty container element (e.g. `<tags/>`) is an empty
            # collection, not an empty string: take the type from the
            # field's declared annotation rather than from the XML.
            if element is root:
                return {}
            if is_list:
                return []
            if is_dict:
                return {}
            # For non-code leaf elements, strip whitespace
            return element.text.strip() if element.text else ""
        else:
            # For branch elements, handle potential lists or nested structures
            # The "all children share a tag" heuristic below misreads a
            # single-entry dict or a single-field nested model as a list,
            # so decide by declared type first where we have one.
            is_mapping = element is root or is_dict or _is_model(annotation)
            # Tell the children what they are expected to be, since they
            # cannot be looked up by tag: a list's items, a mapping's
            # values, or a nested model's same-named field.
            child_type = _child_type(annotation, is_mapping)
            if not is_mapping and all(
                child.tag == element[0].tag for child in element
            ):
                # If all children have the same tag, treat as a list
                return [parse_element(child, None, child_type) for child in element]
            else:
                # Otherwise, treat as a dictionary
                result = {
                    child.tag: parse_element(
                        child, _child_field(annotation, child.tag), child_type
                    )
                    for child in element
                }
                # A nested model is left as a plain dict for the caller's
                # `model_validate` to coerce. Building it here would run
                # its validation before the containing tool's own
                # `mode="before"` validators get to transform the value,
                # and every caller validates the result anyway.
                return result

    result = parse_element(root)
    if not isinstance(result, dict):
        return None
    # Drop only the skipped underscore fields; an empty dict can now be a
    # legitimate argument value, so it must not be filtered out by value.
    return {k: v for k, v in result.items() if not k.startswith("_")}

parse(formatted_string) classmethod

Parses the XML-formatted string and returns an instance of the class.

Parameters:

Name	Type	Description	Default
formatted_string	str	The XML-formatted string to parse.	required

Returns:

Type	Description
Optional[XMLToolMessage]	Optional["XMLToolMessage"]: An instance of the class if parsing succeeds, None otherwise.

Source code in langroid/agent/xml_tool_message.py

@classmethod
def parse(cls, formatted_string: str) -> Optional["XMLToolMessage"]:
    """
    Parses the XML-formatted string and returns an instance of the class.

    Args:
        formatted_string (str): The XML-formatted string to parse.

    Returns:
        Optional["XMLToolMessage"]: An instance of the class if parsing succeeds,
            None otherwise.
    """
    try:
        parsed_data = cls.extract_field_values(formatted_string)
        if parsed_data is None:
            return None

        # Use Pydantic's parse_obj to create and validate the instance
        return cls.model_validate(parsed_data)
    except Exception as e:
        from langroid.exceptions import XMLException

        raise XMLException(f"Error parsing XML: {str(e)}")

format_example()

Format the current instance as an XML example.

Returns:

Name	Type	Description
str	str	A string representation of the current instance in XML format.

Raises:

Type	Description
ValueError	If the result from etree.tostring is not a string.

Source code in langroid/agent/xml_tool_message.py

def format_example(self) -> str:
    """
    Format the current instance as an XML example.

    Returns:
        str: A string representation of the current instance in XML format.

    Raises:
        ValueError: If the result from etree.tostring is not a string.
    """

    def create_element(
        parent: etree._Element, name: str, value: Any, path: str = ""
    ) -> None:
        if value is None:
            return

        elem = etree.SubElement(parent, name)
        current_path = f"{path}.{name}" if path else name

        if isinstance(value, list):
            for item in value:
                create_element(elem, "item", item, current_path)
        elif isinstance(value, dict):
            for k, v in value.items():
                create_element(elem, k, v, current_path)
        elif isinstance(value, BaseModel):
            # Handle nested Pydantic models
            for field_name, field_value in value.model_dump().items():
                create_element(elem, field_name, field_value, current_path)
        else:
            if current_path in self.__class__.find_verbatim_fields():
                elem.text = etree.CDATA(str(value))
            else:
                elem.text = str(value)

    root = etree.Element(self._get_root_element())
    exclude_fields: set[str] = self._get_excluded_fields()
    for name, value in self.model_dump().items():
        if name not in exclude_fields:
            create_element(root, name, value)

    result = etree.tostring(root, encoding="unicode", pretty_print=True)
    if not isinstance(result, str):
        raise ValueError("Unexpected non-string result from etree.tostring")
    return result

find_candidates(text) classmethod

Finds XML-like tool message candidates in text, with relaxed opening tag rules.

Parameters:

Name	Type	Description	Default
text	str	Input text to search for XML structures.	required

Returns:

Type	Description
List[str]	List of XML strings. For fragments missing the root opening tag but having
List[str]	valid XML structure and root closing tag, prepends the root opening tag.

Example

With root_tag="tool", given: "Hello data " Returns: ["data"]

Source code in langroid/agent/xml_tool_message.py

@classmethod
def find_candidates(cls, text: str) -> List[str]:
    """
    Finds XML-like tool message candidates in text, with relaxed opening tag rules.

    Args:
        text: Input text to search for XML structures.

    Returns:
        List of XML strings. For fragments missing the root opening tag but having
        valid XML structure and root closing tag, prepends the root opening tag.

    Example:
        With root_tag="tool", given:
        "Hello <field1>data</field1> </tool>"
        Returns: ["<tool><field1>data</field1></tool>"]
    """

    root_tag = cls._get_root_element()
    opening_tag = f"<{root_tag}>"
    closing_tag = f"</{root_tag}>"

    candidates = []
    pos = 0
    while True:
        # Look for either proper opening tag or closing tag
        start_normal = text.find(opening_tag, pos)
        end = text.find(closing_tag, pos)

        if start_normal == -1 and end == -1:
            break

        if start_normal != -1:
            # Handle normal case (has opening tag)
            end = text.find(closing_tag, start_normal)
            if end != -1:
                candidates.append(text[start_normal : end + len(closing_tag)])
                pos = max(end + len(closing_tag), start_normal + 1)
                continue
            elif start_normal == text.rfind(opening_tag):
                # last fragment - ok to miss closing tag
                candidates.append(text[start_normal:] + closing_tag)
                return candidates
            else:
                pos = start_normal + 1
                continue

        if end != -1:
            # Look backwards for first XML tag
            text_before = text[pos:end]
            first_tag_match = re.search(r"<\w+>", text_before)
            if first_tag_match:
                start = pos + first_tag_match.start()
                candidates.append(
                    opening_tag + text[start : end + len(closing_tag)]
                )
            pos = end + len(closing_tag)

    return candidates