Skip to content

remote_embeds

langroid/embedding_models/remote_embeds.py

If run as a script, starts an RPC server which handles remote embedding requests:

For example: python3 -m langroid.embedding_models.remote_embeds --port port

where port is the port at which the service is exposed. Currently, supports insecure connections only, and this should NOT be exposed to the internet.

serve(bind_address_base='localhost', port=50052, batch_size=512, data_parallel=False, device=None, devices=None, model_name='BAAI/bge-large-en-v1.5') async

Starts the RPC server.

Source code in langroid/embedding_models/remote_embeds.py 
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
async def serve(
    bind_address_base: str = "localhost",
    port: int = 50052,
    batch_size: int = 512,
    data_parallel: bool = False,
    device: Optional[str] = None,
    devices: Optional[list[str]] = None,
    model_name: str = "BAAI/bge-large-en-v1.5",
) -> None:
    """Starts the RPC server."""
    server = grpc.aio.server()
    embeddings_grpc.add_EmbeddingServicer_to_server(
        RemoteEmbeddingRPCs(
            model_name=model_name,
            batch_size=batch_size,
            data_parallel=data_parallel,
            device=device,
            devices=devices,
        ),
        server,
    )  # type: ignore
    url = f"{bind_address_base}:{port}"
    server.add_insecure_port(url)
    await server.start()
    print(f"Embedding server started, listening on {url}")
    await server.wait_for_termination()