Got it! I saw this in the code and assumed you were using embeddings def evaluate(input: shared.EvaluationInput): ds = Dataset.from_list(input.dataset) metrics = [metric_map[metric] for metric in input.metrics]

    llm = ChatOpenAI(
        model_name=shared.LANGUAGE_MODEL,
        base_url=os.environ["OPENAI_API_BASE"],
        api_key=os.environ["OPENAI_API_KEY"],
    )

    embeddings = OpenAIEmbeddings(
        model=shared.EMBEDDING_MODEL,
        base_url=os.environ["OPENAI_API_BASE"],
        api_key=os.environ["OPENAI_API_KEY"],
    )