【AI大模型应用开发】【LangSmith: 生产级AI应用维护平台】1. 快速上手数据集与测试评估过程

{"outlines": "Assistants API\n✅1. OpenAI 给了我们更大空间\n✅2. 原生 API、GPTs、Assistants API、国产/开源大模型选型参考\n✅3. Assistants API 的主要能力\n✅4. 做一个自己的 GPT\n  1. 创建 assistant\n  2. 管理 thread\n  3. 添加 message\n  4. 开始 run\n  5. 中控调度\n  6. Function Calling\n  7. Code Interpreter\n  8. RAG", "user_input": "别进reddit的中文话题，那是最没营养的区域", "label": "N"}
{"outlines": "【神秘嘉宾】大模型时代的AI产品新挑战\n1. AI 能力演进路线\n✅2. LLMs 带来的变化\n✅3. 如何将大模型落地到实际场景中\n✅4. LLMs 存在哪些问题\n✅5. LLMs 落地三要素\n✅6. LLMs 短期、中期和长期落地方向", "user_input": "对话式交互也不是所有场景都合适", "label": "N"}

import json
data = []
with open('D:\GitHub\LEARN_LLM\langsmith\my_annotations.jsonl','r',encoding='utf-8') as fp:
    for line in fp:
        example = json.loads(line.strip())
        item = {
            "input": {
                "outlines": example["outlines"],
                "user_input": example["user_input"]
            },
            "expected_output": example["label"]
        }
        data.append(item)
        
from langsmith import Client
client = Client()
dataset_name = "assistant-001"
dataset = client.create_dataset(
    dataset_name, #数据集名称
    description="AGI课堂的标注数据", #数据集描述
)
client.create_examples(
    inputs=[{"input":item["input"]} for item in data[:50]],  # 只是演示，所以只上传了前50条测试数据
    outputs=[{"output":item["expected_output"]} for item in data[:50]], 
    dataset_id=dataset.id
)

from langchain.evaluation import StringEvaluator
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
import re
from typing import Optional, Any
class AccuracyEvaluator(StringEvaluator):
    def __init__(self):
        pass
    def _evaluate_strings(
        self,
        prediction: str,
        input: Optional[str] = None,
        reference: Optional[str] = None,
        **kwargs: Any
    ) -> dict:
        return {"score": int(prediction==reference)}
from langchain.evaluation import EvaluatorType
from langchain.smith import RunEvalConfig
evaluation_config = RunEvalConfig(
    # 自定义评估标准
    custom_evaluators=[AccuracyEvaluator()],
)

from langchain.prompts import PromptTemplate
need_answer=PromptTemplate.from_template("""
*********
你是AIGC课程的助教，你的工作是从学员的课堂交流中选择出需要老师回答的问题，加以整理以交给老师回答。
 
课程内容:
{outlines}
*********
学员输入:
{user_input}
*********
如果这是一个需要老师答疑的问题，回复Y，否则回复N。
只回复Y或N，不要回复其他内容。""")
model = ChatOpenAI(temperature=0,model_kwargs={"seed":42})
parser = StrOutputParser()
chain_v1 = (
    {
        "outlines":lambda x: x["input"]["outlines"],
        "user_input":lambda x: x["input"]["user_input"],
    }
    | need_answer
    | model
    | parser
)

from langchain.smith import (
    arun_on_dataset,
    run_on_dataset,
)
from langsmith import Client
client = Client()
async def test_run():
    dataset_name = "assistant-001"
    results = await arun_on_dataset(
        dataset_name=dataset_name,
        llm_or_chain_factory=chain_v1,
        evaluation=evaluation_config,
        verbose=True,
        client=client,
        project_name="test-002",
        tags=[
            "prompt_v1",
        ],  # 可选，自定义的标识
    )
    print(results)
    
asyncio.run(test_run())

import asyncio
from langchain_openai import ChatOpenAI
from langchain.prompts import PromptTemplate
from langchain.schema.output_parser import StrOutputParser
from langchain.schema.runnable import RunnablePassthrough
from langchain.schema import HumanMessage
from langchain.prompts.chat import HumanMessagePromptTemplate
from langchain.prompts import ChatPromptTemplate
from langchain.evaluation import StringEvaluator
from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction
import re
from typing import Optional, Any