【AI大模型应用开发】【LangChain系列】2. 一文全览LangChain数据连接模块：从文档加载到向量检索RAG，理论+实战+细节

pip install pypdf

from langchain_community.document_loaders import PyPDFLoader
loader = PyPDFLoader("D:\GitHub\LEARN_LLM\RAG\如何向 ChatGPT 提问以获得高质量答案：提示技巧工程完全指南.pdf")
pages = loader.load_and_split()
print(f"第0页：\n{pages[0]}") ## 也可通过 pages[0].page_content只获取本页内容

pip install -i https://pypi.tuna.tsinghua.edu.cn/simple unstructured
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pdf2image
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple opencv-python
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple unstructured-inference
pip install -i https://pypi.tuna.tsinghua.edu.cn/simple pikepdf

from langchain_community.document_loaders import OnlinePDFLoader
loader = OnlinePDFLoader("https://arxiv.org/pdf/2302.03803.pdf")
data = loader.load()
print(data)

from langchain_community.document_loaders import PyPDFLoader
loader = PyPDFLoader("D:\GitHub\LEARN_LLM\RAG\如何向 ChatGPT 提问以获得高质量答案：提示技巧工程完全指南.pdf")
pages = loader.load_and_split()
print(f"第0页：\n{pages[0].page_content}")
from langchain.text_splitter import RecursiveCharacterTextSplitter
text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=200,
    chunk_overlap=100,
    length_function=len,
    add_start_index=True,
)
paragraphs = text_splitter.create_documents([pages[0].page_content])
for para in paragraphs:
    print(para.page_content)
    print('-------')

from langchain_openai import OpenAIEmbeddings
embeddings_model = OpenAIEmbeddings()  ## OpenAI文本向量化模型接口的封装
embeddings = embeddings_model.embed_documents(
    [
        "Hi there!",
        "Oh, hello!",
        "What's your name?",
        "My friends call me World",
        "Hello World!"
    ]
)
len(embeddings), len(embeddings[0])
##运行结果 (5, 1536)

embedded_query = embeddings_model.embed_query("What was the name mentioned in the conversation?")
embedded_query[:5]
## 运行结果：
## [0.0053587136790156364,
##  -0.0004999046213924885,
##  0.038883671164512634,
##  -0.003001077566295862,
##  -0.00900818221271038]

pip install chromadb

from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma
db = Chroma.from_documents(paragraphs, OpenAIEmbeddings()) ## 一行代码搞定

query = "什么是角色提示？"
docs = db.similarity_search(query) ## 一行代码搞定
for doc in docs:
    print(f"{doc.page_content}\n-------\n")

embedding_vector = OpenAIEmbeddings().embed_query(query)
docs = db.similarity_search_by_vector(embedding_vector)
print(docs[0].page_content)

retriever = db.as_retriever()
docs = retriever.get_relevant_documents("什么是角色提示？")
for doc in docs:
    print(f"{doc.page_content}\n-------\n")

retriever = db.as_retriever(
    search_type="similarity_score_threshold", search_kwargs={"score_threshold": 0.5}
)

retriever = db.as_retriever(search_kwargs={"k": 1})

## 1. 文档加载
    from langchain.document_loaders import PyPDFLoader
    loader = PyPDFLoader("D:\GitHub\LEARN_LLM\RAG\如何向 ChatGPT 提问以获得高质量答案：提示技巧工程完全指南.pdf")
    pages = loader.load_and_split()
    
    ## 2. 文档切分
    from langchain.text_splitter import RecursiveCharacterTextSplitter
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=200,
        chunk_overlap=100,
        length_function=len,
        add_start_index=True,
    )
    paragraphs = []
    for page in pages:
        paragraphs.extend(text_splitter.create_documents([page.page_content]))
    
    ## 3. 文档向量化，向量数据库存储
    from langchain_openai import OpenAIEmbeddings
    from langchain_community.vectorstores import Chroma
    db = Chroma.from_documents(paragraphs, OpenAIEmbeddings())
    
    ## 4. 向量检索
    # query = "什么是角色提示？"
    # docs = db.similarity_search(query)
    # for doc in docs:
    #     print(f"{doc.page_content}\n-------\n")
    
    retriever = db.as_retriever()
    docs = retriever.get_relevant_documents("什么是角色提示？")
    for doc in docs:
        print(f"{doc.page_content}\n-------\n")
    
    ## 5. 组装Prompt模板
    import os
    # 加载 .env 到环境变量
    from dotenv import load_dotenv, find_dotenv
    _ = load_dotenv(find_dotenv())
    from langchain_openai import ChatOpenAI
    
    llm = ChatOpenAI() # 默认是gpt-3.5-turbo
    prompt_template = """
    你是一个问答机器人。
    你的任务是根据下述给定的已知信息回答用户问题。
    确保你的回复完全依据下述已知信息。不要编造答案。
    如果下述已知信息不足以回答用户的问题，请直接回复"我无法回答您的问题"。
    已知信息:
    {info}
    用户问：
    {question}
    请用中文回答用户问题。
    """
    from langchain.prompts import PromptTemplate
    template = PromptTemplate.from_template(prompt_template)
    prompt = template.format(info=docs[0].page_content, question='什么是角色提示？')
    ## 6. 调用LLM
    response = llm.invoke(prompt)
    print(response.content)