LangChain-20 Document Loader 文件加载 加载MD DOCX EXCEL PPT PDF HTML JSON 等多种文件格式 后续可通过FAISS向量化 增强检索

from langchain_community.document_loaders import TextLoader

loader = TextLoader("./index.md")
data = loader.load()
print(data)

from langchain_community.document_loaders.csv_loader import CSVLoader


loader = CSVLoader(file_path='./example_data/mlb_teams_2012.csv')
data = loader.load()
print(data)

loader = CSVLoader(file_path='./example_data/mlb_teams_2012.csv', csv_args={
    'delimiter': ',',
    'quotechar': '"',
    'fieldnames': ['MLB Team', 'Payroll in millions', 'Wins']
})

data = loader.load()
print(data)

from langchain_community.document_loaders import DirectoryLoader

loader = DirectoryLoader('../', glob="**/*.md")
docs = loader.load()
print(docs)

# 显示一个 进度条
loader = DirectoryLoader('../', glob="**/*.md", show_progress=True)

# 多线程加载
loader = DirectoryLoader('../', glob="**/*.md", use_multithreading=True)

# 自动检测编码
text_loader_kwargs={'autodetect_encoding': True}
loader = DirectoryLoader(path, glob="**/*.txt", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)

from langchain_community.document_loaders import UnstructuredHTMLLoader
from langchain_community.document_loaders import BSHTMLLoader

loader = UnstructuredHTMLLoader("example_data/fake-content.html")
data = loader.load()
print(data)

# 如果你会用 BeautifulSoup4 的话，可以用它解析
loader = BSHTMLLoader("example_data/fake-content.html")
data = loader.load()
print(data)

from langchain_community.document_loaders import JSONLoader

import json
from pathlib import Path
from pprint import pprint

# 普通的加载 json.loads
file_path='./example_data/facebook_chat.json'
data = json.loads(Path(file_path).read_text())
pprint(data)

# 使用 JSONLoader
loader = JSONLoader(
    file_path='./example_data/facebook_chat.json',
    jq_schema='.messages[].content',
    text_content=False)

data = loader.load()
pprint(data)

from langchain_community.document_loaders import JSONLoader

import json
from pathlib import Path
from pprint import pprint

file_path = './example_data/facebook_chat_messages.jsonl'
pprint(Path(file_path).read_text())

loader = JSONLoader(
    file_path='./example_data/facebook_chat_messages.jsonl',
    jq_schema='.content',
    text_content=False,
    json_lines=True)

data = loader.load()
pprint(data)

from langchain_community.document_loaders import UnstructuredMarkdownLoader

markdown_path = "../../../../../README.md"
loader = UnstructuredMarkdownLoader(markdown_path)
data = loader.load()

from langchain_community.document_loaders import PyPDFLoader

# 加载方式很多，不止这一个PDF的Loader
loader = PyPDFLoader("example_data/layout-parser-paper.pdf")
pages = loader.load_and_split()
print(pages[0])

# 可以将图片转化为文字
loader = PyPDFLoader("https://arxiv.org/pdf/2103.15348.pdf", extract_images=True)
pages = loader.load()
pages[4].page_content

from langchain_community.vectorstores import FAISS
from langchain_openai import OpenAIEmbeddings

faiss_index = FAISS.from_documents(pages, OpenAIEmbeddings())
docs = faiss_index.similarity_search("How will the community be engaged?", k=2)
for doc in docs:
    print(str(doc.metadata["page"]) + ":", doc.page_content[:300])