Skip to content

LangChain RAG 实践指南 ​

LangChain 是一个强大的框架,用于构建基于大语言模型的应用。本指南将详细介绍如何使用 LangChain 构建 RAG(检索增强生成)应用。

概述 ​

什么是 LangChain? ​

LangChain 是一个开源框架,提供了构建 LLM 应用的工具和组件:

  • 组件化设计:模块化的组件便于组合
  • 链式调用:支持复杂的调用链
  • 记忆管理:内置对话记忆功能
  • 工具集成:丰富的工具和集成
  • RAG 支持:完整的 RAG 实现方案

核心组件 ​

  • Document Loaders: 文档加载器
  • Text Splitters: 文本切分器
  • Embeddings: 向量化模型
  • Vector Stores: 向量存储
  • Retrievers: 检索器
  • Chains: 调用链
  • Memory: 记忆管理

环境准备 ​

安装依赖 ​

bash
pip install langchain langchain-openai langchain-community
pip install chromadb faiss-cpu
pip install pypdf unstructured

配置 API Key ​

python
import os

os.environ["OPENAI_API_KEY"] = "your-api-key"

快速开始 ​

基础 RAG 实现 ​

1. 加载文档 ​

python
from langchain.document_loaders import PyPDFLoader
from langchain.document_loaders import TextLoader
from langchain.document_loaders import DirectoryLoader

def load_documents():
    loader = PyPDFLoader("document.pdf")
    documents = loader.load()
    
    return documents

def load_directory(directory_path):
    loader = DirectoryLoader(
        directory_path,
        glob="**/*.pdf",
        loader_cls=PyPDFLoader
    )
    documents = loader.load()
    
    return documents

2. 文本切分 ​

python
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.text_splitter import CharacterTextSplitter

def split_documents(documents):
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000,
        chunk_overlap=200,
        length_function=len,
        separators=["\n\n", "\n", " ", ""]
    )
    
    chunks = text_splitter.split_documents(documents)
    return chunks

3. 创建向量存储 ​

python
from langchain_openai import OpenAIEmbeddings
from langchain_community.vectorstores import Chroma

def create_vector_store(documents):
    embeddings = OpenAIEmbeddings()
    
    vector_store = Chroma.from_documents(
        documents=documents,
        embedding=embeddings,
        persist_directory="./chroma_db"
    )
    
    return vector_store

4. 创建检索器 ​

python
def create_retriever(vector_store):
    retriever = vector_store.as_retriever(
        search_type="similarity",
        search_kwargs={"k": 5}
    )
    
    return retriever

5. 创建问答链 ​

python
from langchain_openai import ChatOpenAI
from langchain.chains import RetrievalQA

def create_qa_chain(retriever):
    llm = ChatOpenAI(
        model_name="gpt-3.5-turbo",
        temperature=0.7
    )
    
    qa_chain = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=retriever,
        return_source_documents=True
    )
    
    return qa_chain

6. 完整示例 ​

python
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_community.vectorstores import Chroma
from langchain.chains import RetrievalQA

def build_rag_system(file_path):
    loader = PyPDFLoader(file_path)
    documents = loader.load()
    
    text_splitter = RecursiveCharacterTextSplitter(
        chunk_size=1000,
        chunk_overlap=200
    )
    chunks = text_splitter.split_documents(documents)
    
    embeddings = OpenAIEmbeddings()
    vector_store = Chroma.from_documents(
        documents=chunks,
        embedding=embeddings
    )
    
    retriever = vector_store.as_retriever(
        search_kwargs={"k": 5}
    )
    
    llm = ChatOpenAI(temperature=0.7)
    qa_chain = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=retriever,
        return_source_documents=True
    )
    
    return qa_chain

qa_chain = build_rag_system("document.pdf")

result = qa_chain({"query": "文档的主要内容是什么?"})
print(result["result"])

高级功能 ​

自定义 Prompt ​

python
from langchain.prompts import PromptTemplate

def create_custom_prompt():
    prompt_template = """
    基于以下上下文信息回答用户问题。如果上下文中没有相关信息,请明确说明"根据提供的信息无法回答该问题"。

    上下文信息:
    {context}

    用户问题:{question}

    请提供详细、准确的回答:
    """
    
    PROMPT = PromptTemplate(
        template=prompt_template,
        input_variables=["context", "question"]
    )
    
    return PROMPT

def create_custom_qa_chain(retriever, prompt):
    llm = ChatOpenAI(temperature=0.7)
    
    qa_chain = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=retriever,
        chain_type_kwargs={"prompt": prompt},
        return_source_documents=True
    )
    
    return qa_chain

多轮对话 ​

python
from langchain.memory import ConversationBufferMemory
from langchain.chains import ConversationalRetrievalChain

def create_conversational_rag(vector_store):
    llm = ChatOpenAI(temperature=0.7)
    
    memory = ConversationBufferMemory(
        memory_key="chat_history",
        return_messages=True,
        output_key="answer"
    )
    
    qa_chain = ConversationalRetrievalChain.from_llm(
        llm=llm,
        retriever=vector_store.as_retriever(),
        memory=memory,
        return_source_documents=True,
        verbose=True
    )
    
    return qa_chain

qa_chain = create_conversational_rag(vector_store)

chat_history = []
while True:
    query = input("用户: ")
    if query.lower() in ["exit", "quit"]:
        break
    
    result = qa_chain({"question": query, "chat_history": chat_history})
    print(f"AI: {result['answer']}")
    
    chat_history.append((query, result['answer']))

混合检索 ​

python
from langchain.retrievers import EnsembleRetriever
from langchain.retrievers import BM25Retriever

def create_hybrid_retriever(documents, vector_store):
    bm25_retriever = BM25Retriever.from_documents(documents)
    bm25_retriever.k = 5
    
    vector_retriever = vector_store.as_retriever(
        search_kwargs={"k": 5}
    )
    
    ensemble_retriever = EnsembleRetriever(
        retrievers=[bm25_retriever, vector_retriever],
        weights=[0.4, 0.6]
    )
    
    return ensemble_retriever

重排序 ​

python
from langchain.retrievers import ContextualCompressionRetriever
from langchain.retrievers.document_compressors import LLMChainExtractor

def create_reranking_retriever(base_retriever):
    llm = ChatOpenAI(temperature=0)
    
    compressor = LLMChainExtractor.from_llm(llm)
    
    compression_retriever = ContextualCompressionRetriever(
        base_compressor=compressor,
        base_retriever=base_retriever
    )
    
    return compression_retriever

流式输出 ​

python
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler

def create_streaming_qa_chain(retriever):
    llm = ChatOpenAI(
        temperature=0.7,
        streaming=True,
        callbacks=[StreamingStdOutCallbackHandler()]
    )
    
    qa_chain = RetrievalQA.from_chain_type(
        llm=llm,
        chain_type="stuff",
        retriever=retriever
    )
    
    return qa_chain

向量数据库集成 ​

Chroma ​

python
from langchain_community.vectorstores import Chroma

vector_store = Chroma.from_documents(
    documents=chunks,
    embedding=embeddings,
    persist_directory="./chroma_db"
)

vector_store.persist()

FAISS ​

python
from langchain_community.vectorstores import FAISS

vector_store = FAISS.from_documents(
    documents=chunks,
    embedding=embeddings
)

vector_store.save_local("./faiss_index")

vector_store = FAISS.load_local(
    "./faiss_index",
    embeddings
)

Milvus ​

python
from langchain_community.vectorstores import Milvus

vector_store = Milvus.from_documents(
    documents=chunks,
    embedding=embeddings,
    collection_name="langchain_demo",
    connection_args={
        "host": "localhost",
        "port": "19530"
    }
)

Pinecone ​

python
from langchain_community.vectorstores import Pinecone
import pinecone

pinecone.init(
    api_key="your-api-key",
    environment="your-environment"
)

vector_store = Pinecone.from_documents(
    documents=chunks,
    embedding=embeddings,
    index_name="langchain-index"
)

文档加载器 ​

PDF 文档 ​

python
from langchain.document_loaders import PyPDFLoader

loader = PyPDFLoader("document.pdf")
documents = loader.load()

Word 文档 ​

python
from langchain.document_loaders import Docx2txtLoader

loader = Docx2txtLoader("document.docx")
documents = loader.load()

网页内容 ​

python
from langchain.document_loaders import WebBaseLoader

loader = WebBaseLoader("https://example.com")
documents = loader.load()

CSV 文件 ​

python
from langchain.document_loaders import CSVLoader

loader = CSVLoader("data.csv")
documents = loader.load()

JSON 文件 ​

python
from langchain.document_loaders import JSONLoader

loader = JSONLoader(
    file_path="data.json",
    jq_schema=".messages[]",
    content_key="content"
)
documents = loader.load()

文本切分策略 ​

递归字符切分 ​

python
from langchain.text_splitter import RecursiveCharacterTextSplitter

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    length_function=len,
    separators=["\n\n", "\n", " ", ""]
)

字符切分 ​

python
from langchain.text_splitter import CharacterTextSplitter

text_splitter = CharacterTextSplitter(
    separator="\n\n",
    chunk_size=1000,
    chunk_overlap=200
)

代码切分 ​

python
from langchain.text_splitter import PythonCodeTextSplitter

text_splitter = PythonCodeTextSplitter(
    chunk_size=1000,
    chunk_overlap=200
)

Markdown 切分 ​

python
from langchain.text_splitter import MarkdownTextSplitter

text_splitter = MarkdownTextSplitter(
    chunk_size=1000,
    chunk_overlap=200
)

性能优化 ​

缓存机制 ​

python
from langchain.cache import InMemoryCache
from langchain.globals import set_llm_cache

set_llm_cache(InMemoryCache())

llm = ChatOpenAI(temperature=0)

批处理 ​

python
def batch_process_documents(documents, batch_size=100):
    from langchain_community.vectorstores import Chroma
    
    vector_store = None
    
    for i in range(0, len(documents), batch_size):
        batch = documents[i:i + batch_size]
        
        if vector_store is None:
            vector_store = Chroma.from_documents(
                documents=batch,
                embedding=embeddings
            )
        else:
            vector_store.add_documents(batch)
    
    return vector_store

异步处理 ​

python
import asyncio
from langchain_community.vectorstores import Chroma

async def async_add_documents(vector_store, documents):
    await vector_store.aadd_documents(documents)

async def async_search(vector_store, query):
    results = await vector_store.asimilarity_search(query, k=5)
    return results

实战案例 ​

企业知识库问答 ​

python
from langchain.document_loaders import DirectoryLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain_community.vectorstores import Chroma
from langchain.chains import ConversationalRetrievalChain
from langchain.memory import ConversationBufferMemory

class EnterpriseKnowledgeBase:
    def __init__(self, documents_dir):
        self.documents_dir = documents_dir
        self.qa_chain = None
        
    def load_documents(self):
        loader = DirectoryLoader(
            self.documents_dir,
            glob="**/*.pdf",
            loader_cls=PyPDFLoader
        )
        return loader.load()
    
    def build_index(self):
        documents = self.load_documents()
        
        text_splitter = RecursiveCharacterTextSplitter(
            chunk_size=1000,
            chunk_overlap=200
        )
        chunks = text_splitter.split_documents(documents)
        
        embeddings = OpenAIEmbeddings()
        self.vector_store = Chroma.from_documents(
            documents=chunks,
            embedding=embeddings,
            persist_directory="./knowledge_base"
        )
        
    def create_qa_system(self):
        llm = ChatOpenAI(temperature=0.7)
        
        memory = ConversationBufferMemory(
            memory_key="chat_history",
            return_messages=True
        )
        
        self.qa_chain = ConversationalRetrievalChain.from_llm(
            llm=llm,
            retriever=self.vector_store.as_retriever(),
            memory=memory,
            return_source_documents=True
        )
    
    def query(self, question):
        result = self.qa_chain({"question": question})
        return {
            "answer": result["answer"],
            "sources": result["source_documents"]
        }

kb = EnterpriseKnowledgeBase("./documents")
kb.build_index()
kb.create_qa_system()

result = kb.query("公司的年假政策是什么?")
print(result["answer"])

常见问题 ​

1. 内存占用过高 ​

问题: 处理大量文档时内存不足

解决方案:

  • 使用批处理
  • 启用持久化存储
  • 减小 chunk_size

2. 检索质量不佳 ​

问题: 检索到的文档不相关

解决方案:

  • 调整切分策略
  • 使用混合检索
  • 添加重排序

3. 响应速度慢 ​

问题: 系统响应延迟高

解决方案:

  • 启用缓存
  • 使用异步处理
  • 优化向量数据库

最佳实践 ​

1. 文档处理 ​

  • 选择合适的文档加载器
  • 优化切分参数
  • 保留文档元数据

2. 检索优化 ​

  • 使用混合检索策略
  • 配置合理的检索参数
  • 实现重排序机制

3. 用户体验 ​

  • 支持多轮对话
  • 提供来源引用
  • 实现流式输出

参考资料 ​

官方文档 ​

相关教程 ​


提示

LangChain 提供了丰富的组件和灵活的组合方式,建议根据具体需求选择合适的组件。

信息

本指南基于 LangChain 最新版本编写,API 可能随版本更新而变化,请参考官方文档。