From a70517f6d8242cb12886f1be08249428d9356b52 Mon Sep 17 00:00:00 2001 From: Annyfee <2287551746@qq.com> Date: Wed, 19 Nov 2025 23:44:14 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81=EF=BC=8C?= =?UTF-8?q?=E6=8F=90=E9=AB=98=E8=A7=A3=E8=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- 01_agent_introduction/agent-demo.py | 10 +-- .../01_basic_llm_invocation.py | 10 +-- .../02_conversational_agent.py | 8 +-- .../01_custom_function_calling.py | 8 +-- .../02_api_invocation.py | 8 +-- 04_langchain_basics/01_models.py | 9 +-- 04_langchain_basics/02_prompt.py | 7 -- 04_langchain_basics/03_chain.py | 10 +-- 04_langchain_basics/04_memory.py | 42 +----------- 04_langchain_basics/05_practice.py | 13 +--- 05_langchain_advanced/01_define_toolbox.py | 7 -- 05_langchain_advanced/02_general_agent.py | 15 ++-- 05_langchain_advanced/03_sql_agent.py | 10 +-- .../04_memory_general_agent.py | 14 ++-- 05_langchain_advanced/05_caching.py | 12 +--- 05_langchain_advanced/06_streaming.py | 15 ++-- 06_rag_basics/02_embedding.py | 11 ++- 06_rag_basics/03_build_index.py | 19 +----- 06_rag_basics/04_rag_chain_full.py | 26 ++----- .../{build_index.py => 00_build_index.py} | 22 ++---- 07_rag_advanced/01_load_from_chroma.py | 40 +++++------ 07_rag_advanced/02_reranker.py | 46 ++++--------- 07_rag_advanced/03_rag_as_tool.py | 68 ++++++------------- 07_rag_advanced/04_memory_rag_agent.py | 42 ++++++------ requirements.txt | 26 +++---- 25 files changed, 143 insertions(+), 355 deletions(-) rename 07_rag_advanced/{build_index.py => 00_build_index.py} (81%) diff --git a/01_agent_introduction/agent-demo.py b/01_agent_introduction/agent-demo.py index 3a9a2ab..4ad167f 100644 --- a/01_agent_introduction/agent-demo.py +++ b/01_agent_introduction/agent-demo.py @@ -1,17 +1,11 @@ -import os # 导入环境变量 -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - # 让ai说一句话 +from config import OPENAI_API_KEY from langchain_openai import ChatOpenAI # 配置deepseek llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, # 注：在.env里把OPENAI_API_KEY改成你自己的api-key即可 base_url="https://api.deepseek.com" ) diff --git a/02_llm_fundamentals/01_basic_llm_invocation.py b/02_llm_fundamentals/01_basic_llm_invocation.py index 1c1a4d8..7ea4829 100644 --- a/02_llm_fundamentals/01_basic_llm_invocation.py +++ b/02_llm_fundamentals/01_basic_llm_invocation.py @@ -1,14 +1,8 @@ -import os # 导入环境变量 -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - +from config import OPENAI_API_KEY from openai import OpenAI client = OpenAI( - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com") response = client.chat.completions.create( diff --git a/02_llm_fundamentals/02_conversational_agent.py b/02_llm_fundamentals/02_conversational_agent.py index 4c8dc42..e1fe3c7 100644 --- a/02_llm_fundamentals/02_conversational_agent.py +++ b/02_llm_fundamentals/02_conversational_agent.py @@ -1,12 +1,8 @@ -import os # 导入环境变量 -from dotenv import load_dotenv +from config import OPENAI_API_KEY from openai import OpenAI -load_dotenv() - def create_client(): - api_key = os.getenv("OPENAI_API_KEY") - return OpenAI(api_key=api_key,base_url="https://api.deepseek.com") + return OpenAI(api_key=OPENAI_API_KEY,base_url="https://api.deepseek.com") def chat_loop(agent_client): messages = [ diff --git a/03_function_calling_tools/01_custom_function_calling.py b/03_function_calling_tools/01_custom_function_calling.py index ecffdee..740a9c6 100644 --- a/03_function_calling_tools/01_custom_function_calling.py +++ b/03_function_calling_tools/01_custom_function_calling.py @@ -1,14 +1,10 @@ -import os -from dotenv import load_dotenv +from config import OPENAI_API_KEY from openai import OpenAI import json -load_dotenv() - def create_client(): - api_key = os.getenv("OPENAI_API_KEY") - return OpenAI(api_key=api_key, base_url="https://api.deepseek.com") + return OpenAI(api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com") def get_weather(location): # 模拟获得天气信息 diff --git a/03_function_calling_tools/02_api_invocation.py b/03_function_calling_tools/02_api_invocation.py index a9bf95d..4d1c19b 100644 --- a/03_function_calling_tools/02_api_invocation.py +++ b/03_function_calling_tools/02_api_invocation.py @@ -1,14 +1,10 @@ -from dotenv import load_dotenv +from config import OPENAI_API_KEY from openai import OpenAI -import json -import os import requests -load_dotenv() def create_client(): - api_key = os.getenv("OPENAI_API_KEY") - return OpenAI(api_key=api_key, base_url="https://api.deepseek.com") + return OpenAI(api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com") # 通过api调用获得当前ip与位置 def get_addr(): diff --git a/04_langchain_basics/01_models.py b/04_langchain_basics/01_models.py index f1fdda2..073c46f 100644 --- a/04_langchain_basics/01_models.py +++ b/04_langchain_basics/01_models.py @@ -1,15 +1,10 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - +from config import OPENAI_API_KEY from langchain_openai import ChatOpenAI # 初始化模型 llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) diff --git a/04_langchain_basics/02_prompt.py b/04_langchain_basics/02_prompt.py index 8e5dc29..1c508e9 100644 --- a/04_langchain_basics/02_prompt.py +++ b/04_langchain_basics/02_prompt.py @@ -1,10 +1,3 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - from langchain_core.prompts import ChatPromptTemplate # 定义提示词模板(推荐写法) diff --git a/04_langchain_basics/03_chain.py b/04_langchain_basics/03_chain.py index 944b3a7..75236f0 100644 --- a/04_langchain_basics/03_chain.py +++ b/04_langchain_basics/03_chain.py @@ -1,10 +1,4 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - +from config import OPENAI_API_KEY from langchain_core.prompts import ChatPromptTemplate from langchain_openai import ChatOpenAI from langchain_core.output_parsers import StrOutputParser @@ -18,7 +12,7 @@ prompt = ChatPromptTemplate.from_messages([ # 初始化模型 llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) diff --git a/04_langchain_basics/04_memory.py b/04_langchain_basics/04_memory.py index 1a7a342..c41a257 100644 --- a/04_langchain_basics/04_memory.py +++ b/04_langchain_basics/04_memory.py @@ -1,18 +1,9 @@ -import os -from dotenv import load_dotenv - - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - - from langchain_core.prompts import ChatPromptTemplate,MessagesPlaceholder from langchain_openai import ChatOpenAI from langchain_core.output_parsers import StrOutputParser from langchain_community.chat_message_histories import ChatMessageHistory from langchain_core.runnables import RunnableWithMessageHistory - +from config import OPENAI_API_KEY prompt = ChatPromptTemplate.from_messages([ ("system", "你非常可爱，说话末尾会带个喵"), @@ -21,7 +12,7 @@ prompt = ChatPromptTemplate.from_messages([ ]) llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) parser = StrOutputParser() @@ -58,32 +49,3 @@ while 1: config={"configurable":{"session_id":session_id}} ) print(f'AI:{response}') - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/04_langchain_basics/05_practice.py b/04_langchain_basics/05_practice.py index c55c1f5..7e5b22d 100644 --- a/04_langchain_basics/05_practice.py +++ b/04_langchain_basics/05_practice.py @@ -1,13 +1,4 @@ -import os -from dotenv import load_dotenv - -load_dotenv() - -def create_client(): - api_key = os.getenv("OPENAI_API_KEY") - return api_key - - +from config import OPENAI_API_KEY from langchain_core.prompts import ChatPromptTemplate,MessagesPlaceholder from langchain_openai import ChatOpenAI from langchain_core.output_parsers import StrOutputParser @@ -44,7 +35,7 @@ def main(): # 此处集中配置LLM llm = ChatOpenAI( model="deepseek-chat", - api_key=create_client(), + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) prompt = """你是‘小智’，一位专业、耐心且记忆力出色的 AI 助手。 diff --git a/05_langchain_advanced/01_define_toolbox.py b/05_langchain_advanced/01_define_toolbox.py index 987285c..9ecc9b2 100644 --- a/05_langchain_advanced/01_define_toolbox.py +++ b/05_langchain_advanced/01_define_toolbox.py @@ -1,10 +1,3 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - from langchain_core.tools import tool @tool diff --git a/05_langchain_advanced/02_general_agent.py b/05_langchain_advanced/02_general_agent.py index eaaea37..ef46236 100644 --- a/05_langchain_advanced/02_general_agent.py +++ b/05_langchain_advanced/02_general_agent.py @@ -1,20 +1,15 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - -import os +from config import OPENAI_API_KEY from langchain_openai import ChatOpenAI -from langchain.agents import AgentExecutor, create_tool_calling_agent +# 在LangChain 1.0+版本中，以下俩组件移到了langchain-classic包中 +from langchain_classic.agents import AgentExecutor +from langchain_classic.agents import create_tool_calling_agent from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_core.tools import tool # 导入 @tool # 配置LLM llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) diff --git a/05_langchain_advanced/03_sql_agent.py b/05_langchain_advanced/03_sql_agent.py index bf9da6a..8a684ef 100644 --- a/05_langchain_advanced/03_sql_agent.py +++ b/05_langchain_advanced/03_sql_agent.py @@ -1,18 +1,14 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - +from config import OPENAI_API_KEY import sqlite3 from langchain_openai import ChatOpenAI from langchain_community.agent_toolkits import create_sql_agent from langchain_community.utilities import SQLDatabase # 导入 SQLDatabase +import os # 配置llm llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) diff --git a/05_langchain_advanced/04_memory_general_agent.py b/05_langchain_advanced/04_memory_general_agent.py index ac7233c..1c849e5 100644 --- a/05_langchain_advanced/04_memory_general_agent.py +++ b/05_langchain_advanced/04_memory_general_agent.py @@ -1,13 +1,7 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - -import os +from config import OPENAI_API_KEY from langchain_openai import ChatOpenAI -from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_classic.agents import AgentExecutor +from langchain_classic.agents import create_tool_calling_agent from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_core.tools import tool # 导入 @tool from langchain_community.chat_message_histories import ChatMessageHistory @@ -17,7 +11,7 @@ from langchain_core.runnables import RunnableWithMessageHistory # 配置llm llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) # 配置prompt(新增俩占位符 一个为对话历史记录，一个为agent的思考过程) diff --git a/05_langchain_advanced/05_caching.py b/05_langchain_advanced/05_caching.py index ef5c6b3..12b2eb9 100644 --- a/05_langchain_advanced/05_caching.py +++ b/05_langchain_advanced/05_caching.py @@ -1,19 +1,13 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - +from config import OPENAI_API_KEY import time from langchain_openai import ChatOpenAI -from langchain.globals import set_llm_cache +from langchain_core.globals import set_llm_cache from langchain_community.cache import InMemoryCache # 配置llm llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) # 设置全局缓存 diff --git a/05_langchain_advanced/06_streaming.py b/05_langchain_advanced/06_streaming.py index 78faaba..daba014 100644 --- a/05_langchain_advanced/06_streaming.py +++ b/05_langchain_advanced/06_streaming.py @@ -1,23 +1,18 @@ -import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - - +from config import OPENAI_API_KEY from langchain_openai import ChatOpenAI -from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_classic.agents import AgentExecutor +from langchain_classic.agents import create_tool_calling_agent from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder from langchain_core.tools import tool # 导入 @tool from langchain_community.chat_message_histories import ChatMessageHistory from langchain_core.runnables import RunnableWithMessageHistory -from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler +from langchain_core.callbacks.streaming_stdout import StreamingStdOutCallbackHandler # 配置llm llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com", streaming=True, callbacks=[StreamingStdOutCallbackHandler()] diff --git a/06_rag_basics/02_embedding.py b/06_rag_basics/02_embedding.py index bdaecea..f0dae25 100644 --- a/06_rag_basics/02_embedding.py +++ b/06_rag_basics/02_embedding.py @@ -1,12 +1,11 @@ -from langchain_huggingface import HuggingFaceEmbeddings +from embeddings import get_embeddings -# 第一次运行可能时间较久 -print('---正在首次加载本地嵌入模型(bge-small-zh-v1.5)...---') + +# 首次运行可能时间较久 -- 同时运行本文件需要梯子，不然无法加载到本地 +print('---正在加载本地嵌入模型(bge-small-zh-v1.5)...---') # 理论：有embedding的向量模型 -embeddings_model = HuggingFaceEmbeddings( - model_name="BAAI/bge-small-zh-v1.5" # 一个中英双语开源模型 -) +embeddings_model = get_embeddings("bge-small-zh-v1.5") print('嵌入模型载入完毕') # 演示：将文本转换为向量 diff --git a/06_rag_basics/03_build_index.py b/06_rag_basics/03_build_index.py index 4b28d6b..65c1310 100644 --- a/06_rag_basics/03_build_index.py +++ b/06_rag_basics/03_build_index.py @@ -1,10 +1,9 @@ # 仅负责: 切片 -> 向量化 -> 构建索引 -> 保存到磁盘 # 运行一次即可，无需每次检索都运行 - +from embeddings import get_embeddings import os from langchain_community.document_loaders import TextLoader from langchain_text_splitters import RecursiveCharacterTextSplitter -from langchain_huggingface import HuggingFaceEmbeddings from langchain_community.vectorstores import FAISS # 准备知识库内容 @@ -48,7 +47,7 @@ splits = text_splitter.split_documents(docs) # 运行切分器 print(f'p1完成，文档已切分成{len(splits)}个片段\n') # 2. 向量化(Embedding) -embeddings_model = HuggingFaceEmbeddings(model_name="BAAI/bge-small-zh-v1.5") # 载入向量化模型 +embeddings_model = get_embeddings() # 载入向量化模型 print(f'p2完成，Embedding模型已准备\n') # # 3. 存储(Store) @@ -61,17 +60,3 @@ print(f'p3完成，向量数据库{db}已构建') os.remove("knowledge_base.txt") print('---所有阶段已经完成!---') - - - - - - - - - - - - - - diff --git a/06_rag_basics/04_rag_chain_full.py b/06_rag_basics/04_rag_chain_full.py index a16fe61..24abdc9 100644 --- a/06_rag_basics/04_rag_chain_full.py +++ b/06_rag_basics/04_rag_chain_full.py @@ -1,12 +1,8 @@ +from embeddings import get_embeddings +from config import OPENAI_API_KEY import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - from langchain_community.document_loaders import TextLoader from langchain_text_splitters import RecursiveCharacterTextSplitter -from langchain_huggingface import HuggingFaceEmbeddings from langchain_community.vectorstores import FAISS from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate @@ -43,8 +39,8 @@ loader = TextLoader('knowledge_base.txt',encoding='utf8') docs = loader.load() text_splitter = RecursiveCharacterTextSplitter(chunk_size=250,chunk_overlap=40) splits = text_splitter.split_documents(docs) -embedding_model = HuggingFaceEmbeddings(model_name="BAAI/bge-small-zh-v1.5") -db = FAISS.from_documents(splits,embedding_model) +embedding_model = get_embeddings("bge-small-zh-v1.5") +db = FAISS.from_documents(splits,embedding_model) # 在内存中构建向量索引，但不持久化到本地文件 print('---模块A(Indexing)完成---\n') @@ -75,7 +71,7 @@ prompt = ChatPromptTemplate.from_messages([ # 3. G (Generation - 生成) llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) @@ -111,15 +107,3 @@ print(f'回答:{response}\n') # 清理临时文件 os.remove("knowledge_base.txt") - - - - - - - - - - - - diff --git a/07_rag_advanced/build_index.py b/07_rag_advanced/00_build_index.py similarity index 81% rename from 07_rag_advanced/build_index.py rename to 07_rag_advanced/00_build_index.py index 0f707ec..02229c3 100644 --- a/07_rag_advanced/build_index.py +++ b/07_rag_advanced/00_build_index.py @@ -1,17 +1,14 @@ -# pip install chroma -# pip install -U langchain-chroma - import os from langchain_community.document_loaders import TextLoader from langchain_text_splitters import RecursiveCharacterTextSplitter -from langchain_huggingface import HuggingFaceEmbeddings from langchain_chroma import Chroma +from embeddings import get_embeddings knowledge_base_file = "war_and_peace.txt" # 持久化目录: Chroma会把所有数据(向量+文本+元数据)都存到这个文件夹 persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' -embedding_model = 'BAAI/bge-small-en-v1.5' # 如果愿意等待，可以换成模型"BAAI/bge-m3"，效果更好更适合长文，但下载时间也更久(2.2G) +model_name_str = 'BAAI/bge-small-en-v1.5' # 如果愿意等待，可以换成模型"BAAI/bge-m3"，效果更好更适合长文，但下载时间也更久(2.2G) chunk_size = 500 chunk_overlap = 75 @@ -42,9 +39,10 @@ text_splitter = RecursiveCharacterTextSplitter( splits = text_splitter.split_documents(docs) print('分割完成...\n') # 3. 向量化 -- 第一次运行会下载模型,预计耗时2分钟 -embedding_model = HuggingFaceEmbeddings( - model_name=embedding_model, - model_kwargs={'device':'cpu'}, # 强制模型在cpu上运行 +print(f'正在加载/下载模型{model_name_str}...') +embedding_model = get_embeddings( + model_name=model_name_str, + device='cpu', # 强制模型在cpu上运行 encode_kwargs={'batch_size':64} # 每次处理64个文本片段 ) print('Embedding模型加载完成...\n') @@ -63,10 +61,4 @@ for i in range(0, len(splits), batch_size): print(f"已插入 {min(i + batch_size, len(splits))} / {len(splits)} 条") -print(f'✅ 索引构建完毕，共 {len(splits)} 条，已保存到 {persist_directory}') - - - - - - +print(f'✅ 索引构建完毕，共 {len(splits)} 条，已保存到 {persist_directory}') \ No newline at end of file diff --git a/07_rag_advanced/01_load_from_chroma.py b/07_rag_advanced/01_load_from_chroma.py index b2d9607..1f3f0fa 100644 --- a/07_rag_advanced/01_load_from_chroma.py +++ b/07_rag_advanced/01_load_from_chroma.py @@ -1,45 +1,41 @@ -# pip install --upgrade langchain-openai -# pip install --upgrade langchain-huggingface langchain-core langchain-community -# pip install --upgrade langchain-core langchain-community -import os -from dotenv import load_dotenv -from langchain_core.runnables import RunnablePassthrough - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - -import os -from langchain_huggingface import HuggingFaceEmbeddings from langchain_chroma import Chroma -from langchain_openai import ChatOpenAI -from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser +from langchain_openai import ChatOpenAI +from embeddings import get_embeddings +from config import OPENAI_API_KEY +import os +from langchain_core.prompts import ChatPromptTemplate +from langchain_core.runnables import RunnablePassthrough Persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' -Embedding_model = 'BAAI/bge-small-en-v1.5' +model_name_str = 'BAAI/bge-small-en-v1.5' if not os.path.exists(Persist_directory): print(f"错误: 知识库文件 {Persist_directory} 未找到。") - print("请先运行'build_index.py'生成向量数据库，再运行该文件") + print("请先运行'00_build_index.py'生成向量数据库，再运行该文件") exit() print('---加载本地向量数据库---') # 模块A:链接本地Chroma向量数据库 # 1. 加载 Embedding 模型 -embedding_model = HuggingFaceEmbeddings(model_name=Embedding_model) +print(f'正在加载/下载模型{model_name_str}...') +embeddings_model = get_embeddings( + model_name=model_name_str, + device='cpu' +) # 2. 从本地目录加载Chroma DB db = Chroma( persist_directory=Persist_directory, - embedding_function=embedding_model + embedding_function=embeddings_model ) print(f'Chroma数据库已从本地加载(共{db._collection.count()}条)\n') # 模块B:R-A-G Flow # 1. R-检索 -retriever = db.as_retriever(search_kwargs={"k": 3}) # 召回3条相关数据 +retriever = db.as_retriever(search_kwargs={"k": 5}) # 召回5条相关数据 # 2. A-增强 sys_prompt = """ @@ -58,7 +54,7 @@ prompt = ChatPromptTemplate.from_messages([ # 3. G-生成 llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) @@ -79,6 +75,4 @@ print('---正在运行RAG链条---') question = '莫斯科大火发生在小说的哪一部分？有哪些角色亲历了这场灾难？' response = rag_chain.invoke(question) print(f'提问:{question}') -print(f'回答:{response}') - - +print(f'回答:{response}') \ No newline at end of file diff --git a/07_rag_advanced/02_reranker.py b/07_rag_advanced/02_reranker.py index 2567cfe..6411480 100644 --- a/07_rag_advanced/02_reranker.py +++ b/07_rag_advanced/02_reranker.py @@ -1,33 +1,31 @@ import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - -from langchain_huggingface import HuggingFaceEmbeddings +from config import OPENAI_API_KEY +from embeddings import get_embeddings from langchain_chroma import Chroma from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser from langchain_core.runnables import RunnablePassthrough - -# --- Reranker (02) 新增的 import --- -from langchain.retrievers import ContextualCompressionRetriever +from langchain_classic.retrievers import ContextualCompressionRetriever +from langchain_classic.retrievers.document_compressors import CrossEncoderReranker from langchain_community.cross_encoders import HuggingFaceCrossEncoder -from langchain.retrievers.document_compressors import CrossEncoderReranker Persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' -Embedding_model = 'BAAI/bge-small-en-v1.5' +model_name_str = 'BAAI/bge-small-en-v1.5' if not os.path.exists(Persist_directory): print(f"错误: 知识库文件 {Persist_directory} 未找到。") - print("请先运行'build_index.py'生成向量数据库，再运行该文件") + print("请先运行'00_build_index.py'生成向量数据库，再运行该文件") exit() print('---加载本地向量数据库---\n') # 1. 加载 Embedding 模型 -embeddings_model = HuggingFaceEmbeddings(model_name=Embedding_model) +print(f'正在加载/下载模型{model_name_str}...') +embeddings_model = get_embeddings( + model_name=model_name_str, + device='cpu' +) # 2. 加载 Chroma db db = Chroma( @@ -40,11 +38,11 @@ print('---Chroma数据库已加载---\n') # --- 模块 B (R-A-G Flow) --- # 1. R-检索--强化版 # 1.1 基础检索器(Base Retriever) - '粗召回' -base_retriever = db.as_retriever(search_kwargs={"k":5}) # K调大到5 +base_retriever = db.as_retriever(search_kwargs={"k":50}) # K调大到60 # 1.2 Reranker (重排器) - "精排序" -- 首次运行需要耗时下载 -print('正在加载Reranker模型 (bge-reranker-base)...') +print('正在加载 Reranker模型 (bge-reranker-base)...') encoder = HuggingFaceCrossEncoder(model_name="BAAI/bge-reranker-base") # 加载Ranker模型 -reranker = CrossEncoderReranker(model=encoder,top_n=2) # 对检索结果进行精排 +reranker = CrossEncoderReranker(model=encoder,top_n=6) # 对检索结果进行精排 # 1.3 创建管道封装器 compression_retriever = ContextualCompressionRetriever( base_retriever=base_retriever, # 用Chroma做 海选 @@ -71,7 +69,7 @@ prompt = ChatPromptTemplate.from_messages([ # 3. G-生成 llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) @@ -94,17 +92,3 @@ question = '皮埃尔是共济会成员吗？他在其中扮演什么角色？' response = rag_chain.invoke(question) print(f'提问:{question}') print(f'回答:{response}') - - - - - - - - - - - - - - diff --git a/07_rag_advanced/03_rag_as_tool.py b/07_rag_advanced/03_rag_as_tool.py index 471954a..c6b6867 100644 --- a/07_rag_advanced/03_rag_as_tool.py +++ b/07_rag_advanced/03_rag_as_tool.py @@ -1,25 +1,21 @@ import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - -from langchain_huggingface import HuggingFaceEmbeddings +from config import OPENAI_API_KEY +from embeddings import get_embeddings from langchain_chroma import Chroma from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate from langchain_core.output_parsers import StrOutputParser from langchain_core.runnables import RunnablePassthrough -from langchain.retrievers import ContextualCompressionRetriever +from langchain_classic.retrievers import ContextualCompressionRetriever from langchain_community.cross_encoders import HuggingFaceCrossEncoder -from langchain.retrievers.document_compressors import CrossEncoderReranker +from langchain_classic.retrievers.document_compressors import CrossEncoderReranker from langchain_core.tools import tool # 全局 LLM (供Agent和Rag共用) llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) @@ -27,23 +23,26 @@ llm = ChatOpenAI( def build_rag_chain(llm_instance): print('---正在构建RAG链条...---\n') - Persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' - Embedding_model = 'BAAI/bge-small-en-v1.5' - Encoder_model = "BAAI/bge-reranker-base" + persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' + embedding_model_name = 'BAAI/bge-small-en-v1.5' + encoder_model_name = "BAAI/bge-reranker-base" - if not os.path.exists(Persist_directory): - raise FileNotFoundError(f'索引目录{Persist_directory}未找到，请先运行 build_index.py') + if not os.path.exists(persist_directory): + raise FileNotFoundError(f'索引目录{persist_directory}未找到，请先运行 build_index.py') - embeddings_model = HuggingFaceEmbeddings(model_name=Embedding_model) + print(f'正在加载/下载 Embedding模型：{embedding_model_name}') + embeddings_model = get_embeddings(model_name=embedding_model_name,device='cpu') db = Chroma( - persist_directory=Persist_directory, + persist_directory=persist_directory, embedding_function=embeddings_model ) # 1. R-检索--强化版 - base_retriever = db.as_retriever(search_kwargs={"k":5}) - encoder = HuggingFaceCrossEncoder(model_name=Encoder_model) - reranker = CrossEncoderReranker(model=encoder,top_n=2) + base_retriever = db.as_retriever(search_kwargs={"k":50}) + + print(f'正在加载 Reranker模型:{encoder_model_name}...') + encoder = HuggingFaceCrossEncoder(model_name=encoder_model_name) + reranker = CrossEncoderReranker(model=encoder,top_n=6) compression_retriever=ContextualCompressionRetriever( base_retriever=base_retriever, base_compressor=reranker @@ -59,6 +58,7 @@ def build_rag_chain(llm_instance): [上下文]: {context} [问题]: {question} """ + prompt = ChatPromptTemplate.from_messages([ ('system',sys_prompt), ('human','{question}') @@ -105,32 +105,4 @@ if __name__ == '__main__': question = "皮埃尔是共济会成员吗？他在其中扮演什么角色？" res = search_war_and_peace.invoke(question) print(f'问题:{question}') - print(f'回答:{res}') - - - - - - - - - - - - - - - - - - - - - - - - - - - - + print(f'回答:{res}') \ No newline at end of file diff --git a/07_rag_advanced/04_memory_rag_agent.py b/07_rag_advanced/04_memory_rag_agent.py index c362201..d009bcc 100644 --- a/07_rag_advanced/04_memory_rag_agent.py +++ b/07_rag_advanced/04_memory_rag_agent.py @@ -1,21 +1,17 @@ import os -from dotenv import load_dotenv - -load_dotenv() -api_key = os.getenv("OPENAI_API_KEY") - -from langchain_huggingface import HuggingFaceEmbeddings +from config import OPENAI_API_KEY +from embeddings import get_embeddings from langchain_chroma import Chroma from langchain_openai import ChatOpenAI from langchain_core.prompts import ChatPromptTemplate,MessagesPlaceholder from langchain_core.output_parsers import StrOutputParser from langchain_core.runnables import RunnablePassthrough -from langchain.retrievers import ContextualCompressionRetriever +from langchain_classic.retrievers import ContextualCompressionRetriever from langchain_community.cross_encoders import HuggingFaceCrossEncoder -from langchain.retrievers.document_compressors import CrossEncoderReranker +from langchain_classic.retrievers.document_compressors import CrossEncoderReranker from langchain_core.tools import tool - -from langchain.agents import AgentExecutor, create_tool_calling_agent +from langchain_classic.agents import AgentExecutor +from langchain_classic.agents import create_tool_calling_agent from langchain_community.chat_message_histories import ChatMessageHistory from langchain_core.runnables import RunnableWithMessageHistory @@ -25,23 +21,25 @@ from langchain_core.runnables import RunnableWithMessageHistory def build_rag_chain(llm_instance): print('---正在构建RAG链条---') - Persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' - Embedding_model = 'BAAI/bge-small-en-v1.5' - Encoder_model = "BAAI/bge-reranker-base" + persist_directory = './chroma_db_war_and_peace_bge_small_en_v1.5' + embedding_model_name = 'BAAI/bge-small-en-v1.5' + encoder_model_name = "BAAI/bge-reranker-base" - if not os.path.exists(Persist_directory): - raise FileNotFoundError(f'索引目录{Persist_directory}未找到，请先运行 build_index.py') + if not os.path.exists(persist_directory): + raise FileNotFoundError(f'索引目录{persist_directory}未找到，请先运行 build_index.py') # 链接向量数据库 - embedding_model = HuggingFaceEmbeddings(model_name=Embedding_model) + print(f'正在加载/下载 Embedding模型：{embedding_model_name}') + embeddings_model = get_embeddings(model_name=embedding_model_name,device='cpu') db = Chroma( - persist_directory=Persist_directory, - embedding_function=embedding_model + persist_directory=persist_directory, + embedding_function=embeddings_model ) # R - base_retriever = db.as_retriever(search_kwargs={'k':5}) - encoder = HuggingFaceCrossEncoder(model_name=Encoder_model) - reranker = CrossEncoderReranker(model=encoder,top_n=2) + print(f'正在加载 Reranker模型:{encoder_model_name}...') + base_retriever = db.as_retriever(search_kwargs={'k':50}) + encoder = HuggingFaceCrossEncoder(model_name=encoder_model_name) + reranker = CrossEncoderReranker(model=encoder,top_n=6) compression_retriever = ContextualCompressionRetriever( base_retriever=base_retriever, base_compressor=reranker @@ -80,7 +78,7 @@ def create_agent_with_memory(): # LLm llm = ChatOpenAI( model="deepseek-chat", - api_key=api_key, + api_key=OPENAI_API_KEY, base_url="https://api.deepseek.com" ) # Prompt diff --git a/requirements.txt b/requirements.txt index 9a3945f..95d8a2d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,14 @@ -python-dotenv~=1.0.0 -openai>=1.40.0,<2.0.0 -requests~=2.32.4 -langchain~=0.2.16 -langchain-community~=0.2.16 -langchain-openai~=0.1.15 -langchain-core~=0.2.38 -langgraph~=0.2.0 -langchain-huggingface==0.0.3 -langchain-chroma==0.1.1 -chromadb==0.4.22 -sentence-transformers>=2.2.0 \ No newline at end of file +huggingface_hub==0.36.0 +langchain==1.0.8 +langchain_chroma==1.0.0 +langchain_classic==1.0.0 +langchain_community==0.4.1 +langchain_core==1.0.6 +langchain_huggingface==1.0.1 +langchain_openai==1.0.3 +langchain_text_splitters==1.0.0 +langgraph==1.0.3 +langsmith==0.4.43 +openai==2.8.1 +python-dotenv==1.2.1 +Requests==2.32.5