深入聊聊RAG

# 设置本地数据保存目录local_data_dir = pathlib.Path("/Users/jiangdanyang/workspaces/python/MarioPython/src/RAG/dataset/ai-arxiv2")# 加载数据集（如果本地存在则从本地加载，否则下载）dataset = load_dataset("jamescalam/ai-arxiv2", split="train", cache_dir=str(local_data_dir))
# 初始化编码器encoder = OpenAIEncoder(   name="text-embedding-3-small",   openai_api_key=os.getenv("AI_API_KEY"),   openai_base_url=os.getenv("AI_API_BASE_URL"))
chunker = StatisticalChunker(    encoder=encoder,    min_split_tokens=100,    max_split_tokens=500,    plot_chunks=True,    enable_statistics=True,)
chunks_0 = chunker(docs=[dataset["content"][0]], batch_size=500)

Chunking Statistics:  - Total Documents: 474  - Total Chunks: 46  - Chunks by Threshold: 41  - Chunks by Max Chunk Size: 4  - Last Chunk: 1  - Minimum Token Size of Chunk: 54  - Maximum Token Size of Chunk: 495  - Similarity Chunk Ratio: 0.89

DOCUMENT_CONTEXT_PROMPT = """<document>{doc_content}</document>"""
CHUNK_CONTEXT_PROMPT = """Here is the chunk we want to situate within the whole document<chunk>{chunk_content}</chunk>
Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk.Answer only with the succinct context and nothing else."""

Given the following text chunk, generate {n} different questions that this chunk would be a good answer to:
Chunk: {chunk}
Questions (enumarate the questions with 1. 2., etc.):

first_sentence = "直播分享会AI Living第一场"second_sentence = "直播鲁班小组分享第77期"
model = SentenceTransformer("/Users/jiangdanyang/workspaces/python/Model/all-MiniLM-L6-v2")tokenized_first_sentence = model.tokenize([first_sentence])tokenized_second_sentence = model.tokenize([second_sentence])

# Load the chunkscorpus_json = json.load(open('/Users/jiangdanyang/workspaces/python/MarioPython/src/RAG/dataset/corpus.json'))corpus_text = [doc["text"] for doc in corpus_json]
# optional: create a stemmerenglish_stemmer = snowballstemmer.stemmer("english")
# Initialize the Tokenizer with the stemmersparse_tokenizer = Tokenizer(    stemmer=english_stemmer,    lower=True, # lowercase the tokens    stopwords="english",# or pass a list of stopwords    splitter=r"\w+",# by default r"(?u)\b\w\w+\b", can also be a function)
# Tokenize the corpuscorpus_sparse_tokens = (    sparse_tokenizer    .tokenize(        corpus_text,         update_vocab=True, # update the vocab as we tokenize        return_as="ids"    ))
# Create the BM25 retriever and attach your corpus_json to itsparse_index = bm25s.BM25(corpus=corpus_json)# Now, index the corpus_tokens (the corpus_json is not used yet)sparse_index.index(corpus_sparse_tokens)
# Return 10 the most relevant docs according to the querysparse_results, sparse_scores = sparse_index.retrieve(query_tokens, k=10)

#Dense Index# create the vector database clientqdrant = QdrantClient(path="/Users/jiangdanyang/workspaces/python/MarioPython/src/RAG/dataset/qdrant_data")# Create the embedding encoderdense_encoder = SentenceTransformer('/Users/jiangdanyang/workspaces/python/Model/all-MiniLM-L6-v2')
collection_name = "hybrid_search"qdrant.recreate_collection(    collection_name=collection_name,        vectors_config=models.VectorParams(        size=dense_encoder.get_sentence_embedding_dimension(),         distance=models.Distance.COSINE    ))# vectorize!qdrant.upload_points(    collection_name=collection_name,    points=[        models.PointStruct(            id=idx,            vector=dense_encoder.encode(doc["text"]).tolist(),            payload=doc        ) for idx, doc in enumerate(corpus_json) # data is the variable holding all the enriched texts    ])
query_vector = dense_encoder.encode(query).tolist()dense_results = qdrant.search(    collection_name=collection_name,    query_vector=query_vector,    limit=10)

# Normalize the two types of scoresdense_scores = np.array([doc.get("dense_score", 0) for doc in documents_with_scores])sparse_scores = np.array([doc.get("sparse_score", 0) for doc in documents_with_scores])
dense_scores_normalized = (dense_scores - np.min(dense_scores)) / (np.max(dense_scores) - np.min(dense_scores))sparse_scores_normalized = (sparse_scores - np.min(sparse_scores)) / (np.max(sparse_scores) - np.min(sparse_scores))
alpha = 0.2weighted_scores = (1 - alpha) * dense_scores_normalized + alpha * sparse_scores_normalized

from sentence_transformers import CrossEncoder 

cross_encoder = CrossEncoder("/Users/jiangdanyang/workspaces/python/Model/jina-reranker-v1-tiny-en")hybrid_search_results = {}with open('/Users/jiangdanyang/workspaces/python/MarioPython/src/RAG/dataset/dense_results.json') as f: dense_results = json.load(f)    for doc in dense_results:        hybrid_search_results[doc['id']] = doc
with open('/Users/jiangdanyang/workspaces/python/MarioPython/src/RAG/dataset/sparse_results.json') as f:    sparse_results = json.load(f)    for doc in sparse_results:        hybrid_search_results[doc['id']] = doc
console.print(hybrid_search_results)
# This is the query that we used for the retrieval of the above documentsquery = "What is context size of Mixtral?"pairs = [[query, doc['text']] for doc in hybrid_search_results.values()] scores = cross_encoder.predict(pairs) 

client = OpenAI(   api_key=os.getenv("AI_API_KEY"),   base_url=os.getenv("AI_API_BASE_URL"))completion = client.chat.completions.create(    model="qwen_max",    messages=[        {"role": "system", "content": "You are chatbot, an research expert. Your top priority is to help guide users to understand reserach papers."},        {"role": "user", "content": query},        {"role": "assistant", "content": str(search_results)}    ])