"Agents\n42\nSeptember 2024\nEndnotes\n1.\t Shafran, I., Cao, Y. et al., 2022, 'ReAct: Synergizing Reasoning and Acting in Language Models'. Available at: \nhttps://arxiv.org/abs/2210.03629\n2.\t Wei, J., Wang, X. et al., 2023, 'Chain-of-Thought Prompting Elicits Reasoning in Large Language Models'. \nAvailable at: https://arxiv.org/pdf/2201.11903.pdf.\n3.\t Wang, X. et al., 2022, 'Self-Consistency Improves Chain of Thought Reasoning in Language Models'. \nAvailable at: https://arxiv.org/abs/2203.11171.\n4.\t Diao, S. et al., 2023, 'Active Prompting with Chain-of-Thought for Large Language Models'. Available at: \nhttps://arxiv.org/pdf/2302.12246.pdf.\n5.\t Zhang, H. et al., 2023, 'Multimodal Chain-of-Thought Reasoning in Language Models'. Available at: \nhttps://arxiv.org/abs/2302.00923.\n6.\t Yao, S. et al., 2023, 'Tree of Thoughts: Deliberate Problem Solving with Large Language Models'. Available at: \nhttps://arxiv.org/abs/2305.10601.\n7.\t Long, X., 2023, 'Large Language Model Guided Tree-of-Thought'. Available at: \nhttps://arxiv.org/abs/2305.08291.\n8.\t Google. 'Google Gemini Application'. Available at: http://gemini.google.com.\n9.\t Swagger. 'OpenAPI Specification'. Available at: https://swagger.io/specification/.\n10.\tXie, M., 2022, 'How does in-context learning work? A framework for understanding the differences from \ntraditional supervised learning'. Available at: https://ai.stanford.edu/blog/understanding-incontext/.\n11.\t Google Research. 'ScaNN (Scalable Nearest Neighbors)'. Available at: \nhttps://github.com/google-research/google-research/tree/master/scann.\n12.\t LangChain. 'LangChain'. Available at: https://python.langchain.com/v0.2/docs/introduction/.\n"
for i inrange(0, len(text), chunk_size - overlap): chunk_text = text[i:i + chunk_size] chunks.append(chunk_text) return chunks
text_chunks = chunk_text(extract_text, 1000, 200) print("Number of text chunks:", len(text_chunks))
print("\n First text chunk:") print(text_chunks[0])
Number of text chunks: 3
First text chunk: Agents 42 September 2024 Endnotes 1. Shafran, I., Cao, Y. et al., 2022, 'ReAct: Synergizing Reasoning and Acting in Language Models'. Available at: https://arxiv.org/abs/2210.03629 2. Wei, J., Wang, X. et al., 2023, 'Chain-of-Thought Prompting Elicits Reasoning in Large Language Models'. Available at: https://arxiv.org/pdf/2201.11903.pdf. 3. Wang, X. et al., 2022, 'Self-Consistency Improves Chain of Thought Reasoning in Language Models'. Available at: https://arxiv.org/abs/2203.11171. 4. Diao, S. et al., 2023, 'Active Prompting with Chain-of-Thought for Large Language Models'. Available at: https://arxiv.org/pdf/2302.12246.pdf. 5. Zhang, H. et al., 2023, 'Multimodal Chain-of-Thought Reasoning in Language Models'. Available at: https://arxiv.org/abs/2302.00923. 6. Yao, S. et al., 2023, 'Tree of Thoughts: Deliberate Problem Solving with Large Language Models'. Available at: https://arxiv.org/abs/2305.10601. 7. Long, X., 2023, 'Large Language Model Guided Tree-of-Thought'. Av
for i, chunk inenumerate(top_chunks): print(f"Context {i + 1}:\n{chunk}\n==============")
Context 1: {'text': "Agents\n42\nSeptember 2024\nEndnotes\n1.\t Shafran, I., Cao, Y. et al., 2022, 'ReAct: Synergizing Reasoning and Acting in Language Models'. Available at: \nhttps://arxiv.org/abs/2210.03629\n2.\t Wei, J., Wang, X. et al., 2023, 'Chain-of-Thought Prompting Elicits Reasoning in Large Language Models'. \nAvailable at: https://arxiv.org/pdf/2201.11903.pdf.\n3.\t Wang, X. et al., 2022, 'Self-Consistency Improves Chain of Thought Reasoning in Language Models'. \nAvailable at: https://arxiv.org/abs/2203.11171.\n4.\t Diao, S. et al., 2023, 'Active Prompting with Chain-of-Thought for Large Language Models'. Available at: \nhttps://arxiv.org/pdf/2302.12246.pdf.\n5.\t Zhang, H. et al., 2023, 'Multimodal Chain-of-Thought Reasoning in Language Models'. Available at: \nhttps://arxiv.org/abs/2302.00923.\n6.\t Yao, S. et al., 2023, 'Tree of Thoughts: Deliberate Problem Solving with Large Language Models'. Available at: \nhttps://arxiv.org/abs/2305.10601.\n7.\t Long, X., 2023, 'Large Language Model Guided Tree-of-Thought'. Av", 'score': 0.27010855078697205} ============== Context 2: {'text': 'vailable at: https://python.langchain.com/v0.2/docs/introduction/.\n', 'score': 0.15880665183067322} ============== Context 3: {'text': " 2023, 'Tree of Thoughts: Deliberate Problem Solving with Large Language Models'. Available at: \nhttps://arxiv.org/abs/2305.10601.\n7.\t Long, X., 2023, 'Large Language Model Guided Tree-of-Thought'. Available at: \nhttps://arxiv.org/abs/2305.08291.\n8.\t Google. 'Google Gemini Application'. Available at: http://gemini.google.com.\n9.\t Swagger. 'OpenAPI Specification'. Available at: https://swagger.io/specification/.\n10.\tXie, M., 2022, 'How does in-context learning work? A framework for understanding the differences from \ntraditional supervised learning'. Available at: https://ai.stanford.edu/blog/understanding-incontext/.\n11.\t Google Research. 'ScaNN (Scalable Nearest Neighbors)'. Available at: \nhttps://github.com/google-research/google-research/tree/master/scann.\n12.\t LangChain. 'LangChain'. Available at: https://python.langchain.com/v0.2/docs/introduction/.\n", 'score': 0.14151382446289062} ==============
system_prompt = "You are an AI Agent assistant that strictly answers based on the given context. If the answer cannot be derived directly from the provided context, respond with: 'I do not have enough information to answer that.'" # Create the user prompt based on the top chunks user_prompt = "\n".join([f"Context {i + 1}:\n{chunk['text']}\n=====================================\n"for i, chunk inenumerate(top_chunks)]) user_prompt = f"{user_prompt}\nQuestion: {query}"
# Generate AI response ai_response = generate_response(system_prompt, user_prompt) print(f"ai_response:{ai_response}")
ai_response:ChatCompletion(id='chatcmpl-f4693ca5-c995-4af3-b9b9-572ad33258db', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content='I do not have enough information to answer that.', refusal=None, role='assistant', annotations=None, audio=None, function_call=None, tool_calls=None))], created=1762959661, model='qwen-plus', object='chat.completion', service_tier=None, system_fingerprint=None, usage=CompletionUsage(completion_tokens=10, prompt_tokens=723, total_tokens=733, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetails(audio_tokens=None, cached_tokens=0)))
RAG评测
# Define the system prompt for the evaluation system evaluate_system_prompt = "You are an intelligent evaluation system tasked with assessing the AI assistant's responses. If the AI assistant's response is very close to the true response, assign a score of 1. If the response is incorrect or unsatisfactory in relation to the true response, assign a score of 0. If the response is partially aligned with the true response, assign a score of 0.5."
# Create the evaluation prompt by combining the user query, AI response, true response, and evaluation system prompt answer = "Agents are autonomous and can act independently of human intervention, especially when provided with proper goals or objectives they are meant to achieve"
# Generate the evaluation response using the evaluation system prompt and evaluation prompt evaluation_response = generate_response(evaluate_system_prompt, evaluation_prompt)
# Print the evaluation response print(evaluation_response.choices[0].message.content)
粤ICP备14082021号
免费POC，
零成本试错
