重磅！用 Gemini 2.5 搭载 Cohere Embed v4，视觉RAG 终于不用 Markdown 绕路了！

pip install -q cohere

pip install -q google-genai

from google import genaigemini_api_key = "<<YOUR_GEMINI_KEY>>"  #Replace with your Gemini API keyclient = genai.Client(api_key=gemini_api_key)

# Wrap long longs in this Notebookfrom IPython.display import HTML, displaydef set_css():  display(HTML('''  <style>    pre {        white-space: pre-wrap;    }  </style>  '''))get_ipython().events.register('pre_run_cell', set_css)

import requestsimport osimport ioimport base64import PILimport tqdmimport timeimport numpy as np

# Some helper functions to resize images and to convert them to base64 formatmax_pixels = 1568*1568  #Max resolution for images# Resize too large imagesdef resize_image(pil_image):    org_width, org_height = pil_image.size    # Resize image if too large    if org_width * org_height > max_pixels:        scale_factor = (max_pixels / (org_width * org_height)) ** 0.5        new_width = int(org_width * scale_factor)        new_height = int(org_height * scale_factor)        pil_image.thumbnail((new_width, new_height))# Convert images to a base64 string before sending it to the APIdef base64_from_image(img_path):    pil_image = PIL.Image.open(img_path)    img_format = pil_image.format if pil_image.format else "PNG"    resize_image(pil_image)    with io.BytesIO() as img_buffer:        pil_image.save(img_buffer, format=img_format)        img_buffer.seek(0)        img_data = f"data:image/{img_format.lower()};base64,"+base64.b64encode(img_buffer.read()).decode("utf-8")    return img_data

# Several images from https://www.appeconomyinsights.com/images = {    "tesla.png": "https://substackcdn.com/image/fetch/w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fbef936e6-3efa-43b3-88d7-7ec620cdb33b_2744x1539.png",    "netflix.png": "https://substackcdn.com/image/fetch/w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F23bd84c9-5b62-4526-b467-3088e27e4193_2744x1539.png",    "nike.png": "https://substackcdn.com/image/fetch/w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2Fa5cd33ba-ae1a-42a8-a254-d85e690d9870_2741x1541.png",    "google.png": "https://substackcdn.com/image/fetch/f_auto,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F395dd3b9-b38e-4d1f-91bc-d37b642ee920_2741x1541.png",    "accenture.png": "https://substackcdn.com/image/fetch/w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F08b2227c-7dc8-49f7-b3c5-13cab5443ba6_2741x1541.png",    "tecent.png": "https://substackcdn.com/image/fetch/w_1456,c_limit,f_webp,q_auto:good,fl_progressive:steep/https%3A%2F%2Fsubstack-post-media.s3.amazonaws.com%2Fpublic%2Fimages%2F0ec8448c-c4d1-4aab-a8e9-2ddebe0c95fd_2741x1541.png"}

# Download the images and compute an embedding for each imageimg_folder = "img"os.makedirs(img_folder, exist_ok=True)img_paths = []doc_embeddings = []for name, url in tqdm.tqdm(images.items()):    img_path = os.path.join(img_folder, name)    img_paths.append(img_path)    # Download the image    if not os.path.exists(img_path):        response = requests.get(url)        response.raise_for_status()        with open(img_path, "wb") as fOut:            fOut.write(response.content)    # Get the base64 representation of the image    api_input_document = {        "content": [            {"type": "image", "image": base64_from_image(img_path)},        ]    }    # Call the Embed v4.0 model with the image information    api_response = co.embed(        model="embed-v4.0",        input_type="search_document",        embedding_types=["float"],        inputs=[api_input_document],    )    # Append the embedding to our doc_embeddings list    emb = np.asarray(api_response.embeddings.float[0])    doc_embeddings.append(emb)doc_embeddings = np.vstack(doc_embeddings)print("\n\nEmbeddings shape:", doc_embeddings.shape)

# Search allows us to find relevant images for a given question using Cohere Embed v4def search(question, max_img_size=800):    # Compute the embedding for the query    api_response = co.embed(        model="embed-v4.0",        input_type="search_query",        embedding_types=["float"],        texts=[question],    )    query_emb = np.asarray(api_response.embeddings.float[0])    # Compute cosine similarities    cos_sim_scores = np.dot(query_emb, doc_embeddings.T)    # Get the most relevant image    top_idx = np.argmax(cos_sim_scores)    # Show the images    print("Question:", question)    hit_img_path = img_paths[top_idx]    print("Most relevant image:", hit_img_path)    image = PIL.Image.open(hit_img_path)    max_size = (max_img_size, max_img_size)  # Adjust the size as needed    image.thumbnail(max_size)    display(image)    return hit_img_path# Answer the question based on the information from the image# Here we use Gemini 2.5 as powerful Vision-LLMdef answer(question, img_path):    prompt = [f"""Answer the question based on the following image.Don't use markdown.Please provide enough context for your answer.Question: {question}""", PIL.Image.open(img_path)]    response = client.models.generate_content(        model="gemini-2.5-flash-preview-04-17",        contents=prompt    )    answer = response.text    print("LLM Answer:", answer)

# Define the queryquestion = "What is the net profit for Nike?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)

# Define the queryquestion = "What are the 3 largest acquisitions from Google?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)

# Define the queryquestion = "What would be the net profit of Tesla without interest?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)

# Define the queryquestion = "Is GenAI a good business for consulting companies?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)

# Define the queryquestion = "In which region does Netflix generate the highest revenue?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)

# Define the queryquestion = "How much could tecent grow their revenue year-over-year for the last 5 years?"# Search for the most relevant imagetop_image_path = search(question)# Use the image to answer the queryanswer(question, top_image_path)