下载docker-compose.ymlwget https://github.com/milvus-io/milvus/releases/download/v2.6.8/milvus-standalone-docker-compose.yml -O docker-compose.yml启动Milvus（检查端口映射：19530:19530）docker-compose up -d验证服务启动docker ps | grep milvus应该看到3个容器：milvus-standalone, milvus-etcd, milvus-minio

from pymilvus import MilvusClientfrom pymilvus.milvus_client.index import IndexParamsfrom openai import OpenAIfrom langchain_text_splitters import RecursiveCharacterTextSplitterimport umapfrom sklearn.neighbors import NearestNeighborsimport jsonimport numpy as npimport osimport glob--- CONFIG ---MILVUS_URI = "http://localhost:19530"COLLECTION_NAME = "golem_memories"JSON_OUTPUT_PATH = "./golem_cortex.json"数据文件夹（用户把 md 文件放在这里）DATA_DIR = "./data"OpenAI Embedding ConfigOPENAI_API_KEY = os.getenv("OPENAI_API_KEY")OPENAI_BASE_URL = "https://api.openai.com/v1"  #OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"1536 dimensionsEMBEDDING_DIM = 1536颜色映射（自动轮转分配颜色）COLORS = [[0.29, 0.87, 0.50],Green[0.22, 0.74, 0.97],Blue[0.60, 0.20, 0.80],Purple[0.94, 0.94, 0.20],Gold[0.98, 0.55, 0.00],Orange[0.90, 0.30, 0.40],Red[0.40, 0.90, 0.90],Cyan[0.95, 0.50, 0.90],MAgenta]def get_embeddings(texts):"""Batch embedding using OpenAI API"""client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL)embeddings = []batch_size = 100OpenAI allows multiple texts per requestfor i in range(0, len(texts), batch_size):batch = texts[i:i + batch_size]response = client.embeddings.create(model=OPENAI_EMBEDDING_MODEL,input=batch)embeddings.extend([item.embedding for item in response.data])print(f"   ↳ Embedded {min(i + batch_size, len(texts))}/{len(texts)}...")return np.array(embeddings)def load_markdown_files(data_dir):"""Load all markdown files from the data directory"""md_files = glob.glob(os.path.join(data_dir, "**/*.md"), recursive=True)if not md_files:print(f"   ❌ ERROR: No .md files found in '{data_dir}'")print(f"   👉 Create a '{data_dir}' folder and put your markdown files there.")print(f"   👉 Example: {data_dir}/doc1.md, {data_dir}/docs/doc2.md")return Nonedocs = []print(f"\n📚 FOUND {len(md_files)} MARKDOWN FILES:")for i, file_path in enumerate(md_files):filename = os.path.basename(file_path)相对于 data_dir 的路径作为类别rel_path = os.path.relpath(file_path, data_dir)category = os.path.dirname(rel_path) if os.path.dirname(rel_path) else "default"with open(file_path, 'r', encoding='utf-8') as f:content = f.read()docs.append({"title": filename,"text": content,"cat": category,"path": file_path})print(f"   {i+1}. [{category}] {filename}")return docsdef ingest_dense():print(f"🧠 PROJECT GOLEM - NEURAL MEMORY BUILDER")print(f"=" * 50)if not OPENAI_API_KEY:print("   ❌ ERROR: OPENAI_API_KEY environment variable not set!")print("   👉 Run: export OPENAI_API_KEY='your-key-here'")returnprint(f"   ↳ Using OpenAI Embedding: {OPENAI_EMBEDDING_MODEL}")print(f"   ↳ Embedding Dimension: {EMBEDDING_DIM}")print(f"   ↳ Data Directory: {DATA_DIR}")1. Load local markdown filesdocs = load_markdown_files(DATA_DIR)if docs is None:return2. Split documents into chunksprint(f"\n📦 PROCESSING DOCUMENTS...")splitter = RecursiveCharacterTextSplitter(chunk_size=800, chunk_overlap=50)chunks = []raw_texts = []colors = []chunk_titles = []categories = []for doc in docs:doc_chunks = splitter.create_documents([doc['text']])cat_index = hash(doc['cat']) % len(COLORS)for i, chunk in enumerate(doc_chunks):chunks.append({"text": chunk.page_content,"title": doc['title'],"cat": doc['cat']})raw_texts.append(chunk.page_content)colors.append(COLORS[cat_index])chunk_titles.append(f"{doc['title']} (chunk {i+1})")categories.append(doc['cat'])print(f"   ↳ Created {len(chunks)} text chunks from {len(docs)} documents")3. Generate embeddingsprint(f"\n🔮 GENERATING EMBEDDINGS...")vectors = get_embeddings(raw_texts)4. 3D Projection (UMAP)print("\n🎨 CALCULATING 3D MANIFOLD...")reducer = umap.UMAP(n_components=3, n_neighbors=30, min_dist=0.1, metric='cosine')embeddings_3d = reducer.fit_transform(vectors)5. Wiring (KNN)print("   ↳ Wiring Synapses (finding connections)...")nbrs = NearestNeighbors(n_neighbors=8, metric='cosine').fit(vectors)distances, indices = nbrs.kneighbors(vectors)6. Prepare output datacortex_data = []milvus_data = []for i in range(len(chunks)):cortex_data.append({"id": i,"title": chunk_titles[i],"cat": categories[i],"pos": embeddings_3d[i].tolist(),"col": colors[i],"nbs": indices[i][1:].tolist()})milvus_data.append({"id": i,"text": chunks[i]['text'],"title": chunk_titles[i],"category": categories[i],"vector": vectors[i].tolist()})with open(JSON_OUTPUT_PATH, 'w') as f:json.dump(cortex_data, f)7. Store vectors in Milvusprint("\n💾 STORING IN MILVUS...")client = MilvusClient(uri=MILVUS_URI)Drop existing collection if it existsif client.has_collection(COLLECTION_NAME):print(f"   ↳ Dropping existing collection '{COLLECTION_NAME}'...")client.drop_collection(COLLECTION_NAME)Create new collectionprint(f"   ↳ Creating collection '{COLLECTION_NAME}' (dim={EMBEDDING_DIM})...")client.create_collection(collection_name=COLLECTION_NAME,dimension=EMBEDDING_DIM)Insert dataprint(f"   ↳ Inserting {len(milvus_data)} vectors...")client.insert(collection_name=COLLECTION_NAME,data=milvus_data)Create index for faster searchprint("   ↳ Creating index...")index_params = IndexParams()index_params.add_index(field_name="vector",index_type="AUTOINDEX",metric_type="COSINE")client.create_index(collection_name=COLLECTION_NAME,index_params=index_params)print(f"\n✅ CORTEX GENERATED SUCCESSFULLY!")print(f"   📊 {len(chunks)} memory nodes stored in Milvus")print(f"   📁 Cortex data saved to: {JSON_OUTPUT_PATH}")print(f"   🚀 Run 'python GolemServer.py' to start the server")if __name__ == "__main__":ingest_dense()

from flask import Flask, request, jsonify, send_from_directoryfrom openai import OpenAIfrom pymilvus import MilvusClientimport jsonimport osimport sys--- CONFIG ---Explicitly set the folder to where this script is locatedBASE_DIR = os.path.dirname(os.path.abspath(__file__))OpenAI Embedding ConfigOPENAI_API_KEY = os.getenv("OPENAI_API_KEY")OPENAI_BASE_URL = "https://api.openai.com/v1"OPENAI_EMBEDDING_MODEL = "text-embedding-3-small"Milvus ConfigMILVUS_URI = "http://localhost:19530"COLLECTION_NAME = "golem_memories"These match the files generated by ingest.pyJSON_FILE = "golem_cortex.json"UPDATED: Matches your new repo filenameHTML_FILE = "index.html"app = Flask(__name__, static_folder=BASE_DIR)print(f"\n🧠 PROJECT GOLEM SERVER")print(f"   📂 Serving from: {BASE_DIR}")--- DIAGNOSTICS ---Check if files exist before startingmissing_files = []if not os.path.exists(os.path.join(BASE_DIR, JSON_FILE)):missing_files.append(JSON_FILE)if not os.path.exists(os.path.join(BASE_DIR, HTML_FILE)):missing_files.append(HTML_FILE)if missing_files:print(f"   ❌ CRITICAL ERROR: Missing files in this folder:")for f in missing_files:print(f"      - {f}")print("   👉 Did you run 'python ingest.py' successfully?")sys.exit(1)else:print(f"   ✅ Files Verified: Cortex Map found.")Check API Keyif not OPENAI_API_KEY:print(f"   ❌ CRITICAL ERROR: OPENAI_API_KEY environment variable not set!")print("   👉 Run: export OPENAI_API_KEY='your-key-here'")sys.exit(1)print(f"   ↳ Using OpenAI Embedding: {OPENAI_EMBEDDING_MODEL}")print("   ↳ Connecting to Milvus...")milvus_client = MilvusClient(uri=MILVUS_URI)Verify collection existsif not milvus_client.has_collection(COLLECTION_NAME):print(f"   ❌ CRITICAL ERROR: Collection '{COLLECTION_NAME}' not found in Milvus.")print("   👉 Did you run 'python ingest.py' successfully?")sys.exit(1)Initialize OpenAI clientopenai_client = OpenAI(api_key=OPENAI_API_KEY, base_url=OPENAI_BASE_URL)--- ROUTES ---@app.route('/')def root():Force serve the specific HTML filereturn send_from_directory(BASE_DIR, HTML_FILE)@app.route('/')def serve_static(filename):return send_from_directory(BASE_DIR, filename)@app.route('/query', methods=['POST'])def query_brain():data = request.jsontext = data.get('query', '')if not text: return jsonify({"indices": []})print(f"🔎 Query: {text}")Get query embedding from OpenAIresponse = openai_client.embeddings.create(model=OPENAI_EMBEDDING_MODEL,input=text)query_vec = response.data[0].embeddingSearch in Milvusresults = milvus_client.search(collection_name=COLLECTION_NAME,data=[query_vec],limit=50,output_fields=["id"])Extract indices and scoresindices = [r['id'] for r in results[0]]scores = [r['distance'] for r in results[0]]return jsonify({"indices": indices,"scores": scores})if __name__ == '__main__':print("   ✅ SYSTEM ONLINE: http://localhost:8000")app.run(port=8000)

https://github.com/milvus-io/milvus-docs/tree/v2.6.x/site/en

python ingest.py

python GolemServer.py