Firecrawl基于AI的链接内容提取工具

{ "jobId": "1234-5678-9101" }

curl -X GET https://api.firecrawl.dev/v0/crawl/status/1234-5678-9101 \-H 'Content-Type: application/json' \-H 'Authorization: Bearer YOUR_API_KEY'

{"status": "completed","current": 22,"total": 22,"data": [{"content": "Raw Content","markdown": "# Markdown Content","provider": "web-scraper","metadata": {"title": "Mendable | AI for CX and Sales","description": "AI for CX and Sales","language": null,"sourceURL": "https://www.mendable.ai/"}}]}

curl -X POST https://api.firecrawl.dev/v0/scrape \-H 'Content-Type: application/json' \-H 'Authorization: Bearer YOUR_API_KEY' \-d '{"url": "https://mendable.ai"}'

{"success": true,"data": {"content": "Raw Content","markdown": "# Markdown Content","provider": "web-scraper","metadata": {"title": "Mendable | AI for CX and Sales","description": "AI for CX and Sales","language": null,"sourceURL": "https://www.mendable.ai/"}}}

curl -X POST https://api.firecrawl.dev/v0/search \-H 'Content-Type: application/json' \-H 'Authorization: Bearer YOUR_API_KEY' \-d '{"query": "firecrawl","pageOptions": {"fetchPageContent": true // false for a fast serp api}}'

{"success": true,"data": [{"url": "https://mendable.ai","markdown": "# Markdown Content","provider": "web-scraper","metadata": {"title": "Mendable | AI for CX and Sales","description": "AI for CX and Sales","language": null,"sourceURL": "https://www.mendable.ai/"}}]}

curl -X POST https://api.firecrawl.dev/v0/scrape \-H 'Content-Type: application/json' \-H 'Authorization: Bearer YOUR_API_KEY' \-d '{"url": "https://www.mendable.ai/","extractorOptions": {"mode": "llm-extraction","extractionPrompt": "Based on the information on the page, extract the information from the schema.","extractionSchema": {"type": "object","properties": {"company_mission": {"type": "string"},"supports_sso": {"type": "boolean"},"is_open_source": {"type": "boolean"},"is_in_yc": {"type": "boolean"}},"required": ["company_mission", "supports_sso", "is_open_source", "is_in_yc"]}}}'

{"success": true,"data": {"content": "Raw Content","metadata": {"title": "Mendable","description": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide","robots": "follow, index","ogTitle": "Mendable","ogDescription": "Mendable allows you to easily build AI chat applications. Ingest, customize, then deploy with one line of code anywhere you want. Brought to you by SideGuide","ogUrl": "https://mendable.ai/","ogImage": "https://mendable.ai/mendable_new_og1.png","ogLocaleAlternate": [],"ogSiteName": "Mendable","sourceURL": "https://mendable.ai/"},"llm_extraction": {"company_mission": "Train a secure AI on your technical resources that answers customer and employee questions so your team doesn't have to","supports_sso": true,"is_open_source": false,"is_in_yc": true}}}

pip install firecrawl-py

from firecrawl import FirecrawlApp
app = FirecrawlApp(api_key="YOUR_API_KEY")
crawl_result = app.crawl_url('mendable.ai', {'crawlerOptions': {'excludes': ['blog/*']}})for result in crawl_result:print(result['markdown'])

url = 'https://example.com'scraped_data = app.scrape_url(url)

from pydantic import BaseModel, Fieldfrom typing import List
class ArticleSchema(BaseModel):title: strpoints: intby: strcommentsURL: str
class TopArticlesSchema(BaseModel):top: List[ArticleSchema] = Field(..., max_items=5, description="Top 5 stories")
data = app.scrape_url('https://news.ycombinator.com', {'extractorOptions': {'extractionSchema': TopArticlesSchema.schema_json(),'mode': 'llm-extraction'},'pageOptions':{'onlyMainContent': True}})print(data["llm_extraction"])

query = 'What is Mendable?'search_result = app.search(query)

npm install @mendable/firecrawl-js

try {const url = 'https://example.com';const scrapedData = await app.scrapeUrl(url);console.log(scrapedData);} catch (error) {console.error('Error occurred while scraping:',error.message);}

const crawlUrl = 'https://example.com';const params = {crawlerOptions: {excludes: ['blog/'],includes: [], // 留空表示所有页面limit: 1000,},pageOptions: {onlyMainContent: true}};const waitUntilDone = true;const timeout = 5;const crawlResult = await app.crawlUrl(crawlUrl,params,waitUntilDone,timeout);

const status = await app.checkCrawlStatus(jobId);console.log(status);

import FirecrawlApp from "@mendable/firecrawl-js";import { z } from "zod";
const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY",});
// 定义架构以提取内容const schema = z.object({top: z.array(z.object({title: z.string(),points: z.number(),by: z.string(),commentsURL: z.string(),})).length(5).describe("Top 5 stories on Hacker News"),});
const scrapeResult = await app.scrapeUrl("https://news.ycombinator.com", {extractorOptions: { extractionSchema: schema },});
console.log(scrapeResult.data["llm_extraction"]);

const query = 'what is mendable?';const searchResults = await app.search(query, {pageOptions: {fetchPageContent: true // 获取每个搜索结果的页面内容}});