diff --git a/.claude/agents/database-ingestor.md b/.claude/agents/database-ingestor.md index dde35af..da054a4 100644 --- a/.claude/agents/database-ingestor.md +++ b/.claude/agents/database-ingestor.md @@ -10,7 +10,6 @@ description: | 输入参数:{"workspace": ".trending-workspace/..."} model: inherit color: green -tools: Read, Write, Bash --- # 批量入库器 Agent diff --git a/.claude/agents/deduplicator.md b/.claude/agents/deduplicator.md index 86d88c7..6460cd5 100644 --- a/.claude/agents/deduplicator.md +++ b/.claude/agents/deduplicator.md @@ -10,7 +10,6 @@ description: | 输入参数:{"workspace": ".trending-workspace/..."} model: inherit color: purple -tools: Read, Write, Bash --- # 统一去重器 Agent diff --git a/.claude/agents/project-analyzer.md b/.claude/agents/project-analyzer.md index 62de664..48cdb29 100644 --- a/.claude/agents/project-analyzer.md +++ b/.claude/agents/project-analyzer.md @@ -10,7 +10,6 @@ description: | 输入参数:{"workspace": ".trending-workspace/..."} model: inherit color: blue -tools: Read, Write, Bash --- # 项目分析器 Agent @@ -29,21 +28,39 @@ tools: Read, Write, Bash ```json { - "workspace": ".trending-workspace/..." + "workspace": ".trending-workspace/...", + "taskId": 5 } ``` +**参数说明**: +- `workspace`: 工作区路径 +- `taskId`: 要处理的单个任务 ID(对应 task-queue.json 中的任务编号) + ## 执行步骤 ### Step 1: 读取任务队列 -从工作区读取 `task-queue.json`,获取所有 `status: "pending"` 的任务。 +从工作区读取 `task-queue.json`,获取所有任务数据。 -### Step 2: 处理每个项目 +### Step 2: 获取指定任务 -对每个待处理项目: +根据 `taskId` 参数从任务队列中找到对应的任务。 -#### 2.1 访问项目页面 +**验证**: +- 确认任务存在 +- 确认任务状态为 `pending`(避免重复处理) +- 如果任务状态不是 `pending`,直接退出并返回当前状态 + +### Step 3: 更新任务状态为 processing + +将任务状态从 `pending` 更新为 `processing`(防止其他实例重复处理)。 + +### Step 4: 处理单个项目 + +对指定的项目执行以下操作: + +#### 4.1 访问项目页面 使用 chrome-devtools-mcp 访问项目 URL: @@ -51,7 +68,7 @@ tools: Read, Write, Bash - **Hugging Face 模型**: 访问 Hugging Face 模型页面 - **Papers with Code**: 访问项目/论文页面 -#### 2.2 提取详细信息 +#### 4.2 提取详细信息 从页面提取以下信息: @@ -77,7 +94,7 @@ tools: Read, Write, Bash - 任务类别 - 引用数 -#### 2.3 理解项目价值(核心) +#### 4.3 理解项目价值(核心) **项目用途**:项目能做什么? - 核心功能是什么? @@ -95,7 +112,7 @@ tools: Read, Write, Bash 从页面内容中提炼这些信息,用用户友好的语言描述。 -#### 2.4 生成中英双语内容 +#### 4.4 生成中英双语内容 **name / nameEn**: 项目名称翻译 - 通常保持英文名称不变 @@ -113,7 +130,7 @@ tools: Read, Write, Bash - **技术特点**: 技术亮点 - **使用指南**: 快速开始或使用示例 -#### 2.5 提取结构化标签(1-10 个) +#### 4.5 提取结构化标签(1-10 个) 从以下来源提取标签: - GitHub Topics @@ -133,7 +150,7 @@ tools: Read, Write, Bash } ``` -#### 2.6 构造外部链接数组(1-10 个) +#### 4.6 构造外部链接数组(1-10 个) 收集项目相关链接: @@ -159,7 +176,7 @@ tools: Read, Write, Bash } ``` -#### 2.7 计算质量评分 +#### 4.7 计算质量评分 总分 100 分,>= 40 分通过: @@ -192,68 +209,69 @@ function calculateQualityScore(project): number { } ``` -#### 2.8 更新任务状态 +#### 4.8 更新任务状态 将任务状态从 `pending` 更新为 `completed` 或 `failed`(质量不足)。 -### Step 3: 输出分析结果 +### Step 5: 更新任务状态为最终状态 -输出 `analyzed-projects.json`: +将任务状态从 `processing` 更新为: +- `completed` - 质量评分 >= 40 +- `failed` - 质量评分 < 40 或处理出错 +### Step 6: 输出分析结果 + +输出 `analyzed-project-{taskId}.json`(避免多实例文件冲突): + +**成功情况**: ```json { - "metadata": { - "totalTasks": 35, - "processed": 35, - "passed": 32, - "failed": 3, - "failureReasons": { - "lowQuality": 3 - } - }, - "projects": [ - { - "source": "github", - "name": "LangChain", - "nameEn": "LangChain", - "description": "通过组合性构建大型语言模型应用程序的框架,支持链式调用、代理、工具集成等核心功能。", - "descriptionEn": "Building applications with LLMs through composability", - "content": "README 的完整 Markdown 内容...", - "contentEn": "README content...", - "status": "ACTIVE", - "source": "GITHUB_TRENDING", - "tags": [ - { "name": "LLM", "nameEn": "Large Language Model" }, - { "name": "Python", "nameEn": "Python" }, - { "name": "框架", "nameEn": "Framework" } - ], - "links": [ - { - "type": "GITHUB", - "url": "https://github.com/langchain-ai/langchain", - "title": "GitHub 仓库" - }, - { - "type": "WEBSITE", - "url": "https://python.langchain.com", - "title": "官方文档" - } - ], - "qualityScore": 85 - } - ] + "taskId": 5, + "success": true, + "project": { + "source": "github", + "name": "LangChain", + "nameEn": "LangChain", + "description": "通过组合性构建大型语言模型应用程序的框架,支持链式调用、代理、工具集成等核心功能。", + "descriptionEn": "Building applications with LLMs through composability", + "content": "README 的完整 Markdown 内容...", + "contentEn": "README content...", + "status": "ACTIVE", + "source": "GITHUB_TRENDING", + "tags": [ + { "name": "LLM", "nameEn": "Large Language Model" }, + { "name": "Python", "nameEn": "Python" }, + { "name": "框架", "nameEn": "Framework" } + ], + "links": [ + { + "type": "GITHUB", + "url": "https://github.com/langchain-ai/langchain", + "title": "GitHub 仓库" + }, + { + "type": "WEBSITE", + "url": "https://python.langchain.com", + "title": "官方文档" + } + ], + "qualityScore": 85 + } } ``` -## 并行处理策略 - -支持多个 Project Analyzer 实例同时运行: - -1. 每个实例读取 `task-queue.json` -2. 获取 `status: "pending"` 的第一个任务 -3. 将任务状态更新为 `processing`(防止其他实例重复处理) -4. 处理任务 -5. 将任务状态更新为 `completed` 或 `failed` +**失败情况**: +```json +{ + "taskId": 5, + "success": false, + "error": "质量评分不足", + "errorDetails": { + "qualityScore": 25, + "threshold": 40 + } +} +``` ## 错误处理 @@ -266,7 +284,7 @@ function calculateQualityScore(project): number { ## 输出 成功后,返回: -- 处理任务数量 -- 通过质量评分的项目数量 -- 失败项目数量及原因 -- analyzed-projects.json 路径 +- taskId(处理的任务 ID) +- success(是否成功) +- 项目数据(成功时)或错误信息(失败时) +- analyzed-project-{taskId}.json 路径 diff --git a/.claude/agents/scrapers/github-trending.md b/.claude/agents/scrapers/github-trending.md index e8c5b7d..895556c 100644 --- a/.claude/agents/scrapers/github-trending.md +++ b/.claude/agents/scrapers/github-trending.md @@ -10,7 +10,6 @@ description: | 输入参数:{"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."} model: inherit color: black -tools: Read, Write, Bash --- # GitHub Trending 爬虫 @@ -143,9 +142,35 @@ article.Box-row | 无 AI 相关项目 | 返回空项目列表,提示 "未找到 AI 相关项目" | | 数量不足 | 返回找到的所有项目,不报错 | +## 输出文件 + +将结果写入工作区:`scraped-github-projects.json` + +```json +{ + "source": "github", + "count": 25, + "projects": [ + { + "source": "github", + "name": "langchain-ai/langchain", + "url": "https://github.com/langchain-ai/langchain", + "description": "Building applications with LLMs through composability", + "metadata": { + "stars": 85432, + "starsDelta": "+234 today", + "language": "Python", + "forks": 12543 + } + } + ] +} +``` + ## 输出 成功后,返回: +- 输出文件路径:`{workspace}/scraped-github-projects.json` - 爬取的项目数量 - 过滤后的项目数量 - 项目列表 diff --git a/.claude/agents/scrapers/huggingface-trending.md b/.claude/agents/scrapers/huggingface-trending.md index 2a5cf91..ca87e29 100644 --- a/.claude/agents/scrapers/huggingface-trending.md +++ b/.claude/agents/scrapers/huggingface-trending.md @@ -10,7 +10,6 @@ description: | 输入参数:{"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."} model: inherit color: yellow -tools: Read, Write, Bash --- # Hugging Face Trending 爬虫 @@ -135,8 +134,34 @@ Hugging Face 页面结构可能动态变化,需要根据实际情况调整选 | 页面解析失败 | 返回空项目列表,记录错误 | | 无模型数据 | 返回空项目列表 | +## 输出文件 + +将结果写入工作区:`scraped-huggingface-projects.json` + +```json +{ + "source": "huggingface", + "count": 25, + "projects": [ + { + "source": "huggingface", + "name": "meta-llama/Llama-2-7b", + "url": "https://huggingface.co/meta-llama/Llama-2-7b", + "description": "Llama 2 is a collection of pretrained and fine-tuned generative text models...", + "metadata": { + "likes": 15234, + "downloads": 5000000, + "pipeline": "text-generation", + "task": "Text Generation" + } + } + ] +} +``` + ## 输出 成功后,返回: +- 输出文件路径:`{workspace}/scraped-huggingface-projects.json` - 爬取的模型数量 - 模型列表(按点赞数/下载量排序) diff --git a/.claude/agents/scrapers/papers-with-code.md b/.claude/agents/scrapers/papers-with-code.md index 454fa76..5984fe0 100644 --- a/.claude/agents/scrapers/papers-with-code.md +++ b/.claude/agents/scrapers/papers-with-code.md @@ -10,7 +10,6 @@ description: | 输入参数:{"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."} model: inherit color: cyan -tools: Read, Write, Bash --- # Papers with Code 爬虫 @@ -133,8 +132,34 @@ Papers with Code 页面结构可能动态变化,需要根据实际情况调整 | 页面解析失败 | 返回空项目列表,记录错误 | | 无符合条件的论文 | 返回空项目列表 | +## 输出文件 + +将结果写入工作区:`scraped-paperswithcode-projects.json` + +```json +{ + "source": "paperswithcode", + "count": 25, + "projects": [ + { + "source": "paperswithcode", + "name": "YOLOv7: Trainable bag-of-freebies sets new state-of-the-art", + "url": "https://github.com/WongKinYiu/yolov7", + "paperUrl": "https://paperswithcode.com/paper/yolov7-trainable-bag-of-freebies-sets-new", + "description": "YOLOv7 implements bag-of-freebies and bag-of-specials...", + "metadata": { + "stars": 8000, + "tasks": ["Object Detection", "Computer Vision"], + "framework": "PyTorch" + } + } + ] +} +``` + ## 输出 成功后,返回: +- 输出文件路径:`{workspace}/scraped-paperswithcode-projects.json` - 爬取的论文/项目数量 - 项目列表(按 Stars 数量排序) diff --git a/.claude/agents/task-dispatcher.md b/.claude/agents/task-dispatcher.md deleted file mode 100644 index d793e24..0000000 --- a/.claude/agents/task-dispatcher.md +++ /dev/null @@ -1,162 +0,0 @@ ---- -name: task-dispatcher -description: | - 作为数据获取流程的核心协调者。解析执行参数,创建工作区,并行调度所有数据源爬虫,汇总原始数据。使用此 agent 当需要从 GitHub Trending、Hugging Face、Papers with Code 等数据源获取 AI 项目时。 - - 示例场景: - - 定期获取最新的 AI 项目趋势 - - 从多个数据源收集项目数据 - - 输入参数:{"source": "all", "period": "daily", "limit": 25, "workspace": ".trending-workspace/..."} -model: inherit -color: orange -tools: Read, Write, Bash ---- - -# 任务派发器 Agent - -## 职责 - -作为数据获取流程的核心协调者,负责: -1. 解析执行参数 -2. 创建工作区 -3. 并行调度所有数据源爬虫 -4. 汇总原始数据 - -## 输入参数 - -```json -{ - "source": "all", - "period": "daily", - "limit": 25, - "workspace": ".trending-workspace/..." -} -``` - -**参数说明**: -- `source`: 数据源筛选,`all`(默认)/ `github` / `huggingface` / `paperswithcode` - -## 执行步骤 - -### Step 1: 初始化工作区 - -1. 生成时间戳目录名:`{YYYYMMDD-HHMMSS}` -2. 创建完整工作区路径:`.trending-workspace/{timestamp}/` -3. 初始化 `progress.json`: - -```json -{ - "startTime": "2025-01-04T12:00:00Z", - "currentStage": "initializing", - "stages": { - "dispatch": "pending", - "scraping": "pending", - "aggregating": "pending" - } -} -``` - -### Step 2: 根据 source 参数确定要调用的爬虫 - -**所有可用数据源**: - -| 源名称 | 爬虫文件 | URL | -|--------|----------|-----| -| github | `.claude/agents/scrapers/github-trending.md` | https://github.com/trending | -| huggingface | `.claude/agents/scrapers/huggingface-trending.md` | https://huggingface.co/models | -| paperswithcode | `.claude/agents/scrapers/papers-with-code.md` | https://paperswithcode.com/ | - -**根据 source 参数筛选**: -- `source=all`: 调用上述所有爬虫 -- `source=github`: 仅调用 GitHub Trending 爬虫 -- `source=huggingface`: 仅调用 Hugging Face 爬虫 -- `source=paperswithcode`: 仅调用 Papers with Code 爬虫 - -### Step 3: 并行调用筛选后的爬虫 - -**重要**: 必须使用单个消息发送多个 Task 工具调用来实现并行执行。 - -对筛选出的每个数据源爬虫: -- 构造输入参数(period, limit, workspace) -- 调用爬虫 Agent - -### Step 4: 汇总原始数据 - -1. 等待所有爬虫完成 -2. 汇总所有爬虫返回的项目数据 -3. 输出 `raw-projects.json`: - -```json -{ - "metadata": { - "timestamp": "2025-01-04T12:00:00Z", - "period": "daily", - "limit": 25, - "sources": ["github", "huggingface"], - "sourceCounts": { - "github": 25, - "huggingface": 20 - }, - "totalRaw": 45 - }, - "projects": [ - { - "source": "github", - "name": "langchain-ai/langchain", - "url": "https://github.com/langchain-ai/langchain", - "description": "Building applications with LLMs through composability", - "metadata": { - "stars": 85432, - "starsDelta": "+234", - "language": "Python", - "forks": 12543 - } - }, - { - "source": "huggingface", - "name": "meta-llama/Llama-2-7b", - "url": "https://huggingface.co/meta-llama/Llama-2-7b", - "description": "Llama 2 7B parameter model", - "metadata": { - "likes": 15234, - "downloads": 500000, - "pipeline": "text-generation" - } - } - ] -} -``` - -### Step 5: 更新进度 - -更新 `progress.json`: - -```json -{ - "startTime": "2025-01-04T12:00:00Z", - "currentStage": "completed", - "stages": { - "dispatch": "completed", - "scraping": "completed", - "aggregating": "completed" - }, - "endTime": "2025-01-04T12:05:00Z", - "duration": "5m" -} -``` - -## 错误处理 - -| 场景 | 处理方式 | -|------|---------| -| 某个爬虫失败 | 记录失败源到 `errors.json`,其他爬虫继续 | -| 所有爬虫失败 | 错误提示 "所有数据源均失败" | -| 工作区创建失败 | 错误提示并终止 | - -## 输出 - -成功后,返回: -- 工作区路径 -- 汇总的原始项目数量 -- 各数据源的项目数量 diff --git a/.claude/commands/add-trending.md b/.claude/commands/add-trending.md index 203c05d..fd53d39 100644 --- a/.claude/commands/add-trending.md +++ b/.claude/commands/add-trending.md @@ -17,7 +17,7 @@ $ARGUMENTS - **使用 Task 工具调用各个 Agent**,让 Agent 自主完成其职责 - **主窗口仅负责协调 Agent 调用**,不直接处理业务逻辑 -本命令通过任务派发器架构,从多个数据源并行爬取 AI 相关项目,经过去重、分析、质量评分后批量入库。 +本命令从多个数据源并行爬取 AI 相关项目,经过去重、分析、质量评分后批量入库。 **命令格式**: `/add-trending [source] [period] [limit]` @@ -32,11 +32,17 @@ $ARGUMENTS - `/add-trending huggingface daily 10` - 仅 Hugging Face,日榜10个 - `/add-trending all weekly` - 所有数据源,周榜 +## 数据源配置 + +| 源名称 | Agent 名称 | URL | +|--------|-----------|-----| +| github | github-trending | https://github.com/trending | +| huggingface | huggingface-trending | https://huggingface.co/models | +| paperswithcode | papers-with-code | https://paperswithcode.com/ | + ## 执行流程 -**重要**: 本命令使用专用的 Agent 架构处理工作流,由 Agent 之间相互调用完成整个流程。 - -### Stage 1: 初始化与任务派发 +### Stage 1: 初始化工作区与并行爬取 1. **解析参数**: - 从 `$ARGUMENTS` 解析 source、period 和 limit @@ -44,13 +50,100 @@ $ARGUMENTS 2. **创建工作区**: - 生成时间戳目录: `.trending-workspace/{YYYYMMDD-HHMMSS}/` - - 初始化 `progress.json` 文件 + - 初始化 `progress.json` 文件: + ```json + { + "startTime": "2025-01-06T12:00:00Z", + "currentStage": "scraping", + "stages": { + "scraping": "pending", + "deduplicating": "pending", + "analyzing": "pending", + "ingesting": "pending" + }, + "config": { + "source": "all", + "period": "daily", + "limit": 25 + } + } + ``` -3. **执行任务派发器**: - - **使用 Task 工具调用** `task-dispatcher` agent - - 将 source/period/limit/workspace 参数传递给任务派发器 - - 任务派发器并行调度对应的爬虫 - - 输出 `raw-projects.json` +3. **确定要调用的 scrapers**: + 根据 source 参数筛选: + + | source | 调用的 scrapers | + |--------|-----------------| + | all | github-trending, huggingface-trending, papers-with-code | + | github | github-trending | + | huggingface | huggingface-trending | + | paperswithcode | papers-with-code | + +4. **并行调用 scrapers**: + **关键**: 必须在单个消息中发送所有 Task 调用,以实现真正的并行执行。 + + 示例(source=all): + ``` + Task(github-trending, {"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."}) + Task(huggingface-trending, {"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."}) + Task(papers-with-code, {"period": "daily", "limit": 25, "workspace": ".trending-workspace/..."}) + ``` + +5. **等待所有 scraper 完成并汇总**: + 每个 scraper 输出到各自的文件: + - `scraped-github-projects.json` + - `scraped-huggingface-projects.json` + - `scraped-paperswithcode-projects.json` + + 读取所有输出文件,合并生成 `raw-projects.json`: + ```json + { + "metadata": { + "timestamp": "2025-01-06T12:00:00Z", + "period": "daily", + "limit": 25, + "sources": ["github", "huggingface"], + "sourceCounts": { + "github": 25, + "huggingface": 20 + }, + "totalRaw": 45 + }, + "projects": [ + { + "source": "github", + "name": "langchain-ai/langchain", + "url": "https://github.com/langchain-ai/langchain", + "description": "Building applications with LLMs through composability", + "metadata": { + "stars": 85432, + "starsDelta": "+234", + "language": "Python", + "forks": 12543 + } + } + ] + } + ``` + +6. **更新进度**: + ```json + { + "startTime": "2025-01-06T12:00:00Z", + "currentStage": "scraping", + "stages": { + "scraping": "completed", + "deduplicating": "pending", + "analyzing": "pending", + "ingesting": "pending" + }, + "config": { + "source": "all", + "period": "daily", + "limit": 25 + } + } + ``` ### Stage 2: 统一去重 @@ -61,16 +154,38 @@ $ARGUMENTS - 使用 **dbhub PostgreSQL MCP** 执行去重查询 - 输出 `new-projects.json` 和 `task-queue.json` -### Stage 3: 项目分析 +### Stage 3: 项目分析 (并行,固定 3 实例分批处理) -1. **执行项目分析器**: - - **使用 Task 工具调用** `project-analyzer` agent - - 将 workspace 参数传递给分析器 - - 分析器处理任务队列中的项目 - - 使用 **chrome-devtools-mcp** 访问项目页面 - - 生成中英双语内容 - - 计算质量评分 - - 输出 `analyzed-projects.json` +1. **读取任务队列**: + - 读取 `task-queue.json` + - 提取所有 `status: "pending"` 的任务 + - 记录任务数量 N + +2. **分批并行处理**: + - **固定并行度**: 3 个实例 + - **每批处理**: 3 个任务(最后一批可能少于 3 个) + - **批次数**: `Math.ceil(N / 3)` + - **循环执行**以下步骤,直到所有任务完成: + + **对每一批**: + - 选取 3 个 `status: "pending"` 的任务(记录其 taskId) + - **使用单个消息发送 3 个 Task 工具调用**(实现并行): + ``` + Task(project-analyzer, {"workspace": "...", "taskId": 1}) + Task(project-analyzer, {"workspace": "...", "taskId": 2}) + Task(project-analyzer, {"workspace": "...", "taskId": 3}) + ``` + - **等待这 3 个实例完成** + - **检查剩余任务**:重新读取 `task-queue.json`,确认是否还有 `pending` 任务 + - **继续下一批**:如果有 pending 任务,重复上述步骤 + +3. **等待所有批次完成**: + - 确认 `task-queue.json` 中没有 `status: "pending"` 或 `processing` 的任务 + +4. **汇总结果**: + - 读取所有 `analyzed-project-{taskId}.json` 文件 + - 合并生成 `analyzed-projects.json` + - 统计通过/失败的项目数量 ### Stage 4: 批量入库 @@ -86,18 +201,14 @@ $ARGUMENTS 2. **清理旧工作区**(删除 7 天前的) 3. **输出最终报告** -## 数据源爬虫 - -当前支持的数据源(可扩展): -- GitHub Trending: `https://github.com/trending` -- Hugging Face Models: `https://huggingface.co/models` -- Papers with Code: `https://paperswithcode.com/` - ## 工作区文件结构 ``` .trending-workspace/{timestamp}/ -├── raw-projects.json # 所有数据源的原始数据 +├── scraped-github-projects.json # GitHub 原始数据 +├── scraped-huggingface-projects.json # Hugging Face 原始数据 +├── scraped-paperswithcode-projects.json # Papers with Code 原始数据 +├── raw-projects.json # 所有数据源的汇总数据 ├── new-projects.json # 去重后的新项目 ├── task-queue.json # 分析任务队列 ├── analyzed-projects.json # 分析完成的项目 @@ -107,6 +218,7 @@ $ARGUMENTS ## 关键规则 +- **必须使用单个消息发送多个 Task 调用**:实现 scrapers 真正并行执行 - **必须使用 Agent 工作流**:所有数据处理由专用 Agent 完成,不在主窗口执行 - **必须使用 Task 工具**调用 Agent:让 Agent 自主完成其职责 - **必须**先去重再分析,避免处理已存在的项目 @@ -132,7 +244,7 @@ $ARGUMENTS 🚀 多源数据自动入库启动 ⚙️ 配置: source=github, period=daily, limit=25 -✅ Stage 1/4: 任务派发 +✅ Stage 1/4: 数据爬取 🔍 数据源: GitHub Trending 📊 GitHub: 25 个项目 📦 汇总: 25 个原始项目 @@ -156,7 +268,7 @@ $ARGUMENTS - 入库成功: 16 个 - 耗时: 约2分钟 -📁 工作区: .trending-workspace/20250104-120000/ +📁 工作区: .trending-workspace/20250106-120000/ ``` **单数据源测试示例**: diff --git a/.claude/settings.json b/.claude/settings.json index 09737af..ede3347 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,6 +1,6 @@ -{ - "env": { - "HTTP_PROXY": "http://proxy3.bj.petrochina:8080", - "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080" - } -} +// { +// "env": { +// "HTTP_PROXY": "http://proxy3.bj.petrochina:8080", +// "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080" +// } +// } diff --git a/prisma/migrations/migration_lock.toml b/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..044d57c --- /dev/null +++ b/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (e.g., Git) +provider = "postgresql" diff --git a/scripts/scrape-huggingface.js b/scripts/scrape-huggingface.js new file mode 100644 index 0000000..191cf17 --- /dev/null +++ b/scripts/scrape-huggingface.js @@ -0,0 +1,113 @@ +// Simple Hugging Face scraper +const https = require('https'); +const { HttpsProxyAgent } = require('https-proxy-agent'); + +async function scrapeHuggingFace() { + try { + console.log('Fetching Hugging Face models page...'); + + const proxyUrl = process.env.HTTPS_PROXY || process.env.HTTP_PROXY; + const agent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined; + + const html = await new Promise((resolve, reject) => { + const options = { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'en-US,en;q=0.9', + } + }; + + if (agent) { + options.agent = agent; + } + + const req = https.get('https://huggingface.co/models', options, (res) => { + let data = ''; + res.on('data', chunk => data += chunk); + res.on('end', () => { + if (res.statusCode === 200) { + resolve(data); + } else { + reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`)); + } + }); + }); + + req.on('error', reject); + req.setTimeout(30000, () => { + req.destroy(); + reject(new Error('Request timeout')); + }); + }); + + console.log(`Page fetched successfully, size: ${html.length} bytes`); + + // Extract model information + const models = []; + const modelLinkRegex = /]*>/gi; + const seenModels = new Set(); + + let match; + while ((match = modelLinkRegex.exec(html)) !== null) { + const modelId = decodeURIComponent(match[1]); + + // Filter: must have org/model format + if (!modelId.includes('/')) continue; + if (modelId.includes('/discussions')) continue; + if (modelId.includes('/blob')) continue; + if (modelId.includes('/tree')) continue; + if (modelId.includes('/commit')) continue; + if (seenModels.has(modelId)) continue; + + seenModels.add(modelId); + + models.push({ + source: 'huggingface', + name: modelId, + url: `https://huggingface.co/${modelId}`, + description: modelId, + metadata: { + likes: 0, + downloads: 0, + pipeline: '' + } + }); + + if (models.length >= 100) break; + } + + console.log(`Extracted ${models.length} unique models`); + + // Take first 25 (they should be roughly ordered by popularity on the page) + const topModels = models.slice(0, 25); + + const fs = require('fs'); + const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json'; + fs.writeFileSync(outputPath, JSON.stringify(topModels, null, 2), 'utf8'); + console.log(`\nData saved to: ${outputPath}`); + + console.log('\nTop 25 Models:'); + topModels.forEach((m, i) => { + console.log(`${i + 1}. ${m.name}`); + }); + + } catch (error) { + console.error('Error:', error.message); + + // Output empty result on error + const fs = require('fs'); + const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json'; + + const errorResult = { + error: error.message, + projects: [], + timestamp: new Date().toISOString() + }; + + fs.writeFileSync(outputPath, JSON.stringify(errorResult, null, 2), 'utf8'); + console.log(`Error result saved to: ${outputPath}`); + } +} + +scrapeHuggingFace();