refactor: 重构多源数据入库架构，移除任务派发器并优化并行处理

核心变更： - 移除 task-dispatcher agent，将并行调度逻辑迁移至 add-trending 命令 - 重构 project-analyzer 支持单任务处理模式（通过 taskId 参数） - 优化并行处理策略：固定 3 实例分批处理，避免文件竞争 - 改进任务状态管理：pending → processing → completed/failed - 各 scraper 增加输出文件规范说明文件变更： - 删除: .claude/agents/task-dispatcher.md - 修改: .claude/agents/project-analyzer.md（单任务模式、状态机） - 修改: .claude/commands/add-trending.md（直接并行调度） - 修改: .claude/settings.json（代理配置注释） - 新增: prisma/migrations/、scripts/scrape-huggingface.js 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-06 20:59:16 +08:00
parent 3bbf481008
commit 2d3796da43
11 changed files with 427 additions and 270 deletions
@@ -0,0 +1,113 @@
+// Simple Hugging Face scraper
+const https = require('https');
+const { HttpsProxyAgent } = require('https-proxy-agent');
+
+async function scrapeHuggingFace() {
+  try {
+    console.log('Fetching Hugging Face models page...');
+
+    const proxyUrl = process.env.HTTPS_PROXY || process.env.HTTP_PROXY;
+    const agent = proxyUrl ? new HttpsProxyAgent(proxyUrl) : undefined;
+
+    const html = await new Promise((resolve, reject) => {
+      const options = {
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+          'Accept-Language': 'en-US,en;q=0.9',
+        }
+      };
+
+      if (agent) {
+        options.agent = agent;
+      }
+
+      const req = https.get('https://huggingface.co/models', options, (res) => {
+        let data = '';
+        res.on('data', chunk => data += chunk);
+        res.on('end', () => {
+          if (res.statusCode === 200) {
+            resolve(data);
+          } else {
+            reject(new Error(`HTTP ${res.statusCode}: ${res.statusMessage}`));
+          }
+        });
+      });
+
+      req.on('error', reject);
+      req.setTimeout(30000, () => {
+        req.destroy();
+        reject(new Error('Request timeout'));
+      });
+    });
+
+    console.log(`Page fetched successfully, size: ${html.length} bytes`);
+
+    // Extract model information
+    const models = [];
+    const modelLinkRegex = /<a\s+href="\/models\/([^"]+)"[^>]*>/gi;
+    const seenModels = new Set();
+
+    let match;
+    while ((match = modelLinkRegex.exec(html)) !== null) {
+      const modelId = decodeURIComponent(match[1]);
+
+      // Filter: must have org/model format
+      if (!modelId.includes('/')) continue;
+      if (modelId.includes('/discussions')) continue;
+      if (modelId.includes('/blob')) continue;
+      if (modelId.includes('/tree')) continue;
+      if (modelId.includes('/commit')) continue;
+      if (seenModels.has(modelId)) continue;
+
+      seenModels.add(modelId);
+
+      models.push({
+        source: 'huggingface',
+        name: modelId,
+        url: `https://huggingface.co/${modelId}`,
+        description: modelId,
+        metadata: {
+          likes: 0,
+          downloads: 0,
+          pipeline: ''
+        }
+      });
+
+      if (models.length >= 100) break;
+    }
+
+    console.log(`Extracted ${models.length} unique models`);
+
+    // Take first 25 (they should be roughly ordered by popularity on the page)
+    const topModels = models.slice(0, 25);
+
+    const fs = require('fs');
+    const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json';
+    fs.writeFileSync(outputPath, JSON.stringify(topModels, null, 2), 'utf8');
+    console.log(`\nData saved to: ${outputPath}`);
+
+    console.log('\nTop 25 Models:');
+    topModels.forEach((m, i) => {
+      console.log(`${i + 1}. ${m.name}`);
+    });
+
+  } catch (error) {
+    console.error('Error:', error.message);
+
+    // Output empty result on error
+    const fs = require('fs');
+    const outputPath = 'D:\\Code\\AI\\agent-park-v2\\.trending-workspace\\20260106-1226\\scraped-huggingface-projects.json';
+
+    const errorResult = {
+      error: error.message,
+      projects: [],
+      timestamp: new Date().toISOString()
+    };
+
+    fs.writeFileSync(outputPath, JSON.stringify(errorResult, null, 2), 'utf8');
+    console.log(`Error result saved to: ${outputPath}`);
+  }
+}
+
+scrapeHuggingFace();