diff --git a/.claude/agents/database-ingestor.md b/.claude/agents/database-ingestor.md index 5124c62..dde35af 100644 --- a/.claude/agents/database-ingestor.md +++ b/.claude/agents/database-ingestor.md @@ -71,6 +71,9 @@ interface ExternalLink { **API Key**（已配置）：`sk_live_agent_park_webhook_key_2025` +**重要**：必须将请求数据写入 `ingest-request.json` 文件，确保 UTF-8 编码正确。 + +请求格式： ```json { "apiKey": "sk_live_agent_park_webhook_key_2025", @@ -105,21 +108,52 @@ interface ExternalLink { } ``` +将上述内容写入工作区的 `ingest-request.json` 文件。 + ### Step 4: 调用 Webhook API **API Key**：`sk_live_agent_park_webhook_key_2025` **API 端点**：`http://localhost:3000/api/webhook/projects` -使用 Bash 执行： +**重要**：使用 JSON 文件而非命令行参数，避免编码问题。 +1. **写入请求文件** `ingest-request.json`： +```bash +cat > ingest-request.json << 'EOF' +{ + "apiKey": "sk_live_agent_park_webhook_key_2025", + "projects": [ + { + "name": "LangChain", + "nameEn": "LangChain", + "description": "通过组合性构建大型语言模型应用程序的框架...", + "descriptionEn": "Building applications with LLMs through composability", + "status": "ACTIVE", + "source": "GITHUB_TRENDING", + "tags": [ + { "name": "LLM", "nameEn": "Large Language Model" }, + { "name": "Python", "nameEn": "Python" } + ], + "links": [ + { + "type": "GITHUB", + "url": "https://github.com/langchain-ai/langchain", + "title": "GitHub 仓库" + } + ] + } + ] +} +EOF +``` + +2. **发送请求**（使用 `@` 符号读取文件）： ```bash curl -X POST http://localhost:3000/api/webhook/projects \ - -H "Content-Type: application/json" \ - -d '{ - "apiKey": "sk_live_agent_park_webhook_key_2025", - "projects": [...] - }' + -H "Content-Type: application/json; charset=utf-8" \ + -d @ingest-request.json \ + -o ingestion-response.json ``` ### Step 5: 处理响应 diff --git a/.claude/agents/deduplicator.md b/.claude/agents/deduplicator.md index 85bf3f6..86d88c7 100644 --- a/.claude/agents/deduplicator.md +++ b/.claude/agents/deduplicator.md @@ -50,57 +50,82 @@ function normalizeUrl(url: string): string { } ``` -### Step 3: 三级去重检测 +### Step 3: 调用去重检查 API -**重要**：使用 **dbhub PostgreSQL MCP** 查询数据库，而不是 MySQL。 +**重要**：使用 API 接口而非直接访问数据库。 -对每个项目执行以下检测（按优先级）： +#### 3.1 构造请求体 -#### P0: GitHub URL 匹配 -如果项目有 GitHub URL： -```sql -SELECT el.*, p.name as project_name -FROM external_links el -LEFT JOIN projects p ON el.project_id = p.id -WHERE el.type = 'GITHUB' - AND LOWER(el.url) = LOWER($1) -LIMIT 1 +根据原始项目数据构造 API 请求： + +```json +{ + "apiKey": "sk_live_agent_park_webhook_key_2025", + "projects": [ + { + "githubUrl": "https://github.com/langchain-ai/langchain", + "huggingfaceUrl": null, + "websiteUrl": "https://python.langchain.com", + "slug": "langchain" + } + ] +} ``` -参数：`[normalizeUrl(project.githubUrl)]` -#### P1: Hugging Face URL 匹配 -如果项目有 Hugging Face URL： -```sql -SELECT el.*, p.name as project_name -FROM external_links el -LEFT JOIN projects p ON el.project_id = p.id -WHERE el.type = 'HUGGINGFACE' - AND LOWER(el.url) = LOWER($1) -LIMIT 1 +**URL 提取规则**： +- GitHub 项目：`githubUrl` = 项目 URL，`slug` = generateSlug(name) +- Hugging Face 模型：`huggingfaceUrl` = 模型 URL，`slug` = generateSlug(name) +- Papers with Code：`websiteUrl` = 论文/项目 URL，`slug` = generateSlug(name) + +#### 3.2 调用 API + +使用 Bash 执行 curl 请求： + +```bash +curl -X POST http://localhost:3000/api/webhook/check-duplicates \ + -H "Content-Type: application/json" \ + -d @check-request.json \ + -o check-response.json ``` -参数：`[normalizeUrl(project.huggingfaceUrl)]` -#### P2: Website URL 匹配 -如果项目有官网 URL： -```sql -SELECT el.*, p.name as project_name -FROM external_links el -LEFT JOIN projects p ON el.project_id = p.id -WHERE el.type = 'WEBSITE' - AND LOWER(el.url) = LOWER($1) -LIMIT 1 +#### 3.3 处理响应 + +API 响应格式： + +```json +{ + "success": true, + "results": [ + { + "githubUrl": "https://github.com/langchain-ai/langchain", + "exists": true, + "matchType": "GITHUB_URL", + "projectId": "cm2x8k9d10001", + "projectName": "LangChain" + } + ], + "stats": { + "total": 45, + "exists": 10, + "new": 35, + "breakdown": { + "githubUrl": 5, + "huggingfaceUrl": 2, + "websiteUrl": 1, + "slug": 2 + } + } +} ``` -参数：`[normalizeUrl(project.websiteUrl)]` -#### P3: Slug 匹配（兜底） -```sql -SELECT * FROM projects -WHERE slug = $1 -LIMIT 1 -``` -参数：`[generateSlug(project.name)]` +**MatchType 说明**： +- `GITHUB_URL`: 通过 GitHub URL 匹配 +- `HUGGINGFACE_URL`: 通过 Hugging Face URL 匹配 +- `WEBSITE_URL`: 通过官网 URL 匹配 +- `SLUG`: 通过 slug 匹配 +- `NONE`: 未匹配，新项目 -**MCP 工具**：使用 `mcp__dbhub__execute_sql` 执行 SQL 查询。 +根据 `results[i].exists` 判断是否为新项目，仅保留 `exists: false` 的项目。 ### Step 4: 生成新项目列表 @@ -186,7 +211,8 @@ function generateSlug(name: string): string { | 场景 | 处理方式 | |------|---------| | raw-projects.json 不存在 | 错误提示 "请先运行任务派发器" | -| 数据库连接失败 | 错误提示并终止，保存中间结果 | +| API 调用失败 | 错误提示并终止，保存中间结果到 `check-response.json` | +| API 返回 success: false | 错误提示 API 错误详情 | ## 输出 diff --git a/.claude/settings.json b/.claude/settings.json index 09737af..ede3347 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,6 +1,6 @@ -{ - "env": { - "HTTP_PROXY": "http://proxy3.bj.petrochina:8080", - "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080" - } -} +// { +// "env": { +// "HTTP_PROXY": "http://proxy3.bj.petrochina:8080", +// "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080" +// } +// } diff --git a/src/app/api/webhook/check-duplicates/route.ts b/src/app/api/webhook/check-duplicates/route.ts new file mode 100644 index 0000000..426ae6c --- /dev/null +++ b/src/app/api/webhook/check-duplicates/route.ts @@ -0,0 +1,237 @@ +import { NextRequest, NextResponse } from 'next/server' +import { prisma } from '@/lib/prisma' +import { z } from 'zod' + +/** + * 去重检查请求 Schema + */ +const CheckDuplicatesSchema = z.object({ + apiKey: z.string(), + projects: z.array( + z.object({ + githubUrl: z.string().url().optional(), + huggingfaceUrl: z.string().url().optional(), + websiteUrl: z.string().url().optional(), + slug: z.string().optional(), + }) + ), +}) + +/** + * 匹配类型 + */ +type MatchType = + | 'GITHUB_URL' + | 'HUGGINGFACE_URL' + | 'WEBSITE_URL' + | 'SLUG' + | 'NONE' + +/** + * 检查结果 + */ +interface CheckResult { + githubUrl?: string + huggingfaceUrl?: string + websiteUrl?: string + slug?: string + exists: boolean + matchType: MatchType + projectId?: string + projectName?: string +} + +/** + * 多级去重策略：检查项目是否已存在 + * + * 优先级： + * 1. GitHub URL 完全匹配（最准确） + * 2. Hugging Face URL 完全匹配 + * 3. Website URL 完全匹配 + * 4. slug 匹配（兜底） + * + * @param project - 项目待检查信息 + * @returns 检查结果 + */ +async function checkProjectExists(project: { + githubUrl?: string + huggingfaceUrl?: string + websiteUrl?: string + slug?: string +}): Promise { + // 优先级1: GitHub URL 匹配 + if (project.githubUrl) { + const existingByGithub = await prisma.externalLink.findFirst({ + where: { + type: 'GITHUB', + url: project.githubUrl, + }, + include: { + project: true, + }, + }) + + if (existingByGithub) { + return { + githubUrl: project.githubUrl, + exists: true, + matchType: 'GITHUB_URL', + projectId: existingByGithub.project.id, + projectName: existingByGithub.project.name, + } + } + } + + // 优先级2: Hugging Face URL 匹配 + if (project.huggingfaceUrl) { + const existingByHuggingFace = await prisma.externalLink.findFirst({ + where: { + type: 'HUGGINGFACE', + url: project.huggingfaceUrl, + }, + include: { + project: true, + }, + }) + + if (existingByHuggingFace) { + return { + huggingfaceUrl: project.huggingfaceUrl, + exists: true, + matchType: 'HUGGINGFACE_URL', + projectId: existingByHuggingFace.project.id, + projectName: existingByHuggingFace.project.name, + } + } + } + + // 优先级3: Website URL 匹配 + if (project.websiteUrl) { + const existingByWebsite = await prisma.externalLink.findFirst({ + where: { + type: 'WEBSITE', + url: project.websiteUrl, + }, + include: { + project: true, + }, + }) + + if (existingByWebsite) { + return { + websiteUrl: project.websiteUrl, + exists: true, + matchType: 'WEBSITE_URL', + projectId: existingByWebsite.project.id, + projectName: existingByWebsite.project.name, + } + } + } + + // 优先级4: Slug 匹配（兜底） + if (project.slug) { + const existingBySlug = await prisma.project.findUnique({ + where: { slug: project.slug }, + }) + + if (existingBySlug) { + return { + slug: project.slug, + exists: true, + matchType: 'SLUG', + projectId: existingBySlug.id, + projectName: existingBySlug.name, + } + } + } + + // 未找到匹配项 + return { + githubUrl: project.githubUrl, + huggingfaceUrl: project.huggingfaceUrl, + websiteUrl: project.websiteUrl, + slug: project.slug, + exists: false, + matchType: 'NONE', + } +} + +export async function POST(request: NextRequest) { + const startTime = Date.now() + + try { + const body = await request.json() + + // Validate payload + const validationResult = CheckDuplicatesSchema.safeParse(body) + + if (!validationResult.success) { + return NextResponse.json( + { + success: false, + error: 'Validation error', + details: validationResult.error.errors.map((e) => e.message), + }, + { status: 400 } + ) + } + + const payload = validationResult.data + + // Verify API Key + const apiKey = process.env.WEBHOOK_API_KEY + if (payload.apiKey !== apiKey) { + return NextResponse.json( + { + success: false, + error: 'Unauthorized', + details: ['Invalid or missing API Key'], + }, + { status: 401 } + ) + } + + // 并行检查所有项目 + const results = await Promise.all( + payload.projects.map((project) => checkProjectExists(project)) + ) + + // 统计信息 + const stats = { + total: results.length, + exists: results.filter((r) => r.exists).length, + new: results.filter((r) => !r.exists).length, + breakdown: { + githubUrl: results.filter((r) => r.matchType === 'GITHUB_URL').length, + huggingfaceUrl: results.filter( + (r) => r.matchType === 'HUGGINGFACE_URL' + ).length, + websiteUrl: results.filter((r) => r.matchType === 'WEBSITE_URL') + .length, + slug: results.filter((r) => r.matchType === 'SLUG').length, + }, + } + + const duration = Date.now() - startTime + + console.log( + `[CheckDuplicates] Checked ${stats.total} projects in ${duration}ms: ${stats.exists} exist, ${stats.new} new` + ) + + return NextResponse.json({ + success: true, + results, + stats, + }) + } catch (error) { + console.error('[CheckDuplicates] Error:', error) + return NextResponse.json( + { + success: false, + error: 'Internal server error', + details: [error instanceof Error ? error.message : 'Unknown error'], + }, + { status: 500 } + ) + } +} diff --git a/src/app/api/webhook/projects/route.ts b/src/app/api/webhook/projects/route.ts index 6141298..5144295 100644 --- a/src/app/api/webhook/projects/route.ts +++ b/src/app/api/webhook/projects/route.ts @@ -165,22 +165,44 @@ export async function POST(request: NextRequest) { // 多级去重：查找已存在的项目 const existingProject = await findExistingProject(validProject) - // Upsert tags + // Upsert tags with better error handling for name uniqueness const tagConnections = await Promise.all( validProject.tags.map(async (tag) => { const slug = tag.nameEn?.toLowerCase().replace(/\s+/g, '-') || tag.name.toLowerCase().replace(/\s+/g, '-') - return prisma.tag.upsert({ - where: { slug }, - update: {}, - create: { - name: tag.name, - nameEn: tag.nameEn || null, - slug, - }, + // First, try to find by name (handle name uniqueness constraint) + const existingByName = await prisma.tag.findUnique({ + where: { name: tag.name }, }) + + if (existingByName) { + // Tag with this name already exists, use it + return existingByName + } + + // Try upsert by slug (safe now since name doesn't exist) + try { + return await prisma.tag.upsert({ + where: { slug }, + update: {}, + create: { + name: tag.name, + nameEn: tag.nameEn || null, + slug, + }, + }) + } catch (error) { + // If slug conflicts with existing tag, find and use that one + const existingBySlug = await prisma.tag.findUnique({ + where: { slug }, + }) + if (existingBySlug) { + return existingBySlug + } + throw error + } }) ) @@ -204,6 +226,11 @@ export async function POST(request: NextRequest) { `[Webhook] Updating project "${validProject.name}" (matched by ${matchMethod}, id: ${existingProject.id})` ) + // Update tags (delete old ones, create new ones) + await prisma.projectTag.deleteMany({ + where: { projectId: existingProject.id }, + }) + await prisma.project.update({ where: { id: existingProject.id }, data: { @@ -216,7 +243,9 @@ export async function POST(request: NextRequest) { status: validProject.status as any, source: validProject.source || null, tags: { - set: tagConnections.map((t) => ({ id: t.id })), + create: tagConnections.map((t) => ({ + tag: { connect: { id: t.id } }, + })), }, }, }) @@ -254,7 +283,9 @@ export async function POST(request: NextRequest) { status: validProject.status as any, source: validProject.source || null, tags: { - connect: tagConnections.map((t) => ({ id: t.id })), + create: tagConnections.map((t) => ({ + tag: { connect: { id: t.id } }, + })), }, links: { create: validProject.links.map((link) => ({ diff --git a/src/components/project/ProjectCard.tsx b/src/components/project/ProjectCard.tsx index a95f480..c046d37 100644 --- a/src/components/project/ProjectCard.tsx +++ b/src/components/project/ProjectCard.tsx @@ -30,8 +30,10 @@ interface ProjectCardProps { } // Icon mapping for projects based on tags -function getProjectIcon(tags: Array<{ name: string }>): string { - const tagNames = tags.map(t => t.name.toLowerCase()) +function getProjectIcon(tags: Array<{ name: string | null }>): string { + const tagNames = tags + .map(t => t.name?.toLowerCase()) + .filter((name): name is string => Boolean(name)) if (tagNames.some(t => t.includes('automation') || t.includes('workflow'))) return '⚙️' if (tagNames.some(t => t.includes('image') || t.includes('art'))) return '🎨' if (tagNames.some(t => t.includes('chat') || t.includes('assistant'))) return '💬' diff --git a/src/hooks/useProjects.ts b/src/hooks/useProjects.ts index 75fdb9e..3116c56 100644 --- a/src/hooks/useProjects.ts +++ b/src/hooks/useProjects.ts @@ -40,7 +40,11 @@ export async function getProjects(options?: { prisma.project.findMany({ where, include: { - tags: true, + tags: { + include: { + tag: true, + }, + }, links: true, }, orderBy: { @@ -52,8 +56,14 @@ export async function getProjects(options?: { prisma.project.count({ where }), ]) + // Transform tags to flatten the structure + const transformedProjects = projects.map((project) => ({ + ...project, + tags: project.tags.map((pt) => pt.tag), + })) + return { - projects, + projects: transformedProjects, pagination: { page, limit, @@ -64,13 +74,27 @@ export async function getProjects(options?: { } export async function getProjectBySlug(slug: string) { - return prisma.project.findUnique({ + const project = await prisma.project.findUnique({ where: { slug }, include: { - tags: true, + tags: { + include: { + tag: true, + }, + }, links: true, }, }) + + if (!project) { + return null + } + + // Transform tags to flatten the structure + return { + ...project, + tags: project.tags.map((pt) => pt.tag), + } } export async function getAllTags() {