refactor: 去重器改用 API 并修复 tag 唯一性约束处理

- deduplicator: 从直接数据库查询改为调用 /api/webhook/check-duplicates API
- database-ingestor: 使用 JSON 文件传参避免 curl 编码问题
- webhook: 修复 tag name 唯一性约束冲突，更新时删除旧关联重建
- 添加 check-duplicates API 端点用于批量去重检测
- 适配 ProjectTag 显式关联表的数据查询逻辑
- ProjectCard: 修复 getProjectIcon 空值处理

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

.claude/agents/database-ingestor.md

@@ -71,6 +71,9 @@ interface ExternalLink {
 
 **API Key**（已配置）：`sk_live_agent_park_webhook_key_2025`
 
+**重要**：必须将请求数据写入 `ingest-request.json` 文件，确保 UTF-8 编码正确。
+
+请求格式：
 ```json
 {
   "apiKey": "sk_live_agent_park_webhook_key_2025",
@@ -105,21 +108,52 @@ interface ExternalLink {
 }
 ```
 
+将上述内容写入工作区的 `ingest-request.json` 文件。
+
 ### Step 4: 调用 Webhook API
 
 **API Key**：`sk_live_agent_park_webhook_key_2025`
 
 **API 端点**：`http://localhost:3000/api/webhook/projects`
 
-使用 Bash 执行：
+**重要**：使用 JSON 文件而非命令行参数，避免编码问题。
 
+1. **写入请求文件** `ingest-request.json`：
+```bash
+cat > ingest-request.json << 'EOF'
+{
+  "apiKey": "sk_live_agent_park_webhook_key_2025",
+  "projects": [
+    {
+      "name": "LangChain",
+      "nameEn": "LangChain",
+      "description": "通过组合性构建大型语言模型应用程序的框架...",
+      "descriptionEn": "Building applications with LLMs through composability",
+      "status": "ACTIVE",
+      "source": "GITHUB_TRENDING",
+      "tags": [
+        { "name": "LLM", "nameEn": "Large Language Model" },
+        { "name": "Python", "nameEn": "Python" }
+      ],
+      "links": [
+        {
+          "type": "GITHUB",
+          "url": "https://github.com/langchain-ai/langchain",
+          "title": "GitHub 仓库"
+        }
+      ]
+    }
+  ]
+}
+EOF
+```
+
+2. **发送请求**（使用 `@` 符号读取文件）：
 ```bash
 curl -X POST http://localhost:3000/api/webhook/projects \
-  -H "Content-Type: application/json" \
-  -d '{
-    "apiKey": "sk_live_agent_park_webhook_key_2025",
-    "projects": [...]
-  }'
+  -H "Content-Type: application/json; charset=utf-8" \
+  -d @ingest-request.json \
+  -o ingestion-response.json
 ```
 
 ### Step 5: 处理响应

.claude/agents/deduplicator.md

@@ -50,57 +50,82 @@ function normalizeUrl(url: string): string {
 }
 ```
 
-### Step 3: 三级去重检测
+### Step 3: 调用去重检查 API
 
-**重要**：使用 **dbhub PostgreSQL MCP** 查询数据库，而不是 MySQL。
+**重要**：使用 API 接口而非直接访问数据库。
 
-对每个项目执行以下检测（按优先级）：
+#### 3.1 构造请求体
 
-#### P0: GitHub URL 匹配
-如果项目有 GitHub URL：
-```sql
-SELECT el.*, p.name as project_name
-FROM external_links el
-LEFT JOIN projects p ON el.project_id = p.id
-WHERE el.type = 'GITHUB'
-  AND LOWER(el.url) = LOWER($1)
-LIMIT 1
+根据原始项目数据构造 API 请求：
+
+```json
+{
+  "apiKey": "sk_live_agent_park_webhook_key_2025",
+  "projects": [
+    {
+      "githubUrl": "https://github.com/langchain-ai/langchain",
+      "huggingfaceUrl": null,
+      "websiteUrl": "https://python.langchain.com",
+      "slug": "langchain"
+    }
+  ]
+}
 ```
-参数：`[normalizeUrl(project.githubUrl)]`
 
-#### P1: Hugging Face URL 匹配
-如果项目有 Hugging Face URL：
-```sql
-SELECT el.*, p.name as project_name
-FROM external_links el
-LEFT JOIN projects p ON el.project_id = p.id
-WHERE el.type = 'HUGGINGFACE'
-  AND LOWER(el.url) = LOWER($1)
-LIMIT 1
+**URL 提取规则**：
+- GitHub 项目：`githubUrl` = 项目 URL，`slug` = generateSlug(name)
+- Hugging Face 模型：`huggingfaceUrl` = 模型 URL，`slug` = generateSlug(name)
+- Papers with Code：`websiteUrl` = 论文/项目 URL，`slug` = generateSlug(name)
+
+#### 3.2 调用 API
+
+使用 Bash 执行 curl 请求：
+
+```bash
+curl -X POST http://localhost:3000/api/webhook/check-duplicates \
+  -H "Content-Type: application/json" \
+  -d @check-request.json \
+  -o check-response.json
 ```
-参数：`[normalizeUrl(project.huggingfaceUrl)]`
 
-#### P2: Website URL 匹配
-如果项目有官网 URL：
-```sql
-SELECT el.*, p.name as project_name
-FROM external_links el
-LEFT JOIN projects p ON el.project_id = p.id
-WHERE el.type = 'WEBSITE'
-  AND LOWER(el.url) = LOWER($1)
-LIMIT 1
+#### 3.3 处理响应
+
+API 响应格式：
+
+```json
+{
+  "success": true,
+  "results": [
+    {
+      "githubUrl": "https://github.com/langchain-ai/langchain",
+      "exists": true,
+      "matchType": "GITHUB_URL",
+      "projectId": "cm2x8k9d10001",
+      "projectName": "LangChain"
+    }
+  ],
+  "stats": {
+    "total": 45,
+    "exists": 10,
+    "new": 35,
+    "breakdown": {
+      "githubUrl": 5,
+      "huggingfaceUrl": 2,
+      "websiteUrl": 1,
+      "slug": 2
+    }
+  }
+}
 ```
-参数：`[normalizeUrl(project.websiteUrl)]`
 
-#### P3: Slug 匹配（兜底）
-```sql
-SELECT * FROM projects
-WHERE slug = $1
-LIMIT 1
-```
-参数：`[generateSlug(project.name)]`
+**MatchType 说明**：
+- `GITHUB_URL`: 通过 GitHub URL 匹配
+- `HUGGINGFACE_URL`: 通过 Hugging Face URL 匹配
+- `WEBSITE_URL`: 通过官网 URL 匹配
+- `SLUG`: 通过 slug 匹配
+- `NONE`: 未匹配，新项目
 
-**MCP 工具**：使用 `mcp__dbhub__execute_sql` 执行 SQL 查询。
+根据 `results[i].exists` 判断是否为新项目，仅保留 `exists: false` 的项目。
 
 ### Step 4: 生成新项目列表
 
@@ -186,7 +211,8 @@ function generateSlug(name: string): string {
 | 场景 | 处理方式 |
 |------|---------|
 | raw-projects.json 不存在 | 错误提示 "请先运行任务派发器" |
-| 数据库连接失败 | 错误提示并终止，保存中间结果 |
+| API 调用失败 | 错误提示并终止，保存中间结果到 `check-response.json` |
+| API 返回 success: false | 错误提示 API 错误详情 |
 
 ## 输出
 

.claude/settings.json

@@ -1,6 +1,6 @@
-{
-  "env": {
-    "HTTP_PROXY": "http://proxy3.bj.petrochina:8080",
-    "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080"
-  }
-}
+// {
+//   "env": {
+//     "HTTP_PROXY": "http://proxy3.bj.petrochina:8080",
+//     "HTTPS_PROXY": "http://proxy3.bj.petrochina:8080"
+//   }
+// }

src/app/api/webhook/check-duplicates/route.ts

@@ -0,0 +1,237 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { z } from 'zod'
+
+/**
+ * 去重检查请求 Schema
+ */
+const CheckDuplicatesSchema = z.object({
+  apiKey: z.string(),
+  projects: z.array(
+    z.object({
+      githubUrl: z.string().url().optional(),
+      huggingfaceUrl: z.string().url().optional(),
+      websiteUrl: z.string().url().optional(),
+      slug: z.string().optional(),
+    })
+  ),
+})
+
+/**
+ * 匹配类型
+ */
+type MatchType =
+  | 'GITHUB_URL'
+  | 'HUGGINGFACE_URL'
+  | 'WEBSITE_URL'
+  | 'SLUG'
+  | 'NONE'
+
+/**
+ * 检查结果
+ */
+interface CheckResult {
+  githubUrl?: string
+  huggingfaceUrl?: string
+  websiteUrl?: string
+  slug?: string
+  exists: boolean
+  matchType: MatchType
+  projectId?: string
+  projectName?: string
+}
+
+/**
+ * 多级去重策略：检查项目是否已存在
+ *
+ * 优先级：
+ * 1. GitHub URL 完全匹配（最准确）
+ * 2. Hugging Face URL 完全匹配
+ * 3. Website URL 完全匹配
+ * 4. slug 匹配（兜底）
+ *
+ * @param project - 项目待检查信息
+ * @returns 检查结果
+ */
+async function checkProjectExists(project: {
+  githubUrl?: string
+  huggingfaceUrl?: string
+  websiteUrl?: string
+  slug?: string
+}): Promise<CheckResult> {
+  // 优先级1: GitHub URL 匹配
+  if (project.githubUrl) {
+    const existingByGithub = await prisma.externalLink.findFirst({
+      where: {
+        type: 'GITHUB',
+        url: project.githubUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByGithub) {
+      return {
+        githubUrl: project.githubUrl,
+        exists: true,
+        matchType: 'GITHUB_URL',
+        projectId: existingByGithub.project.id,
+        projectName: existingByGithub.project.name,
+      }
+    }
+  }
+
+  // 优先级2: Hugging Face URL 匹配
+  if (project.huggingfaceUrl) {
+    const existingByHuggingFace = await prisma.externalLink.findFirst({
+      where: {
+        type: 'HUGGINGFACE',
+        url: project.huggingfaceUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByHuggingFace) {
+      return {
+        huggingfaceUrl: project.huggingfaceUrl,
+        exists: true,
+        matchType: 'HUGGINGFACE_URL',
+        projectId: existingByHuggingFace.project.id,
+        projectName: existingByHuggingFace.project.name,
+      }
+    }
+  }
+
+  // 优先级3: Website URL 匹配
+  if (project.websiteUrl) {
+    const existingByWebsite = await prisma.externalLink.findFirst({
+      where: {
+        type: 'WEBSITE',
+        url: project.websiteUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByWebsite) {
+      return {
+        websiteUrl: project.websiteUrl,
+        exists: true,
+        matchType: 'WEBSITE_URL',
+        projectId: existingByWebsite.project.id,
+        projectName: existingByWebsite.project.name,
+      }
+    }
+  }
+
+  // 优先级4: Slug 匹配（兜底）
+  if (project.slug) {
+    const existingBySlug = await prisma.project.findUnique({
+      where: { slug: project.slug },
+    })
+
+    if (existingBySlug) {
+      return {
+        slug: project.slug,
+        exists: true,
+        matchType: 'SLUG',
+        projectId: existingBySlug.id,
+        projectName: existingBySlug.name,
+      }
+    }
+  }
+
+  // 未找到匹配项
+  return {
+    githubUrl: project.githubUrl,
+    huggingfaceUrl: project.huggingfaceUrl,
+    websiteUrl: project.websiteUrl,
+    slug: project.slug,
+    exists: false,
+    matchType: 'NONE',
+  }
+}
+
+export async function POST(request: NextRequest) {
+  const startTime = Date.now()
+
+  try {
+    const body = await request.json()
+
+    // Validate payload
+    const validationResult = CheckDuplicatesSchema.safeParse(body)
+
+    if (!validationResult.success) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Validation error',
+          details: validationResult.error.errors.map((e) => e.message),
+        },
+        { status: 400 }
+      )
+    }
+
+    const payload = validationResult.data
+
+    // Verify API Key
+    const apiKey = process.env.WEBHOOK_API_KEY
+    if (payload.apiKey !== apiKey) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Unauthorized',
+          details: ['Invalid or missing API Key'],
+        },
+        { status: 401 }
+      )
+    }
+
+    // 并行检查所有项目
+    const results = await Promise.all(
+      payload.projects.map((project) => checkProjectExists(project))
+    )
+
+    // 统计信息
+    const stats = {
+      total: results.length,
+      exists: results.filter((r) => r.exists).length,
+      new: results.filter((r) => !r.exists).length,
+      breakdown: {
+        githubUrl: results.filter((r) => r.matchType === 'GITHUB_URL').length,
+        huggingfaceUrl: results.filter(
+          (r) => r.matchType === 'HUGGINGFACE_URL'
+        ).length,
+        websiteUrl: results.filter((r) => r.matchType === 'WEBSITE_URL')
+          .length,
+        slug: results.filter((r) => r.matchType === 'SLUG').length,
+      },
+    }
+
+    const duration = Date.now() - startTime
+
+    console.log(
+      `[CheckDuplicates] Checked ${stats.total} projects in ${duration}ms: ${stats.exists} exist, ${stats.new} new`
+    )
+
+    return NextResponse.json({
+      success: true,
+      results,
+      stats,
+    })
+  } catch (error) {
+    console.error('[CheckDuplicates] Error:', error)
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Internal server error',
+        details: [error instanceof Error ? error.message : 'Unknown error'],
+      },
+      { status: 500 }
+    )
+  }
+}

src/app/api/webhook/projects/route.ts

@@ -165,22 +165,44 @@ export async function POST(request: NextRequest) {
         // 多级去重：查找已存在的项目
         const existingProject = await findExistingProject(validProject)
 
-        // Upsert tags
+        // Upsert tags with better error handling for name uniqueness
         const tagConnections = await Promise.all(
           validProject.tags.map(async (tag) => {
             const slug =
               tag.nameEn?.toLowerCase().replace(/\s+/g, '-') ||
               tag.name.toLowerCase().replace(/\s+/g, '-')
 
-            return prisma.tag.upsert({
-              where: { slug },
-              update: {},
-              create: {
-                name: tag.name,
-                nameEn: tag.nameEn || null,
-                slug,
-              },
+            // First, try to find by name (handle name uniqueness constraint)
+            const existingByName = await prisma.tag.findUnique({
+              where: { name: tag.name },
             })
+
+            if (existingByName) {
+              // Tag with this name already exists, use it
+              return existingByName
+            }
+
+            // Try upsert by slug (safe now since name doesn't exist)
+            try {
+              return await prisma.tag.upsert({
+                where: { slug },
+                update: {},
+                create: {
+                  name: tag.name,
+                  nameEn: tag.nameEn || null,
+                  slug,
+                },
+              })
+            } catch (error) {
+              // If slug conflicts with existing tag, find and use that one
+              const existingBySlug = await prisma.tag.findUnique({
+                where: { slug },
+              })
+              if (existingBySlug) {
+                return existingBySlug
+              }
+              throw error
+            }
           })
         )
 
@@ -204,6 +226,11 @@ export async function POST(request: NextRequest) {
             `[Webhook] Updating project "${validProject.name}" (matched by ${matchMethod}, id: ${existingProject.id})`
           )
 
+          // Update tags (delete old ones, create new ones)
+          await prisma.projectTag.deleteMany({
+            where: { projectId: existingProject.id },
+          })
+
           await prisma.project.update({
             where: { id: existingProject.id },
             data: {
@@ -216,7 +243,9 @@ export async function POST(request: NextRequest) {
               status: validProject.status as any,
               source: validProject.source || null,
               tags: {
-                set: tagConnections.map((t) => ({ id: t.id })),
+                create: tagConnections.map((t) => ({
+                  tag: { connect: { id: t.id } },
+                })),
               },
             },
           })
@@ -254,7 +283,9 @@ export async function POST(request: NextRequest) {
               status: validProject.status as any,
               source: validProject.source || null,
               tags: {
-                connect: tagConnections.map((t) => ({ id: t.id })),
+                create: tagConnections.map((t) => ({
+                  tag: { connect: { id: t.id } },
+                })),
               },
               links: {
                 create: validProject.links.map((link) => ({

src/components/project/ProjectCard.tsx

@@ -30,8 +30,10 @@ interface ProjectCardProps {
 }
 
 // Icon mapping for projects based on tags
-function getProjectIcon(tags: Array<{ name: string }>): string {
-  const tagNames = tags.map(t => t.name.toLowerCase())
+function getProjectIcon(tags: Array<{ name: string | null }>): string {
+  const tagNames = tags
+    .map(t => t.name?.toLowerCase())
+    .filter((name): name is string => Boolean(name))
   if (tagNames.some(t => t.includes('automation') || t.includes('workflow'))) return '⚙️'
   if (tagNames.some(t => t.includes('image') || t.includes('art'))) return '🎨'
   if (tagNames.some(t => t.includes('chat') || t.includes('assistant'))) return '💬'

src/hooks/useProjects.ts

@@ -40,7 +40,11 @@ export async function getProjects(options?: {
     prisma.project.findMany({
       where,
       include: {
-        tags: true,
+        tags: {
+          include: {
+            tag: true,
+          },
+        },
         links: true,
       },
       orderBy: {
@@ -52,8 +56,14 @@ export async function getProjects(options?: {
     prisma.project.count({ where }),
   ])
 
+  // Transform tags to flatten the structure
+  const transformedProjects = projects.map((project) => ({
+    ...project,
+    tags: project.tags.map((pt) => pt.tag),
+  }))
+
   return {
-    projects,
+    projects: transformedProjects,
     pagination: {
       page,
       limit,
@@ -64,13 +74,27 @@ export async function getProjects(options?: {
 }
 
 export async function getProjectBySlug(slug: string) {
-  return prisma.project.findUnique({
+  const project = await prisma.project.findUnique({
     where: { slug },
     include: {
-      tags: true,
+      tags: {
+        include: {
+          tag: true,
+        },
+      },
       links: true,
     },
   })
+
+  if (!project) {
+    return null
+  }
+
+  // Transform tags to flatten the structure
+  return {
+    ...project,
+    tags: project.tags.map((pt) => pt.tag),
+  }
 }
 
 export async function getAllTags() {