refactor: 去重器改用 API 并修复 tag 唯一性约束处理

- deduplicator: 从直接数据库查询改为调用 /api/webhook/check-duplicates API - database-ingestor: 使用 JSON 文件传参避免 curl 编码问题 - webhook: 修复 tag name 唯一性约束冲突，更新时删除旧关联重建 - 添加 check-duplicates API 端点用于批量去重检测 - 适配 ProjectTag 显式关联表的数据查询逻辑 - ProjectCard: 修复 getProjectIcon 空值处理 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
2026-01-06 14:21:10 +08:00
parent 7baf547bf3
commit 0b9573072f
7 changed files with 425 additions and 71 deletions
@@ -0,0 +1,237 @@
+import { NextRequest, NextResponse } from 'next/server'
+import { prisma } from '@/lib/prisma'
+import { z } from 'zod'
+
+/**
+ * 去重检查请求 Schema
+ */
+const CheckDuplicatesSchema = z.object({
+  apiKey: z.string(),
+  projects: z.array(
+    z.object({
+      githubUrl: z.string().url().optional(),
+      huggingfaceUrl: z.string().url().optional(),
+      websiteUrl: z.string().url().optional(),
+      slug: z.string().optional(),
+    })
+  ),
+})
+
+/**
+ * 匹配类型
+ */
+type MatchType =
+  | 'GITHUB_URL'
+  | 'HUGGINGFACE_URL'
+  | 'WEBSITE_URL'
+  | 'SLUG'
+  | 'NONE'
+
+/**
+ * 检查结果
+ */
+interface CheckResult {
+  githubUrl?: string
+  huggingfaceUrl?: string
+  websiteUrl?: string
+  slug?: string
+  exists: boolean
+  matchType: MatchType
+  projectId?: string
+  projectName?: string
+}
+
+/**
+ * 多级去重策略：检查项目是否已存在
+ *
+ * 优先级：
+ * 1. GitHub URL 完全匹配（最准确）
+ * 2. Hugging Face URL 完全匹配
+ * 3. Website URL 完全匹配
+ * 4. slug 匹配（兜底）
+ *
+ * @param project - 项目待检查信息
+ * @returns 检查结果
+ */
+async function checkProjectExists(project: {
+  githubUrl?: string
+  huggingfaceUrl?: string
+  websiteUrl?: string
+  slug?: string
+}): Promise<CheckResult> {
+  // 优先级1: GitHub URL 匹配
+  if (project.githubUrl) {
+    const existingByGithub = await prisma.externalLink.findFirst({
+      where: {
+        type: 'GITHUB',
+        url: project.githubUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByGithub) {
+      return {
+        githubUrl: project.githubUrl,
+        exists: true,
+        matchType: 'GITHUB_URL',
+        projectId: existingByGithub.project.id,
+        projectName: existingByGithub.project.name,
+      }
+    }
+  }
+
+  // 优先级2: Hugging Face URL 匹配
+  if (project.huggingfaceUrl) {
+    const existingByHuggingFace = await prisma.externalLink.findFirst({
+      where: {
+        type: 'HUGGINGFACE',
+        url: project.huggingfaceUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByHuggingFace) {
+      return {
+        huggingfaceUrl: project.huggingfaceUrl,
+        exists: true,
+        matchType: 'HUGGINGFACE_URL',
+        projectId: existingByHuggingFace.project.id,
+        projectName: existingByHuggingFace.project.name,
+      }
+    }
+  }
+
+  // 优先级3: Website URL 匹配
+  if (project.websiteUrl) {
+    const existingByWebsite = await prisma.externalLink.findFirst({
+      where: {
+        type: 'WEBSITE',
+        url: project.websiteUrl,
+      },
+      include: {
+        project: true,
+      },
+    })
+
+    if (existingByWebsite) {
+      return {
+        websiteUrl: project.websiteUrl,
+        exists: true,
+        matchType: 'WEBSITE_URL',
+        projectId: existingByWebsite.project.id,
+        projectName: existingByWebsite.project.name,
+      }
+    }
+  }
+
+  // 优先级4: Slug 匹配（兜底）
+  if (project.slug) {
+    const existingBySlug = await prisma.project.findUnique({
+      where: { slug: project.slug },
+    })
+
+    if (existingBySlug) {
+      return {
+        slug: project.slug,
+        exists: true,
+        matchType: 'SLUG',
+        projectId: existingBySlug.id,
+        projectName: existingBySlug.name,
+      }
+    }
+  }
+
+  // 未找到匹配项
+  return {
+    githubUrl: project.githubUrl,
+    huggingfaceUrl: project.huggingfaceUrl,
+    websiteUrl: project.websiteUrl,
+    slug: project.slug,
+    exists: false,
+    matchType: 'NONE',
+  }
+}
+
+export async function POST(request: NextRequest) {
+  const startTime = Date.now()
+
+  try {
+    const body = await request.json()
+
+    // Validate payload
+    const validationResult = CheckDuplicatesSchema.safeParse(body)
+
+    if (!validationResult.success) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Validation error',
+          details: validationResult.error.errors.map((e) => e.message),
+        },
+        { status: 400 }
+      )
+    }
+
+    const payload = validationResult.data
+
+    // Verify API Key
+    const apiKey = process.env.WEBHOOK_API_KEY
+    if (payload.apiKey !== apiKey) {
+      return NextResponse.json(
+        {
+          success: false,
+          error: 'Unauthorized',
+          details: ['Invalid or missing API Key'],
+        },
+        { status: 401 }
+      )
+    }
+
+    // 并行检查所有项目
+    const results = await Promise.all(
+      payload.projects.map((project) => checkProjectExists(project))
+    )
+
+    // 统计信息
+    const stats = {
+      total: results.length,
+      exists: results.filter((r) => r.exists).length,
+      new: results.filter((r) => !r.exists).length,
+      breakdown: {
+        githubUrl: results.filter((r) => r.matchType === 'GITHUB_URL').length,
+        huggingfaceUrl: results.filter(
+          (r) => r.matchType === 'HUGGINGFACE_URL'
+        ).length,
+        websiteUrl: results.filter((r) => r.matchType === 'WEBSITE_URL')
+          .length,
+        slug: results.filter((r) => r.matchType === 'SLUG').length,
+      },
+    }
+
+    const duration = Date.now() - startTime
+
+    console.log(
+      `[CheckDuplicates] Checked ${stats.total} projects in ${duration}ms: ${stats.exists} exist, ${stats.new} new`
+    )
+
+    return NextResponse.json({
+      success: true,
+      results,
+      stats,
+    })
+  } catch (error) {
+    console.error('[CheckDuplicates] Error:', error)
+    return NextResponse.json(
+      {
+        success: false,
+        error: 'Internal server error',
+        details: [error instanceof Error ? error.message : 'Unknown error'],
+      },
+      { status: 500 }
+    )
+  }
+}