refactor: 去重器改用 API 并修复 tag 唯一性约束处理
- deduplicator: 从直接数据库查询改为调用 /api/webhook/check-duplicates API - database-ingestor: 使用 JSON 文件传参避免 curl 编码问题 - webhook: 修复 tag name 唯一性约束冲突，更新时删除旧关联重建 - 添加 check-duplicates API 端点用于批量去重检测 - 适配 ProjectTag 显式关联表的数据查询逻辑 - ProjectCard: 修复 getProjectIcon 空值处理 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Claude
@@ -0,0 +1,237 @@
|
||||
import { NextRequest, NextResponse } from 'next/server'
|
||||
import { prisma } from '@/lib/prisma'
|
||||
import { z } from 'zod'
|
||||
|
||||
/**
|
||||
* 去重检查请求 Schema
|
||||
*/
|
||||
const CheckDuplicatesSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
projects: z.array(
|
||||
z.object({
|
||||
githubUrl: z.string().url().optional(),
|
||||
huggingfaceUrl: z.string().url().optional(),
|
||||
websiteUrl: z.string().url().optional(),
|
||||
slug: z.string().optional(),
|
||||
})
|
||||
),
|
||||
})
|
||||
|
||||
/**
|
||||
* 匹配类型
|
||||
*/
|
||||
type MatchType =
|
||||
| 'GITHUB_URL'
|
||||
| 'HUGGINGFACE_URL'
|
||||
| 'WEBSITE_URL'
|
||||
| 'SLUG'
|
||||
| 'NONE'
|
||||
|
||||
/**
|
||||
* 检查结果
|
||||
*/
|
||||
interface CheckResult {
|
||||
githubUrl?: string
|
||||
huggingfaceUrl?: string
|
||||
websiteUrl?: string
|
||||
slug?: string
|
||||
exists: boolean
|
||||
matchType: MatchType
|
||||
projectId?: string
|
||||
projectName?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* 多级去重策略：检查项目是否已存在
|
||||
*
|
||||
* 优先级：
|
||||
* 1. GitHub URL 完全匹配（最准确）
|
||||
* 2. Hugging Face URL 完全匹配
|
||||
* 3. Website URL 完全匹配
|
||||
* 4. slug 匹配（兜底）
|
||||
*
|
||||
* @param project - 项目待检查信息
|
||||
* @returns 检查结果
|
||||
*/
|
||||
async function checkProjectExists(project: {
|
||||
githubUrl?: string
|
||||
huggingfaceUrl?: string
|
||||
websiteUrl?: string
|
||||
slug?: string
|
||||
}): Promise<CheckResult> {
|
||||
// 优先级1: GitHub URL 匹配
|
||||
if (project.githubUrl) {
|
||||
const existingByGithub = await prisma.externalLink.findFirst({
|
||||
where: {
|
||||
type: 'GITHUB',
|
||||
url: project.githubUrl,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
|
||||
if (existingByGithub) {
|
||||
return {
|
||||
githubUrl: project.githubUrl,
|
||||
exists: true,
|
||||
matchType: 'GITHUB_URL',
|
||||
projectId: existingByGithub.project.id,
|
||||
projectName: existingByGithub.project.name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 优先级2: Hugging Face URL 匹配
|
||||
if (project.huggingfaceUrl) {
|
||||
const existingByHuggingFace = await prisma.externalLink.findFirst({
|
||||
where: {
|
||||
type: 'HUGGINGFACE',
|
||||
url: project.huggingfaceUrl,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
|
||||
if (existingByHuggingFace) {
|
||||
return {
|
||||
huggingfaceUrl: project.huggingfaceUrl,
|
||||
exists: true,
|
||||
matchType: 'HUGGINGFACE_URL',
|
||||
projectId: existingByHuggingFace.project.id,
|
||||
projectName: existingByHuggingFace.project.name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 优先级3: Website URL 匹配
|
||||
if (project.websiteUrl) {
|
||||
const existingByWebsite = await prisma.externalLink.findFirst({
|
||||
where: {
|
||||
type: 'WEBSITE',
|
||||
url: project.websiteUrl,
|
||||
},
|
||||
include: {
|
||||
project: true,
|
||||
},
|
||||
})
|
||||
|
||||
if (existingByWebsite) {
|
||||
return {
|
||||
websiteUrl: project.websiteUrl,
|
||||
exists: true,
|
||||
matchType: 'WEBSITE_URL',
|
||||
projectId: existingByWebsite.project.id,
|
||||
projectName: existingByWebsite.project.name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 优先级4: Slug 匹配（兜底）
|
||||
if (project.slug) {
|
||||
const existingBySlug = await prisma.project.findUnique({
|
||||
where: { slug: project.slug },
|
||||
})
|
||||
|
||||
if (existingBySlug) {
|
||||
return {
|
||||
slug: project.slug,
|
||||
exists: true,
|
||||
matchType: 'SLUG',
|
||||
projectId: existingBySlug.id,
|
||||
projectName: existingBySlug.name,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 未找到匹配项
|
||||
return {
|
||||
githubUrl: project.githubUrl,
|
||||
huggingfaceUrl: project.huggingfaceUrl,
|
||||
websiteUrl: project.websiteUrl,
|
||||
slug: project.slug,
|
||||
exists: false,
|
||||
matchType: 'NONE',
|
||||
}
|
||||
}
|
||||
|
||||
export async function POST(request: NextRequest) {
|
||||
const startTime = Date.now()
|
||||
|
||||
try {
|
||||
const body = await request.json()
|
||||
|
||||
// Validate payload
|
||||
const validationResult = CheckDuplicatesSchema.safeParse(body)
|
||||
|
||||
if (!validationResult.success) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: 'Validation error',
|
||||
details: validationResult.error.errors.map((e) => e.message),
|
||||
},
|
||||
{ status: 400 }
|
||||
)
|
||||
}
|
||||
|
||||
const payload = validationResult.data
|
||||
|
||||
// Verify API Key
|
||||
const apiKey = process.env.WEBHOOK_API_KEY
|
||||
if (payload.apiKey !== apiKey) {
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: 'Unauthorized',
|
||||
details: ['Invalid or missing API Key'],
|
||||
},
|
||||
{ status: 401 }
|
||||
)
|
||||
}
|
||||
|
||||
// 并行检查所有项目
|
||||
const results = await Promise.all(
|
||||
payload.projects.map((project) => checkProjectExists(project))
|
||||
)
|
||||
|
||||
// 统计信息
|
||||
const stats = {
|
||||
total: results.length,
|
||||
exists: results.filter((r) => r.exists).length,
|
||||
new: results.filter((r) => !r.exists).length,
|
||||
breakdown: {
|
||||
githubUrl: results.filter((r) => r.matchType === 'GITHUB_URL').length,
|
||||
huggingfaceUrl: results.filter(
|
||||
(r) => r.matchType === 'HUGGINGFACE_URL'
|
||||
).length,
|
||||
websiteUrl: results.filter((r) => r.matchType === 'WEBSITE_URL')
|
||||
.length,
|
||||
slug: results.filter((r) => r.matchType === 'SLUG').length,
|
||||
},
|
||||
}
|
||||
|
||||
const duration = Date.now() - startTime
|
||||
|
||||
console.log(
|
||||
`[CheckDuplicates] Checked ${stats.total} projects in ${duration}ms: ${stats.exists} exist, ${stats.new} new`
|
||||
)
|
||||
|
||||
return NextResponse.json({
|
||||
success: true,
|
||||
results,
|
||||
stats,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('[CheckDuplicates] Error:', error)
|
||||
return NextResponse.json(
|
||||
{
|
||||
success: false,
|
||||
error: 'Internal server error',
|
||||
details: [error instanceof Error ? error.message : 'Unknown error'],
|
||||
},
|
||||
{ status: 500 }
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -165,22 +165,44 @@ export async function POST(request: NextRequest) {
|
||||
// 多级去重：查找已存在的项目
|
||||
const existingProject = await findExistingProject(validProject)
|
||||
|
||||
// Upsert tags
|
||||
// Upsert tags with better error handling for name uniqueness
|
||||
const tagConnections = await Promise.all(
|
||||
validProject.tags.map(async (tag) => {
|
||||
const slug =
|
||||
tag.nameEn?.toLowerCase().replace(/\s+/g, '-') ||
|
||||
tag.name.toLowerCase().replace(/\s+/g, '-')
|
||||
|
||||
return prisma.tag.upsert({
|
||||
where: { slug },
|
||||
update: {},
|
||||
create: {
|
||||
name: tag.name,
|
||||
nameEn: tag.nameEn || null,
|
||||
slug,
|
||||
},
|
||||
// First, try to find by name (handle name uniqueness constraint)
|
||||
const existingByName = await prisma.tag.findUnique({
|
||||
where: { name: tag.name },
|
||||
})
|
||||
|
||||
if (existingByName) {
|
||||
// Tag with this name already exists, use it
|
||||
return existingByName
|
||||
}
|
||||
|
||||
// Try upsert by slug (safe now since name doesn't exist)
|
||||
try {
|
||||
return await prisma.tag.upsert({
|
||||
where: { slug },
|
||||
update: {},
|
||||
create: {
|
||||
name: tag.name,
|
||||
nameEn: tag.nameEn || null,
|
||||
slug,
|
||||
},
|
||||
})
|
||||
} catch (error) {
|
||||
// If slug conflicts with existing tag, find and use that one
|
||||
const existingBySlug = await prisma.tag.findUnique({
|
||||
where: { slug },
|
||||
})
|
||||
if (existingBySlug) {
|
||||
return existingBySlug
|
||||
}
|
||||
throw error
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
@@ -204,6 +226,11 @@ export async function POST(request: NextRequest) {
|
||||
`[Webhook] Updating project "${validProject.name}" (matched by ${matchMethod}, id: ${existingProject.id})`
|
||||
)
|
||||
|
||||
// Update tags (delete old ones, create new ones)
|
||||
await prisma.projectTag.deleteMany({
|
||||
where: { projectId: existingProject.id },
|
||||
})
|
||||
|
||||
await prisma.project.update({
|
||||
where: { id: existingProject.id },
|
||||
data: {
|
||||
@@ -216,7 +243,9 @@ export async function POST(request: NextRequest) {
|
||||
status: validProject.status as any,
|
||||
source: validProject.source || null,
|
||||
tags: {
|
||||
set: tagConnections.map((t) => ({ id: t.id })),
|
||||
create: tagConnections.map((t) => ({
|
||||
tag: { connect: { id: t.id } },
|
||||
})),
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -254,7 +283,9 @@ export async function POST(request: NextRequest) {
|
||||
status: validProject.status as any,
|
||||
source: validProject.source || null,
|
||||
tags: {
|
||||
connect: tagConnections.map((t) => ({ id: t.id })),
|
||||
create: tagConnections.map((t) => ({
|
||||
tag: { connect: { id: t.id } },
|
||||
})),
|
||||
},
|
||||
links: {
|
||||
create: validProject.links.map((link) => ({
|
||||
|
||||
@@ -30,8 +30,10 @@ interface ProjectCardProps {
|
||||
}
|
||||
|
||||
// Icon mapping for projects based on tags
|
||||
function getProjectIcon(tags: Array<{ name: string }>): string {
|
||||
const tagNames = tags.map(t => t.name.toLowerCase())
|
||||
function getProjectIcon(tags: Array<{ name: string | null }>): string {
|
||||
const tagNames = tags
|
||||
.map(t => t.name?.toLowerCase())
|
||||
.filter((name): name is string => Boolean(name))
|
||||
if (tagNames.some(t => t.includes('automation') || t.includes('workflow'))) return '⚙️'
|
||||
if (tagNames.some(t => t.includes('image') || t.includes('art'))) return '🎨'
|
||||
if (tagNames.some(t => t.includes('chat') || t.includes('assistant'))) return '💬'
|
||||
|
||||
@@ -40,7 +40,11 @@ export async function getProjects(options?: {
|
||||
prisma.project.findMany({
|
||||
where,
|
||||
include: {
|
||||
tags: true,
|
||||
tags: {
|
||||
include: {
|
||||
tag: true,
|
||||
},
|
||||
},
|
||||
links: true,
|
||||
},
|
||||
orderBy: {
|
||||
@@ -52,8 +56,14 @@ export async function getProjects(options?: {
|
||||
prisma.project.count({ where }),
|
||||
])
|
||||
|
||||
// Transform tags to flatten the structure
|
||||
const transformedProjects = projects.map((project) => ({
|
||||
...project,
|
||||
tags: project.tags.map((pt) => pt.tag),
|
||||
}))
|
||||
|
||||
return {
|
||||
projects,
|
||||
projects: transformedProjects,
|
||||
pagination: {
|
||||
page,
|
||||
limit,
|
||||
@@ -64,13 +74,27 @@ export async function getProjects(options?: {
|
||||
}
|
||||
|
||||
export async function getProjectBySlug(slug: string) {
|
||||
return prisma.project.findUnique({
|
||||
const project = await prisma.project.findUnique({
|
||||
where: { slug },
|
||||
include: {
|
||||
tags: true,
|
||||
tags: {
|
||||
include: {
|
||||
tag: true,
|
||||
},
|
||||
},
|
||||
links: true,
|
||||
},
|
||||
})
|
||||
|
||||
if (!project) {
|
||||
return null
|
||||
}
|
||||
|
||||
// Transform tags to flatten the structure
|
||||
return {
|
||||
...project,
|
||||
tags: project.tags.map((pt) => pt.tag),
|
||||
}
|
||||
}
|
||||
|
||||
export async function getAllTags() {
|
||||
|
||||
Reference in New Issue
Block a user