diff --git a/docs/discovery-workflow.md b/docs/discovery-workflow.md index d6a6224..4508f96 100644 --- a/docs/discovery-workflow.md +++ b/docs/discovery-workflow.md @@ -15,14 +15,22 @@ ├─ 筛选符合条件的项目链接 │ ▼ -2. 调用创建任务 API +2. 调用去重检查 API (推荐) + POST /api/discovery/check-duplicates + │ + ├─ 检查 URL 是否已存在任务 + ├─ 检查 URL 对应项目是否已收录 + ├─ 过滤出需要创建的 URL + │ + ▼ +3. 调用创建任务 API POST /api/discovery/tasks │ ├─ 存入 ProjectDiscoveryTask 表 ├─ 状态: PENDING │ ▼ -3. 手动触发本地命令 (定期执行) +4. 手动触发本地命令 (定期执行) /discover-projects │ ├─ 通过 curl 获取待处理任务 @@ -38,7 +46,7 @@ │ └─ 自动重试失败的提交 │ ▼ -4. 完成任务并入库 +5. 完成任务并入库 POST /api/discovery/tasks/{id}/complete │ ├─ 验证数据格式 (Zod Schema) @@ -50,7 +58,7 @@ ├─ 更新任务状态: COMPLETED/FAILED │ ▼ -5. 数据已入库,可在前台展示 +6. 数据已入库,可在前台展示 ``` --- @@ -108,6 +116,109 @@ curl -X POST https://your-domain.com/api/discovery/tasks \ --- +### 步骤 2.5: 调用去重检查 API(推荐) + +**API 端点**: `POST /api/discovery/check-duplicates` + +**调用示例**: +```bash +curl -X POST https://your-domain.com/api/discovery/check-duplicates \ + -H "Content-Type: application/json" \ + -d '{ + "apiKey": "YOUR_API_KEY", + "urls": [ + "https://github.com/langchain-ai/langchain", + "https://github.com/openai/openai-quickstart-python" + ] + }' +``` + +**返回结果**: +```json +{ + "success": true, + "results": [ + { + "url": "https://github.com/langchain-ai/langchain", + "shouldCreate": false, + "reason": "Task already exists with status PENDING", + "existingTask": { + "id": "cmxxxxx", + "status": "PENDING", + "sourceUrl": "https://github.com/langchain-ai/langchain", + "createdAt": "2025-01-18T10:00:00Z", + "projectId": null + } + }, + { + "url": "https://github.com/openai/openai-quickstart-python", + "shouldCreate": true, + "reason": "No existing task or project found" + } + ], + "stats": { + "total": 2, + "shouldCreate": 1, + "duplicate": 1 + } +} +``` + +**去重逻辑**(按优先级): +1. **优先级 1**: 检查是否有 PENDING/IN_PROGRESS 的相同 URL 任务 + - 如果存在 → `shouldCreate: false` + - 原因:任务已在处理中,避免重复探索 + +2. **优先级 2**: 检查是否有 COMPLETED/FAILED 的相同 URL 任务 + - 如果存在 → `shouldCreate: false` + - 原因:任务已探索过,无需重复 + +3. **优先级 3**: 检查 URL 对应的项目是否已存在(通过 ExternalLink) + - 如果存在 → `shouldCreate: false` + - 原因:项目已通过其他来源收录 + +4. **默认**: 允许创建新任务 + - `shouldCreate: true` + +**n8n 集成建议**: +```javascript +// n8n Workflow 示例 +const checkResponse = await fetch('https://your-domain.com/api/discovery/check-duplicates', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + apiKey: 'YOUR_API_KEY', + urls: collectedUrls // 从上一步收集的 URL 列表 + }) +}) + +const { results, stats } = await checkResponse.json() + +// 过滤出应该创建任务的 URL +const urlsToCreate = results + .filter(r => r.shouldCreate) + .map(r => r.url) + +// 只为不重复的 URL 创建任务 +if (urlsToCreate.length > 0) { + await fetch('https://your-domain.com/api/discovery/tasks', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + apiKey: 'YOUR_API_KEY', + tasks: urlsToCreate.map(url => ({ + sourceUrl: url, + sourceType: 'github_trending' + })) + }) + }) +} + +console.log(`创建 ${urlsToCreate.length} 个新任务,跳过 ${stats.duplicate} 个重复任务`) +``` + +--- + ### 步骤 3: 手动触发本地命令 **执行环境**: 本地开发环境 (Local Development) diff --git a/src/app/api/discovery/check-duplicates/route.ts b/src/app/api/discovery/check-duplicates/route.ts new file mode 100644 index 0000000..3f22956 --- /dev/null +++ b/src/app/api/discovery/check-duplicates/route.ts @@ -0,0 +1,247 @@ +import { NextRequest, NextResponse } from 'next/server' +import crypto from 'crypto' +import { prisma } from '@/lib/prisma' +import { CheckTaskDuplicatesSchema } from '@/lib/validations' + +/** + * 检查 URL 是否应该创建新任务 + * + * 去重优先级: + * 1. PENDING/IN_PROGRESS 任务 → 不创建(任务处理中) + * 2. COMPLETED/FAILED 任务 → 不创建(已探索过) + * 3. 已存在的项目(通过 ExternalLink)→ 不创建(已收录) + * 4. 无任何记录 → 允许创建 + * + * @param url - 要检查的 URL + * @returns 检查结果 + */ +async function checkUrlDuplicate(url: string): Promise<{ + url: string + shouldCreate: boolean + reason: string + existingTask?: { + id: string + status: string + sourceUrl: string + createdAt: Date + projectId?: string | null + } + existingProject?: { + id: string + name: string + slug: string + } +}> { + // 优先级 1: 检查是否有 PENDING/IN_PROGRESS 的相同 URL 任务 + const activeTask = await prisma.projectDiscoveryTask.findFirst({ + where: { + sourceUrl: url, + status: { + in: ['PENDING', 'IN_PROGRESS'], + }, + }, + select: { + id: true, + status: true, + sourceUrl: true, + createdAt: true, + projectId: true, + }, + }) + + if (activeTask) { + return { + url, + shouldCreate: false, + reason: `Task already exists with status ${activeTask.status}`, + existingTask: activeTask, + } + } + + // 优先级 2: 检查是否有 COMPLETED/FAILED 的相同 URL 任务 + const finishedTask = await prisma.projectDiscoveryTask.findFirst({ + where: { + sourceUrl: url, + status: { + in: ['COMPLETED', 'FAILED'], + }, + }, + select: { + id: true, + status: true, + sourceUrl: true, + createdAt: true, + projectId: true, + }, + orderBy: { + createdAt: 'desc', + }, + }) + + if (finishedTask) { + // 如果任务已完成,返回项目信息 + let projectInfo + if (finishedTask.status === 'COMPLETED' && finishedTask.projectId) { + const project = await prisma.project.findUnique({ + where: { id: finishedTask.projectId }, + select: { + id: true, + name: true, + slug: true, + }, + }) + if (project) { + projectInfo = project + } + } + + return { + url, + shouldCreate: false, + reason: + finishedTask.status === 'COMPLETED' + ? 'Task already completed' + : 'Task already failed', + existingTask: finishedTask, + existingProject: projectInfo, + } + } + + // 优先级 3: 检查 URL 对应的项目是否已存在(通过 ExternalLink) + const existingLink = await prisma.externalLink.findFirst({ + where: { + url: url, + }, + select: { + project: { + select: { + id: true, + name: true, + slug: true, + }, + }, + }, + }) + + if (existingLink) { + return { + url, + shouldCreate: false, + reason: 'Project already exists with this URL', + existingProject: existingLink.project, + } + } + + // 默认: 允许创建新任务 + return { + url, + shouldCreate: true, + reason: 'No existing task or project found', + } +} + +/** + * POST /api/discovery/check-duplicates + * + * 检查 URL 是否应该创建新的探索任务 + * + * 请求体: + * { + * "apiKey": "xxx", + * "urls": ["https://github.com/user/repo", ...], + * "sourceType": "manual" // 可选 + * } + * + * 返回: + * { + * "success": true, + * "results": [ + * { + * "url": "...", + * "shouldCreate": false, + * "reason": "Task already exists with status PENDING", + * "existingTask": {...}, + * "existingProject": {...} + * } + * ], + * "stats": { + * "total": 10, + * "shouldCreate": 5, + * "duplicate": 5 + * } + * } + */ +export async function POST(request: NextRequest) { + const startTime = Date.now() + + try { + const body = await request.json() + + // 验证请求体 + const validationResult = CheckTaskDuplicatesSchema.safeParse(body) + + if (!validationResult.success) { + return NextResponse.json( + { + success: false, + error: 'Validation error', + details: validationResult.error.errors.map((e) => e.message), + }, + { status: 400 } + ) + } + + const { apiKey, urls, sourceType } = validationResult.data + + // 验证 API Key + const validApiKey = process.env.WEBHOOK_API_KEY + if ( + !validApiKey || + !crypto.timingSafeEqual( + Buffer.from(apiKey), + Buffer.from(validApiKey) + ) + ) { + return NextResponse.json( + { + success: false, + error: 'Unauthorized', + details: ['Invalid or missing API Key'], + }, + { status: 401 } + ) + } + + // 并行检查所有 URL + const results = await Promise.all(urls.map((url) => checkUrlDuplicate(url))) + + // 统计信息 + const stats = { + total: results.length, + shouldCreate: results.filter((r) => r.shouldCreate).length, + duplicate: results.filter((r) => !r.shouldCreate).length, + } + + const duration = Date.now() - startTime + + console.warn( + `[CheckTaskDuplicates] Checked ${stats.total} URLs in ${duration}ms: ${stats.shouldCreate} should create, ${stats.duplicate} duplicate` + ) + + return NextResponse.json({ + success: true, + results, + stats, + }) + } catch (error) { + console.error('[CheckTaskDuplicates] Error:', error) + return NextResponse.json( + { + success: false, + error: 'Internal server error', + details: [error instanceof Error ? error.message : 'Unknown error'], + }, + { status: 500 } + ) + } +} diff --git a/src/lib/validations.ts b/src/lib/validations.ts index 564e971..fcf5dbd 100644 --- a/src/lib/validations.ts +++ b/src/lib/validations.ts @@ -91,6 +91,15 @@ export const GetDiscoveryTasksQuerySchema = z.object({ offset: z.coerce.number().int().nonnegative().default(0), }) +/** + * 检查任务去重 Schema + * 用于在创建任务前检查 URL 是否已存在 + */ +export const CheckTaskDuplicatesSchema = WebhookAuthSchema.extend({ + urls: z.array(z.string().url().max(2000)).min(1).max(100), + sourceType: z.string().max(50).optional(), +}) + // ================================ // Query Schemas // ================================