diff --git a/prisma/migrations/20260421143000_restore_discovery_pipeline/migration.sql b/prisma/migrations/20260421143000_restore_discovery_pipeline/migration.sql new file mode 100644 index 0000000..01fe23c --- /dev/null +++ b/prisma/migrations/20260421143000_restore_discovery_pipeline/migration.sql @@ -0,0 +1,51 @@ +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_type + WHERE typname = 'TaskStatus' + ) THEN + CREATE TYPE "TaskStatus" AS ENUM ('PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'); + END IF; +END +$$; + +CREATE TABLE IF NOT EXISTS "project_discovery_tasks" ( + "id" TEXT NOT NULL, + "status" "TaskStatus" NOT NULL DEFAULT 'PENDING', + "sourceUrl" TEXT NOT NULL, + "sourceType" TEXT NOT NULL DEFAULT 'manual', + "explorationData" JSONB, + "explorationSummary" TEXT, + "errorMessage" TEXT, + "retryCount" INTEGER NOT NULL DEFAULT 0, + "lastRetryAt" TIMESTAMP(3), + "projectId" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "startedAt" TIMESTAMP(3), + "completedAt" TIMESTAMP(3), + "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "project_discovery_tasks_pkey" PRIMARY KEY ("id") +); + +CREATE INDEX IF NOT EXISTS "idx_task_project_id" ON "project_discovery_tasks"("projectId"); +CREATE INDEX IF NOT EXISTS "idx_task_source_url" ON "project_discovery_tasks"("sourceUrl"); +CREATE INDEX IF NOT EXISTS "idx_task_status_created" ON "project_discovery_tasks"("status", "createdAt"); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 + FROM pg_constraint + WHERE conname = 'project_discovery_tasks_projectId_fkey' + ) THEN + ALTER TABLE "project_discovery_tasks" + ADD CONSTRAINT "project_discovery_tasks_projectId_fkey" + FOREIGN KEY ("projectId") + REFERENCES "projects"("id") + ON DELETE SET NULL + ON UPDATE CASCADE; + END IF; +END +$$; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 9b9aab7..08af3df 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -25,6 +25,29 @@ model ExternalLink { @@map("external_links") } +model ProjectDiscoveryTask { + id String @id @default(cuid()) + status TaskStatus @default(PENDING) + sourceUrl String + sourceType String @default("manual") + explorationData Json? + explorationSummary String? + errorMessage String? + retryCount Int @default(0) + lastRetryAt DateTime? + projectId String? + createdAt DateTime @default(now()) + startedAt DateTime? + completedAt DateTime? + updatedAt DateTime @updatedAt + project Project? @relation(fields: [projectId], references: [id]) + + @@index([projectId], map: "idx_task_project_id") + @@index([sourceUrl], map: "idx_task_source_url") + @@index([status, createdAt], map: "idx_task_status_created") + @@map("project_discovery_tasks") +} + model ProjectTag { projectId String tagId String @@ -37,24 +60,25 @@ model ProjectTag { } model Project { - id String @id @default(cuid()) - name String - nameEn String? - slug String @unique - description String - descriptionEn String? - content String? - contentEn String? - githubStars Int @default(0) - githubStarsUpdatedAt DateTime? - status ProjectStatus @default(ACTIVE) - source String? - createdAt DateTime @default(now()) - updatedAt DateTime @updatedAt - embedding Unsupported("vector")? - embeddingUpdatedAt DateTime? - links ExternalLink[] - tags ProjectTag[] + id String @id @default(cuid()) + name String + nameEn String? + slug String @unique + description String + descriptionEn String? + content String? + contentEn String? + githubStars Int @default(0) + githubStarsUpdatedAt DateTime? + status ProjectStatus @default(ACTIVE) + source String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + embedding Unsupported("vector")? + embeddingUpdatedAt DateTime? + links ExternalLink[] + projectDiscoveryTasks ProjectDiscoveryTask[] + tags ProjectTag[] @@index([embedding], map: "idx_project_embedding_cosine") @@index([slug], map: "idx_project_slug") @@ -118,6 +142,13 @@ enum ProjectStatus { ARCHIVED } +enum TaskStatus { + PENDING + IN_PROGRESS + COMPLETED + FAILED +} + enum TagCategory { FIXED_PROJECT_TYPE TECH_STACK diff --git a/src/app/api/discovery/check-duplicates/route.ts b/src/app/api/discovery/check-duplicates/route.ts new file mode 100644 index 0000000..ce4ae5a --- /dev/null +++ b/src/app/api/discovery/check-duplicates/route.ts @@ -0,0 +1,189 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { CheckTaskDuplicatesSchema } from "@/lib/validations"; +import { isValidApiKey } from "@/lib/auth"; + +type DuplicateCheckResult = { + url: string; + shouldCreate: boolean; + reason: string; + existingTask?: { + id: string; + status: string; + sourceUrl: string; + createdAt: Date; + projectId?: string | null; + }; + existingProject?: { + id: string; + name: string; + slug: string; + }; +}; + +async function checkUrlDuplicate(url: string): Promise { + const activeTask = await prisma.projectDiscoveryTask.findFirst({ + where: { + sourceUrl: url, + status: { + in: ["PENDING", "IN_PROGRESS"], + }, + }, + select: { + id: true, + status: true, + sourceUrl: true, + createdAt: true, + projectId: true, + }, + }); + + if (activeTask) { + return { + url, + shouldCreate: false, + reason: `Task already exists with status ${activeTask.status}`, + existingTask: activeTask, + }; + } + + const finishedTask = await prisma.projectDiscoveryTask.findFirst({ + where: { + sourceUrl: url, + status: { + in: ["COMPLETED", "FAILED"], + }, + }, + select: { + id: true, + status: true, + sourceUrl: true, + createdAt: true, + projectId: true, + }, + orderBy: { + createdAt: "desc", + }, + }); + + if (finishedTask) { + let projectInfo; + + if (finishedTask.status === "COMPLETED" && finishedTask.projectId) { + const project = await prisma.project.findUnique({ + where: { id: finishedTask.projectId }, + select: { + id: true, + name: true, + slug: true, + }, + }); + + if (project) { + projectInfo = project; + } + } + + return { + url, + shouldCreate: false, + reason: + finishedTask.status === "COMPLETED" ? "Task already completed" : "Task already failed", + existingTask: finishedTask, + existingProject: projectInfo, + }; + } + + const existingLink = await prisma.externalLink.findFirst({ + where: { + url, + }, + select: { + project: { + select: { + id: true, + name: true, + slug: true, + }, + }, + }, + }); + + if (existingLink) { + return { + url, + shouldCreate: false, + reason: "Project already exists with this URL", + existingProject: existingLink.project, + }; + } + + return { + url, + shouldCreate: true, + reason: "No existing task or project found", + }; +} + +export async function POST(request: NextRequest) { + const startTime = Date.now(); + + try { + const body = await request.json(); + const validationResult = CheckTaskDuplicatesSchema.safeParse(body); + + if (!validationResult.success) { + return NextResponse.json( + { + success: false, + error: "Validation error", + details: validationResult.error.errors.map((error) => error.message), + }, + { status: 400 } + ); + } + + const { apiKey, urls } = validationResult.data; + + if (!isValidApiKey(apiKey)) { + return NextResponse.json( + { + success: false, + error: "Unauthorized", + details: ["Invalid or missing API Key"], + }, + { status: 401 } + ); + } + + const results = await Promise.all(urls.map((url) => checkUrlDuplicate(url))); + const stats = { + total: results.length, + shouldCreate: results.filter((result) => result.shouldCreate).length, + duplicate: results.filter((result) => !result.shouldCreate).length, + }; + + const duration = Date.now() - startTime; + + console.warn( + `[CheckTaskDuplicates] Checked ${stats.total} URLs in ${duration}ms: ${stats.shouldCreate} should create, ${stats.duplicate} duplicate` + ); + + return NextResponse.json({ + success: true, + results, + stats, + }); + } catch (error) { + console.error("[CheckTaskDuplicates] Error:", error); + + return NextResponse.json( + { + success: false, + error: "Internal server error", + details: [error instanceof Error ? error.message : "Unknown error"], + }, + { status: 500 } + ); + } +} diff --git a/src/app/api/discovery/lib/discovery-service.ts b/src/app/api/discovery/lib/discovery-service.ts new file mode 100644 index 0000000..2d8a4d4 --- /dev/null +++ b/src/app/api/discovery/lib/discovery-service.ts @@ -0,0 +1,536 @@ +import { prisma } from "@/lib/prisma"; +import type { ProjectInput } from "@/lib/validations"; +import { generateSlug } from "@/lib/slug"; +import type { Prisma } from "@prisma/client"; +import { + FIXED_PROJECT_TYPE_TAGS, + inferProjectTypeSlug, + inferTagCategory, + type FixedProjectTypeSlug, +} from "@/lib/tag-taxonomy"; + +const PLURAL_NORMALIZATION_MAP = new Map([ + ["agents", "agent"], + ["assistants", "assistant"], + ["tools", "tool"], + ["frameworks", "framework"], + ["models", "model"], + ["servers", "server"], + ["clients", "client"], + ["workflows", "workflow"], + ["plugins", "plugin"], + ["libraries", "library"], + ["datasets", "dataset"], + ["platforms", "platform"], + ["systems", "system"], + ["repositories", "repository"], + ["engines", "engine"], +]); + +const NOISE_TAG_KEYS = new Set([ + "ai", + "artificial intelligence", + "open source", + "requires configuration", + "requires basics", + "low learning curve", + "enterprise", + "complex deployment", + "cloud service", + "cross platform", + "tutorial", + "academic research", + "academic resource", + "ai research resource", + "multi language support", + "self hosted", + "user notification", + "mit license", + "apache 2 0", + "开源", + "需要配置", + "需要基础", + "低学习成本", + "企业级", + "复杂部署", + "云端服务", + "跨平台", + "教程", + "学术研究", + "学术资源", + "多语言支持", + "自托管", + "用户通知", + "许可", +]); + +const TAG_FALLBACK = { + name: "AI开发工具", + nameEn: "AI Development Tool", +} as const; + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function isAsciiText(value: string): boolean { + return /^[\x00-\x7f]+$/.test(value); +} + +function canonicalizeTagKey(value: string): string { + const normalized = normalizeWhitespace(value) + .normalize("NFKC") + .toLowerCase() + .replace(/[+/_&|-]+/g, " ") + .replace(/[^a-z0-9\u4e00-\u9fa5\s]/g, " ") + .replace(/\s+/g, " ") + .trim(); + + if (!normalized) { + return ""; + } + + return normalized + .split(" ") + .map((word) => PLURAL_NORMALIZATION_MAP.get(word) || word) + .join(" ") + .trim(); +} + +function normalizeIncomingTag(tag: ProjectInput["tags"][number]) { + const normalizedName = normalizeWhitespace(tag.name); + const normalizedNameEn = normalizeWhitespace(tag.nameEn || ""); + const resolvedNameEn = normalizedNameEn || (isAsciiText(normalizedName) ? normalizedName : ""); + const canonicalNameKey = canonicalizeTagKey(normalizedName); + const canonicalNameEnKey = canonicalizeTagKey(resolvedNameEn); + + return { + name: normalizedName, + nameEn: resolvedNameEn || null, + canonicalNameKey, + canonicalNameEnKey, + slug: generateSlug(normalizedName, resolvedNameEn || null), + }; +} + +function isMeaningfulTag(tag: ReturnType): boolean { + if (!tag.name) { + return false; + } + + if (NOISE_TAG_KEYS.has(tag.canonicalNameKey)) { + return false; + } + + if (tag.canonicalNameEnKey && NOISE_TAG_KEYS.has(tag.canonicalNameEnKey)) { + return false; + } + + return true; +} + +type TagWithProjectCount = Prisma.TagGetPayload<{ + include: { + _count: { select: { projects: true } }; + }; +}>; + +function chooseBestTag(candidates: TagWithProjectCount[]): TagWithProjectCount { + return [...candidates].sort((a, b) => { + const projectDiff = b._count.projects - a._count.projects; + if (projectDiff !== 0) { + return projectDiff; + } + + return a.createdAt.getTime() - b.createdAt.getTime(); + })[0]!; +} + +type IncomingNormalizedTag = ReturnType; + +type TagLookupMaps = { + tagByExactName: Map; + tagByExactNameEn: Map; + tagBySlug: Map; + tagByCanonicalName: Map; + tagByCanonicalNameEn: Map; +}; + +function pushTagMapEntry( + map: Map, + key: string, + tag: TagWithProjectCount +): void { + if (!key) { + return; + } + + const current = map.get(key); + if (current) { + current.push(tag); + return; + } + + map.set(key, [tag]); +} + +function createTagLookupMaps(tags: TagWithProjectCount[]): TagLookupMaps { + const tagByExactName = new Map(); + const tagByExactNameEn = new Map(); + const tagBySlug = new Map(); + const tagByCanonicalName = new Map(); + const tagByCanonicalNameEn = new Map(); + + for (const tag of tags) { + const exactNameKey = normalizeWhitespace(tag.name); + const exactNameEnKey = normalizeWhitespace(tag.nameEn || ""); + const canonicalNameKey = canonicalizeTagKey(tag.name); + const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || ""); + + pushTagMapEntry(tagByExactName, exactNameKey, tag); + pushTagMapEntry(tagByExactNameEn, exactNameEnKey, tag); + pushTagMapEntry(tagByCanonicalName, canonicalNameKey, tag); + pushTagMapEntry(tagByCanonicalNameEn, canonicalNameEnKey, tag); + tagBySlug.set(tag.slug, tag); + } + + return { + tagByExactName, + tagByExactNameEn, + tagBySlug, + tagByCanonicalName, + tagByCanonicalNameEn, + }; +} + +function addTagToLookupMaps(lookups: TagLookupMaps, tag: TagWithProjectCount): void { + const exactNameKey = normalizeWhitespace(tag.name); + const exactNameEnKey = normalizeWhitespace(tag.nameEn || ""); + const canonicalNameKey = canonicalizeTagKey(tag.name); + const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || ""); + + pushTagMapEntry(lookups.tagByExactName, exactNameKey, tag); + pushTagMapEntry(lookups.tagByExactNameEn, exactNameEnKey, tag); + pushTagMapEntry(lookups.tagByCanonicalName, canonicalNameKey, tag); + pushTagMapEntry(lookups.tagByCanonicalNameEn, canonicalNameEnKey, tag); + lookups.tagBySlug.set(tag.slug, tag); +} + +async function getCandidateTags( + incomingTags: IncomingNormalizedTag[] +): Promise { + const nameValues = Array.from( + new Set( + incomingTags.map((tag) => normalizeWhitespace(tag.name)).filter((name) => name.length > 0) + ) + ); + const slugValues = Array.from( + new Set(incomingTags.map((tag) => tag.slug).filter((slug) => slug.length > 0)) + ); + const nameEnValues = Array.from( + new Set( + incomingTags + .map((tag) => normalizeWhitespace(tag.nameEn || "")) + .filter((nameEn) => nameEn.length > 0) + ) + ); + + const whereOr: Prisma.TagWhereInput[] = []; + if (nameValues.length > 0) { + whereOr.push({ name: { in: nameValues } }); + } + if (slugValues.length > 0) { + whereOr.push({ slug: { in: slugValues } }); + } + if (nameEnValues.length > 0) { + whereOr.push({ nameEn: { in: nameEnValues } }); + } + + if (whereOr.length === 0) { + return []; + } + + return prisma.tag.findMany({ + where: { + OR: whereOr, + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); +} + +async function getFallbackTag(): Promise { + const fallbackSlug = generateSlug(TAG_FALLBACK.name, TAG_FALLBACK.nameEn); + const fallbackCategory = inferTagCategory({ + slug: fallbackSlug, + name: TAG_FALLBACK.name, + nameEn: TAG_FALLBACK.nameEn, + }); + + return prisma.tag.upsert({ + where: { slug: fallbackSlug }, + update: { + name: TAG_FALLBACK.name, + nameEn: TAG_FALLBACK.nameEn, + category: fallbackCategory, + }, + create: { + name: TAG_FALLBACK.name, + nameEn: TAG_FALLBACK.nameEn, + slug: fallbackSlug, + category: fallbackCategory, + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); +} + +const FIXED_PROJECT_TYPE_MAP = new Map( + FIXED_PROJECT_TYPE_TAGS.map((tag) => [tag.slug, tag] as const) +); + +export async function ensureFixedProjectTypeTag( + slug: FixedProjectTypeSlug +): Promise { + const fixedTag = FIXED_PROJECT_TYPE_MAP.get(slug); + if (!fixedTag) { + throw new Error(`Unsupported fixed project type slug: ${slug}`); + } + + return prisma.tag.upsert({ + where: { slug: fixedTag.slug }, + update: { + name: fixedTag.name, + nameEn: fixedTag.nameEn, + category: "FIXED_PROJECT_TYPE", + }, + create: { + name: fixedTag.name, + nameEn: fixedTag.nameEn, + slug: fixedTag.slug, + category: "FIXED_PROJECT_TYPE", + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); +} + +export async function resolveFixedProjectTypeTag( + projectData: Pick +): Promise { + const projectTypeSlug = inferProjectTypeSlug({ + name: projectData.name, + nameEn: projectData.nameEn, + description: projectData.description, + descriptionEn: projectData.descriptionEn, + tags: projectData.tags.map((tag) => ({ + name: tag.name, + nameEn: tag.nameEn, + slug: generateSlug(tag.name, tag.nameEn || null), + })), + }); + + return ensureFixedProjectTypeTag(projectTypeSlug); +} + +export async function findExistingProject(projectData: ProjectInput) { + const githubLink = projectData.links.find((link) => link.type === "GITHUB"); + if (githubLink) { + const existingByGithub = await prisma.externalLink.findFirst({ + where: { + type: "GITHUB", + url: githubLink.url, + }, + include: { + project: { + include: { + links: true, + }, + }, + }, + }); + + if (existingByGithub) { + console.warn(`[Discovery] Found existing project by GitHub URL: ${githubLink.url}`); + return existingByGithub.project; + } + } + + const websiteLink = projectData.links.find((link) => link.type === "WEBSITE"); + if (websiteLink) { + const existingByWebsite = await prisma.externalLink.findFirst({ + where: { + type: "WEBSITE", + url: websiteLink.url, + }, + include: { + project: { + include: { + links: true, + }, + }, + }, + }); + + if (existingByWebsite) { + console.warn(`[Discovery] Found existing project by Website URL: ${websiteLink.url}`); + return existingByWebsite.project; + } + } + + const slug = generateSlug(projectData.name, projectData.nameEn); + const existingBySlug = await prisma.project.findUnique({ + where: { slug }, + include: { + links: true, + }, + }); + + if (existingBySlug) { + console.warn(`[Discovery] Found existing project by slug: ${slug}`); + return existingBySlug; + } + + console.warn("[Discovery] No existing project found, will create new one"); + return null; +} + +export async function upsertTags(tags: ProjectInput["tags"]) { + const meaningfulTags = tags + .map((tag) => normalizeIncomingTag(tag)) + .filter((tag) => isMeaningfulTag(tag)); + + const normalizedIncomingTags = Array.from( + new Map( + meaningfulTags.map((normalized) => [ + normalized.canonicalNameEnKey || normalized.canonicalNameKey || normalized.name, + normalized, + ]) + ).values() + ); + + if (normalizedIncomingTags.length === 0) { + return [await getFallbackTag()]; + } + + const candidateTags = await getCandidateTags(normalizedIncomingTags); + const lookups = createTagLookupMaps(candidateTags); + const resolvedTags: TagWithProjectCount[] = []; + + for (const incomingTag of normalizedIncomingTags) { + const exactNameCandidates = lookups.tagByExactName.get(incomingTag.name) || []; + let matchedTag = exactNameCandidates.length > 0 ? chooseBestTag(exactNameCandidates) : null; + + if (!matchedTag && incomingTag.nameEn) { + const exactNameEnCandidates = lookups.tagByExactNameEn.get(incomingTag.nameEn) || []; + if (exactNameEnCandidates.length > 0) { + matchedTag = chooseBestTag(exactNameEnCandidates); + } + } + + if (!matchedTag && incomingTag.canonicalNameKey) { + const canonicalNameCandidates = + lookups.tagByCanonicalName.get(incomingTag.canonicalNameKey) || []; + if (canonicalNameCandidates.length > 0) { + matchedTag = chooseBestTag(canonicalNameCandidates); + } + } + + if (!matchedTag && incomingTag.canonicalNameEnKey) { + const canonicalNameEnCandidates = + lookups.tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || []; + if (canonicalNameEnCandidates.length > 0) { + matchedTag = chooseBestTag(canonicalNameEnCandidates); + } + } + + if (!matchedTag) { + matchedTag = lookups.tagBySlug.get(incomingTag.slug) || null; + } + + if (matchedTag) { + const inferredCategory = inferTagCategory({ + slug: incomingTag.slug, + name: incomingTag.name, + nameEn: incomingTag.nameEn, + }); + const shouldUpdateNameEn = incomingTag.nameEn && !matchedTag.nameEn; + const shouldUpdateCategory = + matchedTag.category !== inferredCategory && + ["FREE_TAG", "RESOURCE_TYPE", "PROTOCOL_INTERFACE"].includes(matchedTag.category); + + if (shouldUpdateNameEn || shouldUpdateCategory) { + const updatedTag = await prisma.tag.update({ + where: { id: matchedTag.id }, + data: { + ...(shouldUpdateNameEn ? { nameEn: incomingTag.nameEn } : {}), + ...(shouldUpdateCategory ? { category: inferredCategory } : {}), + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); + matchedTag = updatedTag; + addTagToLookupMaps(lookups, matchedTag); + } + + resolvedTags.push(matchedTag); + continue; + } + + const inferredCategory = inferTagCategory({ + slug: incomingTag.slug, + name: incomingTag.name, + nameEn: incomingTag.nameEn, + }); + + try { + const createdTag = await prisma.tag.create({ + data: { + name: incomingTag.name, + nameEn: incomingTag.nameEn, + slug: incomingTag.slug, + category: inferredCategory, + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); + resolvedTags.push(createdTag); + addTagToLookupMaps(lookups, createdTag); + } catch { + const fallbackTag = await prisma.tag.findFirst({ + where: { + OR: [{ name: incomingTag.name }, { slug: incomingTag.slug }], + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }); + + if (fallbackTag) { + resolvedTags.push(fallbackTag); + continue; + } + + throw new Error(`Failed to resolve tag: ${incomingTag.name}`); + } + } + + return Array.from(new Map(resolvedTags.map((tag) => [tag.id, tag])).values()); +} diff --git a/src/app/api/discovery/tasks/[id]/complete/route.ts b/src/app/api/discovery/tasks/[id]/complete/route.ts new file mode 100644 index 0000000..5bdb04b --- /dev/null +++ b/src/app/api/discovery/tasks/[id]/complete/route.ts @@ -0,0 +1,181 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { generateSlug } from "@/lib/slug"; +import { isValidApiKey } from "@/lib/auth"; +import { ProjectInputSchema, type ProjectInput } from "@/lib/validations"; +import { + findExistingProject, + resolveFixedProjectTypeTag, + upsertTags, +} from "@/app/api/discovery/lib/discovery-service"; + +export async function POST(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + const startTime = Date.now(); + const { id: taskId } = await params; + + try { + const body = await request.json(); + const { apiKey, explorationData } = body; + + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const projectValidation = ProjectInputSchema.safeParse(explorationData); + if (!projectValidation.success) { + console.error( + `[Discovery] Invalid exploration data for task ${taskId}:`, + projectValidation.error.errors + ); + return NextResponse.json( + { + success: false, + error: "Invalid exploration data format", + details: projectValidation.error.errors, + }, + { status: 400 } + ); + } + + const projectData = projectValidation.data as ProjectInput; + const existingProject = await findExistingProject(projectData); + + const [dynamicTagConnections, fixedProjectTypeTag] = await Promise.all([ + upsertTags(projectData.tags), + resolveFixedProjectTypeTag(projectData), + ]); + const tagConnections = Array.from( + new Map([...dynamicTagConnections, fixedProjectTypeTag].map((tag) => [tag.id, tag])).values() + ); + + const slug = generateSlug(projectData.name, projectData.nameEn); + + const result = await prisma.$transaction(async (tx) => { + let projectId: string; + + if (existingProject) { + console.warn( + `[Discovery] Updating existing project for task ${taskId}: ${projectData.name}` + ); + + await tx.projectTag.deleteMany({ + where: { projectId: existingProject.id }, + }); + + await tx.project.update({ + where: { id: existingProject.id }, + data: { + name: projectData.name, + nameEn: projectData.nameEn || null, + description: projectData.description, + descriptionEn: projectData.descriptionEn || null, + content: projectData.content || null, + contentEn: projectData.contentEn || null, + status: projectData.status, + source: projectData.source || "discovery", + tags: { + create: tagConnections.map((tag) => ({ + tag: { connect: { id: tag.id } }, + })), + }, + }, + }); + + await tx.externalLink.deleteMany({ + where: { projectId: existingProject.id }, + }); + + await tx.externalLink.createMany({ + data: projectData.links.map((link) => ({ + type: link.type, + url: link.url, + title: link.title || null, + projectId: existingProject.id, + })), + }); + + projectId = existingProject.id; + } else { + console.warn(`[Discovery] Creating new project for task ${taskId}: ${projectData.name}`); + + const newProject = await tx.project.create({ + data: { + name: projectData.name, + nameEn: projectData.nameEn || null, + slug, + description: projectData.description, + descriptionEn: projectData.descriptionEn || null, + content: projectData.content || null, + contentEn: projectData.contentEn || null, + status: projectData.status, + source: projectData.source || "discovery", + tags: { + create: tagConnections.map((tag) => ({ + tag: { connect: { id: tag.id } }, + })), + }, + links: { + create: projectData.links.map((link) => ({ + type: link.type, + url: link.url, + title: link.title || null, + })), + }, + }, + }); + + projectId = newProject.id; + } + + const updatedTask = await tx.projectDiscoveryTask.update({ + where: { id: taskId }, + data: { + status: "COMPLETED", + completedAt: new Date(), + explorationData, + projectId, + }, + }); + + return { projectId, updatedTask }; + }); + + const duration = Date.now() - startTime; + console.warn( + `[Discovery] Completed task ${taskId} in ${duration}ms, project: ${result.projectId}` + ); + + return NextResponse.json({ + success: true, + taskId, + projectId: result.projectId, + action: existingProject ? "updated" : "created", + duration, + }); + } catch (error) { + console.error("[Discovery] Error completing task:", error); + + try { + await prisma.projectDiscoveryTask.update({ + where: { id: taskId }, + data: { + status: "FAILED", + completedAt: new Date(), + errorMessage: error instanceof Error ? error.message : "Unknown error", + retryCount: { increment: 1 }, + }, + }); + } catch (updateError) { + console.error("[Discovery] Failed to update task status:", updateError); + } + + return NextResponse.json( + { + success: false, + error: "Internal server error", + details: [error instanceof Error ? error.message : "Unknown error"], + }, + { status: 500 } + ); + } +} diff --git a/src/app/api/discovery/tasks/[id]/route.ts b/src/app/api/discovery/tasks/[id]/route.ts new file mode 100644 index 0000000..832fafb --- /dev/null +++ b/src/app/api/discovery/tasks/[id]/route.ts @@ -0,0 +1,131 @@ +import { NextRequest, NextResponse } from "next/server"; +import type { Prisma } from "@prisma/client"; +import { prisma } from "@/lib/prisma"; +import { UpdateDiscoveryTaskSchema, type TaskStatus } from "@/lib/validations"; +import { isValidApiKey } from "@/lib/auth"; + +const VALID_STATUS_TRANSITIONS: Record = { + PENDING: ["IN_PROGRESS"], + IN_PROGRESS: ["COMPLETED", "FAILED"], + COMPLETED: [], + FAILED: ["PENDING", "IN_PROGRESS"], +}; + +function isValidStatusTransition(from: TaskStatus, to: TaskStatus): boolean { + return VALID_STATUS_TRANSITIONS[from].includes(to); +} + +export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + try { + const apiKey = request.headers.get("x-api-key") || request.nextUrl.searchParams.get("apiKey"); + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const { id } = await params; + const task = await prisma.projectDiscoveryTask.findUnique({ + where: { id }, + }); + + if (!task) { + return NextResponse.json({ success: false, error: "Task not found" }, { status: 404 }); + } + + return NextResponse.json({ + success: true, + task, + }); + } catch (error) { + console.error("[Discovery] Error fetching task:", error); + return NextResponse.json({ success: false, error: "Internal server error" }, { status: 500 }); + } +} + +export async function PATCH(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { + try { + const { id } = await params; + const body = await request.json(); + const validation = UpdateDiscoveryTaskSchema.safeParse(body); + + if (!validation.success) { + return NextResponse.json( + { + success: false, + error: "Validation error", + details: validation.error.errors.map((error) => error.message), + }, + { status: 400 } + ); + } + + const { apiKey, status, explorationData, explorationSummary, errorMessage } = validation.data; + + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const existingTask = await prisma.projectDiscoveryTask.findUnique({ + where: { id }, + select: { status: true }, + }); + + if (!existingTask) { + return NextResponse.json({ success: false, error: "Task not found" }, { status: 404 }); + } + + if (!isValidStatusTransition(existingTask.status, status)) { + return NextResponse.json( + { + success: false, + error: "Invalid status transition", + details: [ + `Cannot transition from ${existingTask.status} to ${status}. Valid transitions: ${VALID_STATUS_TRANSITIONS[existingTask.status].join(", ")}`, + ], + }, + { status: 400 } + ); + } + + interface TaskUpdateData { + status: TaskStatus; + startedAt?: Date; + completedAt?: Date; + explorationData?: Prisma.InputJsonValue; + explorationSummary?: string | null; + errorMessage?: string | null; + } + + const updateData: TaskUpdateData = { status }; + + if (status === "IN_PROGRESS") { + updateData.startedAt = new Date(); + } else if (status === "COMPLETED" || status === "FAILED") { + updateData.completedAt = new Date(); + } + + if (explorationData !== undefined) { + updateData.explorationData = explorationData as Prisma.InputJsonObject; + } + if (explorationSummary !== undefined) { + updateData.explorationSummary = explorationSummary; + } + if (errorMessage !== undefined) { + updateData.errorMessage = errorMessage; + } + + const task = await prisma.projectDiscoveryTask.update({ + where: { id }, + data: updateData, + }); + + console.warn(`[Discovery] Updated task ${id} to status: ${status}`); + + return NextResponse.json({ + success: true, + task, + }); + } catch (error) { + console.error("[Discovery] Error updating task:", error); + return NextResponse.json({ success: false, error: "Internal server error" }, { status: 500 }); + } +} diff --git a/src/app/api/discovery/tasks/batch-reset/route.ts b/src/app/api/discovery/tasks/batch-reset/route.ts new file mode 100644 index 0000000..d53f54d --- /dev/null +++ b/src/app/api/discovery/tasks/batch-reset/route.ts @@ -0,0 +1,54 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { BatchResetTasksSchema } from "@/lib/validations"; +import { isValidApiKey } from "@/lib/auth"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const validation = BatchResetTasksSchema.safeParse(body); + + if (!validation.success) { + return NextResponse.json( + { + success: false, + error: "Validation error", + details: validation.error.errors.map((error) => error.message), + }, + { status: 400 } + ); + } + + const { apiKey, taskIds, statuses } = validation.data; + + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const where = taskIds + ? { id: { in: taskIds } } + : { status: { in: statuses || ["IN_PROGRESS", "FAILED"] } }; + + const result = await prisma.projectDiscoveryTask.updateMany({ + where, + data: { + status: "PENDING", + startedAt: null, + completedAt: null, + errorMessage: null, + }, + }); + + console.warn( + `[Discovery] Batch reset ${result.count} tasks to PENDING. Condition: ${JSON.stringify(where)}` + ); + + return NextResponse.json({ + success: true, + reset: result.count, + }); + } catch (error) { + console.error("[Discovery] Error batch resetting tasks:", error); + return NextResponse.json({ success: false, error: "Internal server error" }, { status: 500 }); + } +} diff --git a/src/app/api/discovery/tasks/route.ts b/src/app/api/discovery/tasks/route.ts new file mode 100644 index 0000000..5e6af31 --- /dev/null +++ b/src/app/api/discovery/tasks/route.ts @@ -0,0 +1,127 @@ +import { NextRequest, NextResponse } from "next/server"; +import { prisma } from "@/lib/prisma"; +import { CreateDiscoveryTaskSchema, GetDiscoveryTasksQuerySchema } from "@/lib/validations"; +import { isValidApiKey } from "@/lib/auth"; + +export async function POST(request: NextRequest) { + try { + const body = await request.json(); + const validation = CreateDiscoveryTaskSchema.safeParse(body); + + if (!validation.success) { + return NextResponse.json( + { + success: false, + error: "Validation error", + details: validation.error.errors.map((error) => error.message), + }, + { status: 400 } + ); + } + + const { apiKey, tasks } = validation.data; + + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const existingUrls = new Set( + ( + await prisma.projectDiscoveryTask.findMany({ + where: { sourceUrl: { in: tasks.map((task) => task.sourceUrl) } }, + select: { sourceUrl: true }, + }) + ).map((task) => task.sourceUrl) + ); + + const newTasks = tasks.filter((task) => !existingUrls.has(task.sourceUrl)); + + if (newTasks.length === 0) { + return NextResponse.json({ + success: true, + created: 0, + skipped: tasks.length, + total: tasks.length, + message: "All tasks already exist", + }); + } + + const created = await prisma.projectDiscoveryTask.createMany({ + data: newTasks.map((task) => ({ + sourceUrl: task.sourceUrl, + sourceType: task.sourceType, + status: "PENDING", + })), + }); + + console.warn( + `[Discovery] Created ${created.count} tasks, skipped ${tasks.length - created.count} existing tasks` + ); + + return NextResponse.json({ + success: true, + created: created.count, + skipped: tasks.length - created.count, + total: tasks.length, + }); + } catch (error) { + console.error("[Discovery] Error creating tasks:", error); + return NextResponse.json({ success: false, error: "Internal server error" }, { status: 500 }); + } +} + +export async function GET(request: NextRequest) { + try { + const searchParams = request.nextUrl.searchParams; + const apiKey = request.headers.get("x-api-key") || searchParams.get("apiKey"); + + if (!isValidApiKey(apiKey)) { + return NextResponse.json({ success: false, error: "Unauthorized" }, { status: 401 }); + } + + const validation = GetDiscoveryTasksQuerySchema.safeParse({ + status: searchParams.get("status") || undefined, + limit: searchParams.get("limit") || "10", + offset: searchParams.get("offset") || "0", + }); + + if (!validation.success) { + return NextResponse.json( + { + success: false, + error: "Validation error", + details: validation.error.errors.map((error) => error.message), + }, + { status: 400 } + ); + } + + const { status, limit, offset } = validation.data; + + let whereClause = {}; + if (status) { + whereClause = Array.isArray(status) ? { status: { in: status } } : { status }; + } + + const tasks = await prisma.projectDiscoveryTask.findMany({ + where: whereClause, + orderBy: { createdAt: "asc" }, + take: limit, + skip: offset, + }); + + const total = await prisma.projectDiscoveryTask.count({ + where: whereClause, + }); + + return NextResponse.json({ + success: true, + tasks, + total, + hasMore: offset + tasks.length < total, + }); + } catch (error) { + console.error("[Discovery] Error fetching tasks:", error); + return NextResponse.json({ success: false, error: "Internal server error" }, { status: 500 }); + } +} diff --git a/src/lib/validations.test.ts b/src/lib/validations.test.ts index fdef660..fcd94f4 100644 --- a/src/lib/validations.test.ts +++ b/src/lib/validations.test.ts @@ -1,5 +1,10 @@ -import { describe, it, expect } from "vitest"; -import { ProjectInputSchema } from "./validations"; +import { describe, expect, it } from "vitest"; +import { + BatchResetTasksSchema, + CreateDiscoveryTaskSchema, + GetDiscoveryTasksQuerySchema, + ProjectInputSchema, +} from "./validations"; describe("ProjectInputSchema", () => { const baseProjectInput = { @@ -30,4 +35,33 @@ describe("ProjectInputSchema", () => { }) ).toThrow(); }); + + it("should default discovery task sourceType to manual", () => { + const parsed = CreateDiscoveryTaskSchema.parse({ + apiKey: "k".repeat(32), + tasks: [{ sourceUrl: "https://github.com/example/repo" }], + }); + + expect(parsed.tasks[0]?.sourceType).toBe("manual"); + }); + + it("should parse comma-separated discovery statuses", () => { + const parsed = GetDiscoveryTasksQuerySchema.parse({ + status: "PENDING,FAILED", + limit: "20", + offset: "5", + }); + + expect(parsed.status).toEqual(["PENDING", "FAILED"]); + expect(parsed.limit).toBe(20); + expect(parsed.offset).toBe(5); + }); + + it("should reject batch reset without taskIds or statuses", () => { + expect(() => + BatchResetTasksSchema.parse({ + apiKey: "k".repeat(32), + }) + ).toThrow("必须提供 taskIds 或 statuses 之一"); + }); }); diff --git a/src/lib/validations.ts b/src/lib/validations.ts index 6032d9a..b32ce05 100644 --- a/src/lib/validations.ts +++ b/src/lib/validations.ts @@ -61,6 +61,72 @@ export const WebhookAuthSchema = z.object({ apiKey: z.string().min(32, "Invalid API key format"), }); +export const TaskStatusEnum = z.enum(["PENDING", "IN_PROGRESS", "COMPLETED", "FAILED"]); + +const JsonValueSchema: z.ZodType = z.lazy(() => + z.union([ + z.string(), + z.number(), + z.boolean(), + z.null(), + z.array(JsonValueSchema), + z.record(JsonValueSchema), + ]) +); + +export const JsonObjectSchema = z.record(JsonValueSchema); + +export const CreateDiscoveryTaskSchema = WebhookAuthSchema.extend({ + tasks: z + .array( + z.object({ + sourceUrl: z.string().url().max(2000), + sourceType: z.string().max(50).default("manual"), + }) + ) + .min(1), +}); + +export const UpdateDiscoveryTaskSchema = z.object({ + apiKey: z.string().min(32), + status: TaskStatusEnum, + explorationData: JsonObjectSchema.optional(), + explorationSummary: z.string().max(1000).optional(), + errorMessage: z.string().max(2000).optional(), +}); + +export const GetDiscoveryTasksQuerySchema = z.object({ + status: z + .string() + .optional() + .transform((val) => { + if (!val) { + return undefined; + } + + const statuses = val + .split(",") + .map((status) => status.trim() as TaskStatus) + .filter(Boolean); + + return statuses.length === 1 ? statuses[0] : statuses; + }), + limit: z.coerce.number().int().positive().max(100).default(10), + offset: z.coerce.number().int().nonnegative().default(0), +}); + +export const BatchResetTasksSchema = WebhookAuthSchema.extend({ + taskIds: z.array(z.string()).optional(), + statuses: z.array(TaskStatusEnum).optional(), +}).refine((data) => data.taskIds || data.statuses, { + message: "必须提供 taskIds 或 statuses 之一", +}); + +export const CheckTaskDuplicatesSchema = WebhookAuthSchema.extend({ + urls: z.array(z.string().url().max(2000)).min(1).max(100), + sourceType: z.string().max(50).optional(), +}); + // ================================ // Signals Schemas // ================================ @@ -140,6 +206,12 @@ export type ExternalLink = z.infer; export type Tag = z.infer; export type ProjectInput = z.infer; export type ProjectQuery = z.infer; +export type TaskStatus = z.infer; +export type CreateDiscoveryTask = z.infer; +export type UpdateDiscoveryTask = z.infer; +export type GetDiscoveryTasksQuery = z.infer; +export type BatchResetTasks = z.infer; +export type JsonObject = z.infer; export type SignalSource = z.infer; export type SignalSectionStyle = z.infer; export type SignalSectionInput = z.infer;