fix: 清洗标签并收敛入库标签噪音

2026-02-21 12:07:02 +08:00
parent 220e9f8b71
commit c2eabfea97
19 changed files with 4399 additions and 145 deletions
@@ -1,6 +1,166 @@
 import { prisma } from '@/lib/prisma'
 import type { ProjectInput } from '@/lib/validations'
 import { generateSlug } from '@/lib/slug'
+import type { Prisma } from '@prisma/client'
+
+const PLURAL_NORMALIZATION_MAP = new Map([
+  ['agents', 'agent'],
+  ['assistants', 'assistant'],
+  ['tools', 'tool'],
+  ['frameworks', 'framework'],
+  ['models', 'model'],
+  ['servers', 'server'],
+  ['clients', 'client'],
+  ['workflows', 'workflow'],
+  ['plugins', 'plugin'],
+  ['libraries', 'library'],
+  ['datasets', 'dataset'],
+  ['platforms', 'platform'],
+  ['systems', 'system'],
+  ['repositories', 'repository'],
+  ['engines', 'engine'],
+])
+
+const NOISE_TAG_KEYS = new Set([
+  'ai',
+  'artificial intelligence',
+  'open source',
+  'requires configuration',
+  'requires basics',
+  'low learning curve',
+  'enterprise',
+  'complex deployment',
+  'cloud service',
+  'cross platform',
+  'tutorial',
+  'academic research',
+  'academic resource',
+  'ai research resource',
+  'multi language support',
+  'self hosted',
+  'user notification',
+  'mit license',
+  'apache 2 0',
+  '开源',
+  '需要配置',
+  '需要基础',
+  '低学习成本',
+  '企业级',
+  '复杂部署',
+  '云端服务',
+  '跨平台',
+  '教程',
+  '学术研究',
+  '学术资源',
+  '多语言支持',
+  '自托管',
+  '用户通知',
+  '许可',
+])
+
+const TAG_FALLBACK = {
+  name: 'AI开发工具',
+  nameEn: 'AI Development Tool',
+} as const
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim()
+}
+
+function isAsciiText(value: string): boolean {
+  return /^[\x00-\x7f]+$/.test(value)
+}
+
+function canonicalizeTagKey(value: string): string {
+  const normalized = normalizeWhitespace(value)
+    .normalize('NFKC')
+    .toLowerCase()
+    .replace(/[+/_&|-]+/g, ' ')
+    .replace(/[^a-z0-9\u4e00-\u9fa5\s]/g, ' ')
+    .replace(/\s+/g, ' ')
+    .trim()
+
+  if (!normalized) {
+    return ''
+  }
+
+  return normalized
+    .split(' ')
+    .map((word) => PLURAL_NORMALIZATION_MAP.get(word) || word)
+    .join(' ')
+    .trim()
+}
+
+function normalizeIncomingTag(tag: ProjectInput['tags'][number]) {
+  const normalizedName = normalizeWhitespace(tag.name)
+  const normalizedNameEn = normalizeWhitespace(tag.nameEn || '')
+  const resolvedNameEn = normalizedNameEn || (isAsciiText(normalizedName) ? normalizedName : '')
+  const canonicalNameKey = canonicalizeTagKey(normalizedName)
+  const canonicalNameEnKey = canonicalizeTagKey(resolvedNameEn)
+
+  return {
+    name: normalizedName,
+    nameEn: resolvedNameEn || null,
+    canonicalNameKey,
+    canonicalNameEnKey,
+    slug: generateSlug(normalizedName, resolvedNameEn || null),
+  }
+}
+
+function isMeaningfulTag(tag: ReturnType<typeof normalizeIncomingTag>): boolean {
+  if (!tag.name) {
+    return false
+  }
+
+  if (NOISE_TAG_KEYS.has(tag.canonicalNameKey)) {
+    return false
+  }
+
+  if (tag.canonicalNameEnKey && NOISE_TAG_KEYS.has(tag.canonicalNameEnKey)) {
+    return false
+  }
+
+  return true
+}
+
+type TagWithProjectCount = Prisma.TagGetPayload<{
+  include: {
+    _count: { select: { projects: true } }
+  }
+}>
+
+function chooseBestTag(candidates: TagWithProjectCount[]): TagWithProjectCount {
+  return [...candidates].sort((a, b) => {
+    const projectDiff = b._count.projects - a._count.projects
+    if (projectDiff !== 0) {
+      return projectDiff
+    }
+    return a.createdAt.getTime() - b.createdAt.getTime()
+  })[0]!
+}
+
+async function getFallbackTag(): Promise<TagWithProjectCount> {
+  const fallbackSlug = generateSlug(TAG_FALLBACK.name, TAG_FALLBACK.nameEn)
+  const fallbackTag = await prisma.tag.upsert({
+    where: { slug: fallbackSlug },
+    update: {
+      name: TAG_FALLBACK.name,
+      nameEn: TAG_FALLBACK.nameEn,
+    },
+    create: {
+      name: TAG_FALLBACK.name,
+      nameEn: TAG_FALLBACK.nameEn,
+      slug: fallbackSlug,
+    },
+    include: {
+      _count: {
+        select: { projects: true },
+      },
+    },
+  })
+
+  return fallbackTag
+}

 /**
 * 多级去重策略：查找已存在的项目
@@ -90,44 +250,159 @@ export async function findExistingProject(projectData: ProjectInput) {
 * @returns 标签连接对象数组
 */
 export async function upsertTags(tags: ProjectInput['tags']) {
-  // 优化：批量查询所有已存在的标签，避免 N+1 问题
-  const allTagNames = tags.map((t) => t.name)
+  const meaningfulTags = tags
+    .map((tag) => normalizeIncomingTag(tag))
+    .filter((tag) => isMeaningfulTag(tag))
+
+  const normalizedIncomingTags = Array.from(
+    new Map(
+      meaningfulTags.map((normalized) => {
+        return [
+          normalized.canonicalNameEnKey || normalized.canonicalNameKey || normalized.name,
+          normalized,
+        ] as const
+      })
+    ).values()
+  )
+
+  if (normalizedIncomingTags.length === 0) {
+    return [await getFallbackTag()]
+  }
+
  const existingTags = await prisma.tag.findMany({
-    where: { name: { in: allTagNames } },
+    include: {
+      _count: {
+        select: { projects: true },
+      },
+    },
  })
-  const existingTagNames = new Set(existingTags.map((t) => t.name))

-  // Upsert tags - 优化后只查询不存在的标签
-  return await Promise.all(
-    tags.map(async (tag) => {
-      const tagSlug = generateSlug(tag.name, tag.nameEn)
+  const tagByExactName = new Map<string, TagWithProjectCount[]>()
+  const tagBySlug = new Map<string, TagWithProjectCount>()
+  const tagByCanonicalName = new Map<string, TagWithProjectCount[]>()
+  const tagByCanonicalNameEn = new Map<string, TagWithProjectCount[]>()

-      // 首先从批量查询结果中查找
-      if (existingTagNames.has(tag.name)) {
-        return existingTags.find((t) => t.name === tag.name)!
+  for (const tag of existingTags) {
+    const exactNameKey = normalizeWhitespace(tag.name)
+    const existingCanonicalName = canonicalizeTagKey(tag.name)
+    const existingCanonicalNameEn = canonicalizeTagKey(tag.nameEn || '')
+
+    if (!tagByExactName.has(exactNameKey)) {
+      tagByExactName.set(exactNameKey, [])
+    }
+    tagByExactName.get(exactNameKey)!.push(tag)
+
+    if (existingCanonicalName) {
+      if (!tagByCanonicalName.has(existingCanonicalName)) {
+        tagByCanonicalName.set(existingCanonicalName, [])
      }
+      tagByCanonicalName.get(existingCanonicalName)!.push(tag)
+    }

-      // 只有标签不存在时才尝试 upsert
-      try {
-        return await prisma.tag.upsert({
-          where: { slug: tagSlug },
-          update: {},
-          create: {
-            name: tag.name,
-            nameEn: tag.nameEn || null,
-            slug: tagSlug,
-          },
+    if (existingCanonicalNameEn) {
+      if (!tagByCanonicalNameEn.has(existingCanonicalNameEn)) {
+        tagByCanonicalNameEn.set(existingCanonicalNameEn, [])
+      }
+      tagByCanonicalNameEn.get(existingCanonicalNameEn)!.push(tag)
+    }
+
+    tagBySlug.set(tag.slug, tag)
+  }
+
+  const resolvedTags: TagWithProjectCount[] = []
+
+  for (const incomingTag of normalizedIncomingTags) {
+    const exactNameCandidates = tagByExactName.get(incomingTag.name) || []
+    let matchedTag = exactNameCandidates.length > 0 ? chooseBestTag(exactNameCandidates) : null
+
+    if (!matchedTag && incomingTag.canonicalNameKey) {
+      const canonicalNameCandidates = tagByCanonicalName.get(incomingTag.canonicalNameKey) || []
+      if (canonicalNameCandidates.length > 0) {
+        matchedTag = chooseBestTag(canonicalNameCandidates)
+      }
+    }
+
+    if (!matchedTag && incomingTag.canonicalNameEnKey) {
+      const canonicalNameEnCandidates = tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || []
+      if (canonicalNameEnCandidates.length > 0) {
+        matchedTag = chooseBestTag(canonicalNameEnCandidates)
+      }
+    }
+
+    if (!matchedTag) {
+      matchedTag = tagBySlug.get(incomingTag.slug) || null
+    }
+
+    if (matchedTag) {
+      if (incomingTag.nameEn && !matchedTag.nameEn) {
+        await prisma.tag.update({
+          where: { id: matchedTag.id },
+          data: { nameEn: incomingTag.nameEn },
        })
-      } catch (error) {
-        // 如果 slug 冲突，查找并使用已存在的标签
-        const existingBySlug = await prisma.tag.findUnique({
-          where: { slug: tagSlug },
-        })
-        if (existingBySlug) {
-          return existingBySlug
+        matchedTag = {
+          ...matchedTag,
+          nameEn: incomingTag.nameEn,
        }
-        throw error
      }
-    })
+
+      resolvedTags.push(matchedTag)
+      continue
+    }
+
+    try {
+      const createdTag = await prisma.tag.create({
+        data: {
+          name: incomingTag.name,
+          nameEn: incomingTag.nameEn,
+          slug: incomingTag.slug,
+        },
+        include: {
+          _count: {
+            select: { projects: true },
+          },
+        },
+      })
+      resolvedTags.push(createdTag)
+
+      if (!tagByExactName.has(createdTag.name)) {
+        tagByExactName.set(createdTag.name, [])
+      }
+      tagByExactName.get(createdTag.name)!.push(createdTag)
+      tagBySlug.set(createdTag.slug, createdTag)
+      if (createdTag.nameEn) {
+        const canonicalNameEn = canonicalizeTagKey(createdTag.nameEn)
+        if (canonicalNameEn) {
+          if (!tagByCanonicalNameEn.has(canonicalNameEn)) {
+            tagByCanonicalNameEn.set(canonicalNameEn, [])
+          }
+          tagByCanonicalNameEn.get(canonicalNameEn)!.push(createdTag)
+        }
+      }
+    } catch {
+      const fallbackTag = await prisma.tag.findFirst({
+        where: {
+          OR: [
+            { name: incomingTag.name },
+            { slug: incomingTag.slug },
+          ],
+        },
+        include: {
+          _count: {
+            select: { projects: true },
+          },
+        },
+      })
+
+      if (fallbackTag) {
+        resolvedTags.push(fallbackTag)
+        continue
+      }
+
+      throw new Error(`Failed to resolve tag: ${incomingTag.name}`)
+    }
+  }
+
+  return Array.from(
+    new Map(resolvedTags.map((tag) => [tag.id, tag])).values()
  )
 }
@@ -8,7 +8,7 @@ import {
  type WebhookPayload,
  type ProjectInput,
 } from '@/lib/validations'
-import { findExistingProject } from '../../discovery/lib/discovery-service'
+import { findExistingProject, upsertTags } from '../../discovery/lib/discovery-service'
 import { generateSlug } from '@/lib/slug'

 export async function POST(request: NextRequest) {
@@ -90,46 +90,8 @@ export async function POST(request: NextRequest) {
        // 多级去重：查找已存在的项目
        const existingProject = await findExistingProject(validProject)

-        // 优化：批量查询所有已存在的标签，避免 N+1 问题
-        const allTagNames = validProject.tags.map((t) => t.name)
-        const existingTags = await prisma.tag.findMany({
-          where: { name: { in: allTagNames } },
-        })
-        const existingTagNames = new Set(existingTags.map((t) => t.name))
-
-        // Upsert tags - 优化后只查询不存在的标签
-        const tagConnections = await Promise.all(
-          validProject.tags.map(async (tag) => {
-            const tagSlug = generateSlug(tag.name, tag.nameEn)
-
-            // 首先从批量查询结果中查找
-            if (existingTagNames.has(tag.name)) {
-              return existingTags.find((t) => t.name === tag.name)!
-            }
-
-            // 只有标签不存在时才尝试 upsert
-            try {
-              return await prisma.tag.upsert({
-                where: { slug: tagSlug },
-                update: {},
-                create: {
-                  name: tag.name,
-                  nameEn: tag.nameEn || null,
-                  slug: tagSlug,
-                },
-              })
-            } catch (error) {
-              // 如果 slug 冲突，查找并使用已存在的标签
-              const existingBySlug = await prisma.tag.findUnique({
-                where: { slug: tagSlug },
-              })
-              if (existingBySlug) {
-                return existingBySlug
-              }
-              throw error
-            }
-          })
-        )
+        // 标签规范化与自动复用（防止同义标签持续膨胀）
+        const tagConnections = await upsertTags(validProject.tags)

        // Generate slug for project
        const slug = generateSlug(validProject.name, validProject.nameEn)