feat: 完善标签分类与领域场景归一化

This commit is contained in:
2026-02-22 12:03:11 +08:00
parent 285af9a039
commit ef527d69ce
5 changed files with 666 additions and 65 deletions
+3 -1
View File
@@ -9,7 +9,9 @@
"lint": "next lint",
"test": "vitest",
"test:e2e": "playwright test",
"taxonomy:backfill": "ts-node --compiler-options '{\"module\":\"CommonJS\",\"moduleResolution\":\"node\"}' scripts/backfill-tag-taxonomy.ts"
"taxonomy:backfill": "ts-node --compiler-options '{\"module\":\"CommonJS\",\"moduleResolution\":\"node\"}' scripts/backfill-tag-taxonomy.ts",
"domains:sync": "ts-node --compiler-options '{\"module\":\"CommonJS\",\"moduleResolution\":\"node\"}' scripts/sync-domain-scenarios.ts",
"tags:dedupe-free": "ts-node --compiler-options '{\"module\":\"CommonJS\",\"moduleResolution\":\"node\"}' scripts/dedupe-free-tags-to-canonical.ts"
},
"dependencies": {
"@prisma/client": "^6.1.0",
+184
View File
@@ -0,0 +1,184 @@
import { PrismaClient, type TagCategory } from '@prisma/client'
const prisma = new PrismaClient()
type DedupeRule = {
sourceSlug: string
targetSlug: string
reason: string
}
const DEDUPE_RULES: DedupeRule[] = [
{
sourceSlug: 'ai-development-tool',
targetSlug: 'code-dev',
reason: 'AI 开发工具与开发者工具/代码领域语义重复',
},
{
sourceSlug: 'code-generation',
targetSlug: 'code-dev',
reason: '代码生成与开发者工具/代码领域语义重复',
},
{
sourceSlug: 'library',
targetSlug: 'code-dev',
reason: 'Library 与开发者工具/代码领域语义重复',
},
{
sourceSlug: '知识管理',
targetSlug: 'knowledge-rag',
reason: '知识管理与知识管理/检索/RAG 领域语义重复',
},
{
sourceSlug: 'knowledge-graph',
targetSlug: 'knowledge-rag',
reason: '知识图谱归并到知识管理/检索/RAG 领域',
},
{
sourceSlug: '向量数据库',
targetSlug: 'knowledge-rag',
reason: '向量数据库在本项目中归并到知识管理/检索/RAG 领域',
},
{
sourceSlug: 'ai-security',
targetSlug: 'security-privacy',
reason: 'AI 安全与安全/隐私领域语义重复',
},
{
sourceSlug: '隐私保护',
targetSlug: 'security-privacy',
reason: '隐私保护与安全/隐私领域语义重复',
},
{
sourceSlug: 'data-analytics',
targetSlug: 'data-bi',
reason: '数据分析与数据分析/BI/可视化领域语义重复',
},
{
sourceSlug: 'visualization',
targetSlug: 'data-bi',
reason: '可视化与数据分析/BI/可视化领域语义重复',
},
{
sourceSlug: 'computer-vision',
targetSlug: 'vision-multimodal',
reason: '计算机视觉与计算机视觉/多模态领域语义重复',
},
{
sourceSlug: 'enterprise-ai',
targetSlug: 'enterprise-office',
reason: '企业级 AI 应用与企业应用/办公领域语义重复',
},
{
sourceSlug: 'chatbot',
targetSlug: 'enterprise-office',
reason: '聊天机器人与企业应用/办公领域语义重复',
},
]
const CANONICAL_CATEGORIES: TagCategory[] = ['DOMAIN_SCENARIO', 'FIXED_PROJECT_TYPE']
async function dedupeRule(rule: DedupeRule) {
return prisma.$transaction(async (tx) => {
const source = await tx.tag.findUnique({
where: { slug: rule.sourceSlug },
include: { _count: { select: { projects: true } } },
})
if (!source) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: 'source_not_found',
}
}
if (source.category !== 'FREE_TAG') {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: `source_not_free_tag:${source.category}`,
}
}
const target = await tx.tag.findUnique({
where: { slug: rule.targetSlug },
include: { _count: { select: { projects: true } } },
})
if (!target) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: 'target_not_found',
}
}
if (!CANONICAL_CATEGORIES.includes(target.category)) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: `target_not_canonical:${target.category}`,
}
}
const sourceLinks = await tx.projectTag.findMany({
where: { tagId: source.id },
select: { projectId: true },
})
if (sourceLinks.length > 0) {
await tx.projectTag.createMany({
data: sourceLinks.map((link) => ({
projectId: link.projectId,
tagId: target.id,
})),
skipDuplicates: true,
})
}
await tx.tag.delete({
where: { id: source.id },
})
return {
status: 'deduped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
movedProjectLinks: sourceLinks.length,
sourceProjectCount: source._count.projects,
reason: rule.reason,
}
})
}
async function main() {
console.log('[dedupe-free-tags] start')
const results: Array<Record<string, unknown>> = []
for (const rule of DEDUPE_RULES) {
const result = await dedupeRule(rule)
results.push(result)
}
const dedupedCount = results.filter((item) => item.status === 'deduped').length
const skippedCount = results.filter((item) => item.status === 'skipped').length
console.log('[dedupe-free-tags] done', {
totalRules: DEDUPE_RULES.length,
dedupedCount,
skippedCount,
})
console.log('[dedupe-free-tags] results', results)
}
main()
.catch((error) => {
console.error('[dedupe-free-tags] failed', error)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})
+320
View File
@@ -0,0 +1,320 @@
import { PrismaClient } from '@prisma/client'
import {
DOMAIN_SCENARIO_PRESET_TAGS,
type DomainScenarioPresetSlug,
} from '../src/lib/tag-taxonomy'
const prisma = new PrismaClient()
type DomainRule = {
slug: DomainScenarioPresetSlug
aliasSlugs: string[]
keywords: string[]
minScore: number
minKeywordHits: number
}
const DOMAIN_RULES: DomainRule[] = [
{
slug: 'code-dev',
aliasSlugs: [
'code-dev',
'ai-development-tool',
'code-generation',
'tool-calling',
'vs-code-extension',
'cli',
'library',
'agent-framework',
'ai-agent-framework',
],
keywords: ['code', 'coding', 'programming', 'developer', '编程', '代码', '开发工具'],
minScore: 2,
minKeywordHits: 2,
},
{
slug: 'automation-workflow',
aliasSlugs: [
'automation-workflow',
'automation',
'workflow-automation',
'workflow-orchestration',
'web-automation',
'browser-automation',
'rpa',
],
keywords: ['workflow', 'automation', 'orchestration', 'rpa', '自动化', '工作流', '编排'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'knowledge-rag',
aliasSlugs: [
'knowledge-rag',
'knowledge-management',
'知识管理',
'knowledge-base',
'knowledge-graph',
'rag',
'向量数据库',
'vector-database',
'llamaindex',
],
keywords: ['knowledge', 'retrieval', 'memory', 'rag', '知识', '检索', '记忆', '知识库'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'education-research',
aliasSlugs: [
'education-research',
'docs-tutorial',
'ai-research',
'markdown',
'tutorial',
'guide',
'paper',
'benchmark',
],
keywords: ['tutorial', 'guide', 'docs', 'paper', 'course', '教程', '指南', '文档', '论文', '课程'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'model-inference',
aliasSlugs: [
'model-inference',
'inference-model',
'llm',
'大语言模型',
'transformers',
'pytorch',
'machine-learning',
'deep-learning',
'reinforcement-learning',
'vllm',
],
keywords: ['model', 'inference', 'training', 'finetune', '模型', '推理', '训练', '微调'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'api-integration',
aliasSlugs: [
'api-integration',
'model-context-protocol',
'sdk',
'api-gateway',
],
keywords: ['api', 'sdk', 'protocol', 'integration', 'mcp', '接口', '协议', '集成'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'vision-multimodal',
aliasSlugs: ['vision-multimodal', 'computer-vision', 'multimodal', 'multimodal-ai'],
keywords: ['vision', 'image', 'video', 'multimodal', '视觉', '图像', '视频', '多模态'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'data-bi',
aliasSlugs: ['data-bi', 'data-analytics', '数据分析', 'visualization'],
keywords: ['analytics', 'dashboard', 'visualization', 'business intelligence', '数据分析', '可视化'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'security-privacy',
aliasSlugs: ['security-privacy', 'ai-security', '隐私保护'],
keywords: ['security', 'privacy', 'safety', '安全', '隐私'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'enterprise-office',
aliasSlugs: ['enterprise-office', 'enterprise-ai', 'chatbot'],
keywords: ['enterprise', 'office', 'collaboration', 'crm', 'erp', '企业', '办公', '协作'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'finance',
aliasSlugs: ['finance', 'fintech'],
keywords: ['finance', 'financial', 'trading', 'fintech', 'quant', '金融', '交易', '量化', '风控'],
minScore: 1,
minKeywordHits: 1,
},
{
slug: 'medical-biomed',
aliasSlugs: ['medical-biomed', 'medical', 'healthcare', 'biomedical'],
keywords: ['medical', 'healthcare', 'medicine', 'biomed', '医疗', '医学', '医药', '生物医学'],
minScore: 1,
minKeywordHits: 1,
},
]
const FALLBACK_BY_FIXED_PROJECT_TYPE: Partial<Record<string, DomainScenarioPresetSlug>> = {
'agent-tooling': 'code-dev',
'inference-model': 'model-inference',
'docs-tutorial': 'education-research',
}
function normalize(value: string): string {
return value.toLowerCase().trim()
}
function inferDomainSlugs(input: {
name: string
nameEn?: string | null
description: string
descriptionEn?: string | null
tagSlugs: string[]
}): DomainScenarioPresetSlug[] {
const normalizedTagSet = new Set(input.tagSlugs.map((slug) => normalize(slug)))
const text = normalize(
[input.name, input.nameEn || '', input.description, input.descriptionEn || '', ...input.tagSlugs].join(' ')
)
const scoredDomains: Array<{ slug: DomainScenarioPresetSlug; score: number }> = []
for (const rule of DOMAIN_RULES) {
const tagHits = rule.aliasSlugs.reduce(
(acc, slug) => acc + (normalizedTagSet.has(normalize(slug)) ? 1 : 0),
0
)
const keywordHits = rule.keywords.reduce(
(acc, keyword) => acc + (text.includes(normalize(keyword)) ? 1 : 0),
0
)
const score = tagHits * 3 + keywordHits
if (score >= rule.minScore && (tagHits > 0 || keywordHits >= rule.minKeywordHits)) {
scoredDomains.push({ slug: rule.slug, score })
}
}
if (scoredDomains.length > 0) {
return scoredDomains
.sort((a, b) => b.score - a.score)
.slice(0, 3)
.map((item) => item.slug)
}
const fixedProjectType = input.tagSlugs.find((slug) => FALLBACK_BY_FIXED_PROJECT_TYPE[slug])
if (fixedProjectType) {
return [FALLBACK_BY_FIXED_PROJECT_TYPE[fixedProjectType]!]
}
return ['code-dev']
}
async function main() {
console.log('[domain] start sync')
const canonicalTagIdMap = new Map<DomainScenarioPresetSlug, string>()
for (const domainTag of DOMAIN_SCENARIO_PRESET_TAGS) {
const tag = await prisma.tag.upsert({
where: { slug: domainTag.slug },
update: {
name: domainTag.name,
nameEn: domainTag.nameEn,
category: 'DOMAIN_SCENARIO',
},
create: {
name: domainTag.name,
nameEn: domainTag.nameEn,
slug: domainTag.slug,
category: 'DOMAIN_SCENARIO',
},
})
canonicalTagIdMap.set(domainTag.slug, tag.id)
}
const clearedLinks = await prisma.projectTag.deleteMany({
where: {
tagId: {
in: Array.from(canonicalTagIdMap.values()),
},
},
})
const projects = await prisma.project.findMany({
where: { status: 'ACTIVE' },
select: {
id: true,
name: true,
nameEn: true,
description: true,
descriptionEn: true,
tags: {
select: {
tag: {
select: {
slug: true,
},
},
},
},
},
})
const createData: Array<{ projectId: string; tagId: string }> = []
const distribution = new Map<DomainScenarioPresetSlug, number>()
for (const domainTag of DOMAIN_SCENARIO_PRESET_TAGS) {
distribution.set(domainTag.slug, 0)
}
for (const project of projects) {
const tagSlugs = project.tags.map((item) => item.tag.slug)
const domainSlugs = inferDomainSlugs({
name: project.name,
nameEn: project.nameEn,
description: project.description,
descriptionEn: project.descriptionEn,
tagSlugs,
})
for (const domainSlug of domainSlugs) {
const domainTagId = canonicalTagIdMap.get(domainSlug)
if (!domainTagId) {
throw new Error(`Missing canonical domain tag id: ${domainSlug}`)
}
createData.push({
projectId: project.id,
tagId: domainTagId,
})
distribution.set(domainSlug, (distribution.get(domainSlug) || 0) + 1)
}
}
const insertedLinks = await prisma.projectTag.createMany({
data: createData,
skipDuplicates: true,
})
const distributionSummary = DOMAIN_SCENARIO_PRESET_TAGS.map((item) => ({
slug: item.slug,
name: item.name,
projectCount: distribution.get(item.slug) || 0,
}))
console.log('[domain] done', {
canonicalDomainTagCount: DOMAIN_SCENARIO_PRESET_TAGS.length,
clearedCanonicalDomainLinks: clearedLinks.count,
insertedDomainLinks: insertedLinks.count,
activeProjectCount: projects.length,
})
console.log('[domain] distribution', distributionSummary)
}
main()
.catch((error) => {
console.error('[domain] failed', error)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})
@@ -405,7 +405,8 @@ export async function upsertTags(tags: ProjectInput['tags']) {
})
const shouldUpdateNameEn = incomingTag.nameEn && !matchedTag.nameEn
const shouldUpdateCategory =
matchedTag.category === 'FREE_TAG' && inferredCategory !== 'FREE_TAG'
matchedTag.category !== inferredCategory &&
['FREE_TAG', 'RESOURCE_TYPE', 'PROTOCOL_INTERFACE'].includes(matchedTag.category)
if (shouldUpdateNameEn || shouldUpdateCategory) {
const updatedTag = await prisma.tag.update({
+157 -63
View File
@@ -23,6 +23,71 @@ export const FIXED_PROJECT_TYPE_TAGS = [
},
] as const
export const DOMAIN_SCENARIO_PRESET_TAGS = [
{
slug: 'code-dev',
name: '开发者工具/代码',
nameEn: 'Developer Tools & Coding',
},
{
slug: 'automation-workflow',
name: '自动化/工作流/RPA',
nameEn: 'Automation, Workflow & RPA',
},
{
slug: 'knowledge-rag',
name: '知识管理/检索/RAG',
nameEn: 'Knowledge Management, Retrieval & RAG',
},
{
slug: 'education-research',
name: '教育/研究资源',
nameEn: 'Education & Research Resources',
},
{
slug: 'model-inference',
name: '模型训练/推理',
nameEn: 'Model Training & Inference',
},
{
slug: 'api-integration',
name: '协议/API/集成',
nameEn: 'Protocol, API & Integration',
},
{
slug: 'vision-multimodal',
name: '计算机视觉/多模态',
nameEn: 'Computer Vision & Multimodal',
},
{
slug: 'data-bi',
name: '数据分析/BI/可视化',
nameEn: 'Data Analytics, BI & Visualization',
},
{
slug: 'security-privacy',
name: '安全/隐私',
nameEn: 'Security & Privacy',
},
{
slug: 'enterprise-office',
name: '企业应用/办公',
nameEn: 'Enterprise Applications & Office',
},
{
slug: 'finance',
name: '金融',
nameEn: 'Finance',
},
{
slug: 'medical-biomed',
name: '医疗/生物医药',
nameEn: 'Medical & Biomedicine',
},
] as const
export type DomainScenarioPresetSlug = (typeof DOMAIN_SCENARIO_PRESET_TAGS)[number]['slug']
export type FixedProjectTypeSlug = (typeof FIXED_PROJECT_TYPE_TAGS)[number]['slug']
export const FIXED_PROJECT_TYPE_SLUGS = new Set<FixedProjectTypeSlug>(
@@ -117,34 +182,34 @@ const TECH_STACK_SLUGS = new Set([
'bun',
'langchain',
'langgraph',
'pytorch',
'transformers',
'llamaindex',
'agent-framework',
'vllm',
'litellm',
'playwright',
'chromadb',
'gradio',
'streamlit',
'vuejs',
])
const AI_PARADIGM_SLUGS = new Set([
'llm',
'ai-agents',
'multi-agent-system',
'大语言模型',
'rag',
'transformers',
'pytorch',
'machine-learning',
'deep-learning',
'reinforcement-learning',
'multimodal',
'multimodal-ai',
'multi-agent-system',
'multi-agent',
'autonomous-agents',
'vllm',
'llamaindex',
'agent-framework',
'ai-agent-framework',
'ai-agents',
'natural-language-processing',
'model-context-protocol',
'sdk',
])
const PROTOCOL_INTERFACE_SLUGS = new Set([
'model-context-protocol',
'mcp-protocol',
'openai-api',
'sdk',
'api-gateway',
])
@@ -154,28 +219,39 @@ const PRODUCT_FORM_SLUGS = new Set([
'桌面应用',
'vs-code-extension',
'workflow-automation',
'browser-automation',
'knowledge-base',
'automation',
])
const DOMAIN_SCENARIO_SLUGS = new Set([
...DOMAIN_SCENARIO_PRESET_TAGS.map((item) => item.slug),
'data-analytics',
'数据分析',
'knowledge-management',
'知识管理',
'ai-security',
'natural-language-processing',
'nlp',
'knowledge-graph',
'computer-vision',
'visualization',
'enterprise-ai',
'chatbot',
])
const RESOURCE_TYPE_SLUGS = new Set([
'knowledge-base',
'markdown',
'ai-research',
'guide',
'tutorial',
'paper',
])
const RESOURCE_TYPE_SLUGS = new Set<string>([])
const TAG_SLUG_ALIAS_MAP: Record<string, string> = {
mcp: 'model-context-protocol',
'mcp-protocol': 'model-context-protocol',
llm: '大语言模型',
'llm-applications': '大语言模型',
'autonomous-agents': 'ai-agents',
'multi-agent': 'multi-agent-system',
'multimodal-ai': 'multimodal',
'ai-agent-framework': 'agent-framework',
'workflow-orchestration': 'workflow-automation',
'web-automation': 'browser-automation',
}
const TECH_STACK_KEYWORDS = [
'python',
@@ -210,13 +286,14 @@ const AI_PARADIGM_KEYWORDS = [
'multi agent',
'智能体',
'agent',
'mcp',
'sdk',
]
const PROTOCOL_INTERFACE_KEYWORDS = [
'mcp',
'api gateway',
'gateway',
'protocol',
'api',
'sdk',
'接口',
'协议',
]
@@ -235,6 +312,24 @@ const PRODUCT_FORM_KEYWORDS = [
]
const DOMAIN_SCENARIO_KEYWORDS = [
'domain',
'scenario',
'workflow',
'automation',
'rag',
'retrieval',
'developer',
'coding',
'medical',
'health',
'biomed',
'financial',
'trading',
'enterprise',
'office',
'api integration',
'visualization',
'dashboard',
'security',
'analysis',
'knowledge',
@@ -243,30 +338,33 @@ const DOMAIN_SCENARIO_KEYWORDS = [
'speech',
'robot',
'finance',
'integration',
'安全',
'分析',
'知识',
'检索',
'自动化',
'工作流',
'开发者',
'编程',
'代码',
'协议',
'集成',
'企业',
'办公',
'医疗',
'医药',
'医学',
'生物医学',
'金融',
'交易',
'量化',
'隐私',
'语音',
'视觉',
]
const RESOURCE_TYPE_KEYWORDS = [
'guide',
'tutorial',
'paper',
'list',
'awesome',
'docs',
'documentation',
'resource',
'指南',
'教程',
'论文',
'合集',
'文档',
'资源',
'课程',
]
const RESOURCE_TYPE_KEYWORDS: string[] = []
const PROJECT_DOC_KEYWORDS = [
...RESOURCE_TYPE_KEYWORDS,
@@ -301,22 +399,18 @@ const PROJECT_MODEL_TAG_SLUGS = new Set([
const PROJECT_DOC_TAG_SLUGS = new Set([
'knowledge-base',
'markdown',
'ai-research',
])
const PROJECT_TOOL_TAG_SLUGS = new Set([
'ai-development-tool',
'agent-framework',
'ai-agent-framework',
'ai-agents',
'mcp',
'model-context-protocol',
'mcp-protocol',
'cli',
'workflow-automation',
'browser-automation',
'automation',
'openai-api',
'sdk',
'web-application',
'桌面应用',
@@ -358,27 +452,29 @@ export function inferTagCategory(input: {
nameEn?: string | null
}): TagCategory {
const rawSlug = input.slug.trim().toLowerCase()
if (isFixedProjectTypeSlug(rawSlug)) {
const canonicalSlug = TAG_SLUG_ALIAS_MAP[rawSlug] ?? rawSlug
if (isFixedProjectTypeSlug(canonicalSlug)) {
return 'FIXED_PROJECT_TYPE'
}
const normalizedSlug = normalize(rawSlug)
const normalizedSlug = normalize(canonicalSlug)
if (TECH_STACK_SLUGS.has(input.slug)) {
if (TECH_STACK_SLUGS.has(canonicalSlug)) {
return 'TECH_STACK'
}
if (AI_PARADIGM_SLUGS.has(input.slug)) {
if (AI_PARADIGM_SLUGS.has(canonicalSlug)) {
return 'AI_PARADIGM'
}
if (PROTOCOL_INTERFACE_SLUGS.has(input.slug)) {
if (PROTOCOL_INTERFACE_SLUGS.has(canonicalSlug)) {
return 'PROTOCOL_INTERFACE'
}
if (PRODUCT_FORM_SLUGS.has(input.slug)) {
if (PRODUCT_FORM_SLUGS.has(canonicalSlug)) {
return 'PRODUCT_FORM'
}
if (DOMAIN_SCENARIO_SLUGS.has(input.slug)) {
if (DOMAIN_SCENARIO_SLUGS.has(canonicalSlug)) {
return 'DOMAIN_SCENARIO'
}
if (RESOURCE_TYPE_SLUGS.has(input.slug)) {
if (RESOURCE_TYPE_SLUGS.has(canonicalSlug)) {
return 'RESOURCE_TYPE'
}
@@ -386,9 +482,6 @@ export function inferTagCategory(input: {
const normalizedNameEn = normalize(input.nameEn || '')
const mergedText = `${normalizedSlug} ${normalizedName} ${normalizedNameEn}`
if (includesAny(mergedText, RESOURCE_TYPE_KEYWORDS)) {
return 'RESOURCE_TYPE'
}
if (includesAny(mergedText, TECH_STACK_KEYWORDS)) {
return 'TECH_STACK'
}
@@ -405,7 +498,8 @@ export function inferTagCategory(input: {
return 'DOMAIN_SCENARIO'
}
return 'FREE_TAG'
// Free tags are deprecated in this project; default to tech stack for unknown slugs.
return 'TECH_STACK'
}
export function inferProjectTypeSlug(input: {