mzaxd/agent-park
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 完善标签分类与领域场景归一化

Browse Source
This commit is contained in:
mzaxd
2026-02-22 12:03:11 +08:00
parent 285af9a039
commit ef527d69ce
5 changed files with 666 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/dedupe-free-tags-to-canonical.ts
+184
View File
@@ -0,0 +1,184 @@
import { PrismaClient, type TagCategory } from '@prisma/client'
const prisma = new PrismaClient()
type DedupeRule = {
sourceSlug: string
targetSlug: string
reason: string
}
const DEDUPE_RULES: DedupeRule[] = [
{
sourceSlug: 'ai-development-tool',
targetSlug: 'code-dev',
reason: 'AI 开发工具与开发者工具/代码领域语义重复',
},
{
sourceSlug: 'code-generation',
targetSlug: 'code-dev',
reason: '代码生成与开发者工具/代码领域语义重复',
},
{
sourceSlug: 'library',
targetSlug: 'code-dev',
reason: 'Library 与开发者工具/代码领域语义重复',
},
{
sourceSlug: '知识管理',
targetSlug: 'knowledge-rag',
reason: '知识管理与知识管理/检索/RAG 领域语义重复',
},
{
sourceSlug: 'knowledge-graph',
targetSlug: 'knowledge-rag',
reason: '知识图谱归并到知识管理/检索/RAG 领域',
},
{
sourceSlug: '向量数据库',
targetSlug: 'knowledge-rag',
reason: '向量数据库在本项目中归并到知识管理/检索/RAG 领域',
},
{
sourceSlug: 'ai-security',
targetSlug: 'security-privacy',
reason: 'AI 安全与安全/隐私领域语义重复',
},
{
sourceSlug: '隐私保护',
targetSlug: 'security-privacy',
reason: '隐私保护与安全/隐私领域语义重复',
},
{
sourceSlug: 'data-analytics',
targetSlug: 'data-bi',
reason: '数据分析与数据分析/BI/可视化领域语义重复',
},
{
sourceSlug: 'visualization',
targetSlug: 'data-bi',
reason: '可视化与数据分析/BI/可视化领域语义重复',
},
{
sourceSlug: 'computer-vision',
targetSlug: 'vision-multimodal',
reason: '计算机视觉与计算机视觉/多模态领域语义重复',
},
{
sourceSlug: 'enterprise-ai',
targetSlug: 'enterprise-office',
reason: '企业级 AI 应用与企业应用/办公领域语义重复',
},
{
sourceSlug: 'chatbot',
targetSlug: 'enterprise-office',
reason: '聊天机器人与企业应用/办公领域语义重复',
},
]
const CANONICAL_CATEGORIES: TagCategory[] = ['DOMAIN_SCENARIO', 'FIXED_PROJECT_TYPE']
async function dedupeRule(rule: DedupeRule) {
return prisma.$transaction(async (tx) => {
const source = await tx.tag.findUnique({
where: { slug: rule.sourceSlug },
include: { _count: { select: { projects: true } } },
})
if (!source) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: 'source_not_found',
}
}
if (source.category !== 'FREE_TAG') {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: `source_not_free_tag:${source.category}`,
}
}
const target = await tx.tag.findUnique({
where: { slug: rule.targetSlug },
include: { _count: { select: { projects: true } } },
})
if (!target) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: 'target_not_found',
}
}
if (!CANONICAL_CATEGORIES.includes(target.category)) {
return {
status: 'skipped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
message: `target_not_canonical:${target.category}`,
}
}
const sourceLinks = await tx.projectTag.findMany({
where: { tagId: source.id },
select: { projectId: true },
})
if (sourceLinks.length > 0) {
await tx.projectTag.createMany({
data: sourceLinks.map((link) => ({
projectId: link.projectId,
tagId: target.id,
})),
skipDuplicates: true,
})
}
await tx.tag.delete({
where: { id: source.id },
})
return {
status: 'deduped' as const,
sourceSlug: rule.sourceSlug,
targetSlug: rule.targetSlug,
movedProjectLinks: sourceLinks.length,
sourceProjectCount: source._count.projects,
reason: rule.reason,
}
})
}
async function main() {
console.log('[dedupe-free-tags] start')
const results: Array<Record<string, unknown>> = []
for (const rule of DEDUPE_RULES) {
const result = await dedupeRule(rule)
results.push(result)
}
const dedupedCount = results.filter((item) => item.status === 'deduped').length
const skippedCount = results.filter((item) => item.status === 'skipped').length
console.log('[dedupe-free-tags] done', {
totalRules: DEDUPE_RULES.length,
dedupedCount,
skippedCount,
})
console.log('[dedupe-free-tags] results', results)
}
main()
.catch((error) => {
console.error('[dedupe-free-tags] failed', error)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})
scripts/sync-domain-scenarios.ts
+320
View File
@@ -0,0 +1,320 @@
import { PrismaClient } from '@prisma/client'
import {
DOMAIN_SCENARIO_PRESET_TAGS,
type DomainScenarioPresetSlug,
} from '../src/lib/tag-taxonomy'
const prisma = new PrismaClient()
type DomainRule = {
slug: DomainScenarioPresetSlug
aliasSlugs: string[]
keywords: string[]
minScore: number
minKeywordHits: number
}
const DOMAIN_RULES: DomainRule[] = [
{
slug: 'code-dev',
aliasSlugs: [
'code-dev',
'ai-development-tool',
'code-generation',
'tool-calling',
'vs-code-extension',
'cli',
'library',
'agent-framework',
'ai-agent-framework',
],
keywords: ['code', 'coding', 'programming', 'developer', '编程', '代码', '开发工具'],
minScore: 2,
minKeywordHits: 2,
},
{
slug: 'automation-workflow',
aliasSlugs: [
'automation-workflow',
'automation',
'workflow-automation',
'workflow-orchestration',
'web-automation',
'browser-automation',
'rpa',
],
keywords: ['workflow', 'automation', 'orchestration', 'rpa', '自动化', '工作流', '编排'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'knowledge-rag',
aliasSlugs: [
'knowledge-rag',
'knowledge-management',
'知识管理',
'knowledge-base',
'knowledge-graph',
'rag',
'向量数据库',
'vector-database',
'llamaindex',
],
keywords: ['knowledge', 'retrieval', 'memory', 'rag', '知识', '检索', '记忆', '知识库'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'education-research',
aliasSlugs: [
'education-research',
'docs-tutorial',
'ai-research',
'markdown',
'tutorial',
'guide',
'paper',
'benchmark',
],
keywords: ['tutorial', 'guide', 'docs', 'paper', 'course', '教程', '指南', '文档', '论文', '课程'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'model-inference',
aliasSlugs: [
'model-inference',
'inference-model',
'llm',
'大语言模型',
'transformers',
'pytorch',
'machine-learning',
'deep-learning',
'reinforcement-learning',
'vllm',
],
keywords: ['model', 'inference', 'training', 'finetune', '模型', '推理', '训练', '微调'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'api-integration',
aliasSlugs: [
'api-integration',
'model-context-protocol',
'sdk',
'api-gateway',
],
keywords: ['api', 'sdk', 'protocol', 'integration', 'mcp', '接口', '协议', '集成'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'vision-multimodal',
aliasSlugs: ['vision-multimodal', 'computer-vision', 'multimodal', 'multimodal-ai'],
keywords: ['vision', 'image', 'video', 'multimodal', '视觉', '图像', '视频', '多模态'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'data-bi',
aliasSlugs: ['data-bi', 'data-analytics', '数据分析', 'visualization'],
keywords: ['analytics', 'dashboard', 'visualization', 'business intelligence', '数据分析', '可视化'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'security-privacy',
aliasSlugs: ['security-privacy', 'ai-security', '隐私保护'],
keywords: ['security', 'privacy', 'safety', '安全', '隐私'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'enterprise-office',
aliasSlugs: ['enterprise-office', 'enterprise-ai', 'chatbot'],
keywords: ['enterprise', 'office', 'collaboration', 'crm', 'erp', '企业', '办公', '协作'],
minScore: 2,
minKeywordHits: 1,
},
{
slug: 'finance',
aliasSlugs: ['finance', 'fintech'],
keywords: ['finance', 'financial', 'trading', 'fintech', 'quant', '金融', '交易', '量化', '风控'],
minScore: 1,
minKeywordHits: 1,
},
{
slug: 'medical-biomed',
aliasSlugs: ['medical-biomed', 'medical', 'healthcare', 'biomedical'],
keywords: ['medical', 'healthcare', 'medicine', 'biomed', '医疗', '医学', '医药', '生物医学'],
minScore: 1,
minKeywordHits: 1,
},
]
const FALLBACK_BY_FIXED_PROJECT_TYPE: Partial<Record<string, DomainScenarioPresetSlug>> = {
'agent-tooling': 'code-dev',
'inference-model': 'model-inference',
'docs-tutorial': 'education-research',
}
function normalize(value: string): string {
return value.toLowerCase().trim()
}
function inferDomainSlugs(input: {
name: string
nameEn?: string | null
description: string
descriptionEn?: string | null
tagSlugs: string[]
}): DomainScenarioPresetSlug[] {
const normalizedTagSet = new Set(input.tagSlugs.map((slug) => normalize(slug)))
const text = normalize(
[input.name, input.nameEn || '', input.description, input.descriptionEn || '', ...input.tagSlugs].join(' ')
)
const scoredDomains: Array<{ slug: DomainScenarioPresetSlug; score: number }> = []
for (const rule of DOMAIN_RULES) {
const tagHits = rule.aliasSlugs.reduce(
(acc, slug) => acc + (normalizedTagSet.has(normalize(slug)) ? 1 : 0),
0
)
const keywordHits = rule.keywords.reduce(
(acc, keyword) => acc + (text.includes(normalize(keyword)) ? 1 : 0),
0
)
const score = tagHits * 3 + keywordHits
if (score >= rule.minScore && (tagHits > 0 || keywordHits >= rule.minKeywordHits)) {
scoredDomains.push({ slug: rule.slug, score })
}
}
if (scoredDomains.length > 0) {
return scoredDomains
.sort((a, b) => b.score - a.score)
.slice(0, 3)
.map((item) => item.slug)
}
const fixedProjectType = input.tagSlugs.find((slug) => FALLBACK_BY_FIXED_PROJECT_TYPE[slug])
if (fixedProjectType) {
return [FALLBACK_BY_FIXED_PROJECT_TYPE[fixedProjectType]!]
}
return ['code-dev']
}
async function main() {
console.log('[domain] start sync')
const canonicalTagIdMap = new Map<DomainScenarioPresetSlug, string>()
for (const domainTag of DOMAIN_SCENARIO_PRESET_TAGS) {
const tag = await prisma.tag.upsert({
where: { slug: domainTag.slug },
update: {
name: domainTag.name,
nameEn: domainTag.nameEn,
category: 'DOMAIN_SCENARIO',
},
create: {
name: domainTag.name,
nameEn: domainTag.nameEn,
slug: domainTag.slug,
category: 'DOMAIN_SCENARIO',
},
})
canonicalTagIdMap.set(domainTag.slug, tag.id)
}
const clearedLinks = await prisma.projectTag.deleteMany({
where: {
tagId: {
in: Array.from(canonicalTagIdMap.values()),
},
},
})
const projects = await prisma.project.findMany({
where: { status: 'ACTIVE' },
select: {
id: true,
name: true,
nameEn: true,
description: true,
descriptionEn: true,
tags: {
select: {
tag: {
select: {
slug: true,
},
},
},
},
},
})
const createData: Array<{ projectId: string; tagId: string }> = []
const distribution = new Map<DomainScenarioPresetSlug, number>()
for (const domainTag of DOMAIN_SCENARIO_PRESET_TAGS) {
distribution.set(domainTag.slug, 0)
}
for (const project of projects) {
const tagSlugs = project.tags.map((item) => item.tag.slug)
const domainSlugs = inferDomainSlugs({
name: project.name,
nameEn: project.nameEn,
description: project.description,
descriptionEn: project.descriptionEn,
tagSlugs,
})
for (const domainSlug of domainSlugs) {
const domainTagId = canonicalTagIdMap.get(domainSlug)
if (!domainTagId) {
throw new Error(`Missing canonical domain tag id: ${domainSlug}`)
}
createData.push({
projectId: project.id,
tagId: domainTagId,
})
distribution.set(domainSlug, (distribution.get(domainSlug) || 0) + 1)
}
}
const insertedLinks = await prisma.projectTag.createMany({
data: createData,
skipDuplicates: true,
})
const distributionSummary = DOMAIN_SCENARIO_PRESET_TAGS.map((item) => ({
slug: item.slug,
name: item.name,
projectCount: distribution.get(item.slug) || 0,
}))
console.log('[domain] done', {
canonicalDomainTagCount: DOMAIN_SCENARIO_PRESET_TAGS.length,
clearedCanonicalDomainLinks: clearedLinks.count,
insertedDomainLinks: insertedLinks.count,
activeProjectCount: projects.length,
})
console.log('[domain] distribution', distributionSummary)
}
main()
.catch((error) => {
console.error('[domain] failed', error)
process.exit(1)
})
.finally(async () => {
await prisma.$disconnect()
})