mzaxd/agent-park
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: 降低数据库读压并优化去重查询

Browse Source
This commit is contained in:
mzaxd
2026-03-04 19:38:14 +08:00
parent 4f6035f341
commit 9e884ea478
5 changed files with 331 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/[locale]/projects/[id]/page.tsx
+5 -2
View File
@@ -4,11 +4,14 @@ import { getProjectBySlug, getProjects } from '@/hooks/useProjects'
import { ProjectDetail } from '@/components/project/ProjectDetail'
import { ProjectSidebar } from '@/components/project/ProjectSidebar'
import { RelatedProjects } from '@/components/project/RelatedProjects'
import { cache } from 'react'
interface ProjectDetailPageProps {
params: Promise<{ locale: string; id: string }>
}
const getCachedProjectBySlug = cache((slug: string) => getProjectBySlug(slug))
export default async function ProjectDetailPage({
params,
}: ProjectDetailPageProps) {
@@ -18,7 +21,7 @@ export default async function ProjectDetailPage({
// 并行获取项目和相关项目数据
const [project, relatedProjectsResult] = await Promise.all([
getProjectBySlug(id),
getCachedProjectBySlug(id),
getProjects({ limit: 3 }),
])
@@ -72,7 +75,7 @@ export async function generateMetadata({ params }: ProjectDetailPageProps) {
const { id, locale } = resolvedParams
const tProject = await getTranslations('project')
const project = await getProjectBySlug(id)
const project = await getCachedProjectBySlug(id)
if (!project) {
return {
src/app/api/tags/route.ts
+17 -4
View File
@@ -1,9 +1,12 @@
import { NextResponse } from "next/server";
import { prisma } from "@/lib/prisma";
import { unstable_cache } from "next/cache";
export async function GET() {
try {
const tags = await prisma.tag.findMany({
const TAGS_CACHE_REVALIDATE_SECONDS = 300;
const getCachedTags = unstable_cache(
async () =>
prisma.tag.findMany({
include: {
_count: {
select: { projects: true },
@@ -12,7 +15,17 @@ export async function GET() {
orderBy: {
name: "asc",
},
});
}),
["api-tags:v1"],
{
revalidate: TAGS_CACHE_REVALIDATE_SECONDS,
tags: ["api-tags"],
}
);
export async function GET() {
try {
const tags = await getCachedTags();
return NextResponse.json({
success: true,
src/app/api/webhook/check-duplicates/route.ts
+189 -64
View File
@@ -15,7 +15,7 @@ const CheckDuplicatesSchema = z.object({
websiteUrl: z.string().url().optional(),
slug: z.string().optional(),
})
),
).min(1).max(100),
})
/**
@@ -42,99 +42,215 @@ interface CheckResult {
projectName?: string
}
/**
* 多级去重策略：检查项目是否已存在
*
* 优先级：
* 1. GitHub URL 完全匹配（最准确）
* 2. Hugging Face URL 完全匹配
* 3. Website URL 完全匹配
* 4. slug 匹配（兜底）
*
* @param project - 项目待检查信息
* @returns 检查结果
*/
async function checkProjectExists(project: {
type CheckProjectInput = {
githubUrl?: string
huggingfaceUrl?: string
websiteUrl?: string
slug?: string
}): Promise<CheckResult> {
// 优先级1: GitHub URL 匹配
if (project.githubUrl) {
const existingByGithub = await prisma.externalLink.findFirst({
where: {
type: 'GITHUB',
url: project.githubUrl,
},
include: {
project: true,
},
})
}
type ProjectSummary = {
id: string
name: string
}
type DuplicateCheckMaps = {
githubMap: Map<string, ProjectSummary>
huggingfaceMap: Map<string, ProjectSummary>
websiteMap: Map<string, ProjectSummary>
slugMap: Map<string, ProjectSummary>
}
function buildLookupKey(project: CheckProjectInput): string {
return [
project.githubUrl || '',
project.huggingfaceUrl || '',
project.websiteUrl || '',
project.slug || '',
].join('|')
}
function toExternalLinkMap(
rows: Array<{ url: string; project: ProjectSummary }>
): Map<string, ProjectSummary> {
const map = new Map<string, ProjectSummary>()
for (const row of rows) {
if (!map.has(row.url)) {
map.set(row.url, row.project)
}
}
return map
}
function toSlugMap(
rows: Array<{ slug: string; id: string; name: string }>
): Map<string, ProjectSummary> {
const map = new Map<string, ProjectSummary>()
for (const row of rows) {
map.set(row.slug, { id: row.id, name: row.name })
}
return map
}
async function buildDuplicateCheckMaps(
projects: CheckProjectInput[]
): Promise<DuplicateCheckMaps> {
const githubUrls = Array.from(
new Set(
projects
.map((project) => project.githubUrl?.trim())
.filter((url): url is string => Boolean(url))
)
)
const huggingfaceUrls = Array.from(
new Set(
projects
.map((project) => project.huggingfaceUrl?.trim())
.filter((url): url is string => Boolean(url))
)
)
const websiteUrls = Array.from(
new Set(
projects
.map((project) => project.websiteUrl?.trim())
.filter((url): url is string => Boolean(url))
)
)
const slugs = Array.from(
new Set(
projects
.map((project) => project.slug?.trim())
.filter((slug): slug is string => Boolean(slug))
)
)
const [githubRows, huggingfaceRows, websiteRows, slugRows] = await Promise.all([
githubUrls.length > 0
? prisma.externalLink.findMany({
where: {
type: 'GITHUB',
url: {
in: githubUrls,
},
},
select: {
url: true,
project: {
select: {
id: true,
name: true,
},
},
},
})
: Promise.resolve([]),
huggingfaceUrls.length > 0
? prisma.externalLink.findMany({
where: {
type: 'HUGGINGFACE',
url: {
in: huggingfaceUrls,
},
},
select: {
url: true,
project: {
select: {
id: true,
name: true,
},
},
},
})
: Promise.resolve([]),
websiteUrls.length > 0
? prisma.externalLink.findMany({
where: {
type: 'WEBSITE',
url: {
in: websiteUrls,
},
},
select: {
url: true,
project: {
select: {
id: true,
name: true,
},
},
},
})
: Promise.resolve([]),
slugs.length > 0
? prisma.project.findMany({
where: {
slug: {
in: slugs,
},
},
select: {
slug: true,
id: true,
name: true,
},
})
: Promise.resolve([]),
])
return {
githubMap: toExternalLinkMap(githubRows),
huggingfaceMap: toExternalLinkMap(huggingfaceRows),
websiteMap: toExternalLinkMap(websiteRows),
slugMap: toSlugMap(slugRows),
}
}
function checkProjectExists(
project: CheckProjectInput,
duplicateMaps: DuplicateCheckMaps
): CheckResult {
if (project.githubUrl) {
const existingByGithub = duplicateMaps.githubMap.get(project.githubUrl.trim())
if (existingByGithub) {
return {
githubUrl: project.githubUrl,
exists: true,
matchType: 'GITHUB_URL',
projectId: existingByGithub.project.id,
projectName: existingByGithub.project.name,
projectId: existingByGithub.id,
projectName: existingByGithub.name,
}
}
}
// 优先级2: Hugging Face URL 匹配
if (project.huggingfaceUrl) {
const existingByHuggingFace = await prisma.externalLink.findFirst({
where: {
type: 'HUGGINGFACE',
url: project.huggingfaceUrl,
},
include: {
project: true,
},
})
const existingByHuggingFace = duplicateMaps.huggingfaceMap.get(project.huggingfaceUrl.trim())
if (existingByHuggingFace) {
return {
huggingfaceUrl: project.huggingfaceUrl,
exists: true,
matchType: 'HUGGINGFACE_URL',
projectId: existingByHuggingFace.project.id,
projectName: existingByHuggingFace.project.name,
projectId: existingByHuggingFace.id,
projectName: existingByHuggingFace.name,
}
}
}
// 优先级3: Website URL 匹配
if (project.websiteUrl) {
const existingByWebsite = await prisma.externalLink.findFirst({
where: {
type: 'WEBSITE',
url: project.websiteUrl,
},
include: {
project: true,
},
})
const existingByWebsite = duplicateMaps.websiteMap.get(project.websiteUrl.trim())
if (existingByWebsite) {
return {
websiteUrl: project.websiteUrl,
exists: true,
matchType: 'WEBSITE_URL',
projectId: existingByWebsite.project.id,
projectName: existingByWebsite.project.name,
projectId: existingByWebsite.id,
projectName: existingByWebsite.name,
}
}
}
// 优先级4: Slug 匹配（兜底）
if (project.slug) {
const existingBySlug = await prisma.project.findUnique({
where: { slug: project.slug },
})
const existingBySlug = duplicateMaps.slugMap.get(project.slug.trim())
if (existingBySlug) {
return {
slug: project.slug,
@@ -146,7 +262,6 @@ async function checkProjectExists(project: {
}
}
// 未找到匹配项
return {
githubUrl: project.githubUrl,
huggingfaceUrl: project.huggingfaceUrl,
@@ -191,10 +306,20 @@ export async function POST(request: NextRequest) {
)
}
// 并行检查所有项目
const results = await Promise.all(
payload.projects.map((project) => checkProjectExists(project))
)
const duplicateMaps = await buildDuplicateCheckMaps(payload.projects)
const cachedResults = new Map<string, CheckResult>()
const results = payload.projects.map((project) => {
const lookupKey = buildLookupKey(project)
const cached = cachedResults.get(lookupKey)
if (cached) {
return cached
}
const result = checkProjectExists(project, duplicateMaps)
cachedResults.set(lookupKey, result)
return result
})
// 统计信息
const stats = {
src/hooks/useHome.ts
+42 -9
View File
@@ -1,11 +1,13 @@
import { prisma } from '@/lib/prisma'
import { getTopTags } from '@/hooks/useProjects'
import { Prisma } from '@prisma/client'
import { unstable_cache } from 'next/cache'
const ONE_DAY_MS = 24 * 60 * 60 * 1000
const DEFAULT_RANKING_LIMIT = 6
const DEFAULT_TIMELINE_LIMIT = 8
const DEFAULT_TOP_TAG_LIMIT = 12
const HOME_PAGE_REVALIDATE_SECONDS = 300
export type HomeProjectSummary = {
id: string
@@ -137,22 +139,30 @@ async function getTopStarsProjects(limit: number): Promise<HomeProjectSummary[]>
return projects.map(mapProjectSummary)
}
export async function getHomePageData(): Promise<HomePageData> {
function getLatestProjectsByWindow(
projects: HomeProjectSummary[],
createdAfter: Date,
limit: number
): HomeProjectSummary[] {
return projects
.filter((project) => new Date(project.createdAt) >= createdAfter)
.slice(0, limit)
}
async function buildHomePageData(): Promise<HomePageData> {
const now = Date.now()
const last24Hours = new Date(now - ONE_DAY_MS)
const last7Days = new Date(now - ONE_DAY_MS * 7)
const last30Days = new Date(now - ONE_DAY_MS * 30)
const latestProjectsLimit = Math.max(DEFAULT_RANKING_LIMIT, DEFAULT_TIMELINE_LIMIT)
const [
totalProjects,
newProjects30d,
newProjects7d,
newProjects24h,
latest24h,
latest7d,
latest30d,
latestProjects,
topStars,
timeline,
topTags,
] = await Promise.all([
safeQuery('countTotalProjects', 0, () => prisma.project.count()),
@@ -186,14 +196,28 @@ export async function getHomePageData(): Promise<HomePageData> {
},
})
),
getLatestProjects(DEFAULT_RANKING_LIMIT, last24Hours),
getLatestProjects(DEFAULT_RANKING_LIMIT, last7Days),
getLatestProjects(DEFAULT_RANKING_LIMIT, last30Days),
getLatestProjects(latestProjectsLimit),
getTopStarsProjects(DEFAULT_RANKING_LIMIT),
getLatestProjects(DEFAULT_TIMELINE_LIMIT),
getTopTags(DEFAULT_TOP_TAG_LIMIT),
])
const latest24h = getLatestProjectsByWindow(
latestProjects,
last24Hours,
DEFAULT_RANKING_LIMIT
)
const latest7d = getLatestProjectsByWindow(
latestProjects,
last7Days,
DEFAULT_RANKING_LIMIT
)
const latest30d = getLatestProjectsByWindow(
latestProjects,
last30Days,
DEFAULT_RANKING_LIMIT
)
const timeline = latestProjects.slice(0, DEFAULT_TIMELINE_LIMIT)
return {
overview: {
totalProjects,
@@ -221,3 +245,12 @@ export async function getHomePageData(): Promise<HomePageData> {
timeline,
}
}
const getCachedHomePageData = unstable_cache(buildHomePageData, ['home-page-data:v1'], {
revalidate: HOME_PAGE_REVALIDATE_SECONDS,
tags: ['home-page-data'],
})
export async function getHomePageData(): Promise<HomePageData> {
return getCachedHomePageData()
}
src/hooks/useProjects.ts
+78 -16
View File
@@ -1,5 +1,6 @@
import { prisma } from '@/lib/prisma'
import { Prisma, type TagCategory } from '@prisma/client'
import { unstable_cache } from 'next/cache'
import {
FIXED_PROJECT_TYPE_TAGS,
TAG_CATEGORY_META,
@@ -95,6 +96,7 @@ export type ProjectSortOption = (typeof PROJECT_SORT_OPTIONS)[number]
const DEFAULT_PAGE = 1
const DEFAULT_LIMIT = 10
const MAX_LIMIT = 100
const DEFAULT_CACHE_REVALIDATE_SECONDS = 300
export function normalizeProjectSort(value?: string): ProjectSortOption {
const candidate = String(value || '').trim()
@@ -365,8 +367,8 @@ export async function getTagsWithProjectCounts(): Promise<TagWithProjectCount[]>
return tags.filter(tag => tag._count.projects > 0)
}
export async function getTopTags(limit: number = 10): Promise<TagWithProjectCount[]> {
const tags = await withDbRetry('getTopTags', () =>
async function getTopTagsFromDb(limit: number): Promise<TagWithProjectCount[]> {
return withDbRetry('getTopTags', () =>
prisma.tag.findMany({
include: {
_count: {
@@ -378,25 +380,41 @@ export async function getTopTags(limit: number = 10): Promise<TagWithProjectCoun
not: 'FIXED_PROJECT_TYPE',
},
projects: {
some: {}, // 只返回有项目的标签
some: {},
},
},
orderBy: [{ projects: { _count: 'desc' } }, { name: 'asc' }],
take: limit,
})
)
// 按项目数量降序、名称升序排序
return tags
.filter(tag => tag._count.projects > 0)
.sort((a, b) => {
const countDiff = b._count.projects - a._count.projects
if (countDiff !== 0) return countDiff
return a.name.localeCompare(b.name, 'zh')
})
.slice(0, limit)
}
export async function getFixedProjectTypeFilters(
status: 'ACTIVE' | 'ARCHIVED' = 'ACTIVE'
const topTagsCache = new Map<number, () => Promise<TagWithProjectCount[]>>()
function getTopTagsCachedFetcher(limit: number): () => Promise<TagWithProjectCount[]> {
const existing = topTagsCache.get(limit)
if (existing) {
return existing
}
const fetcher = unstable_cache(
async () => getTopTagsFromDb(limit),
[`top-tags:${limit}`],
{
revalidate: DEFAULT_CACHE_REVALIDATE_SECONDS,
tags: ['top-tags'],
}
)
topTagsCache.set(limit, fetcher)
return fetcher
}
export async function getTopTags(limit: number = 10): Promise<TagWithProjectCount[]> {
return getTopTagsCachedFetcher(limit)()
}
async function getFixedProjectTypeFiltersFromDb(
status: 'ACTIVE' | 'ARCHIVED'
): Promise<FixedProjectTypeFilter[]> {
let counts: number[] = []
@@ -434,7 +452,38 @@ export async function getFixedProjectTypeFilters(
}))
}
export async function getTagCategoryGroups(): Promise<FilterTagCategoryGroup[]> {
const fixedProjectTypeFilterCache = new Map<
'ACTIVE' | 'ARCHIVED',
() => Promise<FixedProjectTypeFilter[]>
>()
function getFixedProjectTypeFilterCachedFetcher(
status: 'ACTIVE' | 'ARCHIVED'
): () => Promise<FixedProjectTypeFilter[]> {
const existing = fixedProjectTypeFilterCache.get(status)
if (existing) {
return existing
}
const fetcher = unstable_cache(
async () => getFixedProjectTypeFiltersFromDb(status),
[`fixed-project-type-filters:${status}`],
{
revalidate: DEFAULT_CACHE_REVALIDATE_SECONDS,
tags: ['fixed-project-type-filters'],
}
)
fixedProjectTypeFilterCache.set(status, fetcher)
return fetcher
}
export async function getFixedProjectTypeFilters(
status: 'ACTIVE' | 'ARCHIVED' = 'ACTIVE'
): Promise<FixedProjectTypeFilter[]> {
return getFixedProjectTypeFilterCachedFetcher(status)()
}
async function getTagCategoryGroupsFromDb(): Promise<FilterTagCategoryGroup[]> {
let tags: TagWithProjectCount[] = []
try {
@@ -501,6 +550,19 @@ export async function getTagCategoryGroups(): Promise<FilterTagCategoryGroup[]>
.filter((group) => group.tags.length > 0)
}
const getCachedTagCategoryGroups = unstable_cache(
getTagCategoryGroupsFromDb,
['tag-category-groups:v1'],
{
revalidate: DEFAULT_CACHE_REVALIDATE_SECONDS,
tags: ['tag-category-groups'],
}
)
export async function getTagCategoryGroups(): Promise<FilterTagCategoryGroup[]> {
return getCachedTagCategoryGroups()
}
// 定义 AI 搜索结果类型
export type AISearchResultItem = ProjectWithFlatTags & {
similarity: number