From 359a86f663abe72a1d4eb8b11af0a3b8318a7c9a Mon Sep 17 00:00:00 2001 From: mzaxd Date: Wed, 4 Mar 2026 20:25:54 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=81=BF=E5=85=8Dsignals=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E5=9B=9E=E9=80=80=E5=B9=B6=E4=BC=98=E5=8C=96=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E5=8C=B9=E9=85=8D=E6=9F=A5=E8=AF=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api/discovery/lib/discovery-service.ts | 186 ++++++++++++------ src/app/api/signals/route.ts | 146 +++++++------- src/app/api/webhook/signals/route.ts | 70 ++++--- src/lib/signal-hotness.ts | 60 +++++- 4 files changed, 310 insertions(+), 152 deletions(-) diff --git a/src/app/api/discovery/lib/discovery-service.ts b/src/app/api/discovery/lib/discovery-service.ts index 4f6393b..c97d83e 100644 --- a/src/app/api/discovery/lib/discovery-service.ts +++ b/src/app/api/discovery/lib/discovery-service.ts @@ -145,6 +145,116 @@ function chooseBestTag(candidates: TagWithProjectCount[]): TagWithProjectCount { })[0]! } +type IncomingNormalizedTag = ReturnType + +type TagLookupMaps = { + tagByExactName: Map + tagByExactNameEn: Map + tagBySlug: Map + tagByCanonicalName: Map + tagByCanonicalNameEn: Map +} + +function pushTagMapEntry( + map: Map, + key: string, + tag: TagWithProjectCount +): void { + if (!key) { + return + } + const current = map.get(key) + if (current) { + current.push(tag) + return + } + map.set(key, [tag]) +} + +function createTagLookupMaps(tags: TagWithProjectCount[]): TagLookupMaps { + const tagByExactName = new Map() + const tagByExactNameEn = new Map() + const tagBySlug = new Map() + const tagByCanonicalName = new Map() + const tagByCanonicalNameEn = new Map() + + for (const tag of tags) { + const exactNameKey = normalizeWhitespace(tag.name) + const exactNameEnKey = normalizeWhitespace(tag.nameEn || '') + const canonicalNameKey = canonicalizeTagKey(tag.name) + const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || '') + + pushTagMapEntry(tagByExactName, exactNameKey, tag) + pushTagMapEntry(tagByExactNameEn, exactNameEnKey, tag) + pushTagMapEntry(tagByCanonicalName, canonicalNameKey, tag) + pushTagMapEntry(tagByCanonicalNameEn, canonicalNameEnKey, tag) + tagBySlug.set(tag.slug, tag) + } + + return { + tagByExactName, + tagByExactNameEn, + tagBySlug, + tagByCanonicalName, + tagByCanonicalNameEn, + } +} + +function addTagToLookupMaps(lookups: TagLookupMaps, tag: TagWithProjectCount): void { + const exactNameKey = normalizeWhitespace(tag.name) + const exactNameEnKey = normalizeWhitespace(tag.nameEn || '') + const canonicalNameKey = canonicalizeTagKey(tag.name) + const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || '') + + pushTagMapEntry(lookups.tagByExactName, exactNameKey, tag) + pushTagMapEntry(lookups.tagByExactNameEn, exactNameEnKey, tag) + pushTagMapEntry(lookups.tagByCanonicalName, canonicalNameKey, tag) + pushTagMapEntry(lookups.tagByCanonicalNameEn, canonicalNameEnKey, tag) + lookups.tagBySlug.set(tag.slug, tag) +} + +async function getCandidateTags(incomingTags: IncomingNormalizedTag[]): Promise { + const nameValues = Array.from( + new Set(incomingTags.map((tag) => normalizeWhitespace(tag.name)).filter((name) => name.length > 0)) + ) + const slugValues = Array.from( + new Set(incomingTags.map((tag) => tag.slug).filter((slug) => slug.length > 0)) + ) + const nameEnValues = Array.from( + new Set( + incomingTags + .map((tag) => normalizeWhitespace(tag.nameEn || '')) + .filter((nameEn) => nameEn.length > 0) + ) + ) + + const whereOr: Prisma.TagWhereInput[] = [] + if (nameValues.length > 0) { + whereOr.push({ name: { in: nameValues } }) + } + if (slugValues.length > 0) { + whereOr.push({ slug: { in: slugValues } }) + } + if (nameEnValues.length > 0) { + whereOr.push({ nameEn: { in: nameEnValues } }) + } + + if (whereOr.length === 0) { + return [] + } + + return prisma.tag.findMany({ + where: { + OR: whereOr, + }, + include: { + _count: { + select: { projects: true }, + }, + }, + }) +} + async function getFallbackTag(): Promise { const fallbackSlug = generateSlug(TAG_FALLBACK.name, TAG_FALLBACK.nameEn) const fallbackCategory = inferTagCategory({ @@ -333,68 +443,41 @@ export async function upsertTags(tags: ProjectInput['tags']) { return [await getFallbackTag()] } - const existingTags = await prisma.tag.findMany({ - include: { - _count: { - select: { projects: true }, - }, - }, - }) - - const tagByExactName = new Map() - const tagBySlug = new Map() - const tagByCanonicalName = new Map() - const tagByCanonicalNameEn = new Map() - - for (const tag of existingTags) { - const exactNameKey = normalizeWhitespace(tag.name) - const existingCanonicalName = canonicalizeTagKey(tag.name) - const existingCanonicalNameEn = canonicalizeTagKey(tag.nameEn || '') - - if (!tagByExactName.has(exactNameKey)) { - tagByExactName.set(exactNameKey, []) - } - tagByExactName.get(exactNameKey)!.push(tag) - - if (existingCanonicalName) { - if (!tagByCanonicalName.has(existingCanonicalName)) { - tagByCanonicalName.set(existingCanonicalName, []) - } - tagByCanonicalName.get(existingCanonicalName)!.push(tag) - } - - if (existingCanonicalNameEn) { - if (!tagByCanonicalNameEn.has(existingCanonicalNameEn)) { - tagByCanonicalNameEn.set(existingCanonicalNameEn, []) - } - tagByCanonicalNameEn.get(existingCanonicalNameEn)!.push(tag) - } - - tagBySlug.set(tag.slug, tag) - } + const candidateTags = await getCandidateTags(normalizedIncomingTags) + const lookups = createTagLookupMaps(candidateTags) const resolvedTags: TagWithProjectCount[] = [] for (const incomingTag of normalizedIncomingTags) { - const exactNameCandidates = tagByExactName.get(incomingTag.name) || [] + const exactNameCandidates = lookups.tagByExactName.get(incomingTag.name) || [] let matchedTag = exactNameCandidates.length > 0 ? chooseBestTag(exactNameCandidates) : null + if (!matchedTag && incomingTag.nameEn) { + const exactNameEnCandidates = + lookups.tagByExactNameEn.get(incomingTag.nameEn) || [] + if (exactNameEnCandidates.length > 0) { + matchedTag = chooseBestTag(exactNameEnCandidates) + } + } + if (!matchedTag && incomingTag.canonicalNameKey) { - const canonicalNameCandidates = tagByCanonicalName.get(incomingTag.canonicalNameKey) || [] + const canonicalNameCandidates = + lookups.tagByCanonicalName.get(incomingTag.canonicalNameKey) || [] if (canonicalNameCandidates.length > 0) { matchedTag = chooseBestTag(canonicalNameCandidates) } } if (!matchedTag && incomingTag.canonicalNameEnKey) { - const canonicalNameEnCandidates = tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || [] + const canonicalNameEnCandidates = + lookups.tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || [] if (canonicalNameEnCandidates.length > 0) { matchedTag = chooseBestTag(canonicalNameEnCandidates) } } if (!matchedTag) { - matchedTag = tagBySlug.get(incomingTag.slug) || null + matchedTag = lookups.tagBySlug.get(incomingTag.slug) || null } if (matchedTag) { @@ -422,6 +505,7 @@ export async function upsertTags(tags: ProjectInput['tags']) { }, }) matchedTag = updatedTag + addTagToLookupMaps(lookups, matchedTag) } resolvedTags.push(matchedTag) @@ -449,21 +533,7 @@ export async function upsertTags(tags: ProjectInput['tags']) { }, }) resolvedTags.push(createdTag) - - if (!tagByExactName.has(createdTag.name)) { - tagByExactName.set(createdTag.name, []) - } - tagByExactName.get(createdTag.name)!.push(createdTag) - tagBySlug.set(createdTag.slug, createdTag) - if (createdTag.nameEn) { - const canonicalNameEn = canonicalizeTagKey(createdTag.nameEn) - if (canonicalNameEn) { - if (!tagByCanonicalNameEn.has(canonicalNameEn)) { - tagByCanonicalNameEn.set(canonicalNameEn, []) - } - tagByCanonicalNameEn.get(canonicalNameEn)!.push(createdTag) - } - } + addTagToLookupMaps(lookups, createdTag) } catch { const fallbackTag = await prisma.tag.findFirst({ where: { diff --git a/src/app/api/signals/route.ts b/src/app/api/signals/route.ts index 6eeaa76..b378a11 100644 --- a/src/app/api/signals/route.ts +++ b/src/app/api/signals/route.ts @@ -2,7 +2,12 @@ import { NextRequest, NextResponse } from 'next/server' import { Prisma } from '@prisma/client' import { ZodError } from 'zod' import { prisma } from '@/lib/prisma' -import { computeSignalHotness, isSignalHotColumnMissingError } from '@/lib/signal-hotness' +import { + computeSignalHotness, + isSignalHotColumnMissingError, + markSignalHotColumnsUnsupported, + supportsSignalHotColumns, +} from '@/lib/signal-hotness' import { SignalQuerySchema, type SignalQuery, @@ -34,6 +39,24 @@ interface SignalView { const VALID_SECTION_STYLES: SignalSectionStyle[] = ['focus', 'debate', 'evidence', 'action', 'risk'] +type SignalQueryRow = { + id: string + source: string + sourceUrl: string + title: string + titleEn: string | null + summary: string + summaryEn: string | null + topic: string | null + topicEn: string | null + tags: Prisma.JsonValue + sections: Prisma.JsonValue + engagement: number + hotScore?: number | null + isHot?: boolean | null + publishedAt: Date +} + function isSignalSource(value: string): value is SignalSource { return ( value === 'hacker_news' || @@ -180,6 +203,28 @@ function toSignalView(row: { } } +const SIGNAL_BASE_SELECT = { + id: true, + source: true, + sourceUrl: true, + title: true, + titleEn: true, + summary: true, + summaryEn: true, + topic: true, + topicEn: true, + tags: true, + sections: true, + engagement: true, + publishedAt: true, +} as const + +const SIGNAL_SELECT_WITH_HOT = { + ...SIGNAL_BASE_SELECT, + hotScore: true, + isHot: true, +} as const + export async function GET(request: NextRequest) { try { const { searchParams } = request.nextUrl @@ -227,83 +272,50 @@ export async function GET(request: NextRequest) { : {}), } - const orderBy: Prisma.SignalOrderByWithRelationInput[] = + const hasHotColumns = await supportsSignalHotColumns(prisma) + const orderByWithHot: Prisma.SignalOrderByWithRelationInput[] = parsed.sort === 'hot' ? [{ isHot: 'desc' }, { hotScore: 'desc' }, { engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }] : [{ publishedAt: 'desc' }, { id: 'desc' }] + const orderByFallback: Prisma.SignalOrderByWithRelationInput[] = + parsed.sort === 'hot' + ? [{ engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }] + : [{ publishedAt: 'desc' }, { id: 'desc' }] - let rows: Array<{ - id: string - source: string - sourceUrl: string - title: string - titleEn: string | null - summary: string - summaryEn: string | null - topic: string | null - topicEn: string | null - tags: Prisma.JsonValue - sections: Prisma.JsonValue - engagement: number - hotScore?: number | null - isHot?: boolean | null - publishedAt: Date - }> + let rows: SignalQueryRow[] - try { - rows = await prisma.signal.findMany({ - where, - orderBy, - take: parsed.limit + 1, - cursor: parsed.cursor ? { id: parsed.cursor } : undefined, - skip: parsed.cursor ? 1 : undefined, - select: { - id: true, - source: true, - sourceUrl: true, - title: true, - titleEn: true, - summary: true, - summaryEn: true, - topic: true, - topicEn: true, - tags: true, - sections: true, - engagement: true, - hotScore: true, - isHot: true, - publishedAt: true, - }, - }) - } catch (error) { - if (!isSignalHotColumnMissingError(error)) { - throw error + if (hasHotColumns) { + try { + rows = await prisma.signal.findMany({ + where, + orderBy: orderByWithHot, + take: parsed.limit + 1, + cursor: parsed.cursor ? { id: parsed.cursor } : undefined, + skip: parsed.cursor ? 1 : undefined, + select: SIGNAL_SELECT_WITH_HOT, + }) + } catch (error) { + if (!isSignalHotColumnMissingError(error)) { + throw error + } + markSignalHotColumnsUnsupported() + rows = await prisma.signal.findMany({ + where, + orderBy: orderByFallback, + take: parsed.limit + 1, + cursor: parsed.cursor ? { id: parsed.cursor } : undefined, + skip: parsed.cursor ? 1 : undefined, + select: SIGNAL_BASE_SELECT, + }) } - - const fallbackOrderBy: Prisma.SignalOrderByWithRelationInput[] = - parsed.sort === 'hot' ? [{ engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }] : [{ publishedAt: 'desc' }, { id: 'desc' }] - + } else { rows = await prisma.signal.findMany({ where, - orderBy: fallbackOrderBy, + orderBy: orderByFallback, take: parsed.limit + 1, cursor: parsed.cursor ? { id: parsed.cursor } : undefined, skip: parsed.cursor ? 1 : undefined, - select: { - id: true, - source: true, - sourceUrl: true, - title: true, - titleEn: true, - summary: true, - summaryEn: true, - topic: true, - topicEn: true, - tags: true, - sections: true, - engagement: true, - publishedAt: true, - }, + select: SIGNAL_BASE_SELECT, }) } diff --git a/src/app/api/webhook/signals/route.ts b/src/app/api/webhook/signals/route.ts index 877699b..1693e35 100644 --- a/src/app/api/webhook/signals/route.ts +++ b/src/app/api/webhook/signals/route.ts @@ -2,7 +2,12 @@ import { NextRequest, NextResponse } from 'next/server' import type { Prisma } from '@prisma/client' import { prisma } from '@/lib/prisma' import { isValidApiKey } from '@/lib/auth' -import { computeSignalHotness, isSignalHotColumnMissingError } from '@/lib/signal-hotness' +import { + computeSignalHotness, + isSignalHotColumnMissingError, + markSignalHotColumnsUnsupported, + supportsSignalHotColumns, +} from '@/lib/signal-hotness' import { SignalIngestionInputSchema, SignalWebhookPayloadSchema, @@ -67,6 +72,7 @@ export async function POST(request: NextRequest) { message: string }>, } + let hasHotColumns = await supportsSignalHotColumns(prisma) for (let i = 0; i < payload.signals.length; i++) { const signalData = payload.signals[i] @@ -117,37 +123,49 @@ export async function POST(request: NextRequest) { select: { id: true }, }) - await prisma.signal.upsert({ - where: { - source_sourceUrl: { - source: validSignal.source, - sourceUrl: validSignal.sourceUrl, - }, - }, - update: { - ...baseData, - hotScore, - isHot, - }, - create: { + const upsertWhere = { + source_sourceUrl: { source: validSignal.source, sourceUrl: validSignal.sourceUrl, - ...baseData, - hotScore, - isHot, }, - }).catch(async (error) => { - if (!isSignalHotColumnMissingError(error)) { - throw error - } + } - await prisma.signal.upsert({ - where: { - source_sourceUrl: { + if (hasHotColumns) { + try { + await prisma.signal.upsert({ + where: upsertWhere, + update: { + ...baseData, + hotScore, + isHot, + }, + create: { source: validSignal.source, sourceUrl: validSignal.sourceUrl, + ...baseData, + hotScore, + isHot, }, - }, + }) + } catch (error) { + if (!isSignalHotColumnMissingError(error)) { + throw error + } + markSignalHotColumnsUnsupported() + hasHotColumns = false + await prisma.signal.upsert({ + where: upsertWhere, + update: baseData, + create: { + source: validSignal.source, + sourceUrl: validSignal.sourceUrl, + ...baseData, + }, + }) + } + } else { + await prisma.signal.upsert({ + where: upsertWhere, update: baseData, create: { source: validSignal.source, @@ -155,7 +173,7 @@ export async function POST(request: NextRequest) { ...baseData, }, }) - }) + } if (existing) { results.updated++ diff --git a/src/lib/signal-hotness.ts b/src/lib/signal-hotness.ts index d36df94..22c94e3 100644 --- a/src/lib/signal-hotness.ts +++ b/src/lib/signal-hotness.ts @@ -1,5 +1,5 @@ import type { SignalSource } from '@/lib/validations' -import { Prisma } from '@prisma/client' +import { Prisma, type PrismaClient } from '@prisma/client' interface HotnessInput { source: SignalSource @@ -84,3 +84,61 @@ export function isSignalHotColumnMissingError(error: unknown): boolean { const column = String(error.meta?.column || '') return column.includes('hotScore') || column.includes('isHot') } + +const SIGNAL_HOT_COLUMN_SUPPORT_TTL_MS = 5 * 60 * 1000 + +let signalHotColumnSupportCache: { value: boolean; checkedAt: number } | null = null +let signalHotColumnSupportPending: Promise | null = null + +export function markSignalHotColumnsUnsupported(): void { + signalHotColumnSupportCache = { + value: false, + checkedAt: Date.now(), + } +} + +export async function supportsSignalHotColumns( + prisma: PrismaClient +): Promise { + const now = Date.now() + if ( + signalHotColumnSupportCache && + now - signalHotColumnSupportCache.checkedAt <= SIGNAL_HOT_COLUMN_SUPPORT_TTL_MS + ) { + return signalHotColumnSupportCache.value + } + + if (signalHotColumnSupportPending) { + return signalHotColumnSupportPending + } + + signalHotColumnSupportPending = prisma + .$queryRaw>(Prisma.sql` + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'signals' + AND column_name IN ('hotScore', 'isHot') + `) + .then((rows) => { + const columns = new Set(rows.map((row) => row.column_name)) + const value = columns.has('hotScore') && columns.has('isHot') + signalHotColumnSupportCache = { + value, + checkedAt: Date.now(), + } + return value + }) + .catch(() => { + signalHotColumnSupportCache = { + value: false, + checkedAt: Date.now(), + } + return false + }) + .finally(() => { + signalHotColumnSupportPending = null + }) + + return signalHotColumnSupportPending +}