fix: 避免signals异常回退并优化标签匹配查询

2026-03-04 20:25:54 +08:00
parent 9e884ea478
commit 359a86f663
4 changed files with 310 additions and 152 deletions
@@ -145,6 +145,116 @@ function chooseBestTag(candidates: TagWithProjectCount[]): TagWithProjectCount {
  })[0]!
 }

+type IncomingNormalizedTag = ReturnType<typeof normalizeIncomingTag>
+
+type TagLookupMaps = {
+  tagByExactName: Map<string, TagWithProjectCount[]>
+  tagByExactNameEn: Map<string, TagWithProjectCount[]>
+  tagBySlug: Map<string, TagWithProjectCount>
+  tagByCanonicalName: Map<string, TagWithProjectCount[]>
+  tagByCanonicalNameEn: Map<string, TagWithProjectCount[]>
+}
+
+function pushTagMapEntry(
+  map: Map<string, TagWithProjectCount[]>,
+  key: string,
+  tag: TagWithProjectCount
+): void {
+  if (!key) {
+    return
+  }
+  const current = map.get(key)
+  if (current) {
+    current.push(tag)
+    return
+  }
+  map.set(key, [tag])
+}
+
+function createTagLookupMaps(tags: TagWithProjectCount[]): TagLookupMaps {
+  const tagByExactName = new Map<string, TagWithProjectCount[]>()
+  const tagByExactNameEn = new Map<string, TagWithProjectCount[]>()
+  const tagBySlug = new Map<string, TagWithProjectCount>()
+  const tagByCanonicalName = new Map<string, TagWithProjectCount[]>()
+  const tagByCanonicalNameEn = new Map<string, TagWithProjectCount[]>()
+
+  for (const tag of tags) {
+    const exactNameKey = normalizeWhitespace(tag.name)
+    const exactNameEnKey = normalizeWhitespace(tag.nameEn || '')
+    const canonicalNameKey = canonicalizeTagKey(tag.name)
+    const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || '')
+
+    pushTagMapEntry(tagByExactName, exactNameKey, tag)
+    pushTagMapEntry(tagByExactNameEn, exactNameEnKey, tag)
+    pushTagMapEntry(tagByCanonicalName, canonicalNameKey, tag)
+    pushTagMapEntry(tagByCanonicalNameEn, canonicalNameEnKey, tag)
+    tagBySlug.set(tag.slug, tag)
+  }
+
+  return {
+    tagByExactName,
+    tagByExactNameEn,
+    tagBySlug,
+    tagByCanonicalName,
+    tagByCanonicalNameEn,
+  }
+}
+
+function addTagToLookupMaps(lookups: TagLookupMaps, tag: TagWithProjectCount): void {
+  const exactNameKey = normalizeWhitespace(tag.name)
+  const exactNameEnKey = normalizeWhitespace(tag.nameEn || '')
+  const canonicalNameKey = canonicalizeTagKey(tag.name)
+  const canonicalNameEnKey = canonicalizeTagKey(tag.nameEn || '')
+
+  pushTagMapEntry(lookups.tagByExactName, exactNameKey, tag)
+  pushTagMapEntry(lookups.tagByExactNameEn, exactNameEnKey, tag)
+  pushTagMapEntry(lookups.tagByCanonicalName, canonicalNameKey, tag)
+  pushTagMapEntry(lookups.tagByCanonicalNameEn, canonicalNameEnKey, tag)
+  lookups.tagBySlug.set(tag.slug, tag)
+}
+
+async function getCandidateTags(incomingTags: IncomingNormalizedTag[]): Promise<TagWithProjectCount[]> {
+  const nameValues = Array.from(
+    new Set(incomingTags.map((tag) => normalizeWhitespace(tag.name)).filter((name) => name.length > 0))
+  )
+  const slugValues = Array.from(
+    new Set(incomingTags.map((tag) => tag.slug).filter((slug) => slug.length > 0))
+  )
+  const nameEnValues = Array.from(
+    new Set(
+      incomingTags
+        .map((tag) => normalizeWhitespace(tag.nameEn || ''))
+        .filter((nameEn) => nameEn.length > 0)
+    )
+  )
+
+  const whereOr: Prisma.TagWhereInput[] = []
+  if (nameValues.length > 0) {
+    whereOr.push({ name: { in: nameValues } })
+  }
+  if (slugValues.length > 0) {
+    whereOr.push({ slug: { in: slugValues } })
+  }
+  if (nameEnValues.length > 0) {
+    whereOr.push({ nameEn: { in: nameEnValues } })
+  }
+
+  if (whereOr.length === 0) {
+    return []
+  }
+
+  return prisma.tag.findMany({
+    where: {
+      OR: whereOr,
+    },
+    include: {
+      _count: {
+        select: { projects: true },
+      },
+    },
+  })
+}
+
 async function getFallbackTag(): Promise<TagWithProjectCount> {
  const fallbackSlug = generateSlug(TAG_FALLBACK.name, TAG_FALLBACK.nameEn)
  const fallbackCategory = inferTagCategory({
@@ -333,68 +443,41 @@ export async function upsertTags(tags: ProjectInput['tags']) {
    return [await getFallbackTag()]
  }

-  const existingTags = await prisma.tag.findMany({
-    include: {
-      _count: {
-        select: { projects: true },
-      },
-    },
-  })
-
-  const tagByExactName = new Map<string, TagWithProjectCount[]>()
-  const tagBySlug = new Map<string, TagWithProjectCount>()
-  const tagByCanonicalName = new Map<string, TagWithProjectCount[]>()
-  const tagByCanonicalNameEn = new Map<string, TagWithProjectCount[]>()
-
-  for (const tag of existingTags) {
-    const exactNameKey = normalizeWhitespace(tag.name)
-    const existingCanonicalName = canonicalizeTagKey(tag.name)
-    const existingCanonicalNameEn = canonicalizeTagKey(tag.nameEn || '')
-
-    if (!tagByExactName.has(exactNameKey)) {
-      tagByExactName.set(exactNameKey, [])
-    }
-    tagByExactName.get(exactNameKey)!.push(tag)
-
-    if (existingCanonicalName) {
-      if (!tagByCanonicalName.has(existingCanonicalName)) {
-        tagByCanonicalName.set(existingCanonicalName, [])
-      }
-      tagByCanonicalName.get(existingCanonicalName)!.push(tag)
-    }
-
-    if (existingCanonicalNameEn) {
-      if (!tagByCanonicalNameEn.has(existingCanonicalNameEn)) {
-        tagByCanonicalNameEn.set(existingCanonicalNameEn, [])
-      }
-      tagByCanonicalNameEn.get(existingCanonicalNameEn)!.push(tag)
-    }
-
-    tagBySlug.set(tag.slug, tag)
-  }
+  const candidateTags = await getCandidateTags(normalizedIncomingTags)
+  const lookups = createTagLookupMaps(candidateTags)

  const resolvedTags: TagWithProjectCount[] = []

  for (const incomingTag of normalizedIncomingTags) {
-    const exactNameCandidates = tagByExactName.get(incomingTag.name) || []
+    const exactNameCandidates = lookups.tagByExactName.get(incomingTag.name) || []
    let matchedTag = exactNameCandidates.length > 0 ? chooseBestTag(exactNameCandidates) : null

+    if (!matchedTag && incomingTag.nameEn) {
+      const exactNameEnCandidates =
+        lookups.tagByExactNameEn.get(incomingTag.nameEn) || []
+      if (exactNameEnCandidates.length > 0) {
+        matchedTag = chooseBestTag(exactNameEnCandidates)
+      }
+    }
+
    if (!matchedTag && incomingTag.canonicalNameKey) {
-      const canonicalNameCandidates = tagByCanonicalName.get(incomingTag.canonicalNameKey) || []
+      const canonicalNameCandidates =
+        lookups.tagByCanonicalName.get(incomingTag.canonicalNameKey) || []
      if (canonicalNameCandidates.length > 0) {
        matchedTag = chooseBestTag(canonicalNameCandidates)
      }
    }

    if (!matchedTag && incomingTag.canonicalNameEnKey) {
-      const canonicalNameEnCandidates = tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || []
+      const canonicalNameEnCandidates =
+        lookups.tagByCanonicalNameEn.get(incomingTag.canonicalNameEnKey) || []
      if (canonicalNameEnCandidates.length > 0) {
        matchedTag = chooseBestTag(canonicalNameEnCandidates)
      }
    }

    if (!matchedTag) {
-      matchedTag = tagBySlug.get(incomingTag.slug) || null
+      matchedTag = lookups.tagBySlug.get(incomingTag.slug) || null
    }

    if (matchedTag) {
@@ -422,6 +505,7 @@ export async function upsertTags(tags: ProjectInput['tags']) {
          },
        })
        matchedTag = updatedTag
+        addTagToLookupMaps(lookups, matchedTag)
      }

      resolvedTags.push(matchedTag)
@@ -449,21 +533,7 @@ export async function upsertTags(tags: ProjectInput['tags']) {
        },
      })
      resolvedTags.push(createdTag)
-
-      if (!tagByExactName.has(createdTag.name)) {
-        tagByExactName.set(createdTag.name, [])
-      }
-      tagByExactName.get(createdTag.name)!.push(createdTag)
-      tagBySlug.set(createdTag.slug, createdTag)
-      if (createdTag.nameEn) {
-        const canonicalNameEn = canonicalizeTagKey(createdTag.nameEn)
-        if (canonicalNameEn) {
-          if (!tagByCanonicalNameEn.has(canonicalNameEn)) {
-            tagByCanonicalNameEn.set(canonicalNameEn, [])
-          }
-          tagByCanonicalNameEn.get(canonicalNameEn)!.push(createdTag)
-        }
-      }
+      addTagToLookupMaps(lookups, createdTag)
    } catch {
      const fallbackTag = await prisma.tag.findFirst({
        where: {
@@ -2,7 +2,12 @@ import { NextRequest, NextResponse } from 'next/server'
 import { Prisma } from '@prisma/client'
 import { ZodError } from 'zod'
 import { prisma } from '@/lib/prisma'
-import { computeSignalHotness, isSignalHotColumnMissingError } from '@/lib/signal-hotness'
+import {
+  computeSignalHotness,
+  isSignalHotColumnMissingError,
+  markSignalHotColumnsUnsupported,
+  supportsSignalHotColumns,
+} from '@/lib/signal-hotness'
 import {
  SignalQuerySchema,
  type SignalQuery,
@@ -34,6 +39,24 @@ interface SignalView {

 const VALID_SECTION_STYLES: SignalSectionStyle[] = ['focus', 'debate', 'evidence', 'action', 'risk']

+type SignalQueryRow = {
+  id: string
+  source: string
+  sourceUrl: string
+  title: string
+  titleEn: string | null
+  summary: string
+  summaryEn: string | null
+  topic: string | null
+  topicEn: string | null
+  tags: Prisma.JsonValue
+  sections: Prisma.JsonValue
+  engagement: number
+  hotScore?: number | null
+  isHot?: boolean | null
+  publishedAt: Date
+}
+
 function isSignalSource(value: string): value is SignalSource {
  return (
    value === 'hacker_news' ||
@@ -180,6 +203,28 @@ function toSignalView(row: {
  }
 }

+const SIGNAL_BASE_SELECT = {
+  id: true,
+  source: true,
+  sourceUrl: true,
+  title: true,
+  titleEn: true,
+  summary: true,
+  summaryEn: true,
+  topic: true,
+  topicEn: true,
+  tags: true,
+  sections: true,
+  engagement: true,
+  publishedAt: true,
+} as const
+
+const SIGNAL_SELECT_WITH_HOT = {
+  ...SIGNAL_BASE_SELECT,
+  hotScore: true,
+  isHot: true,
+} as const
+
 export async function GET(request: NextRequest) {
  try {
    const { searchParams } = request.nextUrl
@@ -227,83 +272,50 @@ export async function GET(request: NextRequest) {
        : {}),
    }

-    const orderBy: Prisma.SignalOrderByWithRelationInput[] =
+    const hasHotColumns = await supportsSignalHotColumns(prisma)
+    const orderByWithHot: Prisma.SignalOrderByWithRelationInput[] =
      parsed.sort === 'hot'
        ? [{ isHot: 'desc' }, { hotScore: 'desc' }, { engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }]
        : [{ publishedAt: 'desc' }, { id: 'desc' }]
+    const orderByFallback: Prisma.SignalOrderByWithRelationInput[] =
+      parsed.sort === 'hot'
+        ? [{ engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }]
+        : [{ publishedAt: 'desc' }, { id: 'desc' }]

-    let rows: Array<{
-      id: string
-      source: string
-      sourceUrl: string
-      title: string
-      titleEn: string | null
-      summary: string
-      summaryEn: string | null
-      topic: string | null
-      topicEn: string | null
-      tags: Prisma.JsonValue
-      sections: Prisma.JsonValue
-      engagement: number
-      hotScore?: number | null
-      isHot?: boolean | null
-      publishedAt: Date
-    }>
+    let rows: SignalQueryRow[]

-    try {
-      rows = await prisma.signal.findMany({
-        where,
-        orderBy,
-        take: parsed.limit + 1,
-        cursor: parsed.cursor ? { id: parsed.cursor } : undefined,
-        skip: parsed.cursor ? 1 : undefined,
-        select: {
-          id: true,
-          source: true,
-          sourceUrl: true,
-          title: true,
-          titleEn: true,
-          summary: true,
-          summaryEn: true,
-          topic: true,
-          topicEn: true,
-          tags: true,
-          sections: true,
-          engagement: true,
-          hotScore: true,
-          isHot: true,
-          publishedAt: true,
-        },
-      })
-    } catch (error) {
-      if (!isSignalHotColumnMissingError(error)) {
-        throw error
+    if (hasHotColumns) {
+      try {
+        rows = await prisma.signal.findMany({
+          where,
+          orderBy: orderByWithHot,
+          take: parsed.limit + 1,
+          cursor: parsed.cursor ? { id: parsed.cursor } : undefined,
+          skip: parsed.cursor ? 1 : undefined,
+          select: SIGNAL_SELECT_WITH_HOT,
+        })
+      } catch (error) {
+        if (!isSignalHotColumnMissingError(error)) {
+          throw error
+        }
+        markSignalHotColumnsUnsupported()
+        rows = await prisma.signal.findMany({
+          where,
+          orderBy: orderByFallback,
+          take: parsed.limit + 1,
+          cursor: parsed.cursor ? { id: parsed.cursor } : undefined,
+          skip: parsed.cursor ? 1 : undefined,
+          select: SIGNAL_BASE_SELECT,
+        })
      }
-
-      const fallbackOrderBy: Prisma.SignalOrderByWithRelationInput[] =
-        parsed.sort === 'hot' ? [{ engagement: 'desc' }, { publishedAt: 'desc' }, { id: 'desc' }] : [{ publishedAt: 'desc' }, { id: 'desc' }]
-
+    } else {
      rows = await prisma.signal.findMany({
        where,
-        orderBy: fallbackOrderBy,
+        orderBy: orderByFallback,
        take: parsed.limit + 1,
        cursor: parsed.cursor ? { id: parsed.cursor } : undefined,
        skip: parsed.cursor ? 1 : undefined,
-        select: {
-          id: true,
-          source: true,
-          sourceUrl: true,
-          title: true,
-          titleEn: true,
-          summary: true,
-          summaryEn: true,
-          topic: true,
-          topicEn: true,
-          tags: true,
-          sections: true,
-          engagement: true,
-          publishedAt: true,
-        },
+        select: SIGNAL_BASE_SELECT,
      })
    }

@@ -2,7 +2,12 @@ import { NextRequest, NextResponse } from 'next/server'
 import type { Prisma } from '@prisma/client'
 import { prisma } from '@/lib/prisma'
 import { isValidApiKey } from '@/lib/auth'
-import { computeSignalHotness, isSignalHotColumnMissingError } from '@/lib/signal-hotness'
+import {
+  computeSignalHotness,
+  isSignalHotColumnMissingError,
+  markSignalHotColumnsUnsupported,
+  supportsSignalHotColumns,
+} from '@/lib/signal-hotness'
 import {
  SignalIngestionInputSchema,
  SignalWebhookPayloadSchema,
@@ -67,6 +72,7 @@ export async function POST(request: NextRequest) {
        message: string
      }>,
    }
+    let hasHotColumns = await supportsSignalHotColumns(prisma)

    for (let i = 0; i < payload.signals.length; i++) {
      const signalData = payload.signals[i]
@@ -117,37 +123,49 @@ export async function POST(request: NextRequest) {
          select: { id: true },
        })

-        await prisma.signal.upsert({
-          where: {
-            source_sourceUrl: {
-              source: validSignal.source,
-              sourceUrl: validSignal.sourceUrl,
-            },
-          },
-          update: {
-            ...baseData,
-            hotScore,
-            isHot,
-          },
-          create: {
+        const upsertWhere = {
+          source_sourceUrl: {
            source: validSignal.source,
            sourceUrl: validSignal.sourceUrl,
-            ...baseData,
-            hotScore,
-            isHot,
          },
-        }).catch(async (error) => {
-          if (!isSignalHotColumnMissingError(error)) {
-            throw error
-          }
+        }

-          await prisma.signal.upsert({
-            where: {
-              source_sourceUrl: {
+        if (hasHotColumns) {
+          try {
+            await prisma.signal.upsert({
+              where: upsertWhere,
+              update: {
+                ...baseData,
+                hotScore,
+                isHot,
+              },
+              create: {
                source: validSignal.source,
                sourceUrl: validSignal.sourceUrl,
+                ...baseData,
+                hotScore,
+                isHot,
              },
-            },
+            })
+          } catch (error) {
+            if (!isSignalHotColumnMissingError(error)) {
+              throw error
+            }
+            markSignalHotColumnsUnsupported()
+            hasHotColumns = false
+            await prisma.signal.upsert({
+              where: upsertWhere,
+              update: baseData,
+              create: {
+                source: validSignal.source,
+                sourceUrl: validSignal.sourceUrl,
+                ...baseData,
+              },
+            })
+          }
+        } else {
+          await prisma.signal.upsert({
+            where: upsertWhere,
            update: baseData,
            create: {
              source: validSignal.source,
@@ -155,7 +173,7 @@ export async function POST(request: NextRequest) {
              ...baseData,
            },
          })
-        })
+        }

        if (existing) {
          results.updated++
@@ -1,5 +1,5 @@
 import type { SignalSource } from '@/lib/validations'
-import { Prisma } from '@prisma/client'
+import { Prisma, type PrismaClient } from '@prisma/client'

 interface HotnessInput {
  source: SignalSource
@@ -84,3 +84,61 @@ export function isSignalHotColumnMissingError(error: unknown): boolean {
  const column = String(error.meta?.column || '')
  return column.includes('hotScore') || column.includes('isHot')
 }
+
+const SIGNAL_HOT_COLUMN_SUPPORT_TTL_MS = 5 * 60 * 1000
+
+let signalHotColumnSupportCache: { value: boolean; checkedAt: number } | null = null
+let signalHotColumnSupportPending: Promise<boolean> | null = null
+
+export function markSignalHotColumnsUnsupported(): void {
+  signalHotColumnSupportCache = {
+    value: false,
+    checkedAt: Date.now(),
+  }
+}
+
+export async function supportsSignalHotColumns(
+  prisma: PrismaClient
+): Promise<boolean> {
+  const now = Date.now()
+  if (
+    signalHotColumnSupportCache &&
+    now - signalHotColumnSupportCache.checkedAt <= SIGNAL_HOT_COLUMN_SUPPORT_TTL_MS
+  ) {
+    return signalHotColumnSupportCache.value
+  }
+
+  if (signalHotColumnSupportPending) {
+    return signalHotColumnSupportPending
+  }
+
+  signalHotColumnSupportPending = prisma
+    .$queryRaw<Array<{ column_name: string }>>(Prisma.sql`
+      SELECT column_name
+      FROM information_schema.columns
+      WHERE table_schema = 'public'
+        AND table_name = 'signals'
+        AND column_name IN ('hotScore', 'isHot')
+    `)
+    .then((rows) => {
+      const columns = new Set(rows.map((row) => row.column_name))
+      const value = columns.has('hotScore') && columns.has('isHot')
+      signalHotColumnSupportCache = {
+        value,
+        checkedAt: Date.now(),
+      }
+      return value
+    })
+    .catch(() => {
+      signalHotColumnSupportCache = {
+        value: false,
+        checkedAt: Date.now(),
+      }
+      return false
+    })
+    .finally(() => {
+      signalHotColumnSupportPending = null
+    })
+
+  return signalHotColumnSupportPending
+}