feat: 新增免费演出雷达工具与定投评估功能，优化周报提醒逻辑

1. 新增免费演出雷达工具：含秀动/猫眼/大麦平台抓取、本地代理鉴权配置 2. 新增建行定投评估模块：四大指数PE-TTM估值与定投档位建议 3. 新增推送告警白名单过滤功能，限制推送指定指数告警 4. 重构周报首页提醒逻辑，移至服务端处理并减少前端请求 5. 新增多套调试探针脚本与单元测试用例
2026-08-04 17:33:14 +08:00
parent af5d0f43a5
commit 6a7a2b089a
25 changed files with 3499 additions and 121 deletions
@@ -0,0 +1,486 @@
+// ============================================================
+// 免费演出雷达 · 本地抓取代理（零依赖单文件 · v2 常驻浏览器版）
+//
+// 用途：大麦对服务器 IP（尤其阿里云 ECS）风控极严，
+//       本代理在家里电脑运行，用家庭宽带 IP 抓大麦，
+//       工具箱页面（127.0.0.1 探测）自动把大麦请求路由到这里。
+//
+// 用法：node free_show_local_agent.js [端口，默认 8978] [--headless] [--cdp <端口，默认 9222>]
+// 依赖：仅 Node 内置模块 + 本机已安装的 Chrome 或 Edge（无需 npm install）
+//
+// v2 工作方式（解决 headless 被大麦风控识别的问题）：
+//   ① 启动代理 → 自动唤起一个【可视化】原生 Edge 窗口（默认不 headless），
+//      使用持久化用户配置目录（Cookie / 登录态 / 过验证状态落盘保存）；
+//   ② 首次使用请人工在该窗口完成滑块验证 / 登录（只需一次，会话自动保存）；
+//   ③ 代理通过 CDP（remote-debugging-port）复用【同一个浏览器实例】执行抓取，
+//      每次任务只开/关标签页，绝不新建空白浏览器环境、不重启浏览器进程；
+//   ④ 抓取前等待异步列表渲染完成；若检测到滑块弹窗，保留标签页并明确提示
+//      「需要人工完成滑块验证」，由你在可见窗口处理后重试。
+//
+// 反自动化特征参数（内置）：
+//   --disable-blink-features=AutomationControlled
+//   --exclude-switches=enable-automation
+// ============================================================
+'use strict'
+
+const http = require('http')
+const net = require('net')
+const fs = require('fs')
+const os = require('os')
+const path = require('path')
+const crypto = require('crypto')
+const { spawn } = require('child_process')
+
+// ============ CLI 参数 ============
+const argv = process.argv.slice(2)
+let PORT = 8978
+let CDP_PORT = 9222
+let HEADLESS = false // 默认可视化窗口；传 --headless 才无头（兼容旧模式）
+for (let i = 0; i < argv.length; i++) {
+  const a = argv[i]
+  if (a === '--headless') HEADLESS = true
+  else if (a === '--cdp' && argv[i + 1]) CDP_PORT = Number(argv[++i]) || 9222
+  else if (/^\d+$/.test(a)) PORT = Number(a)
+}
+
+const DAMAI_CITY_CODE = '310100' // 仅上海（业务约定）
+const START_URL = 'https://search.damai.cn/search.htm?city=' + DAMAI_CITY_CODE
+// 持久化配置目录：放在脚本同级，Cookie/登录态/验证状态长期保存
+const PROFILE_DIR = path.join(__dirname, 'edge_profile_free_show')
+
+const sleep = ms => new Promise(r => setTimeout(r, ms))
+
+// ============ 浏览器探测 ============
+
+function findBrowser() {
+  const candidates = [
+    'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
+    'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
+    'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+    'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe'
+  ]
+  for (const p of candidates) {
+    if (fs.existsSync(p)) return p
+  }
+  return null
+}
+
+// ============ CDP HTTP 端点 ============
+
+function cdpHttp(method, pathAndQuery, timeoutMs) {
+  return new Promise((resolve, reject) => {
+    const req = http.request({
+      host: '127.0.0.1', port: CDP_PORT, path: pathAndQuery, method,
+      timeout: timeoutMs || 5000
+    }, res => {
+      let body = ''
+      res.on('data', c => { body += c })
+      res.on('end', () => {
+        try { resolve(JSON.parse(body)) } catch (e) { reject(new Error('CDP 响应解析失败')) }
+      })
+    })
+    req.on('timeout', () => req.destroy(new Error('CDP 连接超时')))
+    req.on('error', reject)
+    req.end()
+  })
+}
+
+const cdpVersion = () => cdpHttp('GET', '/json/version').catch(() => null)
+
+// ============ 浏览器生命周期：只拉起一次，长期复用 ============
+
+async function ensureBrowser() {
+  const ver = await cdpVersion()
+  if (ver) return { up: true, reused: true }
+
+  const browserPath = findBrowser()
+  if (!browserPath) throw new Error('未找到本机 Chrome/Edge，无法启动抓取浏览器')
+
+  const args = [
+    '--remote-debugging-port=' + CDP_PORT,
+    '--user-data-dir=' + PROFILE_DIR,
+    '--disable-blink-features=AutomationControlled', // 反自动化特征
+    '--exclude-switches=enable-automation',           // 去掉"受自动化控制"提示条
+    '--no-first-run',
+    '--no-default-browser-check'
+  ]
+  if (HEADLESS) args.push('--headless=new', '--disable-gpu')
+  args.push(START_URL) // 首次直接打开大麦上海列表页，方便人工过滑块/登录
+
+  const child = spawn(browserPath, args, { detached: true, stdio: 'ignore' })
+  child.unref() // 代理退出后浏览器继续活着，会话不丢
+
+  for (let i = 0; i < 30; i++) {
+    await sleep(500)
+    const v = await cdpVersion()
+    if (v) return { up: true, reused: false }
+  }
+  throw new Error('浏览器已拉起但调试端口未就绪（' + CDP_PORT + ' 被占用或启动失败）')
+}
+
+async function openTab(url) {
+  const p = '/json/new?' + encodeURIComponent(url)
+  try { return await cdpHttp('PUT', p) } catch (e) { return await cdpHttp('GET', p) }
+}
+
+const closeTab = id => cdpHttp('GET', '/json/close/' + id).catch(() => null)
+
+// ============ 最小 WebSocket 客户端（Node 内置 net/crypto 手写，RFC6455） ============
+
+function parseWsFrame(buf) {
+  if (buf.length < 2) return null
+  const fin = (buf[0] & 0x80) !== 0
+  const opcode = buf[0] & 0x0f
+  const masked = (buf[1] & 0x80) !== 0
+  let len = buf[1] & 0x7f
+  let off = 2
+  if (len === 126) {
+    if (buf.length < 4) return null
+    len = buf.readUInt16BE(2); off = 4
+  } else if (len === 127) {
+    if (buf.length < 10) return null
+    len = Number(buf.readBigUInt64BE(2)); off = 10
+  }
+  let mask = null
+  if (masked) {
+    if (buf.length < off + 4) return null
+    mask = buf.slice(off, off + 4); off += 4
+  }
+  if (buf.length < off + len) return null
+  let payload = buf.slice(off, off + len)
+  if (mask) {
+    payload = Buffer.from(payload)
+    for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i % 4]
+  }
+  return { fin, opcode, payload, total: off + len }
+}
+
+function encodeWsFrame(text) {
+  const payload = Buffer.from(text, 'utf8')
+  const len = payload.length
+  let header
+  if (len < 126) header = Buffer.from([0x81, 0x80 | len])
+  else if (len < 65536) { header = Buffer.from([0x81, 0x80 | 126, 0, 0]); header.writeUInt16BE(len, 2) }
+  else { header = Buffer.from([0x81, 0x80 | 127, 0, 0, 0, 0, 0, 0, 0, 0]); header.writeBigUInt64BE(BigInt(len), 2) }
+  const mask = crypto.randomBytes(4)
+  const masked = Buffer.from(payload)
+  for (let i = 0; i < masked.length; i++) masked[i] ^= mask[i % 4]
+  return Buffer.concat([header, mask, masked])
+}
+
+function wsConnect(wsUrl) {
+  return new Promise((resolve, reject) => {
+    const u = new URL(wsUrl)
+    const sock = net.connect(Number(u.port) || 80, u.hostname || '127.0.0.1')
+    const key = crypto.randomBytes(16).toString('base64')
+    const handshake = [
+      'GET ' + (u.pathname || '/') + (u.search || '') + ' HTTP/1.1',
+      'Host: ' + u.host,
+      'Upgrade: websocket',
+      'Connection: Upgrade',
+      'Sec-WebSocket-Key: ' + key,
+      'Sec-WebSocket-Version: 13',
+      '', ''
+    ].join('\r\n')
+
+    let upgraded = false
+    let buf = Buffer.alloc(0)
+    let fragBuf = null
+    const handlers = { message: null, close: null }
+    const client = {
+      send(text) { try { sock.write(encodeWsFrame(text)) } catch (e) {} },
+      close() { try { sock.end() } catch (e) {} },
+      onMessage(fn) { handlers.message = fn },
+      onClose(fn) { handlers.close = fn }
+    }
+
+    sock.on('connect', () => sock.write(handshake))
+    sock.on('data', chunk => {
+      buf = Buffer.concat([buf, chunk])
+      if (!upgraded) {
+        const idx = buf.indexOf('\r\n\r\n')
+        if (idx < 0) return
+        const head = buf.slice(0, idx).toString()
+        if (!/^HTTP\/1\.1\s+101/.test(head)) {
+          reject(new Error('WebSocket 握手失败'))
+          sock.end()
+          return
+        }
+        upgraded = true
+        buf = buf.slice(idx + 4)
+        resolve(client)
+      }
+      while (true) {
+        const frame = parseWsFrame(buf)
+        if (!frame) break
+        buf = buf.slice(frame.total)
+        if (frame.opcode === 0x8) { if (handlers.close) handlers.close(); return }
+        if (frame.opcode === 0x9) { sock.write(Buffer.from([0x8a, 0x80, 0, 0, 0, 0])); continue } // ping→pong
+        if (frame.opcode === 0xA) continue // pong
+        if (frame.opcode === 0x1 || frame.opcode === 0x2) {
+          if (frame.fin) {
+            const full = fragBuf ? Buffer.concat([fragBuf, frame.payload]) : frame.payload
+            fragBuf = null
+            if (handlers.message) handlers.message(full.toString('utf8'))
+          } else fragBuf = frame.payload
+        } else if (frame.opcode === 0x0) {
+          fragBuf = fragBuf ? Buffer.concat([fragBuf, frame.payload]) : frame.payload
+          if (frame.fin) {
+            const full = fragBuf
+            fragBuf = null
+            if (handlers.message) handlers.message(full.toString('utf8'))
+          }
+        }
+      }
+    })
+    sock.on('error', e => { if (!upgraded) reject(e); else if (handlers.close) handlers.close() })
+    sock.on('close', () => { if (handlers.close) handlers.close() })
+  })
+}
+
+// ============ CDP 会话封装 ============
+
+function makeCdp(ws) {
+  let nextId = 1
+  const pending = new Map()
+  ws.onMessage(text => {
+    let msg = null
+    try { msg = JSON.parse(text) } catch (e) { return }
+    if (msg && msg.id && pending.has(msg.id)) {
+      const cb = pending.get(msg.id)
+      pending.delete(msg.id)
+      cb(msg)
+    }
+  })
+  return {
+    send(method, params, timeoutMs) {
+      return new Promise((resolve, reject) => {
+        const id = nextId++
+        const timer = setTimeout(() => {
+          if (pending.has(id)) { pending.delete(id); reject(new Error('CDP 超时: ' + method)) }
+        }, timeoutMs || 30000)
+        pending.set(id, msg => { clearTimeout(timer); resolve(msg) })
+        ws.send(JSON.stringify({ id, method, params: params || {} }))
+      })
+    }
+  }
+}
+
+// 页面状态探测：滑块特征 / 列表渲染完成特征
+const CHECK_EXPR = '(() => { try { const txt = (document.body && document.body.innerText) ? document.body.innerText.slice(0, 3000) : ""; const slider = !!(document.querySelector("#nc_1_wrapper, .nc-container, #baxia-dialog-content, #nocaptcha, .slidetounlock")) || /拖动滑块|请按照说明拖动滑块|滑动验证/.test(txt) || /rgv587|punish/.test(location.href); const ready = document.querySelectorAll("a[href*=\'detail.damai.cn\']").length > 0 || /共\\s*\\d+\\s*个商品/.test(txt); return { slider: slider, ready: ready }; } catch (e) { return { slider: false, ready: false }; } })()'
+
+async function waitForListOrSlider(cdp, timeoutMs) {
+  const t0 = Date.now()
+  while (Date.now() - t0 < timeoutMs) {
+    try {
+      const r = await cdp.send('Runtime.evaluate', { expression: CHECK_EXPR, returnByValue: true }, 5000)
+      const v = r && r.result && r.result.result ? r.result.result.value : null
+      if (v) {
+        if (v.slider) return { slider: true }
+        if (v.ready) return { ready: true }
+      }
+    } catch (e) { /* 页面导航中 evaluate 可能短暂失败，忽略后重试 */ }
+    await sleep(600)
+  }
+  return { timeout: true }
+}
+
+// ============ 大麦 DOM 解析（字符串版，无 DOM 库） ============
+
+// 价格提取：大麦列表价格形如 "130-2620元" / "380.0元" / "¥130"（无 ¥ 符号是常态）
+// 区间价取下限用于 ≤ 过滤；返回 { price, priceText }，提取不到返回 null
+function extractPrice(win) {
+  let text = ''
+  const blockM = win.match(/class="[^"]*price[^"]*"[^>]*>([\s\S]{0,120}?)<\/(?:div|span|p|em|b|i)>/i)
+  if (blockM) text = blockM[1].replace(/<[^>]+>/g, '').trim()
+  if (!text) {
+    const symM = win.match(/[¥￥]\s*[\d.]+(?:\s*[-~至]\s*[\d.]+)?/)
+    if (symM) text = symM[0]
+  }
+  if (!text) {
+    const yuanM = win.match(/[\d.]+(?:\s*[-~至]\s*[\d.]+)?\s*元/)
+    if (yuanM) text = yuanM[0]
+  }
+  if (!text) return null
+  const rangeM = text.match(/([\d.]+)\s*[-~至]\s*[\d.]+/)
+  const singleM = text.match(/([\d.]+)/)
+  const low = rangeM ? parseFloat(rangeM[1]) : (singleM ? parseFloat(singleM[1]) : NaN)
+  if (!isFinite(low)) return null
+  return { price: low, priceText: text.replace(/\s+/g, '') }
+}
+
+function parseDamaiHtml(html, city) {
+  if (/rgv587|punish|captcha|滑动验证|拖动滑块/i.test(html)) {
+    throw new Error('需要人工完成滑块验证：请在自动打开的 Edge 窗口中完成滑块/登录（会话将自动保存），完成后重试')
+  }
+  const events = []
+  const seen = new Set()
+  const anchorRe = /<a\b([^>]*)>/g
+  let m
+  while ((m = anchorRe.exec(html))) {
+    const attrs = m[1]
+    const hrefM = attrs.match(/href="([^"]+)"/)
+    if (!hrefM) continue
+    let href = hrefM[1]
+    const idM = href.match(/id=(\d+)/)
+    if (!/damai\.cn/.test(href) || !idM) continue
+    if (href.startsWith('//')) href = 'https:' + href
+
+    const titleM = attrs.match(/title="([^"]+)"/)
+    let title = titleM ? titleM[1] : ''
+    const win = html.substr(m.index, 3000)
+    if (!title) {
+      const t2 = win.match(/<[^>]*class="[^"]*title[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/)
+      if (t2) title = t2[1].replace(/<[^>]+>/g, '').trim()
+    }
+    if (!title || seen.has(title)) continue
+    seen.add(title)
+
+    const priceInfo = extractPrice(win)
+    const timeM = win.match(/\d{4}[.\-/]\d{1,2}[.\-/]\d{1,2}[^<|]*/)
+    const imgM = win.match(/<img[^>]*src="([^"]+)"/)
+
+    events.push({
+      platform: 'damai',
+      id: idM[1],
+      title,
+      price: priceInfo ? priceInfo.price : 9999,
+      priceText: priceInfo ? priceInfo.priceText : '价格待定',
+      showTime: timeM ? timeM[0].trim() : '',
+      venue: '',
+      city,
+      poster: imgM ? imgM[1] : '',
+      url: href
+    })
+  }
+  return events
+}
+
+// ============ 抓取主流程：复用常驻浏览器，只开/关标签页 ============
+
+async function fetchDamai(city, pages) {
+  await ensureBrowser()
+
+  const maxPages = Math.min(Number(pages) || 2, 3)
+  let allEvents = []
+  for (let p = 1; p <= maxPages; p++) {
+    const url = `https://search.damai.cn/search.htm?keyword=&city=${DAMAI_CITY_CODE}&ctl=0&order=1&page=${p}`
+    // 先开空白标签再 Page.navigate，避免对大麦页面发起两次请求
+    const tab = await openTab('about:blank')
+    if (!tab || !tab.webSocketDebuggerUrl) throw new Error('无法在常驻浏览器中打开新标签页')
+
+    let ws = null
+    try {
+      ws = await wsConnect(tab.webSocketDebuggerUrl)
+      const cdp = makeCdp(ws)
+      await cdp.send('Page.enable').catch(() => {})
+      await cdp.send('Runtime.enable').catch(() => {})
+      await cdp.send('Page.navigate', { url })
+
+      // 等待异步列表渲染完成（单页最多 20s）
+      const state = await waitForListOrSlider(cdp, 20000)
+      if (state.slider) {
+        // 标签页保留在可见窗口，交给人工拖滑块；会话过验证后自动落盘
+        try { ws.close() } catch (e) {}
+        return { events: allEvents, sliderNeeded: true, sliderPage: p }
+      }
+
+      const htmlRes = await cdp.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true })
+      const html = htmlRes && htmlRes.result && htmlRes.result.result ? String(htmlRes.result.result.value || '') : ''
+      try { ws.close() } catch (e) {}
+      await closeTab(tab.id)
+
+      if (!html) { if (p === 1) throw new Error('页面 DOM 为空（可能仍在加载或被拦截）'); break }
+      const items = parseDamaiHtml(html, city)
+      if (items.length === 0) break
+      allEvents = allEvents.concat(items)
+    } catch (e) {
+      try { if (ws) ws.close() } catch (e2) {}
+      await closeTab(tab.id)
+      throw e
+    }
+  }
+  return { events: allEvents, sliderNeeded: false, sliderPage: 0 }
+}
+
+// ============ HTTP 服务（仅 127.0.0.1） ============
+
+function sendJson(res, code, obj) {
+  const body = JSON.stringify(obj)
+  res.writeHead(code, {
+    'Content-Type': 'application/json; charset=utf-8',
+    'Access-Control-Allow-Origin': '*',
+    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
+    'Access-Control-Allow-Headers': 'Content-Type'
+  })
+  res.end(body)
+}
+
+const server = http.createServer(async (req, res) => {
+  const u = new URL(req.url, `http://127.0.0.1:${PORT}`)
+
+  if (req.method === 'OPTIONS') return sendJson(res, 204, {})
+
+  if (u.pathname === '/ping') {
+    const ver = await cdpVersion()
+    return sendJson(res, 200, {
+      ok: true,
+      agent: 'free_show_local_agent',
+      version: 2,
+      browser: !!findBrowser(),
+      browser_up: !!ver,
+      headless: HEADLESS,
+      cdp_port: CDP_PORT,
+      profile_dir: PROFILE_DIR
+    })
+  }
+
+  if (u.pathname === '/fetch' && req.method === 'GET') {
+    const city = String(u.searchParams.get('city') || '上海')
+    const pages = u.searchParams.get('pages') || 2
+    if (city !== '上海') return sendJson(res, 400, { success: false, error: '本地代理仅支持上海' })
+    try {
+      const result = await fetchDamai(city, pages)
+      if (result.sliderNeeded && result.events.length === 0) {
+        return sendJson(res, 200, {
+          success: false,
+          code: 'need_manual_slider',
+          error: '需要人工完成滑块验证：请在自动打开的 Edge 窗口中完成滑块/登录（会话将自动保存），完成后重试'
+        })
+      }
+      const warning = result.sliderNeeded
+        ? ('第 ' + result.sliderPage + ' 页起触发滑块验证，该标签页已保留在 Edge 窗口，请人工处理后重试以补齐后续页')
+        : ''
+      return sendJson(res, 200, {
+        success: true, platform: 'damai', city,
+        newCount: result.events.length, events: result.events, warning
+      })
+    } catch (e) {
+      return sendJson(res, 500, { success: false, error: String(e.message || e) })
+    }
+  }
+
+  return sendJson(res, 404, { ok: false, error: 'not_found' })
+})
+
+server.listen(PORT, '127.0.0.1', () => {
+  console.log('==================================================')
+  console.log(' 免费演出雷达 · 本地抓取代理 v2（常驻浏览器版）')
+  console.log(` 监听: http://127.0.0.1:${PORT} (仅本机)`)
+  console.log(` 浏览器调试端口: ${CDP_PORT} | 模式: ${HEADLESS ? 'headless' : '可视化窗口'}`)
+  console.log(' 浏览器: ' + (findBrowser() || '未找到 Chrome/Edge！'))
+  console.log(' 配置目录: ' + PROFILE_DIR)
+  console.log(' ------------------------------------------------')
+  console.log(' 正在自动唤起 Edge 窗口...')
+  console.log(' 首次使用请在该窗口人工完成滑块验证/登录（一次即可，会话自动保存）。')
+  console.log(' 之后回工具箱页面点「Go」，代理复用同一浏览器抓取。')
+  console.log(' 保持本窗口开启；Ctrl+C 退出（浏览器会继续保留会话）')
+  console.log('==================================================')
+  // 启动即拉起浏览器（用户要求①：启动代理自动唤起可视化 Edge），
+  // 失败不阻塞 HTTP 服务，首次 /fetch 时会再重试
+  ensureBrowser()
+    .then(info => console.log(info.reused
+      ? ' 已复用正在运行的 Edge 实例（调试端口 ' + CDP_PORT + '）'
+      : ' Edge 窗口已打开，可在其中人工过滑块/登录'))
+    .catch(e => console.log(' [警告] 自动唤起浏览器失败: ' + (e.message || e) + '（首次抓取时会重试）'))
+})