Toolbox/public/tools/free_show/free_show_local_agent.js

// ============================================================
// 免费演出雷达 · 本地抓取代理（零依赖单文件 · v2 常驻浏览器版）
//
// 用途：大麦对服务器 IP（尤其阿里云 ECS）风控极严，
//       本代理在家里电脑运行，用家庭宽带 IP 抓大麦，
//       工具箱页面（127.0.0.1 探测）自动把大麦请求路由到这里。
//
// 用法：node free_show_local_agent.js [端口，默认 8978] [--headless] [--cdp <端口，默认 9222>]
// 依赖：仅 Node 内置模块 + 本机已安装的 Chrome 或 Edge（无需 npm install）
//
// v2 工作方式（解决 headless 被大麦风控识别的问题）：
//   ① 启动代理 → 自动唤起一个【可视化】原生 Edge 窗口（默认不 headless），
//      使用持久化用户配置目录（Cookie / 登录态 / 过验证状态落盘保存）；
//   ② 首次使用请人工在该窗口完成滑块验证 / 登录（只需一次，会话自动保存）；
//   ③ 代理通过 CDP（remote-debugging-port）复用【同一个浏览器实例】执行抓取，
//      每次任务只开/关标签页，绝不新建空白浏览器环境、不重启浏览器进程；
//   ④ 抓取前等待异步列表渲染完成；若检测到滑块弹窗，保留标签页并明确提示
//      「需要人工完成滑块验证」，由你在可见窗口处理后重试。
//
// 反自动化特征参数（内置）：
//   --disable-blink-features=AutomationControlled
//   --exclude-switches=enable-automation
// ============================================================
'use strict'

const http = require('http')
const net = require('net')
const fs = require('fs')
const os = require('os')
const path = require('path')
const crypto = require('crypto')
const { spawn } = require('child_process')

// ============ CLI 参数 ============
const argv = process.argv.slice(2)
let PORT = 8978
let CDP_PORT = 9222
let HEADLESS = false // 默认可视化窗口；传 --headless 才无头（兼容旧模式）
for (let i = 0; i < argv.length; i++) {
  const a = argv[i]
  if (a === '--headless') HEADLESS = true
  else if (a === '--cdp' && argv[i + 1]) CDP_PORT = Number(argv[++i]) || 9222
  else if (/^\d+$/.test(a)) PORT = Number(a)
}

const DAMAI_CITY_CODE = '310100' // 仅上海（业务约定）
const START_URL = 'https://search.damai.cn/search.htm?city=' + DAMAI_CITY_CODE
// 持久化配置目录：放在脚本同级，Cookie/登录态/验证状态长期保存
const PROFILE_DIR = path.join(__dirname, 'edge_profile_free_show')

const sleep = ms => new Promise(r => setTimeout(r, ms))

// ============ 浏览器探测 ============

function findBrowser() {
  const candidates = [
    'C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe',
    'C:\\Program Files\\Microsoft\\Edge\\Application\\msedge.exe',
    'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
    'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe'
  ]
  for (const p of candidates) {
    if (fs.existsSync(p)) return p
  }
  return null
}

// ============ CDP HTTP 端点 ============

function cdpHttp(method, pathAndQuery, timeoutMs) {
  return new Promise((resolve, reject) => {
    const req = http.request({
      host: '127.0.0.1', port: CDP_PORT, path: pathAndQuery, method,
      timeout: timeoutMs || 5000
    }, res => {
      let body = ''
      res.on('data', c => { body += c })
      res.on('end', () => {
        try { resolve(JSON.parse(body)) } catch (e) { reject(new Error('CDP 响应解析失败')) }
      })
    })
    req.on('timeout', () => req.destroy(new Error('CDP 连接超时')))
    req.on('error', reject)
    req.end()
  })
}

const cdpVersion = () => cdpHttp('GET', '/json/version').catch(() => null)

// ============ 浏览器生命周期：只拉起一次，长期复用 ============

async function ensureBrowser() {
  const ver = await cdpVersion()
  if (ver) return { up: true, reused: true }

  const browserPath = findBrowser()
  if (!browserPath) throw new Error('未找到本机 Chrome/Edge，无法启动抓取浏览器')

  const args = [
    '--remote-debugging-port=' + CDP_PORT,
    '--user-data-dir=' + PROFILE_DIR,
    '--disable-blink-features=AutomationControlled', // 反自动化特征
    '--exclude-switches=enable-automation',           // 去掉"受自动化控制"提示条
    '--no-first-run',
    '--no-default-browser-check'
  ]
  if (HEADLESS) args.push('--headless=new', '--disable-gpu')
  args.push(START_URL) // 首次直接打开大麦上海列表页，方便人工过滑块/登录

  const child = spawn(browserPath, args, { detached: true, stdio: 'ignore' })
  child.unref() // 代理退出后浏览器继续活着，会话不丢

  for (let i = 0; i < 30; i++) {
    await sleep(500)
    const v = await cdpVersion()
    if (v) return { up: true, reused: false }
  }
  throw new Error('浏览器已拉起但调试端口未就绪（' + CDP_PORT + ' 被占用或启动失败）')
}

async function openTab(url) {
  const p = '/json/new?' + encodeURIComponent(url)
  try { return await cdpHttp('PUT', p) } catch (e) { return await cdpHttp('GET', p) }
}

const closeTab = id => cdpHttp('GET', '/json/close/' + id).catch(() => null)

// ============ 最小 WebSocket 客户端（Node 内置 net/crypto 手写，RFC6455） ============

function parseWsFrame(buf) {
  if (buf.length < 2) return null
  const fin = (buf[0] & 0x80) !== 0
  const opcode = buf[0] & 0x0f
  const masked = (buf[1] & 0x80) !== 0
  let len = buf[1] & 0x7f
  let off = 2
  if (len === 126) {
    if (buf.length < 4) return null
    len = buf.readUInt16BE(2); off = 4
  } else if (len === 127) {
    if (buf.length < 10) return null
    len = Number(buf.readBigUInt64BE(2)); off = 10
  }
  let mask = null
  if (masked) {
    if (buf.length < off + 4) return null
    mask = buf.slice(off, off + 4); off += 4
  }
  if (buf.length < off + len) return null
  let payload = buf.slice(off, off + len)
  if (mask) {
    payload = Buffer.from(payload)
    for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i % 4]
  }
  return { fin, opcode, payload, total: off + len }
}

function encodeWsFrame(text) {
  const payload = Buffer.from(text, 'utf8')
  const len = payload.length
  let header
  if (len < 126) header = Buffer.from([0x81, 0x80 | len])
  else if (len < 65536) { header = Buffer.from([0x81, 0x80 | 126, 0, 0]); header.writeUInt16BE(len, 2) }
  else { header = Buffer.from([0x81, 0x80 | 127, 0, 0, 0, 0, 0, 0, 0, 0]); header.writeBigUInt64BE(BigInt(len), 2) }
  const mask = crypto.randomBytes(4)
  const masked = Buffer.from(payload)
  for (let i = 0; i < masked.length; i++) masked[i] ^= mask[i % 4]
  return Buffer.concat([header, mask, masked])
}

function wsConnect(wsUrl) {
  return new Promise((resolve, reject) => {
    const u = new URL(wsUrl)
    const sock = net.connect(Number(u.port) || 80, u.hostname || '127.0.0.1')
    const key = crypto.randomBytes(16).toString('base64')
    const handshake = [
      'GET ' + (u.pathname || '/') + (u.search || '') + ' HTTP/1.1',
      'Host: ' + u.host,
      'Upgrade: websocket',
      'Connection: Upgrade',
      'Sec-WebSocket-Key: ' + key,
      'Sec-WebSocket-Version: 13',
      '', ''
    ].join('\r\n')

    let upgraded = false
    let buf = Buffer.alloc(0)
    let fragBuf = null
    const handlers = { message: null, close: null }
    const client = {
      send(text) { try { sock.write(encodeWsFrame(text)) } catch (e) {} },
      close() { try { sock.end() } catch (e) {} },
      onMessage(fn) { handlers.message = fn },
      onClose(fn) { handlers.close = fn }
    }

    sock.on('connect', () => sock.write(handshake))
    sock.on('data', chunk => {
      buf = Buffer.concat([buf, chunk])
      if (!upgraded) {
        const idx = buf.indexOf('\r\n\r\n')
        if (idx < 0) return
        const head = buf.slice(0, idx).toString()
        if (!/^HTTP\/1\.1\s+101/.test(head)) {
          reject(new Error('WebSocket 握手失败'))
          sock.end()
          return
        }
        upgraded = true
        buf = buf.slice(idx + 4)
        resolve(client)
      }
      while (true) {
        const frame = parseWsFrame(buf)
        if (!frame) break
        buf = buf.slice(frame.total)
        if (frame.opcode === 0x8) { if (handlers.close) handlers.close(); return }
        if (frame.opcode === 0x9) { sock.write(Buffer.from([0x8a, 0x80, 0, 0, 0, 0])); continue } // ping→pong
        if (frame.opcode === 0xA) continue // pong
        if (frame.opcode === 0x1 || frame.opcode === 0x2) {
          if (frame.fin) {
            const full = fragBuf ? Buffer.concat([fragBuf, frame.payload]) : frame.payload
            fragBuf = null
            if (handlers.message) handlers.message(full.toString('utf8'))
          } else fragBuf = frame.payload
        } else if (frame.opcode === 0x0) {
          fragBuf = fragBuf ? Buffer.concat([fragBuf, frame.payload]) : frame.payload
          if (frame.fin) {
            const full = fragBuf
            fragBuf = null
            if (handlers.message) handlers.message(full.toString('utf8'))
          }
        }
      }
    })
    sock.on('error', e => { if (!upgraded) reject(e); else if (handlers.close) handlers.close() })
    sock.on('close', () => { if (handlers.close) handlers.close() })
  })
}

// ============ CDP 会话封装 ============

function makeCdp(ws) {
  let nextId = 1
  const pending = new Map()
  ws.onMessage(text => {
    let msg = null
    try { msg = JSON.parse(text) } catch (e) { return }
    if (msg && msg.id && pending.has(msg.id)) {
      const cb = pending.get(msg.id)
      pending.delete(msg.id)
      cb(msg)
    }
  })
  return {
    send(method, params, timeoutMs) {
      return new Promise((resolve, reject) => {
        const id = nextId++
        const timer = setTimeout(() => {
          if (pending.has(id)) { pending.delete(id); reject(new Error('CDP 超时: ' + method)) }
        }, timeoutMs || 30000)
        pending.set(id, msg => { clearTimeout(timer); resolve(msg) })
        ws.send(JSON.stringify({ id, method, params: params || {} }))
      })
    }
  }
}

// 页面状态探测：滑块特征 / 列表渲染完成特征
const CHECK_EXPR = '(() => { try { const txt = (document.body && document.body.innerText) ? document.body.innerText.slice(0, 3000) : ""; const slider = !!(document.querySelector("#nc_1_wrapper, .nc-container, #baxia-dialog-content, #nocaptcha, .slidetounlock")) || /拖动滑块|请按照说明拖动滑块|滑动验证/.test(txt) || /rgv587|punish/.test(location.href); const ready = document.querySelectorAll("a[href*=\'detail.damai.cn\']").length > 0 || /共\\s*\\d+\\s*个商品/.test(txt); return { slider: slider, ready: ready }; } catch (e) { return { slider: false, ready: false }; } })()'

async function waitForListOrSlider(cdp, timeoutMs) {
  const t0 = Date.now()
  while (Date.now() - t0 < timeoutMs) {
    try {
      const r = await cdp.send('Runtime.evaluate', { expression: CHECK_EXPR, returnByValue: true }, 5000)
      const v = r && r.result && r.result.result ? r.result.result.value : null
      if (v) {
        if (v.slider) return { slider: true }
        if (v.ready) return { ready: true }
      }
    } catch (e) { /* 页面导航中 evaluate 可能短暂失败，忽略后重试 */ }
    await sleep(600)
  }
  return { timeout: true }
}

// ============ 大麦 DOM 解析（字符串版，无 DOM 库） ============

// 价格提取：大麦列表价格形如 "130-2620元" / "380.0元" / "¥130"（无 ¥ 符号是常态）
// 区间价取下限用于 ≤ 过滤；返回 { price, priceText }，提取不到返回 null
function extractPrice(win) {
  let text = ''
  const blockM = win.match(/class="[^"]*price[^"]*"[^>]*>([\s\S]{0,120}?)<\/(?:div|span|p|em|b|i)>/i)
  if (blockM) text = blockM[1].replace(/<[^>]+>/g, '').trim()
  if (!text) {
    const symM = win.match(/[¥￥]\s*[\d.]+(?:\s*[-~至]\s*[\d.]+)?/)
    if (symM) text = symM[0]
  }
  if (!text) {
    const yuanM = win.match(/[\d.]+(?:\s*[-~至]\s*[\d.]+)?\s*元/)
    if (yuanM) text = yuanM[0]
  }
  if (!text) return null
  const rangeM = text.match(/([\d.]+)\s*[-~至]\s*[\d.]+/)
  const singleM = text.match(/([\d.]+)/)
  const low = rangeM ? parseFloat(rangeM[1]) : (singleM ? parseFloat(singleM[1]) : NaN)
  if (!isFinite(low)) return null
  return { price: low, priceText: text.replace(/\s+/g, '') }
}

function parseDamaiHtml(html, city) {
  if (/rgv587|punish|captcha|滑动验证|拖动滑块/i.test(html)) {
    throw new Error('需要人工完成滑块验证：请在自动打开的 Edge 窗口中完成滑块/登录（会话将自动保存），完成后重试')
  }
  const events = []
  const seen = new Set()
  const anchorRe = /<a\b([^>]*)>/g
  let m
  while ((m = anchorRe.exec(html))) {
    const attrs = m[1]
    const hrefM = attrs.match(/href="([^"]+)"/)
    if (!hrefM) continue
    let href = hrefM[1]
    const idM = href.match(/id=(\d+)/)
    if (!/damai\.cn/.test(href) || !idM) continue
    if (href.startsWith('//')) href = 'https:' + href

    const titleM = attrs.match(/title="([^"]+)"/)
    let title = titleM ? titleM[1] : ''
    const win = html.substr(m.index, 3000)
    if (!title) {
      const t2 = win.match(/<[^>]*class="[^"]*title[^"]*"[^>]*>([\s\S]*?)<\/[^>]+>/)
      if (t2) title = t2[1].replace(/<[^>]+>/g, '').trim()
    }
    if (!title || seen.has(title)) continue
    seen.add(title)

    const priceInfo = extractPrice(win)
    const timeM = win.match(/\d{4}[.\-/]\d{1,2}[.\-/]\d{1,2}[^<|]*/)
    const imgM = win.match(/<img[^>]*src="([^"]+)"/)

    events.push({
      platform: 'damai',
      id: idM[1],
      title,
      price: priceInfo ? priceInfo.price : 9999,
      priceText: priceInfo ? priceInfo.priceText : '价格待定',
      showTime: timeM ? timeM[0].trim() : '',
      venue: '',
      city,
      poster: imgM ? imgM[1] : '',
      url: href
    })
  }
  return events
}

// ============ 抓取主流程：复用常驻浏览器，只开/关标签页 ============

async function fetchDamai(city, pages) {
  await ensureBrowser()

  const maxPages = Math.min(Number(pages) || 2, 3)
  let allEvents = []
  for (let p = 1; p <= maxPages; p++) {
    const url = `https://search.damai.cn/search.htm?keyword=&city=${DAMAI_CITY_CODE}&ctl=0&order=1&page=${p}`
    // 先开空白标签再 Page.navigate，避免对大麦页面发起两次请求
    const tab = await openTab('about:blank')
    if (!tab || !tab.webSocketDebuggerUrl) throw new Error('无法在常驻浏览器中打开新标签页')

    let ws = null
    try {
      ws = await wsConnect(tab.webSocketDebuggerUrl)
      const cdp = makeCdp(ws)
      await cdp.send('Page.enable').catch(() => {})
      await cdp.send('Runtime.enable').catch(() => {})
      await cdp.send('Page.navigate', { url })

      // 等待异步列表渲染完成（单页最多 20s）
      const state = await waitForListOrSlider(cdp, 20000)
      if (state.slider) {
        // 标签页保留在可见窗口，交给人工拖滑块；会话过验证后自动落盘
        try { ws.close() } catch (e) {}
        return { events: allEvents, sliderNeeded: true, sliderPage: p }
      }

      const htmlRes = await cdp.send('Runtime.evaluate', { expression: 'document.documentElement.outerHTML', returnByValue: true })
      const html = htmlRes && htmlRes.result && htmlRes.result.result ? String(htmlRes.result.result.value || '') : ''
      try { ws.close() } catch (e) {}
      await closeTab(tab.id)

      if (!html) { if (p === 1) throw new Error('页面 DOM 为空（可能仍在加载或被拦截）'); break }
      const items = parseDamaiHtml(html, city)
      if (items.length === 0) break
      allEvents = allEvents.concat(items)
    } catch (e) {
      try { if (ws) ws.close() } catch (e2) {}
      await closeTab(tab.id)
      throw e
    }
  }
  return { events: allEvents, sliderNeeded: false, sliderPage: 0 }
}

// ============ HTTP 服务（仅 127.0.0.1） ============

function sendJson(res, code, obj) {
  const body = JSON.stringify(obj)
  res.writeHead(code, {
    'Content-Type': 'application/json; charset=utf-8',
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
    'Access-Control-Allow-Headers': 'Content-Type'
  })
  res.end(body)
}

const server = http.createServer(async (req, res) => {
  const u = new URL(req.url, `http://127.0.0.1:${PORT}`)

  if (req.method === 'OPTIONS') return sendJson(res, 204, {})

  if (u.pathname === '/ping') {
    const ver = await cdpVersion()
    return sendJson(res, 200, {
      ok: true,
      agent: 'free_show_local_agent',
      version: 2,
      browser: !!findBrowser(),
      browser_up: !!ver,
      headless: HEADLESS,
      cdp_port: CDP_PORT,
      profile_dir: PROFILE_DIR
    })
  }

  if (u.pathname === '/fetch' && req.method === 'GET') {
    const city = String(u.searchParams.get('city') || '上海')
    const pages = u.searchParams.get('pages') || 2
    if (city !== '上海') return sendJson(res, 400, { success: false, error: '本地代理仅支持上海' })
    try {
      const result = await fetchDamai(city, pages)
      if (result.sliderNeeded && result.events.length === 0) {
        return sendJson(res, 200, {
          success: false,
          code: 'need_manual_slider',
          error: '需要人工完成滑块验证：请在自动打开的 Edge 窗口中完成滑块/登录（会话将自动保存），完成后重试'
        })
      }
      const warning = result.sliderNeeded
        ? ('第 ' + result.sliderPage + ' 页起触发滑块验证，该标签页已保留在 Edge 窗口，请人工处理后重试以补齐后续页')
        : ''
      return sendJson(res, 200, {
        success: true, platform: 'damai', city,
        newCount: result.events.length, events: result.events, warning
      })
    } catch (e) {
      return sendJson(res, 500, { success: false, error: String(e.message || e) })
    }
  }

  return sendJson(res, 404, { ok: false, error: 'not_found' })
})

server.listen(PORT, '127.0.0.1', () => {
  console.log('==================================================')
  console.log(' 免费演出雷达 · 本地抓取代理 v2（常驻浏览器版）')
  console.log(` 监听: http://127.0.0.1:${PORT} (仅本机)`)
  console.log(` 浏览器调试端口: ${CDP_PORT} | 模式: ${HEADLESS ? 'headless' : '可视化窗口'}`)
  console.log(' 浏览器: ' + (findBrowser() || '未找到 Chrome/Edge！'))
  console.log(' 配置目录: ' + PROFILE_DIR)
  console.log(' ------------------------------------------------')
  console.log(' 正在自动唤起 Edge 窗口...')
  console.log(' 首次使用请在该窗口人工完成滑块验证/登录（一次即可，会话自动保存）。')
  console.log(' 之后回工具箱页面点「Go」，代理复用同一浏览器抓取。')
  console.log(' 保持本窗口开启；Ctrl+C 退出（浏览器会继续保留会话）')
  console.log('==================================================')
  // 启动即拉起浏览器（用户要求①：启动代理自动唤起可视化 Edge），
  // 失败不阻塞 HTTP 服务，首次 /fetch 时会再重试
  ensureBrowser()
    .then(info => console.log(info.reused
      ? ' 已复用正在运行的 Edge 实例（调试端口 ' + CDP_PORT + '）'
      : ' Edge 窗口已打开，可在其中人工过滑块/登录'))
    .catch(e => console.log(' [警告] 自动唤起浏览器失败: ' + (e.message || e) + '（首次抓取时会重试）'))
})