chore: 初始提交 - 移除所有硬编码凭据,统一从环境变量读取

This commit is contained in:
yangxiangyuan
2026-06-11 17:10:58 +08:00
commit c223c79b36
1699 changed files with 313940 additions and 0 deletions
+899
View File
@@ -0,0 +1,899 @@
const axios = require('axios')
const iconv = require('iconv-lite')
const fs = require('fs')
const path = require('path')
const { logJSON } = require('./logger')
const sinaMapSymbol = s => {
const x = String(s).toLowerCase()
if (x === '000001' || x === 'sh000001') return 'sh000001'
if (x === '399001' || x === 'sz399001') return 'sz399001'
if (x === '399006' || x === 'sz399006') return 'sz399006'
if (x === '000300' || x === 'sh000300') return 'sh000300'
if (x === '899050' || x.startsWith('bj899050')) return 'bj899050'
if (x === '000688' || x === 'sh000688') return 'sh000688'
if (x === 'ixic' || x === '.ixic' || x === 'gb_ixic') return 'gb_ixic'
if (x === 'inx' || x === '.inx' || x === 'gb_inx') return 'gb_inx'
if (x === 'hsi.hk' || x === 'hsi' || x === 'hk_hsi' || x === 'hkhsi') return 'rt_hkHSI'
if (x === '03153.hk' || x === 'hk03153' || x === '03153') return 'rt_hk03153'
if (x === 'inda' || x === 'amex:inda' || x === 'us_inda' || x === 'gb_inda') return 'gb_inda'
if (x === 'ewg' || x === 'nysearca:ewg' || x === 'us_ewg' || x === 'gb_ewg') return 'gb_ewg'
return s
}
const parseSinaLine = line => {
const m = line.match(/^var\s+hq_str_(\w+)=\"([^\"]*)/)
if (!m) return null
const symbol = m[1]
const body = m[2]
const parts = body.split(',')
if (symbol.startsWith('gb_')) {
const name = parts[0] || symbol
const price = parseFloat(parts[1] || '0')
// find percent in any field
const pctField = parts.find(p => /%/.test(p)) || ''
const pctMatch = String(pctField).match(/(-?\d+(?:\.\d+)?)%/)
const pct = pctMatch ? parseFloat(pctMatch[1]) : 0
let prevClose = 0, change = 0
if (pct !== 0 && price) { prevClose = price / (1 + pct / 100); change = price - prevClose }
return { symbol, name, price, prevClose, change, change_percent: pct }
}
if (symbol.startsWith('hk')) {
const name = parts[0] || symbol
const price = parseFloat(parts[6] || parts[1] || '0')
const prevClose = parseFloat(parts[3] || parts[2] || '0')
const change = price - prevClose
const pct = prevClose ? (change / prevClose) * 100 : 0
return { symbol, name, price, prevClose, change, change_percent: pct }
}
const name = parts[0] || ''
const price = parseFloat(parts[3] || parts[1] || '0')
const prevClose = parseFloat(parts[2] || '0')
const change = price - prevClose
const pct = prevClose ? (change / prevClose) * 100 : 0
return { symbol, name, price, prevClose, change, change_percent: pct }
}
const fetchSina = async symbols => {
if (!symbols || symbols.length === 0) return []
const chunkSize = 6
const outMap = new Map()
for (let i = 0; i < symbols.length; i += chunkSize) {
const chunk = symbols.slice(i, i + chunkSize).map(sinaMapSymbol)
const list = chunk.join(',')
const url = `http://hq.sinajs.cn/list=${list}`
logJSON('markets.http.get', { source: 'sina', url, symbols: chunk, chunk_index: Math.floor(i / chunkSize), chunk_size: chunk.length }, 'markets')
try {
const res = await axios.get(url, {
headers: {
'Referer': 'https://finance.sina.com.cn/',
'User-Agent': 'Mozilla/5.0',
'Accept-Language': 'zh-CN,zh;q=0.9'
},
responseType: 'arraybuffer',
timeout: 10000,
validateStatus: () => true
})
if (!res || res.status >= 400) {
logJSON('markets.http.status', { source: 'sina', url, status: res && res.status, chunk_index: Math.floor(i / chunkSize) }, 'markets')
continue
}
const text = iconv.decode(Buffer.from(res.data), 'gb18030')
const lines = String(text || '').split(/;\s*/).filter(Boolean)
let parsedCount = 0
for (const line of lines) {
const r = parseSinaLine(line)
if (r) {
parsedCount += 1
outMap.set(r.symbol, { ...r, source: 'sina' })
}
}
logJSON('markets.http.parse', { source: 'sina', chunk_index: Math.floor(i / chunkSize), count: parsedCount }, 'markets')
} catch (e) {
logJSON('markets.http.error', { source: 'sina', url, chunk_index: Math.floor(i / chunkSize), error: String(e.message || e) }, 'markets')
}
}
return Array.from(outMap.values())
}
const marketsConfigPath = path.join(process.cwd(), 'config', 'markets.json')
const yahooProbeState = {
quoteCache: new Map(),
chartCache: new Map(),
quotePendingSymbols: new Set(),
quotePendingWaiters: [],
quoteFlushTimer: null,
quoteFlushRunning: false,
chartInflight: new Map(),
lastRequestAt: { quote: 0, chart: 0 },
finnhubLastRequestAt: 0,
finnhubRequestTimes: []
}
const YAHOO_PROBE_SYMBOL_ALIAS = Object.freeze({
usINX: '^GSPC',
usIXIC: '^IXIC',
usDJI: '^DJI',
hkHSI: '^HSI',
zsN225: '^N225',
zsGDAXI: '^GDAXI',
zstwii: '^TWII',
zskospi: '^KS11',
zsBVSP: '^BVSP',
zsSTI: '^STI',
usEWG: 'EWG',
usFLSA: 'FLSA',
usNFTY: 'NFTY',
usREMX: 'REMX',
usEWJ: 'EWJ',
usEWY: 'EWY',
usEWZ: 'EWZ',
usASEA: 'ASEA',
usEWT: 'EWT',
usINDA: 'INDA',
hf_GC: 'GC=F',
hf_SI: 'SI=F'
})
const FINNHUB_TIMEZONE_ALIAS = Object.freeze({
'^GSPC': 'America/New_York',
'^IXIC': 'America/New_York',
'^DJI': 'America/New_York',
'^HSI': 'Asia/Hong_Kong',
'^N225': 'Asia/Tokyo',
'^GDAXI': 'Europe/Berlin',
'^TWII': 'Asia/Taipei',
'^KS11': 'Asia/Seoul',
'^BVSP': 'America/Sao_Paulo',
'^STI': 'Asia/Singapore'
})
const FINNHUB_DEFAULT_TZ = 'America/New_York'
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
const toFiniteNumber = value => {
const n = Number(value)
return Number.isFinite(n) ? n : null
}
const pushYahooTrace = (trace, stage, details = {}) => {
if (!Array.isArray(trace)) return
trace.push({
time: new Date().toISOString(),
stage,
...details
})
}
const readMarketsConfig = () => {
try {
if (!fs.existsSync(marketsConfigPath)) return {}
return JSON.parse(fs.readFileSync(marketsConfigPath, 'utf-8'))
} catch {
return {}
}
}
const getYahooProbeConfig = () => {
const cfg = readMarketsConfig()
const raw = cfg && typeof cfg.yahoo_probe === 'object' ? cfg.yahoo_probe : {}
const baseUrl = String(raw.base_url || 'http://47.252.30.127:19071').trim().replace(/\/$/, '')
return {
enabled: raw.enabled !== false,
base_url: baseUrl || 'http://47.252.30.127:19071',
token: String(raw.token || '').trim(),
quote_endpoint: String(raw.quote_endpoint || raw.endpoint || '/probe/yahoo').trim() || '/probe/yahoo',
chart_endpoint: String(raw.chart_endpoint || raw.endpoint || '/probe/yahoo').trim() || '/probe/yahoo',
finnhub_quote_endpoint: String(raw.finnhub_quote_endpoint || '/probe/finnhub/quote').trim() || '/probe/finnhub/quote',
finnhub_candle_endpoint: String(raw.finnhub_candle_endpoint || '/probe/finnhub/candle').trim() || '/probe/finnhub/candle',
request_timeout_ms: Math.max(3000, Number(raw.request_timeout_ms || 15000)),
quote_ttl_seconds: Math.max(15, Number(raw.quote_ttl_seconds || 120)),
chart_ttl_seconds: Math.max(300, Number(raw.chart_ttl_seconds || 21600)),
quote_batch_size: Math.max(1, Math.min(20, Number(raw.quote_batch_size || 4))),
quote_min_interval_ms: Math.max(0, Number(raw.quote_min_interval_ms || raw.min_request_interval_ms || 2500)),
chart_min_interval_ms: Math.max(0, Number(raw.chart_min_interval_ms || raw.min_request_interval_ms || 2500)),
finnhub_min_interval_ms: Math.max(1200, Number(raw.finnhub_min_interval_ms || 1300)),
finnhub_max_requests_per_minute: Math.max(1, Math.min(55, Number(raw.finnhub_max_requests_per_minute || 50)))
}
}
const getYahooProbeCache = (cache, key) => {
const hit = cache.get(key)
if (!hit) return null
if (Number(hit.expires_at || 0) <= Date.now()) {
cache.delete(key)
return null
}
return hit.value
}
const setYahooProbeCache = (cache, key, value, ttlSeconds) => {
cache.set(key, {
value,
expires_at: Date.now() + Math.max(1, ttlSeconds) * 1000
})
}
const getYahooProbeSymbol = symbol => {
const raw = String(symbol || '').trim()
if (!raw) return ''
if (YAHOO_PROBE_SYMBOL_ALIAS[raw]) return YAHOO_PROBE_SYMBOL_ALIAS[raw]
if (/^us[A-Z][A-Z0-9._-]+$/.test(raw)) return raw.slice(2)
return raw
}
const getFinnhubSymbol = symbol => getYahooProbeSymbol(symbol)
const getFinnhubTimeZone = symbol => {
const key = String(symbol || '').trim()
return FINNHUB_TIMEZONE_ALIAS[key] || FINNHUB_DEFAULT_TZ
}
const sleepUntilNextMinuteWindow = async cfg => {
const windowMs = 60 * 1000
const now = Date.now()
yahooProbeState.finnhubRequestTimes = yahooProbeState.finnhubRequestTimes.filter(ts => (now - ts) < windowMs)
const maxPerMinute = Math.max(1, Number(cfg.finnhub_max_requests_per_minute || 50))
if (yahooProbeState.finnhubRequestTimes.length < maxPerMinute) return
const oldest = Number(yahooProbeState.finnhubRequestTimes[0] || 0)
const waitMs = Math.max(0, windowMs - (now - oldest) + 50)
if (waitMs > 0) await sleep(waitMs)
}
const throttleFinnhubProbe = async (cfg, trace, reason) => {
await sleepUntilNextMinuteWindow(cfg)
const minInterval = Math.max(0, Number(cfg.finnhub_min_interval_ms || 1300))
const waitMs = Math.max(0, minInterval - (Date.now() - Number(yahooProbeState.finnhubLastRequestAt || 0)))
if (waitMs > 0) {
pushYahooTrace(trace, 'finnhub.throttle_wait', { wait_ms: waitMs, reason: String(reason || '') })
await sleep(waitMs)
}
const now = Date.now()
yahooProbeState.finnhubLastRequestAt = now
yahooProbeState.finnhubRequestTimes.push(now)
yahooProbeState.finnhubRequestTimes = yahooProbeState.finnhubRequestTimes.filter(ts => (now - ts) < 60 * 1000)
}
const shiftDateByRange = (date, amount, unit) => {
const next = new Date(date.getTime())
if (unit === 'd') next.setUTCDate(next.getUTCDate() - amount)
if (unit === 'mo') next.setUTCMonth(next.getUTCMonth() - amount)
if (unit === 'y') next.setUTCFullYear(next.getUTCFullYear() - amount)
return next
}
const convertYahooRangeToUnixWindow = range => {
const now = new Date()
const text = String(range || '1y').trim().toLowerCase()
let fromDate = shiftDateByRange(now, 1, 'y')
if (text === 'ytd') {
fromDate = new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0))
} else {
const match = text.match(/^(\d+)(d|mo|y)$/)
if (match) {
fromDate = shiftDateByRange(now, Number(match[1] || 1), match[2])
}
}
return {
from: Math.max(0, Math.floor(fromDate.getTime() / 1000)),
to: Math.max(0, Math.floor(now.getTime() / 1000))
}
}
const mapYahooIntervalToFinnhubResolution = interval => {
const text = String(interval || '1d').trim().toLowerCase()
if (text === '1d' || text === '1day' || text === 'd') return 'D'
if (text === '1wk' || text === '1w' || text === 'wk' || text === 'w') return 'W'
if (text === '1mo' || text === '1mth' || text === 'mo' || text === 'm') return 'M'
if (text === '60m' || text === '1h' || text === '60') return '60'
if (text === '30m' || text === '30') return '30'
if (text === '15m' || text === '15') return '15'
if (text === '5m' || text === '5') return '5'
if (text === '1m' || text === '1') return '1'
return 'D'
}
const scheduleYahooQuoteFlush = () => {
if (yahooProbeState.quoteFlushTimer) return
yahooProbeState.quoteFlushTimer = setTimeout(() => {
yahooProbeState.quoteFlushTimer = null
flushYahooQuoteQueue().catch(() => {})
}, 15)
}
const requestYahooQuoteBatch = async (batch, cfg) => {
const mappings = batch.map(original => ({ original, yahoo: getYahooProbeSymbol(original) }))
await throttleYahooProbe('quote', cfg, null)
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.quote_endpoint}`
// #region debug-point D:quote-batch-request
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'D', location: 'marketsClient.js:requestYahooQuoteBatch:request', msg: '[DEBUG] yahoo quote batch request', data: { url, batch, yahoo_symbols: mappings.map(item => item.yahoo), batch_size: batch.length }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
logJSON('markets.http.get', {
source: 'yahoo_probe',
url,
symbols: batch,
yahoo_symbols: mappings.map(item => item.yahoo),
batch_size: batch.length
}, 'markets')
const resp = await axios.get(url, {
timeout: cfg.request_timeout_ms,
validateStatus: () => true,
params: {
token: cfg.token,
type: 'quote',
symbols: mappings.map(item => item.yahoo).join(',')
}
})
if (resp && resp.status === 429) {
pushYahooTrace(null, 'yahoo.quote.http_429', { symbols: batch, yahoo_symbols: mappings.map(item => item.yahoo) })
logJSON('markets.http.status', { source: 'yahoo_probe', url, status: 429, symbols: batch, fallback: 'finnhub_probe_quote' }, 'markets')
return requestFinnhubQuoteBatchSerial(mappings, cfg)
}
if (!resp || resp.status >= 400) {
throw new Error(`yahoo_probe_http_${resp && resp.status ? resp.status : 'ERR'}`)
}
if (!resp.data) throw new Error('yahoo_probe_invalid_response')
const rows = parseYahooQuotePayload(resp.data, mappings)
// #region debug-point D:quote-batch-response
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'D', location: 'marketsClient.js:requestYahooQuoteBatch:response', msg: '[DEBUG] yahoo quote batch response', data: { batch, status: resp.status, rows_count: rows.length, row_symbols: rows.map(item => item.symbol), row_yahoo_symbols: rows.map(item => item.yahoo_symbol) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
return rows
}
const requestFinnhubQuoteSingle = async (mapping, cfg) => {
const finnhubSymbol = getFinnhubSymbol(mapping && mapping.yahoo)
if (!finnhubSymbol) throw new Error('finnhub_symbol_required')
await throttleFinnhubProbe(cfg, null, 'quote')
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.finnhub_quote_endpoint}`
logJSON('markets.http.get', {
source: 'finnhub_probe_quote',
url,
symbol: mapping.original,
yahoo_symbol: mapping.yahoo,
finnhub_symbol: finnhubSymbol
}, 'markets')
const resp = await axios.get(url, {
timeout: cfg.request_timeout_ms,
validateStatus: () => true,
params: {
token: cfg.token,
symbol: finnhubSymbol
}
})
if (!resp || resp.status >= 400) throw new Error(`finnhub_probe_quote_http_${resp && resp.status ? resp.status : 'ERR'}`)
const data = resp.data && typeof resp.data === 'object' ? resp.data : {}
const price = toFiniteNumber(data && data.c)
if (!Number.isFinite(price) || price <= 0) throw new Error('finnhub_probe_quote_invalid_response')
const prevClose = toFiniteNumber(data && data.pc)
const change = toFiniteNumber(data && data.d)
const pct = toFiniteNumber(data && data.dp)
const ts = toFiniteNumber(data && data.t)
return normalizeYahooQuoteItem({
symbol: mapping.original,
yahoo_symbol: mapping.yahoo,
name: mapping.original,
price,
prevClose: prevClose != null ? prevClose : 0,
change: change != null ? change : (prevClose != null ? price - prevClose : 0),
change_percent: pct != null ? pct : (prevClose ? ((price - prevClose) / prevClose) * 100 : 0),
regular_market_time: ts,
exchange_timezone_name: getFinnhubTimeZone(finnhubSymbol),
source: 'finnhub_probe'
}, {
originalSymbol: mapping.original,
yahooSymbol: mapping.yahoo,
source: 'finnhub_probe'
})
}
const requestFinnhubQuoteBatchSerial = async (mappings, cfg) => {
const rows = []
for (const mapping of (Array.isArray(mappings) ? mappings : [])) {
try {
const row = await requestFinnhubQuoteSingle(mapping, cfg)
if (row && row.symbol) rows.push(row)
} catch (error) {
const symbol = String(mapping && mapping.original || '').trim()
const cached = symbol ? getYahooProbeCache(yahooProbeState.quoteCache, symbol) : null
const message = String(error && error.message ? error.message : error)
logJSON('markets.http.error', {
source: 'finnhub_probe_quote',
symbol,
yahoo_symbol: String(mapping && mapping.yahoo || '').trim(),
error: message,
fallback: cached && cached.symbol ? 'quote_cache' : 'skip_symbol'
}, 'markets')
if (cached && cached.symbol) rows.push(cached)
}
}
logJSON('markets.http.parse', { source: 'finnhub_probe_quote', count: rows.length, requested: Array.isArray(mappings) ? mappings.length : 0 }, 'markets')
return rows
}
const flushYahooQuoteQueue = async () => {
if (yahooProbeState.quoteFlushRunning) return
if (yahooProbeState.quotePendingWaiters.length === 0) return
yahooProbeState.quoteFlushRunning = true
try {
while (yahooProbeState.quotePendingWaiters.length > 0) {
const waiters = yahooProbeState.quotePendingWaiters.splice(0)
const pendingSymbols = Array.from(yahooProbeState.quotePendingSymbols)
yahooProbeState.quotePendingSymbols.clear()
// #region debug-point A:quote-flush-begin
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'A', location: 'marketsClient.js:flushYahooQuoteQueue:begin', msg: '[DEBUG] yahoo quote flush begin', data: { waiter_count: waiters.length, pending_symbol_count: pendingSymbols.length, pending_symbols: pendingSymbols }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
if (pendingSymbols.length === 0) {
for (const waiter of waiters) waiter.resolve([])
continue
}
const cfg = getYahooProbeConfig()
const batchSize = Math.max(1, Number(cfg.quote_batch_size || 4))
const outMap = new Map()
for (let i = 0; i < pendingSymbols.length; i += batchSize) {
const batch = pendingSymbols.slice(i, i + batchSize)
try {
const rows = await requestYahooQuoteBatch(batch, cfg)
for (const row of rows) {
if (!row || !row.symbol) continue
outMap.set(row.symbol, row)
setYahooProbeCache(yahooProbeState.quoteCache, row.symbol, row, cfg.quote_ttl_seconds)
}
} catch (e) {
// #region debug-point A:quote-batch-error
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e2 = fs.readFileSync(p, 'utf8'); u = (e2.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e2.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'A', location: 'marketsClient.js:flushYahooQuoteQueue:error', msg: '[DEBUG] yahoo quote batch error', data: { batch, error: String(e && e.message ? e.message : e) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
logJSON('markets.http.error', { source: 'yahoo_probe', symbols: batch, error: String(e.message || e) }, 'markets')
}
}
for (const waiter of waiters) {
waiter.resolve(waiter.symbols.map(symbol => outMap.get(symbol)).filter(Boolean))
}
}
} finally {
yahooProbeState.quoteFlushRunning = false
if (yahooProbeState.quotePendingWaiters.length > 0) scheduleYahooQuoteFlush()
}
}
const enqueueYahooQuoteRequest = symbols => new Promise(resolve => {
const wanted = Array.from(new Set((symbols || []).map(s => String(s || '').trim()).filter(Boolean)))
yahooProbeState.quotePendingWaiters.push({ symbols: wanted, resolve })
for (const symbol of wanted) yahooProbeState.quotePendingSymbols.add(symbol)
scheduleYahooQuoteFlush()
})
const formatDateTime = (date, timeZone) => {
try {
const parts = new Intl.DateTimeFormat('sv-SE', {
timeZone,
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
hour12: false
}).formatToParts(date)
const get = type => parts.find(x => x.type === type)?.value || '00'
return `${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}:${get('second')}`
} catch {
return ''
}
}
const unixToDateTime = (sec, timeZone) => {
const s = Number(sec)
if (!Number.isFinite(s) || s <= 0) return ''
return formatDateTime(new Date(s * 1000), timeZone || 'America/New_York')
}
const unixToDate = (sec, timeZone) => {
const dt = unixToDateTime(sec, timeZone)
return dt ? dt.slice(0, 10) : ''
}
const approxAmount = (close, volume) => {
const c = toFiniteNumber(close)
const v = toFiniteNumber(volume)
if (!Number.isFinite(c) || !Number.isFinite(v) || c <= 0 || v <= 0) return null
return c * v
}
const throttleYahooProbe = async (kind, cfg, trace) => {
const key = kind === 'chart' ? 'chart' : 'quote'
const minInterval = key === 'chart' ? cfg.chart_min_interval_ms : cfg.quote_min_interval_ms
const lastAt = Number(yahooProbeState.lastRequestAt[key] || 0)
const waitMs = Math.max(0, minInterval - (Date.now() - lastAt))
if (waitMs > 0) {
pushYahooTrace(trace, 'yahoo.throttle_wait', { kind: key, wait_ms: waitMs })
await sleep(waitMs)
}
yahooProbeState.lastRequestAt[key] = Date.now()
}
const normalizeYahooQuoteItem = (item, options = {}) => {
const originalSymbol = String(options.originalSymbol || (item && item.symbol) || '').trim()
const yahooSymbol = String(options.yahooSymbol || (item && item.yahoo_symbol) || (item && item.symbol) || '').trim()
const price = toFiniteNumber(item && (item.price != null ? item.price : item.regularMarketPrice))
const prevClose = toFiniteNumber(item && (item.prevClose != null ? item.prevClose : item.regularMarketPreviousClose))
const change = toFiniteNumber(item && (item.change != null ? item.change : item.regularMarketChange))
const pct = toFiniteNumber(item && (item.change_percent != null ? item.change_percent : item.regularMarketChangePercent))
const epoch = toFiniteNumber(item && (item.regular_market_time != null ? item.regular_market_time : item.regularMarketTime))
const exchangeTz = String(item && (item.exchange_timezone_name || item.exchangeTimezoneName || 'America/New_York') || 'America/New_York')
const tradeAt = String(item && item.last_trade_at || '').trim() || unixToDateTime(epoch, exchangeTz)
const beijingAt = String(item && item.last_trade_beijing_at || '').trim() || unixToDateTime(epoch, 'Asia/Shanghai')
return {
symbol: originalSymbol,
yahoo_symbol: yahooSymbol,
name: String(item && (item.name || item.longName || item.shortName || item.displayName || item.yahoo_symbol || item.symbol) || '').trim(),
price: price != null ? price : 0,
prevClose: prevClose != null ? prevClose : 0,
change: change != null ? change : (price != null && prevClose != null ? price - prevClose : 0),
change_percent: pct != null ? pct : (price != null && prevClose ? ((price - prevClose) / prevClose) * 100 : 0),
last_trade_date: String(item && item.last_trade_date || '').trim() || (tradeAt ? tradeAt.slice(0, 10) : ''),
last_trade_at: tradeAt,
last_trade_beijing_at: beijingAt,
captured_at: String(item && item.captured_at || '').trim(),
source: String(item && item.source || options.source || 'yahoo_probe').trim() || 'yahoo_probe'
}
}
const attachYahooPrevClose = rows => {
let prevClose = null
return (Array.isArray(rows) ? rows : []).map(row => {
const close = toFiniteNumber(row && row.close)
const output = {
...row,
open: toFiniteNumber(row && row.open),
high: toFiniteNumber(row && row.high),
low: toFiniteNumber(row && row.low),
close,
volume: toFiniteNumber(row && row.volume),
amount: toFiniteNumber(row && row.amount),
prev_close: prevClose,
source: 'yahoo_probe'
}
if (Number.isFinite(close)) prevClose = close
return output
})
}
const buildYahooChartPrevClose = items => {
const rows = (Array.isArray(items) ? items : []).map(row => ({
symbol: String(row && row.symbol || '').trim(),
yahoo_symbol: String(row && row.yahoo_symbol || '').trim(),
trade_date: String(row && row.trade_date || '').trim(),
trade_at: String(row && row.trade_at || '').trim(),
beijing_date: String(row && row.beijing_date || '').trim(),
beijing_at: String(row && row.beijing_at || '').trim(),
open: toFiniteNumber(row && row.open),
high: toFiniteNumber(row && row.high),
low: toFiniteNumber(row && row.low),
close: toFiniteNumber(row && row.close),
volume: toFiniteNumber(row && row.volume),
amount: toFiniteNumber(row && row.amount),
source: 'yahoo_probe'
}))
return attachYahooPrevClose(rows)
}
const parseYahooQuotePayload = (payload, mappings) => {
if (payload && Array.isArray(payload.items)) {
return payload.items.map(item => normalizeYahooQuoteItem(item))
}
const rows = payload && payload.quoteResponse && Array.isArray(payload.quoteResponse.result)
? payload.quoteResponse.result
: []
const reverse = new Map((Array.isArray(mappings) ? mappings : []).map(item => [String(item.yahoo || '').toUpperCase(), item]))
return rows.map(item => {
const hit = reverse.get(String(item && item.symbol || '').toUpperCase())
return normalizeYahooQuoteItem(item, {
originalSymbol: hit ? hit.original : String(item && item.symbol || '').trim(),
yahooSymbol: hit ? hit.yahoo : String(item && item.symbol || '').trim()
})
})
}
const fetchYahoo = async symbols => {
if (!symbols || symbols.length === 0) return []
const cfg = getYahooProbeConfig()
const trace = Array.isArray(arguments[1] && arguments[1].trace) ? arguments[1].trace : null
if (!cfg.enabled) {
pushYahooTrace(trace, 'yahoo.disabled', { reason: 'probe_disabled' })
return []
}
const wanted = Array.from(new Set((symbols || []).map(s => String(s || '').trim()).filter(Boolean)))
const outMap = new Map()
const missing = []
for (const symbol of wanted) {
const cached = getYahooProbeCache(yahooProbeState.quoteCache, symbol)
if (cached) {
outMap.set(symbol, cached)
pushYahooTrace(trace, 'yahoo.quote.cache_hit', { symbol })
} else {
missing.push(symbol)
}
}
if (missing.length > 0) {
pushYahooTrace(trace, 'yahoo.quote.enqueue', { symbols: missing, requested: wanted.length })
const rows = await enqueueYahooQuoteRequest(missing)
for (const row of rows) {
if (!row || !row.symbol) continue
outMap.set(row.symbol, row)
}
pushYahooTrace(trace, 'yahoo.quote.batch_ok', { symbols: missing, count: rows.length, cached: false })
}
const parsed = wanted.map(symbol => outMap.get(symbol)).filter(Boolean)
logJSON('markets.http.parse', { source: 'yahoo_probe', count: parsed.length, requested: wanted.length }, 'markets')
return parsed
}
const parseFinnhubCandlePayload = (payload, mapping) => {
const status = String(payload && payload.s || '').trim().toLowerCase()
if (status !== 'ok') throw new Error(`finnhub_probe_candle_invalid_${status || 'response'}`)
const timestamps = Array.isArray(payload && payload.t) ? payload.t : []
const openArr = Array.isArray(payload && payload.o) ? payload.o : []
const highArr = Array.isArray(payload && payload.h) ? payload.h : []
const lowArr = Array.isArray(payload && payload.l) ? payload.l : []
const closeArr = Array.isArray(payload && payload.c) ? payload.c : []
const volumeArr = Array.isArray(payload && payload.v) ? payload.v : []
const tz = getFinnhubTimeZone(mapping && mapping.yahoo)
const rows = []
for (let i = 0; i < timestamps.length; i += 1) {
const ts = toFiniteNumber(timestamps[i])
const close = toFiniteNumber(closeArr[i])
if (!Number.isFinite(ts) || !Number.isFinite(close)) continue
const tradeAt = unixToDateTime(ts, tz)
const beijingAt = unixToDateTime(ts, 'Asia/Shanghai')
rows.push({
symbol: mapping.original,
yahoo_symbol: mapping.yahoo,
trade_date: unixToDate(ts, tz),
trade_at: tradeAt,
beijing_date: beijingAt ? beijingAt.slice(0, 10) : '',
beijing_at: beijingAt,
open: toFiniteNumber(openArr[i]),
high: toFiniteNumber(highArr[i]),
low: toFiniteNumber(lowArr[i]),
close,
volume: toFiniteNumber(volumeArr[i]),
amount: approxAmount(close, volumeArr[i]),
source: 'finnhub_probe'
})
}
return attachYahooPrevClose(rows.map(row => ({ ...row, source: 'finnhub_probe' })))
}
const requestFinnhubCandle = async (mapping, options, cfg) => {
const finnhubSymbol = getFinnhubSymbol(mapping && mapping.yahoo)
if (!finnhubSymbol) throw new Error('finnhub_symbol_required')
await throttleFinnhubProbe(cfg, options && options.trace, 'chart')
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.finnhub_candle_endpoint}`
const unixRange = convertYahooRangeToUnixWindow(options && options.range)
const resolution = mapYahooIntervalToFinnhubResolution(options && options.interval)
logJSON('markets.http.get', {
source: 'finnhub_probe_candle',
url,
symbol: mapping.original,
yahoo_symbol: mapping.yahoo,
finnhub_symbol: finnhubSymbol,
resolution,
from: unixRange.from,
to: unixRange.to
}, 'markets')
const resp = await axios.get(url, {
timeout: cfg.request_timeout_ms,
validateStatus: () => true,
params: {
token: cfg.token,
symbol: finnhubSymbol,
resolution,
from: unixRange.from,
to: unixRange.to
}
})
if (!resp || resp.status >= 400) throw new Error(`finnhub_probe_candle_http_${resp && resp.status ? resp.status : 'ERR'}`)
const data = resp.data && typeof resp.data === 'object' ? resp.data : {}
const rows = parseFinnhubCandlePayload(data, mapping)
logJSON('markets.http.parse', { source: 'finnhub_probe_candle', symbol: mapping.original, count: rows.length }, 'markets')
return rows
}
const parseYahooChartPayload = (payload, mapping) => {
if (payload && Array.isArray(payload.items)) return buildYahooChartPrevClose(payload.items)
const result = payload && payload.chart && Array.isArray(payload.chart.result) ? payload.chart.result[0] : null
const error = payload && payload.chart ? payload.chart.error : null
if (!result) {
throw new Error(`yahoo_probe_chart_invalid_${String(error && error.description || 'response')}`)
}
const meta = result.meta || {}
const tz = String(meta.exchangeTimezoneName || 'America/New_York')
const timestamps = Array.isArray(result.timestamp) ? result.timestamp : []
const quote = result.indicators && Array.isArray(result.indicators.quote) ? result.indicators.quote[0] : {}
const openArr = Array.isArray(quote.open) ? quote.open : []
const highArr = Array.isArray(quote.high) ? quote.high : []
const lowArr = Array.isArray(quote.low) ? quote.low : []
const closeArr = Array.isArray(quote.close) ? quote.close : []
const volumeArr = Array.isArray(quote.volume) ? quote.volume : []
const rows = []
for (let i = 0; i < timestamps.length; i += 1) {
const ts = toFiniteNumber(timestamps[i])
const close = toFiniteNumber(closeArr[i])
if (!Number.isFinite(ts) || !Number.isFinite(close)) continue
const tradeAt = unixToDateTime(ts, tz)
const beijingAt = unixToDateTime(ts, 'Asia/Shanghai')
rows.push({
symbol: mapping.original,
yahoo_symbol: mapping.yahoo,
trade_date: unixToDate(ts, tz),
trade_at: tradeAt,
beijing_date: beijingAt ? beijingAt.slice(0, 10) : '',
beijing_at: beijingAt,
open: toFiniteNumber(openArr[i]),
high: toFiniteNumber(highArr[i]),
low: toFiniteNumber(lowArr[i]),
close,
volume: toFiniteNumber(volumeArr[i]),
amount: approxAmount(close, volumeArr[i]),
source: 'yahoo_probe'
})
}
return attachYahooPrevClose(rows)
}
const fetchYahooChart = async (symbol, options = {}) => {
const rawSymbol = String(symbol || '').trim()
if (!rawSymbol) return []
const yahooSymbol = getYahooProbeSymbol(rawSymbol)
const cfg = getYahooProbeConfig()
const trace = Array.isArray(options.trace) ? options.trace : null
if (!cfg.enabled) {
pushYahooTrace(trace, 'yahoo.chart.disabled', { symbol: rawSymbol, reason: 'probe_disabled' })
return []
}
const range = String(options.range || '1y').trim() || '1y'
const interval = String(options.interval || '1d').trim() || '1d'
const includePrePost = options.includePrePost === true || String(options.includePrePost || '') === '1'
const cacheKey = `${rawSymbol}|${range}|${interval}|${includePrePost ? 1 : 0}`
const cached = getYahooProbeCache(yahooProbeState.chartCache, cacheKey)
if (cached) {
pushYahooTrace(trace, 'yahoo.chart.cache_hit', { symbol: rawSymbol, range, interval })
return cached
}
let promise = yahooProbeState.chartInflight.get(cacheKey)
if (!promise) {
promise = (async () => {
await throttleYahooProbe('chart', cfg, trace)
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.chart_endpoint}`
// #region debug-point B:yahoo-chart-request
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'B', location: 'marketsClient.js:fetchYahooChart:request', msg: '[DEBUG] yahoo chart request', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, includePrePost, url }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
logJSON('markets.http.get', { source: 'yahoo_probe_chart', url, symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval }, 'markets')
const resp = await axios.get(url, {
timeout: cfg.request_timeout_ms,
validateStatus: () => true,
params: {
token: cfg.token,
type: 'chart',
symbol: yahooSymbol,
range,
interval,
includePrePost: includePrePost ? '1' : '0'
}
})
if (resp && resp.status === 429) {
pushYahooTrace(trace, 'yahoo.chart.http_429', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval })
logJSON('markets.http.status', { source: 'yahoo_probe_chart', url, status: 429, symbol: rawSymbol, yahoo_symbol: yahooSymbol, fallback: 'finnhub_probe_candle' }, 'markets')
return requestFinnhubCandle({ original: rawSymbol, yahoo: yahooSymbol }, { range, interval, trace }, cfg)
}
if (!resp || resp.status >= 400) throw new Error(`yahoo_probe_chart_http_${resp && resp.status ? resp.status : 'ERR'}`)
if (!resp.data) throw new Error('yahoo_probe_chart_invalid_response')
const rows = parseYahooChartPayload(resp.data, { original: rawSymbol, yahoo: yahooSymbol })
// #region debug-point C:yahoo-chart-response
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'C', location: 'marketsClient.js:fetchYahooChart:response', msg: '[DEBUG] yahoo chart response', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, status: resp.status, count: rows.length, sample_trade_date: (((rows || [])[0]) || {}).trade_date || '' }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
pushYahooTrace(trace, 'yahoo.chart.remote_ok', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, count: rows.length })
return rows
})()
yahooProbeState.chartInflight.set(cacheKey, promise)
promise
.then(
() => yahooProbeState.chartInflight.delete(cacheKey),
() => yahooProbeState.chartInflight.delete(cacheKey)
)
.catch(() => {})
} else {
pushYahooTrace(trace, 'yahoo.chart.inflight_reuse', { symbol: rawSymbol, range, interval })
}
try {
const rows = await promise
setYahooProbeCache(yahooProbeState.chartCache, cacheKey, rows, cfg.chart_ttl_seconds)
return rows
} catch (e) {
pushYahooTrace(trace, 'yahoo.chart.remote_error', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e.message || e) })
// #region debug-point C:yahoo-chart-error
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'C', location: 'marketsClient.js:fetchYahooChart:error', msg: '[DEBUG] yahoo chart error', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e && e.message ? e.message : e) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
// #endregion
logJSON('markets.http.error', { source: 'yahoo_probe_chart', symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e.message || e) }, 'markets')
throw e
}
}
const parseTencentLine = line => {
const m = line.match(/^v_(\w+)="([^"]*)/)
if (!m) return null
const symbol = m[1]
const body = m[2]
let parts = body.split('~')
if (parts.length < 3 && body.includes(',')) {
parts = body.split(',')
}
if (symbol.startsWith('hf_')) {
// Futures format: Price, Change%, ?, ?, ?, ?, Time, PrevClose, Open, ... , Name
// e.g. 5098.95,3.32,5097.20,5097.90,5109.30,4930.40,15:24:08,4935.00,4966.10,0,4,2,2026-02-04,纽约黄金
const name = parts[13] || symbol
const price = parseFloat(parts[0] || '0')
const prevClose = parseFloat(parts[7] || '0')
const change = price - prevClose
const pct = prevClose ? (change / prevClose) * 100 : 0
const last_trade_date = /^\d{4}-\d{2}-\d{2}$/.test(String(parts[12] || '')) ? String(parts[12]) : ''
return { symbol, name, price, prevClose, change, change_percent: pct, last_trade_date }
}
const name = parts[1] || symbol
const price = parseFloat(parts[3] || '0')
const prevClose = parseFloat(parts[4] || '0')
const change = price - prevClose
const pct = prevClose ? (change / prevClose) * 100 : 0
return { symbol, name, price, prevClose, change, change_percent: pct }
}
const fetchTencent = async symbols => {
const url = `http://qt.gtimg.cn/q=${symbols.join(',')}`
logJSON('markets.http.get', { source: 'tencent', url, symbols }, 'markets')
try {
const res = await axios.get(url, { responseType: 'arraybuffer', timeout: 10000, validateStatus: () => true })
if (!res || res.status >= 400) { logJSON('markets.http.status', { source: 'tencent', url, status: res && res.status }, 'markets'); return [] }
const text = iconv.decode(Buffer.from(res.data), 'gb18030')
const lines = String(text || '').split(/;\s*/).filter(Boolean)
const out = []
for (const line of lines) { const r = parseTencentLine(line); if (r) out.push({ ...r, source: 'tencent' }) }
logJSON('markets.http.parse', { source: 'tencent', url, count: out.length }, 'markets')
return out
} catch (e) {
logJSON('markets.http.error', { source: 'tencent', url, error: String(e.message || e) }, 'markets')
return []
}
}
const fetchEastmoneyComexGold = async () => {
const url = 'https://quote.eastmoney.com/globalfuture/GC00Y.html'
logJSON('markets.http.get', { source: 'eastmoney', url }, 'markets')
try {
const res = await axios.get(url, { timeout: 10000, validateStatus: () => true })
if (!res || res.status >= 400 || !res.data) {
logJSON('markets.http.status', { source: 'eastmoney', url, status: res && res.status }, 'markets')
return null
}
const html = String(res.data || '')
let m = html.match(/最新[::]\s*([0-9]+(?:\.[0-9]+)?)/)
if (!m) m = html.match(/最新价[::]\s*([0-9]+(?:\.[0-9]+)?)/)
const priceStr = m ? m[1] : null
if (!priceStr) {
logJSON('markets.http.parse', { source: 'eastmoney', url, matched: false }, 'markets')
return null
}
const price = parseFloat(priceStr)
if (!Number.isFinite(price)) return null
return { symbol: 'hf_GC', name: 'COMEX黄金', price, prevClose: 0, change: 0, change_percent: 0, source: 'eastmoney' }
} catch (e) {
logJSON('markets.http.error', { source: 'eastmoney', url, error: String(e.message || e) }, 'markets')
return null
}
}
module.exports = { fetchSina, fetchYahoo, fetchYahooChart, fetchTencent, fetchEastmoneyComexGold, getYahooProbeConfig }