yangxiangyuan
900 lines
41 KiB
JavaScript
900 lines
41 KiB
JavaScript
const axios = require('axios')
|
||
const iconv = require('iconv-lite')
|
||
const fs = require('fs')
|
||
const path = require('path')
|
||
const { logJSON } = require('./logger')
|
||
|
||
const sinaMapSymbol = s => {
|
||
const x = String(s).toLowerCase()
|
||
if (x === '000001' || x === 'sh000001') return 'sh000001'
|
||
if (x === '399001' || x === 'sz399001') return 'sz399001'
|
||
if (x === '399006' || x === 'sz399006') return 'sz399006'
|
||
if (x === '000300' || x === 'sh000300') return 'sh000300'
|
||
if (x === '899050' || x.startsWith('bj899050')) return 'bj899050'
|
||
if (x === '000688' || x === 'sh000688') return 'sh000688'
|
||
if (x === 'ixic' || x === '.ixic' || x === 'gb_ixic') return 'gb_ixic'
|
||
if (x === 'inx' || x === '.inx' || x === 'gb_inx') return 'gb_inx'
|
||
if (x === 'hsi.hk' || x === 'hsi' || x === 'hk_hsi' || x === 'hkhsi') return 'rt_hkHSI'
|
||
if (x === '03153.hk' || x === 'hk03153' || x === '03153') return 'rt_hk03153'
|
||
if (x === 'inda' || x === 'amex:inda' || x === 'us_inda' || x === 'gb_inda') return 'gb_inda'
|
||
if (x === 'ewg' || x === 'nysearca:ewg' || x === 'us_ewg' || x === 'gb_ewg') return 'gb_ewg'
|
||
return s
|
||
}
|
||
|
||
const parseSinaLine = line => {
|
||
const m = line.match(/^var\s+hq_str_(\w+)=\"([^\"]*)/)
|
||
if (!m) return null
|
||
const symbol = m[1]
|
||
const body = m[2]
|
||
const parts = body.split(',')
|
||
if (symbol.startsWith('gb_')) {
|
||
const name = parts[0] || symbol
|
||
const price = parseFloat(parts[1] || '0')
|
||
// find percent in any field
|
||
const pctField = parts.find(p => /%/.test(p)) || ''
|
||
const pctMatch = String(pctField).match(/(-?\d+(?:\.\d+)?)%/)
|
||
const pct = pctMatch ? parseFloat(pctMatch[1]) : 0
|
||
let prevClose = 0, change = 0
|
||
if (pct !== 0 && price) { prevClose = price / (1 + pct / 100); change = price - prevClose }
|
||
return { symbol, name, price, prevClose, change, change_percent: pct }
|
||
}
|
||
if (symbol.startsWith('hk')) {
|
||
const name = parts[0] || symbol
|
||
const price = parseFloat(parts[6] || parts[1] || '0')
|
||
const prevClose = parseFloat(parts[3] || parts[2] || '0')
|
||
const change = price - prevClose
|
||
const pct = prevClose ? (change / prevClose) * 100 : 0
|
||
return { symbol, name, price, prevClose, change, change_percent: pct }
|
||
}
|
||
const name = parts[0] || ''
|
||
const price = parseFloat(parts[3] || parts[1] || '0')
|
||
const prevClose = parseFloat(parts[2] || '0')
|
||
const change = price - prevClose
|
||
const pct = prevClose ? (change / prevClose) * 100 : 0
|
||
return { symbol, name, price, prevClose, change, change_percent: pct }
|
||
}
|
||
|
||
const fetchSina = async symbols => {
|
||
if (!symbols || symbols.length === 0) return []
|
||
const chunkSize = 6
|
||
const outMap = new Map()
|
||
for (let i = 0; i < symbols.length; i += chunkSize) {
|
||
const chunk = symbols.slice(i, i + chunkSize).map(sinaMapSymbol)
|
||
const list = chunk.join(',')
|
||
const url = `http://hq.sinajs.cn/list=${list}`
|
||
logJSON('markets.http.get', { source: 'sina', url, symbols: chunk, chunk_index: Math.floor(i / chunkSize), chunk_size: chunk.length }, 'markets')
|
||
try {
|
||
const res = await axios.get(url, {
|
||
headers: {
|
||
'Referer': 'https://finance.sina.com.cn/',
|
||
'User-Agent': 'Mozilla/5.0',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9'
|
||
},
|
||
responseType: 'arraybuffer',
|
||
timeout: 10000,
|
||
validateStatus: () => true
|
||
})
|
||
if (!res || res.status >= 400) {
|
||
logJSON('markets.http.status', { source: 'sina', url, status: res && res.status, chunk_index: Math.floor(i / chunkSize) }, 'markets')
|
||
continue
|
||
}
|
||
const text = iconv.decode(Buffer.from(res.data), 'gb18030')
|
||
const lines = String(text || '').split(/;\s*/).filter(Boolean)
|
||
let parsedCount = 0
|
||
for (const line of lines) {
|
||
const r = parseSinaLine(line)
|
||
if (r) {
|
||
parsedCount += 1
|
||
outMap.set(r.symbol, { ...r, source: 'sina' })
|
||
}
|
||
}
|
||
logJSON('markets.http.parse', { source: 'sina', chunk_index: Math.floor(i / chunkSize), count: parsedCount }, 'markets')
|
||
} catch (e) {
|
||
logJSON('markets.http.error', { source: 'sina', url, chunk_index: Math.floor(i / chunkSize), error: String(e.message || e) }, 'markets')
|
||
}
|
||
}
|
||
return Array.from(outMap.values())
|
||
}
|
||
|
||
const marketsConfigPath = path.join(process.cwd(), 'config', 'markets.json')
|
||
const yahooProbeState = {
|
||
quoteCache: new Map(),
|
||
chartCache: new Map(),
|
||
quotePendingSymbols: new Set(),
|
||
quotePendingWaiters: [],
|
||
quoteFlushTimer: null,
|
||
quoteFlushRunning: false,
|
||
chartInflight: new Map(),
|
||
lastRequestAt: { quote: 0, chart: 0 },
|
||
finnhubLastRequestAt: 0,
|
||
finnhubRequestTimes: []
|
||
}
|
||
const YAHOO_PROBE_SYMBOL_ALIAS = Object.freeze({
|
||
usINX: '^GSPC',
|
||
usIXIC: '^IXIC',
|
||
usDJI: '^DJI',
|
||
hkHSI: '^HSI',
|
||
zsN225: '^N225',
|
||
zsGDAXI: '^GDAXI',
|
||
zstwii: '^TWII',
|
||
zskospi: '^KS11',
|
||
zsBVSP: '^BVSP',
|
||
zsSTI: '^STI',
|
||
usEWG: 'EWG',
|
||
usFLSA: 'FLSA',
|
||
usNFTY: 'NFTY',
|
||
usREMX: 'REMX',
|
||
usEWJ: 'EWJ',
|
||
usEWY: 'EWY',
|
||
usEWZ: 'EWZ',
|
||
usASEA: 'ASEA',
|
||
usEWT: 'EWT',
|
||
usINDA: 'INDA',
|
||
hf_GC: 'GC=F',
|
||
hf_SI: 'SI=F'
|
||
})
|
||
const FINNHUB_TIMEZONE_ALIAS = Object.freeze({
|
||
'^GSPC': 'America/New_York',
|
||
'^IXIC': 'America/New_York',
|
||
'^DJI': 'America/New_York',
|
||
'^HSI': 'Asia/Hong_Kong',
|
||
'^N225': 'Asia/Tokyo',
|
||
'^GDAXI': 'Europe/Berlin',
|
||
'^TWII': 'Asia/Taipei',
|
||
'^KS11': 'Asia/Seoul',
|
||
'^BVSP': 'America/Sao_Paulo',
|
||
'^STI': 'Asia/Singapore'
|
||
})
|
||
const FINNHUB_DEFAULT_TZ = 'America/New_York'
|
||
|
||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))
|
||
|
||
const toFiniteNumber = value => {
|
||
const n = Number(value)
|
||
return Number.isFinite(n) ? n : null
|
||
}
|
||
|
||
const pushYahooTrace = (trace, stage, details = {}) => {
|
||
if (!Array.isArray(trace)) return
|
||
trace.push({
|
||
time: new Date().toISOString(),
|
||
stage,
|
||
...details
|
||
})
|
||
}
|
||
|
||
const readMarketsConfig = () => {
|
||
try {
|
||
if (!fs.existsSync(marketsConfigPath)) return {}
|
||
return JSON.parse(fs.readFileSync(marketsConfigPath, 'utf-8'))
|
||
} catch {
|
||
return {}
|
||
}
|
||
}
|
||
|
||
const getYahooProbeConfig = () => {
|
||
const cfg = readMarketsConfig()
|
||
const raw = cfg && typeof cfg.yahoo_probe === 'object' ? cfg.yahoo_probe : {}
|
||
const baseUrl = String(raw.base_url || 'http://47.252.30.127:19071').trim().replace(/\/$/, '')
|
||
return {
|
||
enabled: raw.enabled !== false,
|
||
base_url: baseUrl || 'http://47.252.30.127:19071',
|
||
token: String(raw.token || '').trim(),
|
||
quote_endpoint: String(raw.quote_endpoint || raw.endpoint || '/probe/yahoo').trim() || '/probe/yahoo',
|
||
chart_endpoint: String(raw.chart_endpoint || raw.endpoint || '/probe/yahoo').trim() || '/probe/yahoo',
|
||
finnhub_quote_endpoint: String(raw.finnhub_quote_endpoint || '/probe/finnhub/quote').trim() || '/probe/finnhub/quote',
|
||
finnhub_candle_endpoint: String(raw.finnhub_candle_endpoint || '/probe/finnhub/candle').trim() || '/probe/finnhub/candle',
|
||
request_timeout_ms: Math.max(3000, Number(raw.request_timeout_ms || 15000)),
|
||
quote_ttl_seconds: Math.max(15, Number(raw.quote_ttl_seconds || 120)),
|
||
chart_ttl_seconds: Math.max(300, Number(raw.chart_ttl_seconds || 21600)),
|
||
quote_batch_size: Math.max(1, Math.min(20, Number(raw.quote_batch_size || 4))),
|
||
quote_min_interval_ms: Math.max(0, Number(raw.quote_min_interval_ms || raw.min_request_interval_ms || 2500)),
|
||
chart_min_interval_ms: Math.max(0, Number(raw.chart_min_interval_ms || raw.min_request_interval_ms || 2500)),
|
||
finnhub_min_interval_ms: Math.max(1200, Number(raw.finnhub_min_interval_ms || 1300)),
|
||
finnhub_max_requests_per_minute: Math.max(1, Math.min(55, Number(raw.finnhub_max_requests_per_minute || 50)))
|
||
}
|
||
}
|
||
|
||
const getYahooProbeCache = (cache, key) => {
|
||
const hit = cache.get(key)
|
||
if (!hit) return null
|
||
if (Number(hit.expires_at || 0) <= Date.now()) {
|
||
cache.delete(key)
|
||
return null
|
||
}
|
||
return hit.value
|
||
}
|
||
|
||
const setYahooProbeCache = (cache, key, value, ttlSeconds) => {
|
||
cache.set(key, {
|
||
value,
|
||
expires_at: Date.now() + Math.max(1, ttlSeconds) * 1000
|
||
})
|
||
}
|
||
|
||
const getYahooProbeSymbol = symbol => {
|
||
const raw = String(symbol || '').trim()
|
||
if (!raw) return ''
|
||
if (YAHOO_PROBE_SYMBOL_ALIAS[raw]) return YAHOO_PROBE_SYMBOL_ALIAS[raw]
|
||
if (/^us[A-Z][A-Z0-9._-]+$/.test(raw)) return raw.slice(2)
|
||
return raw
|
||
}
|
||
|
||
const getFinnhubSymbol = symbol => getYahooProbeSymbol(symbol)
|
||
|
||
const getFinnhubTimeZone = symbol => {
|
||
const key = String(symbol || '').trim()
|
||
return FINNHUB_TIMEZONE_ALIAS[key] || FINNHUB_DEFAULT_TZ
|
||
}
|
||
|
||
const sleepUntilNextMinuteWindow = async cfg => {
|
||
const windowMs = 60 * 1000
|
||
const now = Date.now()
|
||
yahooProbeState.finnhubRequestTimes = yahooProbeState.finnhubRequestTimes.filter(ts => (now - ts) < windowMs)
|
||
const maxPerMinute = Math.max(1, Number(cfg.finnhub_max_requests_per_minute || 50))
|
||
if (yahooProbeState.finnhubRequestTimes.length < maxPerMinute) return
|
||
const oldest = Number(yahooProbeState.finnhubRequestTimes[0] || 0)
|
||
const waitMs = Math.max(0, windowMs - (now - oldest) + 50)
|
||
if (waitMs > 0) await sleep(waitMs)
|
||
}
|
||
|
||
const throttleFinnhubProbe = async (cfg, trace, reason) => {
|
||
await sleepUntilNextMinuteWindow(cfg)
|
||
const minInterval = Math.max(0, Number(cfg.finnhub_min_interval_ms || 1300))
|
||
const waitMs = Math.max(0, minInterval - (Date.now() - Number(yahooProbeState.finnhubLastRequestAt || 0)))
|
||
if (waitMs > 0) {
|
||
pushYahooTrace(trace, 'finnhub.throttle_wait', { wait_ms: waitMs, reason: String(reason || '') })
|
||
await sleep(waitMs)
|
||
}
|
||
const now = Date.now()
|
||
yahooProbeState.finnhubLastRequestAt = now
|
||
yahooProbeState.finnhubRequestTimes.push(now)
|
||
yahooProbeState.finnhubRequestTimes = yahooProbeState.finnhubRequestTimes.filter(ts => (now - ts) < 60 * 1000)
|
||
}
|
||
|
||
const shiftDateByRange = (date, amount, unit) => {
|
||
const next = new Date(date.getTime())
|
||
if (unit === 'd') next.setUTCDate(next.getUTCDate() - amount)
|
||
if (unit === 'mo') next.setUTCMonth(next.getUTCMonth() - amount)
|
||
if (unit === 'y') next.setUTCFullYear(next.getUTCFullYear() - amount)
|
||
return next
|
||
}
|
||
|
||
const convertYahooRangeToUnixWindow = range => {
|
||
const now = new Date()
|
||
const text = String(range || '1y').trim().toLowerCase()
|
||
let fromDate = shiftDateByRange(now, 1, 'y')
|
||
if (text === 'ytd') {
|
||
fromDate = new Date(Date.UTC(now.getUTCFullYear(), 0, 1, 0, 0, 0))
|
||
} else {
|
||
const match = text.match(/^(\d+)(d|mo|y)$/)
|
||
if (match) {
|
||
fromDate = shiftDateByRange(now, Number(match[1] || 1), match[2])
|
||
}
|
||
}
|
||
return {
|
||
from: Math.max(0, Math.floor(fromDate.getTime() / 1000)),
|
||
to: Math.max(0, Math.floor(now.getTime() / 1000))
|
||
}
|
||
}
|
||
|
||
const mapYahooIntervalToFinnhubResolution = interval => {
|
||
const text = String(interval || '1d').trim().toLowerCase()
|
||
if (text === '1d' || text === '1day' || text === 'd') return 'D'
|
||
if (text === '1wk' || text === '1w' || text === 'wk' || text === 'w') return 'W'
|
||
if (text === '1mo' || text === '1mth' || text === 'mo' || text === 'm') return 'M'
|
||
if (text === '60m' || text === '1h' || text === '60') return '60'
|
||
if (text === '30m' || text === '30') return '30'
|
||
if (text === '15m' || text === '15') return '15'
|
||
if (text === '5m' || text === '5') return '5'
|
||
if (text === '1m' || text === '1') return '1'
|
||
return 'D'
|
||
}
|
||
|
||
const scheduleYahooQuoteFlush = () => {
|
||
if (yahooProbeState.quoteFlushTimer) return
|
||
yahooProbeState.quoteFlushTimer = setTimeout(() => {
|
||
yahooProbeState.quoteFlushTimer = null
|
||
flushYahooQuoteQueue().catch(() => {})
|
||
}, 15)
|
||
}
|
||
|
||
const requestYahooQuoteBatch = async (batch, cfg) => {
|
||
const mappings = batch.map(original => ({ original, yahoo: getYahooProbeSymbol(original) }))
|
||
await throttleYahooProbe('quote', cfg, null)
|
||
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.quote_endpoint}`
|
||
// #region debug-point D:quote-batch-request
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'D', location: 'marketsClient.js:requestYahooQuoteBatch:request', msg: '[DEBUG] yahoo quote batch request', data: { url, batch, yahoo_symbols: mappings.map(item => item.yahoo), batch_size: batch.length }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
logJSON('markets.http.get', {
|
||
source: 'yahoo_probe',
|
||
url,
|
||
symbols: batch,
|
||
yahoo_symbols: mappings.map(item => item.yahoo),
|
||
batch_size: batch.length
|
||
}, 'markets')
|
||
const resp = await axios.get(url, {
|
||
timeout: cfg.request_timeout_ms,
|
||
validateStatus: () => true,
|
||
params: {
|
||
token: cfg.token,
|
||
type: 'quote',
|
||
symbols: mappings.map(item => item.yahoo).join(',')
|
||
}
|
||
})
|
||
if (resp && resp.status === 429) {
|
||
pushYahooTrace(null, 'yahoo.quote.http_429', { symbols: batch, yahoo_symbols: mappings.map(item => item.yahoo) })
|
||
logJSON('markets.http.status', { source: 'yahoo_probe', url, status: 429, symbols: batch, fallback: 'finnhub_probe_quote' }, 'markets')
|
||
return requestFinnhubQuoteBatchSerial(mappings, cfg)
|
||
}
|
||
if (!resp || resp.status >= 400) {
|
||
throw new Error(`yahoo_probe_http_${resp && resp.status ? resp.status : 'ERR'}`)
|
||
}
|
||
if (!resp.data) throw new Error('yahoo_probe_invalid_response')
|
||
const rows = parseYahooQuotePayload(resp.data, mappings)
|
||
// #region debug-point D:quote-batch-response
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'D', location: 'marketsClient.js:requestYahooQuoteBatch:response', msg: '[DEBUG] yahoo quote batch response', data: { batch, status: resp.status, rows_count: rows.length, row_symbols: rows.map(item => item.symbol), row_yahoo_symbols: rows.map(item => item.yahoo_symbol) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
return rows
|
||
}
|
||
|
||
const requestFinnhubQuoteSingle = async (mapping, cfg) => {
|
||
const finnhubSymbol = getFinnhubSymbol(mapping && mapping.yahoo)
|
||
if (!finnhubSymbol) throw new Error('finnhub_symbol_required')
|
||
await throttleFinnhubProbe(cfg, null, 'quote')
|
||
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.finnhub_quote_endpoint}`
|
||
logJSON('markets.http.get', {
|
||
source: 'finnhub_probe_quote',
|
||
url,
|
||
symbol: mapping.original,
|
||
yahoo_symbol: mapping.yahoo,
|
||
finnhub_symbol: finnhubSymbol
|
||
}, 'markets')
|
||
const resp = await axios.get(url, {
|
||
timeout: cfg.request_timeout_ms,
|
||
validateStatus: () => true,
|
||
params: {
|
||
token: cfg.token,
|
||
symbol: finnhubSymbol
|
||
}
|
||
})
|
||
if (!resp || resp.status >= 400) throw new Error(`finnhub_probe_quote_http_${resp && resp.status ? resp.status : 'ERR'}`)
|
||
const data = resp.data && typeof resp.data === 'object' ? resp.data : {}
|
||
const price = toFiniteNumber(data && data.c)
|
||
if (!Number.isFinite(price) || price <= 0) throw new Error('finnhub_probe_quote_invalid_response')
|
||
const prevClose = toFiniteNumber(data && data.pc)
|
||
const change = toFiniteNumber(data && data.d)
|
||
const pct = toFiniteNumber(data && data.dp)
|
||
const ts = toFiniteNumber(data && data.t)
|
||
return normalizeYahooQuoteItem({
|
||
symbol: mapping.original,
|
||
yahoo_symbol: mapping.yahoo,
|
||
name: mapping.original,
|
||
price,
|
||
prevClose: prevClose != null ? prevClose : 0,
|
||
change: change != null ? change : (prevClose != null ? price - prevClose : 0),
|
||
change_percent: pct != null ? pct : (prevClose ? ((price - prevClose) / prevClose) * 100 : 0),
|
||
regular_market_time: ts,
|
||
exchange_timezone_name: getFinnhubTimeZone(finnhubSymbol),
|
||
source: 'finnhub_probe'
|
||
}, {
|
||
originalSymbol: mapping.original,
|
||
yahooSymbol: mapping.yahoo,
|
||
source: 'finnhub_probe'
|
||
})
|
||
}
|
||
|
||
const requestFinnhubQuoteBatchSerial = async (mappings, cfg) => {
|
||
const rows = []
|
||
for (const mapping of (Array.isArray(mappings) ? mappings : [])) {
|
||
try {
|
||
const row = await requestFinnhubQuoteSingle(mapping, cfg)
|
||
if (row && row.symbol) rows.push(row)
|
||
} catch (error) {
|
||
const symbol = String(mapping && mapping.original || '').trim()
|
||
const cached = symbol ? getYahooProbeCache(yahooProbeState.quoteCache, symbol) : null
|
||
const message = String(error && error.message ? error.message : error)
|
||
logJSON('markets.http.error', {
|
||
source: 'finnhub_probe_quote',
|
||
symbol,
|
||
yahoo_symbol: String(mapping && mapping.yahoo || '').trim(),
|
||
error: message,
|
||
fallback: cached && cached.symbol ? 'quote_cache' : 'skip_symbol'
|
||
}, 'markets')
|
||
if (cached && cached.symbol) rows.push(cached)
|
||
}
|
||
}
|
||
logJSON('markets.http.parse', { source: 'finnhub_probe_quote', count: rows.length, requested: Array.isArray(mappings) ? mappings.length : 0 }, 'markets')
|
||
return rows
|
||
}
|
||
|
||
const flushYahooQuoteQueue = async () => {
|
||
if (yahooProbeState.quoteFlushRunning) return
|
||
if (yahooProbeState.quotePendingWaiters.length === 0) return
|
||
yahooProbeState.quoteFlushRunning = true
|
||
try {
|
||
while (yahooProbeState.quotePendingWaiters.length > 0) {
|
||
const waiters = yahooProbeState.quotePendingWaiters.splice(0)
|
||
const pendingSymbols = Array.from(yahooProbeState.quotePendingSymbols)
|
||
yahooProbeState.quotePendingSymbols.clear()
|
||
// #region debug-point A:quote-flush-begin
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'A', location: 'marketsClient.js:flushYahooQuoteQueue:begin', msg: '[DEBUG] yahoo quote flush begin', data: { waiter_count: waiters.length, pending_symbol_count: pendingSymbols.length, pending_symbols: pendingSymbols }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
if (pendingSymbols.length === 0) {
|
||
for (const waiter of waiters) waiter.resolve([])
|
||
continue
|
||
}
|
||
const cfg = getYahooProbeConfig()
|
||
const batchSize = Math.max(1, Number(cfg.quote_batch_size || 4))
|
||
const outMap = new Map()
|
||
for (let i = 0; i < pendingSymbols.length; i += batchSize) {
|
||
const batch = pendingSymbols.slice(i, i + batchSize)
|
||
try {
|
||
const rows = await requestYahooQuoteBatch(batch, cfg)
|
||
for (const row of rows) {
|
||
if (!row || !row.symbol) continue
|
||
outMap.set(row.symbol, row)
|
||
setYahooProbeCache(yahooProbeState.quoteCache, row.symbol, row, cfg.quote_ttl_seconds)
|
||
}
|
||
} catch (e) {
|
||
// #region debug-point A:quote-batch-error
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'markets-yahoo-429.env'); let u = 'http://127.0.0.1:7777/event', s = 'markets-yahoo-429'; try { const e2 = fs.readFileSync(p, 'utf8'); u = (e2.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e2.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'post-fix', hypothesisId: 'A', location: 'marketsClient.js:flushYahooQuoteQueue:error', msg: '[DEBUG] yahoo quote batch error', data: { batch, error: String(e && e.message ? e.message : e) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
logJSON('markets.http.error', { source: 'yahoo_probe', symbols: batch, error: String(e.message || e) }, 'markets')
|
||
}
|
||
}
|
||
for (const waiter of waiters) {
|
||
waiter.resolve(waiter.symbols.map(symbol => outMap.get(symbol)).filter(Boolean))
|
||
}
|
||
}
|
||
} finally {
|
||
yahooProbeState.quoteFlushRunning = false
|
||
if (yahooProbeState.quotePendingWaiters.length > 0) scheduleYahooQuoteFlush()
|
||
}
|
||
}
|
||
|
||
const enqueueYahooQuoteRequest = symbols => new Promise(resolve => {
|
||
const wanted = Array.from(new Set((symbols || []).map(s => String(s || '').trim()).filter(Boolean)))
|
||
yahooProbeState.quotePendingWaiters.push({ symbols: wanted, resolve })
|
||
for (const symbol of wanted) yahooProbeState.quotePendingSymbols.add(symbol)
|
||
scheduleYahooQuoteFlush()
|
||
})
|
||
|
||
const formatDateTime = (date, timeZone) => {
|
||
try {
|
||
const parts = new Intl.DateTimeFormat('sv-SE', {
|
||
timeZone,
|
||
year: 'numeric',
|
||
month: '2-digit',
|
||
day: '2-digit',
|
||
hour: '2-digit',
|
||
minute: '2-digit',
|
||
second: '2-digit',
|
||
hour12: false
|
||
}).formatToParts(date)
|
||
const get = type => parts.find(x => x.type === type)?.value || '00'
|
||
return `${get('year')}-${get('month')}-${get('day')} ${get('hour')}:${get('minute')}:${get('second')}`
|
||
} catch {
|
||
return ''
|
||
}
|
||
}
|
||
|
||
const unixToDateTime = (sec, timeZone) => {
|
||
const s = Number(sec)
|
||
if (!Number.isFinite(s) || s <= 0) return ''
|
||
return formatDateTime(new Date(s * 1000), timeZone || 'America/New_York')
|
||
}
|
||
|
||
const unixToDate = (sec, timeZone) => {
|
||
const dt = unixToDateTime(sec, timeZone)
|
||
return dt ? dt.slice(0, 10) : ''
|
||
}
|
||
|
||
const approxAmount = (close, volume) => {
|
||
const c = toFiniteNumber(close)
|
||
const v = toFiniteNumber(volume)
|
||
if (!Number.isFinite(c) || !Number.isFinite(v) || c <= 0 || v <= 0) return null
|
||
return c * v
|
||
}
|
||
|
||
const throttleYahooProbe = async (kind, cfg, trace) => {
|
||
const key = kind === 'chart' ? 'chart' : 'quote'
|
||
const minInterval = key === 'chart' ? cfg.chart_min_interval_ms : cfg.quote_min_interval_ms
|
||
const lastAt = Number(yahooProbeState.lastRequestAt[key] || 0)
|
||
const waitMs = Math.max(0, minInterval - (Date.now() - lastAt))
|
||
if (waitMs > 0) {
|
||
pushYahooTrace(trace, 'yahoo.throttle_wait', { kind: key, wait_ms: waitMs })
|
||
await sleep(waitMs)
|
||
}
|
||
yahooProbeState.lastRequestAt[key] = Date.now()
|
||
}
|
||
|
||
const normalizeYahooQuoteItem = (item, options = {}) => {
|
||
const originalSymbol = String(options.originalSymbol || (item && item.symbol) || '').trim()
|
||
const yahooSymbol = String(options.yahooSymbol || (item && item.yahoo_symbol) || (item && item.symbol) || '').trim()
|
||
const price = toFiniteNumber(item && (item.price != null ? item.price : item.regularMarketPrice))
|
||
const prevClose = toFiniteNumber(item && (item.prevClose != null ? item.prevClose : item.regularMarketPreviousClose))
|
||
const change = toFiniteNumber(item && (item.change != null ? item.change : item.regularMarketChange))
|
||
const pct = toFiniteNumber(item && (item.change_percent != null ? item.change_percent : item.regularMarketChangePercent))
|
||
const epoch = toFiniteNumber(item && (item.regular_market_time != null ? item.regular_market_time : item.regularMarketTime))
|
||
const exchangeTz = String(item && (item.exchange_timezone_name || item.exchangeTimezoneName || 'America/New_York') || 'America/New_York')
|
||
const tradeAt = String(item && item.last_trade_at || '').trim() || unixToDateTime(epoch, exchangeTz)
|
||
const beijingAt = String(item && item.last_trade_beijing_at || '').trim() || unixToDateTime(epoch, 'Asia/Shanghai')
|
||
return {
|
||
symbol: originalSymbol,
|
||
yahoo_symbol: yahooSymbol,
|
||
name: String(item && (item.name || item.longName || item.shortName || item.displayName || item.yahoo_symbol || item.symbol) || '').trim(),
|
||
price: price != null ? price : 0,
|
||
prevClose: prevClose != null ? prevClose : 0,
|
||
change: change != null ? change : (price != null && prevClose != null ? price - prevClose : 0),
|
||
change_percent: pct != null ? pct : (price != null && prevClose ? ((price - prevClose) / prevClose) * 100 : 0),
|
||
last_trade_date: String(item && item.last_trade_date || '').trim() || (tradeAt ? tradeAt.slice(0, 10) : ''),
|
||
last_trade_at: tradeAt,
|
||
last_trade_beijing_at: beijingAt,
|
||
captured_at: String(item && item.captured_at || '').trim(),
|
||
source: String(item && item.source || options.source || 'yahoo_probe').trim() || 'yahoo_probe'
|
||
}
|
||
}
|
||
|
||
const attachYahooPrevClose = rows => {
|
||
let prevClose = null
|
||
return (Array.isArray(rows) ? rows : []).map(row => {
|
||
const close = toFiniteNumber(row && row.close)
|
||
const output = {
|
||
...row,
|
||
open: toFiniteNumber(row && row.open),
|
||
high: toFiniteNumber(row && row.high),
|
||
low: toFiniteNumber(row && row.low),
|
||
close,
|
||
volume: toFiniteNumber(row && row.volume),
|
||
amount: toFiniteNumber(row && row.amount),
|
||
prev_close: prevClose,
|
||
source: 'yahoo_probe'
|
||
}
|
||
if (Number.isFinite(close)) prevClose = close
|
||
return output
|
||
})
|
||
}
|
||
|
||
const buildYahooChartPrevClose = items => {
|
||
const rows = (Array.isArray(items) ? items : []).map(row => ({
|
||
symbol: String(row && row.symbol || '').trim(),
|
||
yahoo_symbol: String(row && row.yahoo_symbol || '').trim(),
|
||
trade_date: String(row && row.trade_date || '').trim(),
|
||
trade_at: String(row && row.trade_at || '').trim(),
|
||
beijing_date: String(row && row.beijing_date || '').trim(),
|
||
beijing_at: String(row && row.beijing_at || '').trim(),
|
||
open: toFiniteNumber(row && row.open),
|
||
high: toFiniteNumber(row && row.high),
|
||
low: toFiniteNumber(row && row.low),
|
||
close: toFiniteNumber(row && row.close),
|
||
volume: toFiniteNumber(row && row.volume),
|
||
amount: toFiniteNumber(row && row.amount),
|
||
source: 'yahoo_probe'
|
||
}))
|
||
return attachYahooPrevClose(rows)
|
||
}
|
||
|
||
const parseYahooQuotePayload = (payload, mappings) => {
|
||
if (payload && Array.isArray(payload.items)) {
|
||
return payload.items.map(item => normalizeYahooQuoteItem(item))
|
||
}
|
||
const rows = payload && payload.quoteResponse && Array.isArray(payload.quoteResponse.result)
|
||
? payload.quoteResponse.result
|
||
: []
|
||
const reverse = new Map((Array.isArray(mappings) ? mappings : []).map(item => [String(item.yahoo || '').toUpperCase(), item]))
|
||
return rows.map(item => {
|
||
const hit = reverse.get(String(item && item.symbol || '').toUpperCase())
|
||
return normalizeYahooQuoteItem(item, {
|
||
originalSymbol: hit ? hit.original : String(item && item.symbol || '').trim(),
|
||
yahooSymbol: hit ? hit.yahoo : String(item && item.symbol || '').trim()
|
||
})
|
||
})
|
||
}
|
||
|
||
const fetchYahoo = async symbols => {
|
||
if (!symbols || symbols.length === 0) return []
|
||
const cfg = getYahooProbeConfig()
|
||
const trace = Array.isArray(arguments[1] && arguments[1].trace) ? arguments[1].trace : null
|
||
if (!cfg.enabled) {
|
||
pushYahooTrace(trace, 'yahoo.disabled', { reason: 'probe_disabled' })
|
||
return []
|
||
}
|
||
|
||
const wanted = Array.from(new Set((symbols || []).map(s => String(s || '').trim()).filter(Boolean)))
|
||
const outMap = new Map()
|
||
const missing = []
|
||
for (const symbol of wanted) {
|
||
const cached = getYahooProbeCache(yahooProbeState.quoteCache, symbol)
|
||
if (cached) {
|
||
outMap.set(symbol, cached)
|
||
pushYahooTrace(trace, 'yahoo.quote.cache_hit', { symbol })
|
||
} else {
|
||
missing.push(symbol)
|
||
}
|
||
}
|
||
|
||
if (missing.length > 0) {
|
||
pushYahooTrace(trace, 'yahoo.quote.enqueue', { symbols: missing, requested: wanted.length })
|
||
const rows = await enqueueYahooQuoteRequest(missing)
|
||
for (const row of rows) {
|
||
if (!row || !row.symbol) continue
|
||
outMap.set(row.symbol, row)
|
||
}
|
||
pushYahooTrace(trace, 'yahoo.quote.batch_ok', { symbols: missing, count: rows.length, cached: false })
|
||
}
|
||
|
||
const parsed = wanted.map(symbol => outMap.get(symbol)).filter(Boolean)
|
||
logJSON('markets.http.parse', { source: 'yahoo_probe', count: parsed.length, requested: wanted.length }, 'markets')
|
||
return parsed
|
||
}
|
||
|
||
const parseFinnhubCandlePayload = (payload, mapping) => {
|
||
const status = String(payload && payload.s || '').trim().toLowerCase()
|
||
if (status !== 'ok') throw new Error(`finnhub_probe_candle_invalid_${status || 'response'}`)
|
||
const timestamps = Array.isArray(payload && payload.t) ? payload.t : []
|
||
const openArr = Array.isArray(payload && payload.o) ? payload.o : []
|
||
const highArr = Array.isArray(payload && payload.h) ? payload.h : []
|
||
const lowArr = Array.isArray(payload && payload.l) ? payload.l : []
|
||
const closeArr = Array.isArray(payload && payload.c) ? payload.c : []
|
||
const volumeArr = Array.isArray(payload && payload.v) ? payload.v : []
|
||
const tz = getFinnhubTimeZone(mapping && mapping.yahoo)
|
||
const rows = []
|
||
for (let i = 0; i < timestamps.length; i += 1) {
|
||
const ts = toFiniteNumber(timestamps[i])
|
||
const close = toFiniteNumber(closeArr[i])
|
||
if (!Number.isFinite(ts) || !Number.isFinite(close)) continue
|
||
const tradeAt = unixToDateTime(ts, tz)
|
||
const beijingAt = unixToDateTime(ts, 'Asia/Shanghai')
|
||
rows.push({
|
||
symbol: mapping.original,
|
||
yahoo_symbol: mapping.yahoo,
|
||
trade_date: unixToDate(ts, tz),
|
||
trade_at: tradeAt,
|
||
beijing_date: beijingAt ? beijingAt.slice(0, 10) : '',
|
||
beijing_at: beijingAt,
|
||
open: toFiniteNumber(openArr[i]),
|
||
high: toFiniteNumber(highArr[i]),
|
||
low: toFiniteNumber(lowArr[i]),
|
||
close,
|
||
volume: toFiniteNumber(volumeArr[i]),
|
||
amount: approxAmount(close, volumeArr[i]),
|
||
source: 'finnhub_probe'
|
||
})
|
||
}
|
||
return attachYahooPrevClose(rows.map(row => ({ ...row, source: 'finnhub_probe' })))
|
||
}
|
||
|
||
const requestFinnhubCandle = async (mapping, options, cfg) => {
|
||
const finnhubSymbol = getFinnhubSymbol(mapping && mapping.yahoo)
|
||
if (!finnhubSymbol) throw new Error('finnhub_symbol_required')
|
||
await throttleFinnhubProbe(cfg, options && options.trace, 'chart')
|
||
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.finnhub_candle_endpoint}`
|
||
const unixRange = convertYahooRangeToUnixWindow(options && options.range)
|
||
const resolution = mapYahooIntervalToFinnhubResolution(options && options.interval)
|
||
logJSON('markets.http.get', {
|
||
source: 'finnhub_probe_candle',
|
||
url,
|
||
symbol: mapping.original,
|
||
yahoo_symbol: mapping.yahoo,
|
||
finnhub_symbol: finnhubSymbol,
|
||
resolution,
|
||
from: unixRange.from,
|
||
to: unixRange.to
|
||
}, 'markets')
|
||
const resp = await axios.get(url, {
|
||
timeout: cfg.request_timeout_ms,
|
||
validateStatus: () => true,
|
||
params: {
|
||
token: cfg.token,
|
||
symbol: finnhubSymbol,
|
||
resolution,
|
||
from: unixRange.from,
|
||
to: unixRange.to
|
||
}
|
||
})
|
||
if (!resp || resp.status >= 400) throw new Error(`finnhub_probe_candle_http_${resp && resp.status ? resp.status : 'ERR'}`)
|
||
const data = resp.data && typeof resp.data === 'object' ? resp.data : {}
|
||
const rows = parseFinnhubCandlePayload(data, mapping)
|
||
logJSON('markets.http.parse', { source: 'finnhub_probe_candle', symbol: mapping.original, count: rows.length }, 'markets')
|
||
return rows
|
||
}
|
||
|
||
const parseYahooChartPayload = (payload, mapping) => {
|
||
if (payload && Array.isArray(payload.items)) return buildYahooChartPrevClose(payload.items)
|
||
const result = payload && payload.chart && Array.isArray(payload.chart.result) ? payload.chart.result[0] : null
|
||
const error = payload && payload.chart ? payload.chart.error : null
|
||
if (!result) {
|
||
throw new Error(`yahoo_probe_chart_invalid_${String(error && error.description || 'response')}`)
|
||
}
|
||
const meta = result.meta || {}
|
||
const tz = String(meta.exchangeTimezoneName || 'America/New_York')
|
||
const timestamps = Array.isArray(result.timestamp) ? result.timestamp : []
|
||
const quote = result.indicators && Array.isArray(result.indicators.quote) ? result.indicators.quote[0] : {}
|
||
const openArr = Array.isArray(quote.open) ? quote.open : []
|
||
const highArr = Array.isArray(quote.high) ? quote.high : []
|
||
const lowArr = Array.isArray(quote.low) ? quote.low : []
|
||
const closeArr = Array.isArray(quote.close) ? quote.close : []
|
||
const volumeArr = Array.isArray(quote.volume) ? quote.volume : []
|
||
const rows = []
|
||
for (let i = 0; i < timestamps.length; i += 1) {
|
||
const ts = toFiniteNumber(timestamps[i])
|
||
const close = toFiniteNumber(closeArr[i])
|
||
if (!Number.isFinite(ts) || !Number.isFinite(close)) continue
|
||
const tradeAt = unixToDateTime(ts, tz)
|
||
const beijingAt = unixToDateTime(ts, 'Asia/Shanghai')
|
||
rows.push({
|
||
symbol: mapping.original,
|
||
yahoo_symbol: mapping.yahoo,
|
||
trade_date: unixToDate(ts, tz),
|
||
trade_at: tradeAt,
|
||
beijing_date: beijingAt ? beijingAt.slice(0, 10) : '',
|
||
beijing_at: beijingAt,
|
||
open: toFiniteNumber(openArr[i]),
|
||
high: toFiniteNumber(highArr[i]),
|
||
low: toFiniteNumber(lowArr[i]),
|
||
close,
|
||
volume: toFiniteNumber(volumeArr[i]),
|
||
amount: approxAmount(close, volumeArr[i]),
|
||
source: 'yahoo_probe'
|
||
})
|
||
}
|
||
return attachYahooPrevClose(rows)
|
||
}
|
||
|
||
const fetchYahooChart = async (symbol, options = {}) => {
|
||
const rawSymbol = String(symbol || '').trim()
|
||
if (!rawSymbol) return []
|
||
const yahooSymbol = getYahooProbeSymbol(rawSymbol)
|
||
const cfg = getYahooProbeConfig()
|
||
const trace = Array.isArray(options.trace) ? options.trace : null
|
||
if (!cfg.enabled) {
|
||
pushYahooTrace(trace, 'yahoo.chart.disabled', { symbol: rawSymbol, reason: 'probe_disabled' })
|
||
return []
|
||
}
|
||
const range = String(options.range || '1y').trim() || '1y'
|
||
const interval = String(options.interval || '1d').trim() || '1d'
|
||
const includePrePost = options.includePrePost === true || String(options.includePrePost || '') === '1'
|
||
const cacheKey = `${rawSymbol}|${range}|${interval}|${includePrePost ? 1 : 0}`
|
||
const cached = getYahooProbeCache(yahooProbeState.chartCache, cacheKey)
|
||
if (cached) {
|
||
pushYahooTrace(trace, 'yahoo.chart.cache_hit', { symbol: rawSymbol, range, interval })
|
||
return cached
|
||
}
|
||
|
||
let promise = yahooProbeState.chartInflight.get(cacheKey)
|
||
if (!promise) {
|
||
promise = (async () => {
|
||
await throttleYahooProbe('chart', cfg, trace)
|
||
const url = `${cfg.base_url.replace(/\/$/, '')}${cfg.chart_endpoint}`
|
||
// #region debug-point B:yahoo-chart-request
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'B', location: 'marketsClient.js:fetchYahooChart:request', msg: '[DEBUG] yahoo chart request', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, includePrePost, url }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
logJSON('markets.http.get', { source: 'yahoo_probe_chart', url, symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval }, 'markets')
|
||
const resp = await axios.get(url, {
|
||
timeout: cfg.request_timeout_ms,
|
||
validateStatus: () => true,
|
||
params: {
|
||
token: cfg.token,
|
||
type: 'chart',
|
||
symbol: yahooSymbol,
|
||
range,
|
||
interval,
|
||
includePrePost: includePrePost ? '1' : '0'
|
||
}
|
||
})
|
||
if (resp && resp.status === 429) {
|
||
pushYahooTrace(trace, 'yahoo.chart.http_429', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval })
|
||
logJSON('markets.http.status', { source: 'yahoo_probe_chart', url, status: 429, symbol: rawSymbol, yahoo_symbol: yahooSymbol, fallback: 'finnhub_probe_candle' }, 'markets')
|
||
return requestFinnhubCandle({ original: rawSymbol, yahoo: yahooSymbol }, { range, interval, trace }, cfg)
|
||
}
|
||
if (!resp || resp.status >= 400) throw new Error(`yahoo_probe_chart_http_${resp && resp.status ? resp.status : 'ERR'}`)
|
||
if (!resp.data) throw new Error('yahoo_probe_chart_invalid_response')
|
||
const rows = parseYahooChartPayload(resp.data, { original: rawSymbol, yahoo: yahooSymbol })
|
||
// #region debug-point C:yahoo-chart-response
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'C', location: 'marketsClient.js:fetchYahooChart:response', msg: '[DEBUG] yahoo chart response', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, status: resp.status, count: rows.length, sample_trade_date: (((rows || [])[0]) || {}).trade_date || '' }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
pushYahooTrace(trace, 'yahoo.chart.remote_ok', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, count: rows.length })
|
||
return rows
|
||
})()
|
||
yahooProbeState.chartInflight.set(cacheKey, promise)
|
||
promise
|
||
.then(
|
||
() => yahooProbeState.chartInflight.delete(cacheKey),
|
||
() => yahooProbeState.chartInflight.delete(cacheKey)
|
||
)
|
||
.catch(() => {})
|
||
} else {
|
||
pushYahooTrace(trace, 'yahoo.chart.inflight_reuse', { symbol: rawSymbol, range, interval })
|
||
}
|
||
|
||
try {
|
||
const rows = await promise
|
||
setYahooProbeCache(yahooProbeState.chartCache, cacheKey, rows, cfg.chart_ttl_seconds)
|
||
return rows
|
||
} catch (e) {
|
||
pushYahooTrace(trace, 'yahoo.chart.remote_error', { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e.message || e) })
|
||
// #region debug-point C:yahoo-chart-error
|
||
;(() => { try { const p = path.join(process.cwd(), '.dbg', 'weekly-yahoo-zero.env'); let u = 'http://127.0.0.1:7777/event', s = 'weekly-yahoo-zero'; try { const e = fs.readFileSync(p, 'utf8'); u = (e.match(/DEBUG_SERVER_URL=(.+)/) || [])[1] || u; s = (e.match(/DEBUG_SESSION_ID=(.+)/) || [])[1] || s } catch {} fetch(u, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ sessionId: s, runId: 'pre-fix', hypothesisId: 'C', location: 'marketsClient.js:fetchYahooChart:error', msg: '[DEBUG] yahoo chart error', data: { symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e && e.message ? e.message : e) }, ts: Date.now() }) }).catch(() => {}) } catch {} })()
|
||
// #endregion
|
||
logJSON('markets.http.error', { source: 'yahoo_probe_chart', symbol: rawSymbol, yahoo_symbol: yahooSymbol, range, interval, error: String(e.message || e) }, 'markets')
|
||
throw e
|
||
}
|
||
}
|
||
|
||
const parseTencentLine = line => {
|
||
const m = line.match(/^v_(\w+)="([^"]*)/)
|
||
if (!m) return null
|
||
const symbol = m[1]
|
||
const body = m[2]
|
||
let parts = body.split('~')
|
||
if (parts.length < 3 && body.includes(',')) {
|
||
parts = body.split(',')
|
||
}
|
||
|
||
if (symbol.startsWith('hf_')) {
|
||
// Futures format: Price, Change%, ?, ?, ?, ?, Time, PrevClose, Open, ... , Name
|
||
// e.g. 5098.95,3.32,5097.20,5097.90,5109.30,4930.40,15:24:08,4935.00,4966.10,0,4,2,2026-02-04,纽约黄金
|
||
const name = parts[13] || symbol
|
||
const price = parseFloat(parts[0] || '0')
|
||
const prevClose = parseFloat(parts[7] || '0')
|
||
const change = price - prevClose
|
||
const pct = prevClose ? (change / prevClose) * 100 : 0
|
||
const last_trade_date = /^\d{4}-\d{2}-\d{2}$/.test(String(parts[12] || '')) ? String(parts[12]) : ''
|
||
return { symbol, name, price, prevClose, change, change_percent: pct, last_trade_date }
|
||
}
|
||
|
||
const name = parts[1] || symbol
|
||
const price = parseFloat(parts[3] || '0')
|
||
const prevClose = parseFloat(parts[4] || '0')
|
||
const change = price - prevClose
|
||
const pct = prevClose ? (change / prevClose) * 100 : 0
|
||
return { symbol, name, price, prevClose, change, change_percent: pct }
|
||
}
|
||
|
||
const fetchTencent = async symbols => {
|
||
const url = `http://qt.gtimg.cn/q=${symbols.join(',')}`
|
||
logJSON('markets.http.get', { source: 'tencent', url, symbols }, 'markets')
|
||
try {
|
||
const res = await axios.get(url, { responseType: 'arraybuffer', timeout: 10000, validateStatus: () => true })
|
||
if (!res || res.status >= 400) { logJSON('markets.http.status', { source: 'tencent', url, status: res && res.status }, 'markets'); return [] }
|
||
const text = iconv.decode(Buffer.from(res.data), 'gb18030')
|
||
const lines = String(text || '').split(/;\s*/).filter(Boolean)
|
||
const out = []
|
||
for (const line of lines) { const r = parseTencentLine(line); if (r) out.push({ ...r, source: 'tencent' }) }
|
||
logJSON('markets.http.parse', { source: 'tencent', url, count: out.length }, 'markets')
|
||
return out
|
||
} catch (e) {
|
||
logJSON('markets.http.error', { source: 'tencent', url, error: String(e.message || e) }, 'markets')
|
||
return []
|
||
}
|
||
}
|
||
|
||
const fetchEastmoneyComexGold = async () => {
|
||
const url = 'https://quote.eastmoney.com/globalfuture/GC00Y.html'
|
||
logJSON('markets.http.get', { source: 'eastmoney', url }, 'markets')
|
||
try {
|
||
const res = await axios.get(url, { timeout: 10000, validateStatus: () => true })
|
||
if (!res || res.status >= 400 || !res.data) {
|
||
logJSON('markets.http.status', { source: 'eastmoney', url, status: res && res.status }, 'markets')
|
||
return null
|
||
}
|
||
const html = String(res.data || '')
|
||
let m = html.match(/最新[：:]\s*([0-9]+(?:\.[0-9]+)?)/)
|
||
if (!m) m = html.match(/最新价[：:]\s*([0-9]+(?:\.[0-9]+)?)/)
|
||
const priceStr = m ? m[1] : null
|
||
if (!priceStr) {
|
||
logJSON('markets.http.parse', { source: 'eastmoney', url, matched: false }, 'markets')
|
||
return null
|
||
}
|
||
const price = parseFloat(priceStr)
|
||
if (!Number.isFinite(price)) return null
|
||
return { symbol: 'hf_GC', name: 'COMEX黄金', price, prevClose: 0, change: 0, change_percent: 0, source: 'eastmoney' }
|
||
} catch (e) {
|
||
logJSON('markets.http.error', { source: 'eastmoney', url, error: String(e.message || e) }, 'markets')
|
||
return null
|
||
}
|
||
}
|
||
|
||
module.exports = { fetchSina, fetchYahoo, fetchYahooChart, fetchTencent, fetchEastmoneyComexGold, getYahooProbeConfig }
|