feat: 初始提交(仅核心代码,已排除大文件)

This commit is contained in:
yangxiangyuan
2026-07-08 10:29:50 +08:00
commit be7c5842f1
623 changed files with 332897 additions and 0 deletions
+85
View File
@@ -0,0 +1,85 @@
// ============================================================
// data_gateway/auth.js - 对外 API 鉴权模块
// 职责:校验 X-API-Id + X-API-Key,匹配 skill config.json 中的哈希
// ============================================================
const crypto = require('crypto')
const fs = require('fs')
const path = require('path')
const SKILLS_DIR = path.join(__dirname, 'skills')
// 缓存已加载的 skill 配置
const skillConfigCache = new Map()
const loadSkillConfig = (skillId) => {
if (skillConfigCache.has(skillId)) return skillConfigCache.get(skillId)
const cfgPath = path.join(SKILLS_DIR, skillId, 'config.json')
if (!fs.existsSync(cfgPath)) return null
try {
const cfg = JSON.parse(fs.readFileSync(cfgPath, 'utf-8'))
skillConfigCache.set(skillId, cfg)
return cfg
} catch {
return null
}
}
// 刷新缓存(用于配置热更新)
const refreshCache = (skillId) => {
skillConfigCache.delete(skillId)
return loadSkillConfig(skillId)
}
// 枚举所有已启用的 skill
const listEnabledSkills = () => {
const result = []
if (!fs.existsSync(SKILLS_DIR)) return result
const dirs = fs.readdirSync(SKILLS_DIR, { withFileTypes: true })
for (const d of dirs) {
if (!d.isDirectory()) continue
const cfg = loadSkillConfig(d.name)
if (cfg && cfg.enabled !== false) {
result.push({ id: cfg.id, name: cfg.name })
}
}
return result
}
// 校验外部 API 请求
const verifyApiKey = (req) => {
const apiId = String(req.headers['x-api-id'] || '').trim()
const apiKey = String(req.headers['x-api-key'] || '').trim()
if (!apiId || !apiKey) return { ok: false, error: 'missing api id or key' }
const cfg = loadSkillConfig(apiId)
if (!cfg) return { ok: false, error: 'skill not found' }
if (cfg.enabled === false) return { ok: false, error: 'skill disabled' }
const expectedHash = String(cfg.api_key_hash || '')
if (!expectedHash) return { ok: false, error: 'skill not configured' }
const actualHash = 'sha256:' + crypto.createHash('sha256').update(apiKey).digest('hex')
if (actualHash !== expectedHash) return { ok: false, error: 'invalid api key' }
return { ok: true, skill: cfg }
}
// 校验内部 Tool 读取请求
const verifyInternalToken = (req) => {
const token = String(req.headers['x-internal-token'] || '').trim()
const skillId = String(req.headers['x-skill-id'] || req.params?.skillId || '').trim()
if (!token || !skillId) return { ok: false, error: 'missing token or skill id' }
const cfg = loadSkillConfig(skillId)
if (!cfg) return { ok: false, error: 'skill not found' }
const expectedHash = String(cfg.read_token_hash || '')
if (!expectedHash) return { ok: false, error: 'skill not configured for internal read' }
const actualHash = 'sha256:' + crypto.createHash('sha256').update(token).digest('hex')
if (actualHash !== expectedHash) return { ok: false, error: 'invalid internal token' }
return { ok: true, skill: cfg }
}
module.exports = { loadSkillConfig, refreshCache, listEnabledSkills, verifyApiKey, verifyInternalToken }
+219
View File
@@ -0,0 +1,219 @@
// ============================================================
// data_gateway/index.js - 对外 API 数据网关主入口
// 暴露两个路由:
// POST /api/v1/ingest/:skillId 外部数据写入(公网,需 API Key)
// GET /api/v1/data/:skillId/query 内部数据读取(仅 localhost,需 Internal Token)
// ============================================================
const { logJSON } = require('../logger')
const { verifyApiKey, verifyInternalToken, listEnabledSkills } = require('./auth')
const { insert, upsertLatest, queryLatest, queryList, queryByTimeRange, count } = require('./store')
const { loadSkillConfig } = require('./auth')
const { bindRoutes: bindExternalStorageRoutes } = require('./skills/external_storage')
const { bindRoutes: bindEmailSenderRoutes } = require('./skills/email_sender')
const setNoCache = (res) => {
try {
res.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate')
res.set('Pragma', 'no-cache')
res.set('Expires', '0')
} catch {}
}
// ============================================================
// 安全工具函数
// ============================================================
// 原型污染防护:移除 __proto__、constructor、prototype 等危险键
const PROTO_KEYS = new Set(['__proto__', 'constructor', 'prototype', '__defineGetter__', '__defineSetter__', '__lookupGetter__', '__lookupSetter__', '__proto__'])
const sanitizeObject = (obj, depth = 0) => {
if (depth > 10) return {} // 防止无限递归
if (obj === null || typeof obj !== 'object') return obj
if (Array.isArray(obj)) return obj.map(item => sanitizeObject(item, depth + 1))
const clean = {}
for (const [key, val] of Object.entries(obj)) {
if (PROTO_KEYS.has(key)) continue
clean[key] = sanitizeObject(val, depth + 1)
}
return clean
}
// 全局错误处理:隐藏详细堆栈,避免泄露路径和技术栈
const errorHandler = (err, req, res, _next) => {
try { logJSON('data_gateway.error', { path: req.path, error: String(err.message || err) }, 'data_gateway') } catch {}
if (!res.headersSent) {
res.status(500).json({ ok: false, error: 'internal_error' })
}
}
// Ingest 接口速率限制:同一 IP 每分钟最多 60 次请求
const ingestRateLimit = new Map()
const INGEST_MAX_PER_MIN = 60
const INGEST_WINDOW_MS = 60 * 1000
const checkIngestRate = (ip) => {
const rec = ingestRateLimit.get(ip)
if (!rec || Date.now() - rec.start > INGEST_WINDOW_MS) {
ingestRateLimit.set(ip, { start: Date.now(), count: 1 })
return { ok: true }
}
rec.count++
if (rec.count > INGEST_MAX_PER_MIN) {
const waitSec = Math.ceil((rec.start + INGEST_WINDOW_MS - Date.now()) / 1000)
return { ok: false, retryAfter: Math.max(1, waitSec) }
}
return { ok: true }
}
// 每分钟清理一次
setInterval(() => {
const now = Date.now()
for (const [ip, rec] of ingestRateLimit.entries()) {
if (now - rec.start > INGEST_WINDOW_MS + 10000) ingestRateLimit.delete(ip)
}
}, 60000)
const SOURCE_MAX_LEN = 255
const DATA_MAX_BYTES = 10 * 1024 // 10KB
const bindRoutes = (app) => {
// ============================================================
// 外部存储 skill(第三方 OSS 读写)
// 路由挂载在 /api/v1/skills/external_storage/*
// ============================================================
try {
bindExternalStorageRoutes(app)
} catch (e) {
console.error(`[data_gateway] Failed to bind external_storage routes: ${e.message}`)
}
// ============================================================
// 邮件发送 skill(对外发邮件)
// 路由挂载在 /api/v1/ingest/email_sender/*
// ============================================================
try {
bindEmailSenderRoutes(app)
} catch (e) {
console.error(`[data_gateway] Failed to bind email_sender routes: ${e.message}`)
}
// ============================================================
// 外部写入接口(公网暴露)
// POST /api/v1/ingest/:skillId
// Headers: X-API-Id, X-API-Key
// Body: { "source": "...", "data": { ... } }
// ============================================================
app.post('/api/v1/ingest/:skillId', (req, res) => {
setNoCache(res)
const skillId = String(req.params.skillId || '').trim()
if (!skillId) return res.status(400).json({ ok: false, error: 'missing skill id' })
const auth = verifyApiKey(req)
if (!auth.ok) {
try { logJSON('data_gateway.auth.fail', { skillId, error: auth.error, ip: req.ip }, 'data_gateway') } catch {}
return res.status(401).json({ ok: false, error: auth.error })
}
// 速率限制
const clientIp = req.ip || req.connection.remoteAddress || 'unknown'
const rateCheck = checkIngestRate(clientIp)
if (!rateCheck.ok) {
return res.status(429).json({ ok: false, error: '请求过于频繁,请在 ' + rateCheck.retryAfter + ' 秒后重试' })
}
const body = req.body || {}
let source = String(body.source || '').trim()
let data = body.data
// source 长度限制
if (source.length > SOURCE_MAX_LEN) {
return res.status(400).json({ ok: false, error: 'source 超过长度限制(' + SOURCE_MAX_LEN + ' 字符)' })
}
// 如果没有 data 字段,将整个 body 视为 data(排除 source)
if (data === undefined) {
const bodyData = { ...body }
delete bodyData.source
data = Object.keys(bodyData).length > 0 ? bodyData : body
}
// data 类型验证:必须是对象
if (typeof data !== 'object' || data === null || Array.isArray(data)) {
return res.status(400).json({ ok: false, error: 'data 必须是 JSON 对象' })
}
// data 大小限制
const dataBytes = Buffer.byteLength(JSON.stringify(data), 'utf8')
if (dataBytes > DATA_MAX_BYTES) {
return res.status(413).json({ ok: false, error: 'data 超过大小限制(' + Math.round(DATA_MAX_BYTES / 1024) + 'KB)' })
}
// 原型污染防护
data = sanitizeObject(data)
const record = insert(skillId, { source, data })
try { logJSON('data_gateway.ingest.ok', { skillId, source, recordId: record.id }, 'data_gateway') } catch {}
return res.json({ ok: true, id: record.id, created_at: record.created_at })
})
// ============================================================
// 内部读取接口(仅 localhost)
// GET /api/v1/data/:skillId/query?mode=latest|list&limit=100&offset=0
// Headers: X-Internal-Token, X-Skill-Id
// ============================================================
app.get('/api/v1/data/:skillId/query', (req, res) => {
setNoCache(res)
const skillId = String(req.params.skillId || '').trim()
if (!skillId) return res.status(400).json({ ok: false, error: 'missing skill id' })
const auth = verifyInternalToken(req)
if (!auth.ok) return res.status(401).json({ ok: false, error: auth.error })
const mode = String(req.query.mode || 'latest').trim()
const limit = parseInt(req.query.limit, 10) || 100
const offset = parseInt(req.query.offset, 10) || 0
const source = req.query.source || null
const start = req.query.start || null
const end = req.query.end || null
try {
if (mode === 'list') {
if (start || end) {
const result = queryByTimeRange(skillId, { start, end, limit })
return res.json({ ok: true, ...result })
}
const result = queryList(skillId, { limit, offset, source })
return res.json({ ok: true, ...result })
}
// mode === 'latest'(默认)
const row = queryLatest(skillId)
if (!row) return res.json({ ok: true, rows: [], total: 0 })
return res.json({ ok: true, rows: [row], total: 1 })
} catch (e) {
try { logJSON('data_gateway.query.error', { skillId, error: String(e.message || e) }, 'data_gateway') } catch {}
return res.status(500).json({ ok: false, error: String(e.message || e) })
}
})
// ============================================================
// 管理接口:列出所有已启用的 skill(仅 localhost)
// GET /api/v1/data/skills
// ============================================================
app.get('/api/v1/data/skills', (req, res) => {
setNoCache(res)
const skills = listEnabledSkills()
const result = skills.map(s => {
const cfg = loadSkillConfig(s.id)
return {
id: s.id,
name: s.name,
mode: String(cfg && cfg.mode || 'append'),
record_count: count(s.id)
}
})
res.json({ ok: true, skills: result })
})
// 全局错误处理:统一返回简洁错误,隐藏堆栈和路径信息
app.use(errorHandler)
}
module.exports = { bindRoutes, listEnabledSkills, loadSkillConfig }
@@ -0,0 +1,29 @@
{
"_comment": "邮件发送 skill - 对外邮件发送 API",
"id": "email_sender",
"name": "邮件发送",
"enabled": true,
"api_key_comment": "对外发送邮件的 API Key,妥善保管",
"api_key": "es_hAtXkry70lHtCVMfr0TaQ",
"api_key_hash": "sha256:88aee9d336c64b3b9bd86ab97e635604137c5de971b10419bb1801c7bd4eb7b9",
"smtp_comment": "SMTP 服务配置(优先级:环境变量 > 此处配置",
"smtp": {
"host_comment": "SMTP 服务器主机名",
"host": "smtp.gmail.com",
"port_comment": "SMTP 服务器端口号",
"port": 465,
"secure_comment": "是否使用 TLS/SSL 加密连接",
"secure": true,
"auth": {
"user_comment": "SMTP 登录用户名(建议通过环境变量 EMAIL_SENDER_SMTP_USER 配置)",
"user": "yangdafe@gmail.com",
"pass_comment": "SMTP 登录密码或应用专用密码(请通过环境变量 EMAIL_SENDER_SMTP_PASS 配置)",
"pass": ""
}
},
"from_comment": "发件人邮箱地址",
"from": "yangdafe@gmail.com",
"default_to_comment": "默认收件人(当请求参数中未提供时使用)",
"default_to": "yangxiangyuan@umer.com.cn",
"env_mapping_comment": "环境变量映射(优先级最高)\n EMAIL_SENDER_SMTP_HOST\n EMAIL_SENDER_SMTP_PORT\n EMAIL_SENDER_SMTP_SECURE (true/false)\n EMAIL_SENDER_SMTP_USER\n EMAIL_SENDER_SMTP_PASS\n EMAIL_SENDER_FROM\n EMAIL_SENDER_DEFAULT_TO"
}
@@ -0,0 +1,180 @@
const path = require('path')
const Database = require('better-sqlite3')
const SKILL_DIR = __dirname
const DB_PATH = path.join(SKILL_DIR, 'email.db')
let db = null
/**
* 脱敏 SMTP 响应信息,过滤文件路径、堆栈等敏感内容
*/
const sanitizeSmtpResponse = (text) => {
if (!text || typeof text !== 'string') return null
let result = String(text)
// 过滤堆栈跟踪(先做,避免路径被部分替换后残留)
result = result.replace(/at\s+\(?.*?\)?(\n|$)/g, '')
// 过滤 Windows 文件路径 D:\xxx 或 C:\xxx
result = result.replace(/[A-Z]:\\(?:[^\\]|\\.)*/gi, '[path]')
// 过滤 Unix 长路径
result = result.replace(/(?:^|\s)\/[\w.\-\/]{8,}/g, (m) => m.replace(/\/[\w.\-\/]{8,}/, ' [path]'))
return result.trim() || null
}
/**
* 获取或创建独立的 SQLite 数据库连接
* 不共享 data_gateway/store.js,完全独立
*/
const getDB = () => {
if (db) return db
db = new Database(DB_PATH)
db.pragma('journal_mode = WAL')
db.pragma('synchronous = NORMAL')
db.pragma('foreign_keys = ON')
initSchema(db)
return db
}
/**
* 初始化数据库表结构
*/
const initSchema = (database) => {
database.exec(`
CREATE TABLE IF NOT EXISTS email_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
api_key_hash TEXT NOT NULL,
to_addr TEXT NOT NULL,
subject TEXT NOT NULL,
text_preview TEXT,
html_preview TEXT,
attachments TEXT,
status TEXT NOT NULL CHECK(status IN ('sent', 'failed')),
smtp_response TEXT,
created_at TEXT NOT NULL
)
`)
database.exec(`
CREATE INDEX IF NOT EXISTS idx_email_logs_created_at ON email_logs(created_at DESC)
`)
database.exec(`
CREATE INDEX IF NOT EXISTS idx_email_logs_status ON email_logs(status)
`)
database.exec(`
CREATE INDEX IF NOT EXISTS idx_email_logs_to_addr ON email_logs(to_addr)
`)
}
/**
* 插入邮件发送日志
*/
const insertLog = (database, data) => {
const stmt = database.prepare(`
INSERT INTO email_logs (api_key_hash, to_addr, subject, text_preview, html_preview, attachments, status, smtp_response, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`)
const result = stmt.run(
data.apiKeyHash,
data.to,
data.subject,
data.textPreview,
data.htmlPreview,
data.attachments || null,
data.status,
data.smtpResponse || null,
data.createdAt
)
return result.lastInsertRowid
}
/**
* 查询日志列表(支持分页和筛选)
*/
const queryLogs = (database, filters) => {
const { limit = 20, offset = 0, status, toAddr, fromDate, toDate, apiKeyHash } = filters
const safeLimit = Math.min(Math.max(Number(limit) || 20, 1), 100)
const safeOffset = Math.max(Number(offset) || 0, 0)
let whereClauses = ['1=1']
const params = []
if (apiKeyHash) {
whereClauses.push('api_key_hash = ?')
params.push(apiKeyHash)
}
if (status) {
whereClauses.push('status = ?')
params.push(status)
}
if (toAddr) {
whereClauses.push('to_addr = ?')
params.push(toAddr)
}
if (fromDate) {
whereClauses.push('created_at >= ?')
params.push(`${fromDate}T00:00:00.000Z`)
}
if (toDate) {
whereClauses.push('created_at <= ?')
params.push(`${toDate}T23:59:59.999Z`)
}
const whereSQL = whereClauses.join(' AND ')
// Count total
const countStmt = database.prepare(`SELECT COUNT(*) as total FROM email_logs WHERE ${whereSQL}`)
const { total } = countStmt.get(...params)
// Query with pagination
const queryStmt = database.prepare(`
SELECT id, api_key_hash, to_addr, subject, text_preview, html_preview, attachments, status, smtp_response, created_at
FROM email_logs
WHERE ${whereSQL}
ORDER BY created_at DESC
LIMIT ? OFFSET ?
`)
const logs = queryStmt.all(...params, safeLimit, safeOffset)
return {
logs: logs.map(l => ({
id: l.id,
to: l.to_addr,
subject: l.subject,
text_preview: l.text_preview,
html_preview: l.html_preview,
attachments: l.attachments ? JSON.parse(l.attachments) : null,
status: l.status,
smtp_response: sanitizeSmtpResponse(l.smtp_response),
created_at: l.created_at
})),
total,
limit: safeLimit,
offset: safeOffset
}
}
/**
* 查询单条日志详情
*/
const getLogById = (database, id, apiKeyHash) => {
const stmt = database.prepare(`
SELECT id, api_key_hash, to_addr, subject, text_preview, html_preview, attachments, status, smtp_response, created_at
FROM email_logs
WHERE id = ? AND api_key_hash = ?
`)
const row = stmt.get(id, apiKeyHash)
if (!row) return null
return {
id: row.id,
to: row.to_addr,
subject: row.subject,
text_preview: row.text_preview,
html_preview: row.html_preview,
attachments: row.attachments ? JSON.parse(row.attachments) : null,
status: row.status,
smtp_response: sanitizeSmtpResponse(row.smtp_response),
created_at: row.created_at
}
}
module.exports = { getDB, insertLog, queryLogs, getLogById }
Binary file not shown.
@@ -0,0 +1,269 @@
const express = require('express')
const crypto = require('crypto')
const { verifyApiKey } = require('../../auth')
const { sendEmail, loadConfig } = require('./mail_client')
const { getDB, insertLog, queryLogs, getLogById } = require('./db')
const { logJSON } = require('../../../logger')
// ============================================================
// Email Sender Skill — 对外邮件发送 API
// 路由挂载在 /api/v1/ingest/email_sender/*
// POST /send
// GET /logs
// GET /logs/:id
// ============================================================
const SKILL_ID = 'email_sender'
const RATE_LIMIT_MAX = 30 // 每分钟 30 封
const RATE_LIMIT_WINDOW_MS = 60 * 1000
const MAX_SUBJECT_LENGTH = 200
const MAX_TEXT_LENGTH = 50000 // 50KB
const MAX_HTML_LENGTH = 100000 // 100KB
const MAX_ATTACHMENTS = 3
const EMAIL_REGEX = /^[^\s@]+@[^\s@]+\.[^\s@]+$/
const PREVIEW_LENGTH = 500
// 速率限制存储
const rateLimitMap = new Map()
const checkRate = (ip) => {
const rec = rateLimitMap.get(ip)
if (!rec || Date.now() - rec.start > RATE_LIMIT_WINDOW_MS) {
rateLimitMap.set(ip, { start: Date.now(), count: 1 })
return { ok: true }
}
rec.count++
if (rec.count > RATE_LIMIT_MAX) {
const waitSec = Math.ceil((rec.start + RATE_LIMIT_WINDOW_MS - Date.now()) / 1000)
return { ok: false, retryAfter: Math.max(1, waitSec) }
}
return { ok: true }
}
setInterval(() => {
const now = Date.now()
for (const [ip, rec] of rateLimitMap.entries()) {
if (now - rec.start > RATE_LIMIT_WINDOW_MS + 10000) rateLimitMap.delete(ip)
}
}, 60000)
/**
* 邮箱格式校验
*/
const isValidEmail = (email) => {
if (typeof email !== 'string' || !email) return false
return EMAIL_REGEX.test(email.trim())
}
/**
* 内容预览截取
*/
const makePreview = (text, maxLength) => {
if (!text) return null
const str = String(text)
if (str.length <= maxLength) return str
return str.slice(0, maxLength) + '...'
}
/**
* 绑定路由到 Express 应用
*/
const bindRoutes = (app) => {
const router = express.Router()
// API Key 鉴权 + skill ID 校验
router.use((req, res, next) => {
const auth = verifyApiKey(req)
if (!auth.ok) {
try { logJSON('email_sender.auth.fail', { error: auth.error, ip: req.ip }, 'email_sender') } catch {}
return res.status(401).json({ ok: false, error: auth.error })
}
const apiId = String(req.headers['x-api-id'] || '').trim()
if (apiId !== SKILL_ID) {
return res.status(403).json({ ok: false, error: 'skill id mismatch' })
}
next()
})
// 无缓存
router.use((req, res, next) => {
try {
res.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate')
res.set('Pragma', 'no-cache')
res.set('Expires', '0')
} catch {}
next()
})
// ============================================================
// 发送邮件
// POST /api/v1/ingest/email_sender/send
// ============================================================
router.post('/send', async (req, res) => {
try {
const body = req.body || {}
// 解析收件人
let to = body.to
const cfg = loadConfig()
if (!to || String(to).trim() === '') {
to = cfg.defaultTo
} else {
to = String(to).trim()
}
// CRLF 注入防护:清除换行符
to = to.replace(/[\r\n]/g, '')
// 校验
if (!isValidEmail(to)) {
return res.status(400).json({ ok: false, error: '收件人邮箱格式无效' })
}
const subject = body.subject
if (!subject || typeof subject !== 'string' || subject.trim() === '') {
return res.status(400).json({ ok: false, error: '邮件主题不能为空' })
}
// CRLF 注入防护:清除换行符
const cleanSubject = String(subject).replace(/[\r\n]/g, '')
if (cleanSubject.length > MAX_SUBJECT_LENGTH) {
return res.status(400).json({ ok: false, error: `主题不能超过 ${MAX_SUBJECT_LENGTH} 字符` })
}
const text = body.text !== undefined ? body.text : null
const html = body.html !== undefined ? body.html : null
if ((text === null || text === '') && (html === null || html === '')) {
return res.status(400).json({ ok: false, error: 'text 和 html 内容至少需要提供一个' })
}
if (text !== null && typeof text === 'string' && text.length > MAX_TEXT_LENGTH) {
return res.status(400).json({ ok: false, error: `纯文本内容不能超过 ${MAX_TEXT_LENGTH} 字符` })
}
if (html !== null && typeof html === 'string' && html.length > MAX_HTML_LENGTH) {
return res.status(400).json({ ok: false, error: `HTML 内容不能超过 ${MAX_HTML_LENGTH} 字符` })
}
const attachments = body.attachments || []
if (!Array.isArray(attachments)) {
return res.status(400).json({ ok: false, error: 'attachments 必须是数组' })
}
if (attachments.length > MAX_ATTACHMENTS) {
return res.status(400).json({ ok: false, error: `附件数量不能超过 ${MAX_ATTACHMENTS} 个` })
}
for (const att of attachments) {
if (!att.filename || !att.path) {
return res.status(400).json({ ok: false, error: '每个附件必须包含 filename 和 path' })
}
if (typeof att.filename !== 'string' || typeof att.path !== 'string') {
return res.status(400).json({ ok: false, error: '附件 filename 和 path 必须是字符串' })
}
if (att.path.includes('..')) {
return res.status(400).json({ ok: false, error: '附件路径不允许包含 ".."' })
}
}
// 速率限制
const clientIp = req.ip || req.connection.remoteAddress || 'unknown'
const rateCheck = checkRate(clientIp)
if (!rateCheck.ok) {
return res.status(429).json({ ok: false, error: '发送过于频繁,请在 ' + rateCheck.retryAfter + ' 秒后重试' })
}
// 发送邮件
const result = await sendEmail({
to,
subject: cleanSubject.trim(),
text: text ? String(text) : undefined,
html: html ? String(html) : undefined,
attachments: attachments.length > 0 ? attachments : undefined
})
// 记录日志
const database = getDB()
const apiKeyHash = crypto.createHash('sha256').update(String(req.headers['x-api-key'] || '')).digest('hex')
const emailId = insertLog(database, {
apiKeyHash,
to,
subject: cleanSubject.trim(),
textPreview: makePreview(text, PREVIEW_LENGTH),
htmlPreview: makePreview(html, PREVIEW_LENGTH),
attachments: attachments.length > 0 ? JSON.stringify(attachments.map(a => ({ filename: a.filename }))) : null,
status: result.ok ? 'sent' : 'failed',
smtpResponse: result.response || result.error || null,
createdAt: new Date().toISOString()
})
if (result.ok) {
return res.json({
ok: true,
email_id: emailId,
accepted: result.accepted,
rejected: result.rejected
})
} else {
return res.status(500).json({
ok: false,
error: result.error,
email_id: emailId
})
}
} catch (e) {
try { logJSON('email_sender.send.error', { error: String(e.message || e) }, 'email_sender') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// ============================================================
// 查询发送日志列表
// GET /api/v1/ingest/email_sender/logs?limit=20&offset=0&status=sent&to=xxx&from_date=2026-01-01&to_date=2026-06-30
// ============================================================
router.get('/logs', async (req, res) => {
try {
const apiKeyHash = crypto.createHash('sha256').update(String(req.headers['x-api-key'] || '')).digest('hex')
const filters = {
limit: req.query.limit,
offset: req.query.offset,
status: req.query.status || undefined,
toAddr: req.query.to || undefined,
fromDate: req.query.from_date || undefined,
toDate: req.query.to_date || undefined,
apiKeyHash
}
const result = queryLogs(getDB(), filters)
res.json({ ok: true, ...result })
} catch (e) {
try { logJSON('email_sender.logs.error', { error: String(e.message || e) }, 'email_sender') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// ============================================================
// 查询单条日志详情
// GET /api/v1/ingest/email_sender/logs/:id
// ============================================================
router.get('/logs/:id', async (req, res) => {
try {
const id = Number(req.params.id)
if (!id || isNaN(id)) {
return res.status(400).json({ ok: false, error: '无效的日志 ID' })
}
const apiKeyHash = crypto.createHash('sha256').update(String(req.headers['x-api-key'] || '')).digest('hex')
const log = getLogById(getDB(), id, apiKeyHash)
if (!log) {
return res.status(404).json({ ok: false, error: '日志不存在或无权访问' })
}
res.json({ ok: true, log })
} catch (e) {
try { logJSON('email_sender.log_detail.error', { error: String(e.message || e) }, 'email_sender') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// 挂载路由
app.use('/api/v1/ingest/email_sender', router)
}
module.exports = { bindRoutes }
@@ -0,0 +1,133 @@
const nodemailer = require('nodemailer')
const fs = require('fs')
const path = require('path')
/**
* 从配置文件 + 环境变量加载 SMTP 配置
* config.json 只存空模板,实际凭证从 process.env 读取
* 优先级:process.env > config.json 空值
*/
const loadConfig = () => {
const configPath = path.join(__dirname, 'config.json')
const config = JSON.parse(fs.readFileSync(configPath, 'utf-8'))
// 从环境变量覆盖敏感字段
const smtpHost = process.env.EMAIL_SENDER_SMTP_HOST || config.smtp?.host || ''
const smtpPort = Number(process.env.EMAIL_SENDER_SMTP_PORT || config.smtp?.port || 0)
const smtpSecure = process.env.EMAIL_SENDER_SMTP_SECURE !== 'false'
const smtpUser = process.env.EMAIL_SENDER_SMTP_USER || config.smtp?.auth?.user || ''
const smtpPass = process.env.EMAIL_SENDER_SMTP_PASS || config.smtp?.auth?.pass || ''
const from = process.env.EMAIL_SENDER_FROM || config.from || ''
const defaultTo = process.env.EMAIL_SENDER_DEFAULT_TO || config.default_to || 'yangxiangyuan@umer.com.cn'
return {
smtp: {
host: smtpHost,
port: smtpPort,
secure: smtpSecure,
auth: { user: smtpUser, pass: smtpPass }
},
from,
defaultTo
}
}
/**
* 创建 nodemailer transporter
* 每次调用时新建,避免复用失败的连接
*/
const createTransporter = (config) => {
return nodemailer.createTransport({
host: config.smtp.host,
port: config.smtp.port,
secure: config.smtp.secure,
auth: {
user: config.smtp.auth.user,
pass: config.smtp.auth.pass
},
// 连接超时
connectionTimeout: 10000,
// 发送邮件超时
socketTimeout: 30000,
// Greeting 超时
greetingTimeout: 10000
})
}
/**
* 发送邮件
* @param {Object} params
* @param {string} params.to - 收件人
* @param {string} params.subject - 主题
* @param {string} [params.text] - 纯文本内容
* @param {string} [params.html] - HTML 内容
* @param {Array} [params.attachments] - 附件数组 [{ filename, path }]
* @returns {Promise<{ ok: boolean, accepted: string[], rejected: string[], response: string, error: string }>}
*/
const sendEmail = async (params) => {
const config = loadConfig()
// 验证配置完整性
if (!config.smtp.host || !config.smtp.port || !config.smtp.auth.user || !config.smtp.auth.pass) {
return { ok: false, error: 'SMTP 配置不完整' }
}
if (!config.from) {
return { ok: false, error: '发件人地址未配置' }
}
const transporter = createTransporter(config)
try {
// 验证 SMTP 连接
await transporter.verify()
const mailOptions = {
from: config.from,
to: params.to,
subject: params.subject,
text: params.text || undefined,
html: params.html || undefined,
attachments: params.attachments || undefined
}
const info = await transporter.sendMail(mailOptions)
return {
ok: true,
accepted: info.accepted || [],
rejected: info.rejected || [],
response: String(info.response || '')
}
} catch (e) {
// 通用化错误信息,不暴露 SMTP 凭证
const msg = String(e.message || e)
let userError = '发送失败'
if (msg.includes('ENOTFOUND') || msg.includes('getaddrinfo')) {
userError = 'SMTP 服务器地址无法解析'
} else if (msg.includes('ECONNREFUSED')) {
userError = 'SMTP 服务器连接被拒绝'
} else if (msg.includes('ETIMEDOUT') || msg.includes('Timeout')) {
userError = 'SMTP 连接超时'
} else if (msg.includes('Authentication') || msg.includes('AUTH')) {
userError = 'SMTP 认证失败'
} else if (msg.includes('recipient')) {
userError = '收件人地址被拒绝'
} else if (msg.includes('550') || msg.includes('553')) {
userError = '邮件被 SMTP 服务器拒绝'
}
return {
ok: false,
error: userError,
accepted: [],
rejected: [],
response: msg
}
} finally {
// 关闭连接
try { transporter.close() } catch {}
}
}
module.exports = { loadConfig, sendEmail }
@@ -0,0 +1,14 @@
{
"_comment": "外部存储 skill - 供第三方程序读写 OSS external_storage 区域",
"id": "external_storage",
"name": "外部存储",
"enabled": true,
"api_key_comment": "这个发给对方,API Key,用于上传文件到外部存储区域",
"api_key": "4a211084d059Pk0C6t8dlk3ew61d46",
"api_key_hash": "sha256:17eb4c895064b955d80d007b15af992d7bb7bd1525ce5726f2f63f06229857c9",
"oss_bucket": "android-o1-images",
"oss_endpoint_backup1": "oss-accelerate.aliyuncs.com 传输加速域名(全地域上传下载加速)",
"oss_endpoint_backup2": "oss-cn-shanghai.aliyuncs.com 上海区域",
"oss_endpoint": "oss-accelerate.aliyuncs.com",
"oss_prefix": "550e8400-e29b-41d4-a716-446655442082/external_storage"
}
@@ -0,0 +1,380 @@
// ============================================================
// external_storage/index.js - 外部存储 skill 路由入口
// 提供第三方程序读写 OSS external_storage 区域的 API
// ============================================================
const express = require('express')
const multer = require('multer')
const path = require('path')
const archiver = require('archiver')
const { createOssClient } = require('./oss_client')
const { sanitizePath, createSizeLimitMiddleware } = require('./middleware')
const { loadSkillConfig } = require('../../auth')
const { verifyApiKey } = require('../../auth')
const { logJSON } = require('../../../logger')
const MAX_UPLOAD_BYTES = 4 * 1024 * 1024 * 1024 // 4GB
const fs = require('fs')
const os = require('os')
// 确保临时目录存在
const TEMP_DIR = path.join(os.tmpdir(), 'external_storage_uploads')
if (!fs.existsSync(TEMP_DIR)) {
try { fs.mkdirSync(TEMP_DIR, { recursive: true }) } catch (e) {}
}
// 允许上传的文件扩展名白名单(禁止 .html/.js/.exe/.php 等可执行/脚本文件)
const ALLOWED_EXTENSIONS = new Set([
'jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg', 'ico', // 图片
'pdf', 'doc', 'docx', 'xls', 'xlsx', 'ppt', 'pptx', // 文档
'txt', 'csv', 'log', 'json', 'xml', 'yaml', 'yml', // 文本
'zip', 'rar', '7z', 'tar', 'gz', 'bz2', // 压缩
'mp3', 'mp4', 'wav', 'avi', 'mov', 'mkv', // 音视频
])
const validateFileExtension = (filename) => {
const ext = path.extname(filename).replace(/^\./, '').toLowerCase()
if (!ext) return false // 无扩展名
return ALLOWED_EXTENSIONS.has(ext)
}
// 磁盘临时存储上传文件
const upload = multer({
dest: TEMP_DIR,
limits: { fileSize: MAX_UPLOAD_BYTES, files: 20 }
})
// MIME 类型推断(支持中文文件名 UTF-8)
const contentTypeByExt = (filename) => {
const ext = path.extname(filename).toLowerCase().slice(1)
const map = {
jpg: 'image/jpeg', jpeg: 'image/jpeg', png: 'image/png', gif: 'image/gif', webp: 'image/webp', svg: 'image/svg+xml', ico: 'image/x-icon', bmp: 'image/bmp', avif: 'image/avif',
txt: 'text/plain; charset=utf-8', md: 'text/plain; charset=utf-8', html: 'text/html; charset=utf-8',
js: 'application/javascript', ts: 'application/javascript', css: 'text/css', json: 'application/json',
xml: 'text/xml', log: 'text/plain; charset=utf-8', csv: 'text/csv',
pdf: 'application/pdf',
doc: 'application/msword', docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
xls: 'application/vnd.ms-excel', xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
ppt: 'application/vnd.ms-powerpoint', pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
mp4: 'video/mp4', mov: 'video/quicktime', avi: 'video/x-msvideo', webm: 'video/webm', mkv: 'video/x-matroska',
mp3: 'audio/mpeg', wav: 'audio/wav', ogg: 'audio/ogg', flac: 'audio/flac',
zip: 'application/zip', rar: 'application/x-rar-compressed', '7z': 'application/x-7z-compressed',
tar: 'application/x-tar', gz: 'application/gzip',
exe: 'application/x-msdownload', dmg: 'application/x-apple-diskimage',
font: 'font/woff', woff: 'font/woff', woff2: 'font/woff2', ttf: 'font/ttf', otf: 'font/otf'
}
return map[ext] || 'application/octet-stream'
}
// 安全文件名清理(保留中文,移除危险字符)
const safeName = (name) => {
let s = String(name || 'untitled')
// 尝试修复编码(UTF-8 被误读为 latin1 的情况)
try {
const recovered = Buffer.from(s, 'latin1').toString('utf8')
if (recovered !== s && !/\ufffd/.test(recovered)) s = recovered
} catch {}
// 移除危险字符,但保留中文、英文、数字、常见符号
return s.replace(/[<>\":/\\|?*\x00-\x1F]/g, '_').trim() || 'untitled'
}
// 设置无缓存响应头
const setNoCache = (res) => {
try {
res.set('Cache-Control', 'no-store, no-cache, must-revalidate, proxy-revalidate')
res.set('Pragma', 'no-cache')
res.set('Expires', '0')
} catch {}
}
/**
* 绑定路由到 Express 应用
* @param {express.Application} app
*/
const bindRoutes = (app) => {
const skillId = 'external_storage'
const skillConfig = loadSkillConfig(skillId)
if (!skillConfig) {
console.warn(`[external_storage] config.json not found, skill disabled`)
return
}
let ossClient
try {
ossClient = createOssClient(skillConfig)
} catch (e) {
console.error(`[external_storage] Failed to init OSS client: ${e.message}`)
return
}
const router = express.Router()
// API Key 鉴权中间件 — 强制 X-API-Id 必须为 external_storage,防止跨 skill 鉴权泄露
router.use((req, res, next) => {
const auth = verifyApiKey(req)
if (!auth.ok) {
try { logJSON('external_storage.auth.fail', { error: auth.error, ip: req.ip }, 'external_storage') } catch {}
return res.status(401).json({ ok: false, error: auth.error })
}
// 校验 skill ID 必须匹配
const apiId = String(req.headers['x-api-id'] || '').trim()
if (apiId !== 'external_storage') {
return res.status(403).json({ ok: false, error: 'skill id mismatch' })
}
next()
})
// 全局无缓存
router.use((req, res, next) => {
setNoCache(res)
next()
})
// ============================================================
// 上传文件
// POST /api/v1/skills/external_storage/upload?path=subdir/
// Headers: X-API-Id, X-API-Key
// Body: multipart/form-data, field: "files"
// ============================================================
router.post('/upload', createSizeLimitMiddleware(MAX_UPLOAD_BYTES), upload.array('files', 20), async (req, res) => {
try {
const basePath = sanitizePath(req.query.path || '')
if (!req.files || !Array.isArray(req.files) || req.files.length === 0) {
return res.status(400).json({ ok: false, error: '未收到文件' })
}
const results = []
const rejected = []
const tempFiles = [] // 记录临时文件路径,最后统一删除
for (const f of req.files) {
if (f.path) tempFiles.push(f.path)
const originalName = safeName(f.originalname || 'file')
// 文件扩展名白名单校验
if (!validateFileExtension(originalName)) {
rejected.push({ name: originalName, reason: 'unsupported_extension' })
continue
}
const keyPath = basePath ? `${basePath}/${originalName}` : originalName
const contentType = f.mimetype || contentTypeByExt(originalName)
const { key, raw_url } = await ossClient.putObject(keyPath, f.path || f.buffer, contentType)
results.push({
path: keyPath,
oss_key: key,
raw_url,
name: originalName,
size: f.size,
content_type: contentType
})
}
// 删除临时文件
for (const fp of tempFiles) {
try { fs.unlinkSync(fp) } catch (e) {}
}
if (rejected.length > 0 && results.length === 0) {
return res.status(400).json({ ok: false, error: '所有文件的扩展名均不被允许', rejected })
}
res.json({ ok: true, count: results.length, files: results, rejected: rejected.length > 0 ? rejected : undefined })
} catch (e) {
// 不管成功失败,都删除临时文件
if (req.files && Array.isArray(req.files)) {
for (const f of req.files) {
if (f.path) try { fs.unlinkSync(f.path) } catch (ee) {}
}
}
if (String(e.message || '').includes('File too large')) {
return res.status(413).json({ ok: false, error: '文件超过 4GB 上限' })
}
try { logJSON('external_storage.upload.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '上传失败' })
}
})
// ============================================================
// 下载文件(代理流)
// GET /api/v1/skills/external_storage/download?path=xxx
// Headers: X-API-Id, X-API-Key
// ============================================================
router.get('/download', async (req, res) => {
try {
const filePath = sanitizePath(req.query.path)
if (!filePath) return res.status(400).json({ ok: false, error: '缺少 path 参数' })
const displayName = path.basename(filePath)
const { stream, headers } = await ossClient.getObjectStream(filePath)
// 使用 RFC 5987 格式支持 UTF-8 文件名
res.set('Content-Disposition', `attachment; filename*=UTF-8''${encodeURIComponent(displayName)}`)
if (headers['content-type']) res.set('Content-Type', headers['content-type'])
if (headers['content-length']) res.set('Content-Length', headers['content-length'])
stream.pipe(res)
} catch (e) {
try { logJSON('external_storage.download.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '下载失败' })
}
})
// ============================================================
// 获取原始 URL
// GET /api/v1/skills/external_storage/raw_url?path=xxx
// Headers: X-API-Id, X-API-Key
// ============================================================
router.get('/raw_url', async (req, res) => {
try {
const filePath = sanitizePath(req.query.path)
if (!filePath) return res.status(400).json({ ok: false, error: '缺少 path 参数' })
const raw_url = ossClient.getRawUrl(filePath)
res.json({ ok: true, path: filePath, raw_url })
} catch (e) {
try { logJSON('external_storage.raw_url.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// ============================================================
// 列出文件
// GET /api/v1/skills/external_storage/list?prefix=subdir/&recursive=true|false
// Headers: X-API-Id, X-API-Key
// ============================================================
router.get('/list', async (req, res) => {
try {
const prefix = sanitizePath(req.query.prefix || '')
const recursive = req.query.recursive === 'true'
let result
if (recursive) {
const files = await ossClient.listRecursive(prefix)
result = { files, folders: [] }
} else {
result = await ossClient.listObjects(prefix)
}
res.json({ ok: true, prefix, recursive, ...result })
} catch (e) {
try { logJSON('external_storage.list.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// ============================================================
// 删除文件
// DELETE /api/v1/skills/external_storage/delete?path=xxx
// Headers: X-API-Id, X-API-Key
// ============================================================
router.delete('/delete', async (req, res) => {
try {
const filePath = sanitizePath(req.query.path)
if (!filePath) return res.status(400).json({ ok: false, error: '缺少 path 参数' })
await ossClient.deleteObject(filePath)
res.json({ ok: true, path: filePath })
} catch (e) {
try { logJSON('external_storage.delete.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '删除失败' })
}
})
// ============================================================
// 文件夹打包下载
// GET /api/v1/skills/external_storage/folder/download?path=subdir/
// Headers: X-API-Id, X-API-Key
// ============================================================
router.get('/folder/download', async (req, res) => {
try {
const folderPath = sanitizePath(req.query.path)
if (!folderPath) return res.status(400).json({ ok: false, error: '缺少 path 参数' })
const folderKey = folderPath.endsWith('/') ? folderPath : folderPath + '/'
const folderName = path.basename(folderKey.replace(/\/+$/, '')) || 'folder'
const files = await ossClient.listRecursive(folderKey)
res.set('Content-Type', 'application/zip')
res.set('Content-Disposition', `attachment; filename*=UTF-8''${encodeURIComponent(folderName + '.zip')}`)
const archive = archiver('zip', { zlib: { level: 6 } })
archive.on('error', err => {
try { res.status(500).end() } catch {}
})
archive.pipe(res)
const prefixClean = folderKey.replace(/\/+$/, '') + '/'
for (const f of files) {
try {
// 提取相对于文件夹前缀的路径
const fullRel = f.key
const relPart = fullRel.slice(prefixClean.length)
const { stream } = await ossClient.getObjectStream(relPart)
archive.append(stream, { name: relPart })
} catch {}
}
archive.finalize()
} catch (e) {
try { logJSON('external_storage.folder_download.error', { error: String(e.message || e) }, 'external_storage') } catch {}
if (!res.headersSent) res.status(500).json({ ok: false, error: '打包下载失败' })
}
})
// ============================================================
// 获取文件信息
// GET /api/v1/skills/external_storage/info?path=xxx
// Headers: X-API-Id, X-API-Key
// ============================================================
router.get('/info', async (req, res) => {
try {
const filePath = sanitizePath(req.query.path)
if (!filePath) return res.status(400).json({ ok: false, error: '缺少 path 参数' })
const isFolder = filePath.endsWith('/')
const fullKey = ossClient.fullKey(filePath)
const raw_url = ossClient.getRawUrl(filePath)
let size = 0
let lastModified = 0
if (!isFolder) {
try {
const parent = filePath.includes('/') ? filePath.split('/').slice(0, -1).join('/') : ''
const result = await ossClient.listObjects(parent)
const targetKey = fullKey
const hit = result.files.find(f => f.key === targetKey)
if (hit) {
size = hit.size
lastModified = hit.lastModified
}
} catch {}
} else {
try {
const files = await ossClient.listRecursive(filePath)
size = files.reduce((s, f) => s + (Number(f.size) || 0), 0)
} catch {}
}
res.json({
ok: true,
info: {
path: filePath,
oss_key: fullKey,
raw_url,
isFolder,
size,
lastModified: lastModified ? new Date(lastModified).toISOString() : null
}
})
} catch (e) {
try { logJSON('external_storage.info.error', { error: String(e.message || e) }, 'external_storage') } catch {}
res.status(500).json({ ok: false, error: '操作失败' })
}
})
// 挂载路由
app.use('/api/v1/skills/external_storage', router)
console.log('[external_storage] Routes mounted at /api/v1/skills/external_storage')
}
module.exports = { bindRoutes }
@@ -0,0 +1,92 @@
// ============================================================
// external_storage/middleware.js - 路径安全校验中间件
// 职责:防止路径遍历攻击,确保所有操作限定在 external_storage 范围内
// ============================================================
/**
* 校验并清理路径
* - 拒绝包含 ../ 的路径遍历
* - 拒绝绝对路径
* - 拒绝控制字符
* - 统一使用 UTF-8 编码,正确处理中文
* @param {string} rawPath - 原始路径参数
* @returns {string} 清理后的安全路径
* @throws {Error} 如果路径不合法
*/
const sanitizePath = (rawPath) => {
if (rawPath === undefined || rawPath === null) return ''
// 确保使用 UTF-8 字符串
let p = String(rawPath)
// 拒绝路径遍历(在规范化之前检查原始字符串中的 ..)
if (p.includes('..')) {
throw new Error('路径不允许包含 ".."')
}
// Unicode 规范化:NFKC 将全角/兼容字符转为标准形式
// 如 \uff0e (全角点 .) → . (ASCII 点)
p = p.normalize('NFKC')
// 规范化后再次检查路径遍历(防止 Unicode 全角字符绕过)
if (p.includes('..')) {
throw new Error('路径不允许包含 ".."')
}
// 拒绝控制字符(除了正常的换行和制表符)
if (/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/.test(p)) {
throw new Error('路径包含非法控制字符')
}
// 拒绝绝对路径
if (p.startsWith('/') || p.startsWith('\\')) {
throw new Error('路径不允许以 "/" 或 "\\" 开头')
}
// 统一斜杠
p = p.replace(/[\\]+/g, '/')
// 去除连续斜杠
p = p.replace(/\/+/g, '/')
// 去除首尾斜杠
p = p.replace(/^\/+|\/+$/g, '')
return p
}
/**
* 校验路径是否在允许的前缀范围内
* @param {string} ossKey - 完整 OSS key
* @param {string} allowedPrefix - 允许的前缀
* @returns {boolean}
*/
const isPathWithinPrefix = (ossKey, allowedPrefix) => {
const key = String(ossKey || '')
const prefix = String(allowedPrefix || '').replace(/\/+$/, '')
return key === prefix || key.startsWith(prefix + '/')
}
/**
* 文件大小校验中间件
* @param {number} maxBytes - 最大字节数
* @returns {Function} express 中间件
*/
const createSizeLimitMiddleware = (maxBytes = 50 * 1024 * 1024) => {
return (req, res, next) => {
const contentLength = parseInt(req.headers['content-length'] || '0', 10)
if (contentLength > maxBytes) {
return res.status(413).json({
ok: false,
error: `文件超过大小限制 (${Math.round(maxBytes / 1024 / 1024)}MB)`
})
}
next()
}
}
module.exports = {
sanitizePath,
isPathWithinPrefix,
createSizeLimitMiddleware
}
@@ -0,0 +1,345 @@
// ============================================================
// external_storage/oss_client.js - 阿里云 OSS 客户端封装
// 职责:提供上传、下载、删除、列表、获取URL等基础操作
// 所有操作限定在 config.json 配置的 oss_prefix 范围内
// ============================================================
const crypto = require('crypto')
const https = require('https')
const http = require('http')
const fs = require('fs')
const path = require('path')
const { URLSearchParams } = require('url')
/**
* 初始化 OSS 客户端配置
* @param {Object} skillConfig - skill 的 config.json 内容
* @returns {Object} 客户端实例
*/
const createOssClient = (skillConfig) => {
const bucket = skillConfig.oss_bucket || 'android-o1-images'
const endpoint = skillConfig.oss_endpoint || 'oss-cn-shanghai.aliyuncs.com'
const prefix = (skillConfig.oss_prefix || '550e8400-e29b-41d4-a716-446655442082/external_storage').replace(/\/+$/, '')
// 从 process.env 读取(index.js 启动时已从 Toolbox_local_creds.env.local 加载)
let accessKeyId = process.env.EXTERNAL_STORAGE_OSS_ACCESS_KEY_ID || ''
let accessKeySecret = process.env.EXTERNAL_STORAGE_OSS_ACCESS_KEY_SECRET || ''
if (!accessKeyId || !accessKeySecret) {
throw new Error('OSS credentials not found. Please set oss_access_key_id and oss_access_key_secret in config.json')
}
/**
* 规范化 OSS key:处理斜杠、去除危险字符
*/
const normalizeKey = (relativePath) => {
let k = String(relativePath || '').replace(/[\\]+/g, '/').replace(/\/+/g, '/')
if (k.startsWith('/')) k = k.slice(1)
return k
}
/**
* 拼接完整 OSS key(自动加前缀)
*/
const fullKey = (relativePath) => {
const rel = normalizeKey(relativePath)
return rel ? `${prefix}/${rel}` : prefix
}
/**
* URL 编码 OSS key(用于 HTTP 路径)
*/
const encodeKey = (key) => String(key || '').split('/').map(s => encodeURIComponent(s)).join('/')
/**
* 构建 OSS 资源标识
*/
const ossResource = (key) => `/${bucket}/${String(key || '').replace(/^\/+/, '')}`
/**
* OSS 请求签名(HMAC-SHA1)
*/
const signRequest = ({ method, contentType = '', date, key, extraHeaders = {} }) => {
const canonicalHeaders = Object.keys(extraHeaders)
.filter(k => k && k.toLowerCase().startsWith('x-oss-'))
.sort((a, b) => a.toLowerCase().localeCompare(b.toLowerCase()))
.map(k => `${k.toLowerCase()}:${String(extraHeaders[k] || '').trim()}`)
.join('\n')
const resource = ossResource(key)
const stringToSign = `${method.toUpperCase()}\n\n${contentType}\n${date}\n${canonicalHeaders ? canonicalHeaders + '\n' : ''}${resource}`
return crypto.createHmac('sha1', accessKeySecret).update(stringToSign).digest('base64')
}
/**
* 发起 HTTP 请求(支持 body 为流)
*/
const httpRequest = (options, body = null, timeoutMs = 600000) => new Promise((resolve, reject) => {
const lib = options.protocol === 'https:' ? https : http
const req = lib.request(options, (res) => {
const chunks = []
res.on('data', chunk => chunks.push(chunk))
res.on('end', () => {
const buf = Buffer.concat(chunks)
resolve({ statusCode: res.statusCode, headers: res.headers, body: buf })
})
})
req.on('error', reject)
req.setTimeout(timeoutMs, () => req.destroy(new Error('request timeout')))
if (body !== null && body !== undefined) {
if (body.pipe) {
body.pipe(req)
} else if (Buffer.isBuffer(body) || typeof body === 'string') {
req.write(body)
req.end()
} else {
req.end()
}
} else {
req.end()
}
})
/**
* 获取文件流
*/
const getObjectStream = (relativePath) => new Promise((resolve, reject) => {
const key = fullKey(relativePath)
const date = new Date().toUTCString()
const authorization = `OSS ${accessKeyId}:${signRequest({ method: 'GET', date, key })}`
const options = {
hostname: `${bucket}.${endpoint}`,
path: `/${encodeKey(key)}`,
method: 'GET',
headers: {
Host: `${bucket}.${endpoint}`,
Date: date,
Authorization: authorization
}
}
const req = https.request(options, (res) => {
if (res.statusCode < 200 || res.statusCode >= 300) {
const chunks = []
res.on('data', c => chunks.push(c))
res.on('end', () => reject(new Error(`OSS get failed: ${res.statusCode} ${Buffer.concat(chunks).toString('utf8').slice(0, 500)}`)))
return
}
resolve({ stream: res, headers: res.headers })
})
req.on('error', reject)
req.setTimeout(60000, () => req.destroy(new Error('request timeout')))
req.end()
})
/**
* 上传文件
* @param {string} relativePath - 相对路径(自动加前缀)
* @param {Buffer|string} body - 文件内容 Buffer 或本地文件路径
* @param {string} contentType - MIME 类型
* @returns {Object} { key, raw_url }
*/
const putObject = async (relativePath, body, contentType = 'application/octet-stream') => {
const key = fullKey(relativePath)
const date = new Date().toUTCString()
const extraHeaders = { 'x-oss-object-acl': 'public-read' }
const authorization = `OSS ${accessKeyId}:${signRequest({ method: 'PUT', contentType, date, key, extraHeaders })}`
let contentLength = 0
let stream = null
let sendBody = body
// 如果是字符串,假设是本地文件路径
if (typeof body === 'string' && fs.existsSync(body)) {
const stat = fs.statSync(body)
contentLength = stat.size
stream = fs.createReadStream(body)
sendBody = stream
} else if (Buffer.isBuffer(body)) {
contentLength = body.length
sendBody = body
}
const options = {
hostname: `${bucket}.${endpoint}`,
path: `/${encodeKey(key)}`,
method: 'PUT',
headers: {
Host: `${bucket}.${endpoint}`,
Date: date,
'Content-Type': contentType,
'Content-Length': contentLength,
...extraHeaders,
Authorization: authorization
}
}
const resp = await httpRequest(options, sendBody, 3600000) // 1小时超时,大文件需要
if (resp.statusCode < 200 || resp.statusCode >= 300) {
throw new Error(`OSS put failed: ${resp.statusCode} ${resp.body.toString('utf8').slice(0, 500)}`)
}
const raw_url = `https://${bucket}.${endpoint}/${encodeKey(key)}`
return { key, raw_url }
}
/**
* 删除文件
*/
const deleteObject = async (relativePath) => {
const key = fullKey(relativePath)
const date = new Date().toUTCString()
const authorization = `OSS ${accessKeyId}:${signRequest({ method: 'DELETE', date, key })}`
const options = {
hostname: `${bucket}.${endpoint}`,
path: `/${encodeKey(key)}`,
method: 'DELETE',
headers: {
Host: `${bucket}.${endpoint}`,
Date: date,
Authorization: authorization
}
}
const resp = await httpRequest(options)
if (resp.statusCode < 200 || resp.statusCode >= 300) {
throw new Error(`OSS delete failed: ${resp.statusCode} ${resp.body.toString('utf8').slice(0, 500)}`)
}
return true
}
/**
* 解析 OSS ListObjects XML 响应
*/
const parseListXml = (xml) => {
const extractTag = (tag, src) => {
const open = `<${tag}>`
const close = `</${tag}>`
const result = []
let idx = 0
while (true) {
const s = src.indexOf(open, idx)
if (s < 0) break
const e = src.indexOf(close, s + open.length)
if (e < 0) break
result.push(src.slice(s + open.length, e))
idx = e + close.length
}
return result
}
const extractInSection = (tag, sectionOpen, sectionClose, src) => {
const result = []
let searchFrom = 0
while (true) {
const sOpen = src.indexOf(sectionOpen, searchFrom)
if (sOpen < 0) break
const sClose = src.indexOf(sectionClose, sOpen)
if (sClose < 0) break
const section = src.slice(sOpen, sClose)
result.push(...extractTag(tag, section))
searchFrom = sClose + sectionClose.length
}
return result
}
const folders = extractInSection('Prefix', '<CommonPrefixes>', '</CommonPrefixes>', xml).map(p => ({
key: p,
name: p.split('/').filter(Boolean).pop() + '/',
isFolder: true,
size: 0
}))
const fileKeys = extractTag('Key', xml)
const fileSizes = extractTag('Size', xml)
const fileLms = extractTag('LastModified', xml)
const files = fileKeys.map((k, i) => ({
key: k,
name: k.split('/').filter(Boolean).pop(),
isFolder: false,
size: parseInt(fileSizes[i] || '0', 10) || 0,
lastModified: fileLms[i] ? Date.parse(fileLms[i]) : 0
})).filter(f => f.key && !f.key.endsWith('/'))
return { folders, files }
}
/**
* 列出文件(递归)
* @param {string} relativePrefix - 相对前缀
* @returns {Array} 文件列表
*/
const listRecursive = async (relativePrefix = '') => {
const all = []
const stack = [relativePrefix]
while (stack.length > 0) {
const current = stack.pop()
const keyPrefix = fullKey(current)
const date = new Date().toUTCString()
const authorization = `OSS ${accessKeyId}:${signRequest({ method: 'GET', date, key: '' })}`
const query = new URLSearchParams()
query.set('prefix', keyPrefix)
query.set('delimiter', '/')
query.set('max-keys', '1000')
const options = {
hostname: `${bucket}.${endpoint}`,
path: `/?${query.toString()}`,
method: 'GET',
headers: {
Host: `${bucket}.${endpoint}`,
Date: date,
Authorization: authorization
}
}
const resp = await httpRequest(options)
if (resp.statusCode < 200 || resp.statusCode >= 300) {
throw new Error(`OSS list failed: ${resp.statusCode} ${resp.body.toString('utf8').slice(0, 500)}`)
}
const parsed = parseListXml(resp.body.toString('utf8'))
all.push(...parsed.files)
for (const f of parsed.folders) {
stack.push(f.key.slice(keyPrefix.length > 0 ? (fullKey('').length) : 0))
}
}
return all
}
/**
* 列出对象(单层)
*/
const listObjects = async (relativePrefix = '') => {
const keyPrefix = fullKey(relativePrefix)
const date = new Date().toUTCString()
const authorization = `OSS ${accessKeyId}:${signRequest({ method: 'GET', date, key: '' })}`
const query = new URLSearchParams()
query.set('prefix', keyPrefix)
query.set('delimiter', '/')
query.set('max-keys', '1000')
const options = {
hostname: `${bucket}.${endpoint}`,
path: `/?${query.toString()}`,
method: 'GET',
headers: {
Host: `${bucket}.${endpoint}`,
Date: date,
Authorization: authorization
}
}
const resp = await httpRequest(options)
if (resp.statusCode < 200 || resp.statusCode >= 300) {
throw new Error(`OSS list failed: ${resp.statusCode} ${resp.body.toString('utf8').slice(0, 500)}`)
}
return parseListXml(resp.body.toString('utf8'))
}
/**
* 获取文件原始 URL(公共读可直接访问)
*/
const getRawUrl = (relativePath) => {
const key = fullKey(relativePath)
return `https://${bucket}.${endpoint}/${encodeKey(key)}`
}
return {
fullKey,
normalizeKey,
putObject,
deleteObject,
getObjectStream,
listObjects,
listRecursive,
getRawUrl
}
}
module.exports = { createOssClient }
@@ -0,0 +1,10 @@
{
"_comment": "天气数据 skill - 由外部 Hermes Agent 每30分钟写入一次",
"id": "weather",
"name": "天气数据",
"enabled": true,
"mode": "append",
"mode_comment": "append: 增量追加保留所有历史;latest: 只保留最新一条",
"api_key_hash": "sha256:7b4035f766bf66e39cff1a58a5e8c73f2fcbf75f3f2a368bf0003e1110a56255",
"read_token_hash": "sha256:aa40833cd71a76b2fc558edb1384f4d993c9e8d966a730005f7c99d56ab19037"
}
+123
View File
@@ -0,0 +1,123 @@
// ============================================================
// data_gateway/store.js - 数据存储模块
// 职责:动态创建/管理每个 skill 的独立 SQLite 数据库
// 表结构:id | source | data(JSON) | created_at
// 支持 mode: "append"(增量追加)和 "latest"(覆盖最新)
// ============================================================
const Database = require('better-sqlite3')
const fs = require('fs')
const path = require('path')
const DATA_DIR = path.join(process.cwd(), 'data', 'ingest')
if (!fs.existsSync(DATA_DIR)) fs.mkdirSync(DATA_DIR, { recursive: true })
// 已打开的数据库连接缓存
const dbCache = new Map()
const getDbPath = (skillId) => path.join(DATA_DIR, `${skillId}.db`)
const getOrOpenDb = (skillId) => {
if (dbCache.has(skillId)) return dbCache.get(skillId)
const dbPath = getDbPath(skillId)
const db = new Database(dbPath)
// 启用 WAL 模式提高并发写入性能
db.pragma('journal_mode = WAL')
db.pragma('synchronous = NORMAL')
db.exec(
`CREATE TABLE IF NOT EXISTS records (
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL DEFAULT '',
data TEXT NOT NULL,
created_at TEXT NOT NULL DEFAULT (datetime('now'))
)`
)
db.exec(`CREATE INDEX IF NOT EXISTS idx_records_created ON records(created_at DESC)`)
dbCache.set(skillId, db)
return db
}
// 写入一条记录
const insert = (skillId, { source = '', data = {} }) => {
const db = getOrOpenDb(skillId)
const dataStr = JSON.stringify(data)
const info = db.prepare(`INSERT INTO records (source, data) VALUES (?, ?)`).run(
String(source || ''), dataStr
)
return { id: info.lastInsertRowid, source, data, created_at: new Date().toISOString() }
}
// 覆盖写入(mode: "latest")- 先删后插,保证只有一条
const upsertLatest = (skillId, { source = '', data = {} }) => {
const db = getOrOpenDb(skillId)
const dataStr = JSON.stringify(data)
const tx = db.transaction(() => {
db.prepare(`DELETE FROM records`).run()
return db.prepare(`INSERT INTO records (source, data) VALUES (?, ?)`).run(
String(source || ''), dataStr
)
})
const info = tx()
return { id: info.lastInsertRowid, source, data, created_at: new Date().toISOString() }
}
// 查询最新一条记录
const queryLatest = (skillId) => {
const db = getOrOpenDb(skillId)
const row = db.prepare(`SELECT * FROM records ORDER BY id DESC LIMIT 1`).get()
if (!row) return null
try { row.data = JSON.parse(row.data) } catch { /* keep as string */ }
return row
}
// 查询记录列表(分页)
const queryList = (skillId, { limit = 100, offset = 0, source = null } = {}) => {
const db = getOrOpenDb(skillId)
let sql = `SELECT * FROM records`
const params = []
if (source) {
sql += ` WHERE source = ?`
params.push(String(source))
}
sql += ` ORDER BY id DESC LIMIT ? OFFSET ?`
params.push(Math.min(Number(limit) || 100, 1000), Math.max(0, Number(offset) || 0))
const rows = db.prepare(sql).all(...params)
for (const row of rows) {
try { row.data = JSON.parse(row.data) } catch { /* keep as string */ }
}
const total = db.prepare(`SELECT COUNT(*) as cnt FROM records`).get().cnt
return { rows, total }
}
// 按时间范围查询
const queryByTimeRange = (skillId, { start, end, limit = 100 } = {}) => {
const db = getOrOpenDb(skillId)
let sql = `SELECT * FROM records WHERE 1=1`
const params = []
if (start) { sql += ` AND created_at >= ?`; params.push(String(start)) }
if (end) { sql += ` AND created_at <= ?`; params.push(String(end)) }
sql += ` ORDER BY id DESC LIMIT ?`
params.push(Math.min(Number(limit) || 100, 1000))
const rows = db.prepare(sql).all(...params)
for (const row of rows) {
try { row.data = JSON.parse(row.data) } catch { /* keep as string */ }
}
return { rows, total: rows.length }
}
// 获取记录总数
const count = (skillId) => {
const db = getOrOpenDb(skillId)
return db.prepare(`SELECT COUNT(*) as cnt FROM records`).get().cnt
}
// 关闭所有数据库连接
const closeAll = () => {
for (const [skillId, db] of dbCache) {
try { db.close() } catch {}
dbCache.delete(skillId)
}
}
module.exports = {
insert, upsertLatest, queryLatest, queryList, queryByTimeRange, count, closeAll, getDbPath, getOrOpenDb
}