storage: 服务端存储加固(原子落盘fsync/损坏隔离/墓碑14天GC/空同步不落盘/当日首份快照不可变/每周深快照/大批删除前置快照/health增强)

2026-10-26 20:51:12 +08:00
parent d48330e5c4
commit 1afc236d88
10 changed files with 216 additions and 40 deletions
@@ -62,8 +62,13 @@ Caddy 会自动为 daywk.linyux.cn 申请并续期 HTTPS 证书（需 DNS 已生

 - 数据目录：/opt/daywork/server/data
  - registry.json          用户注册表（用户名 / 角色 / PBKDF2 哈希）
-  - users/<id>.json        每用户九张表数据 + 墓碑
-  - backups/<日期>/        **每日自动本地备份**（启动与每次注册表变更也会触发，保留最近 14 天）
+  - users/<id>.json        每用户全部业务表数据 + 墓碑
+  - backups/<日期>/        每日快照。**当日首份不可变**（完整证据链），同日再备份写入 `<日期>_t<时分秒>` 副本
+  - backups/weekly/<日期>/ 每周一额外深快照，保留 8 份
+  - 保留策略：每日快照 14 天；写入均为「临时文件 + fsync + 原子改名」，断电不会留下半截文件
+  - 用户数据文件若解析失败会被自动隔离为 `<文件>.corrupt-<时间戳>` 并在日志中大声报错，绝不静默当作空数据
+  - 墓碑自动回收：删除标记保留 14 天后自动清除（足够覆盖离线设备），文件不会无限膨胀
+  - 单次同步携带 ≥20 条墓碑（大批删除）时，服务端会**先自动快照再应用删除**，可随时回滚
 - **异地备份（推荐开启）**：在 docker-compose.yml 的 internlog-sync 服务加环境变量，备份自动上传腾讯云 COS：

      environment:
@@ -11,6 +11,11 @@ const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, 'data')
 const PORT = Number(process.env.PORT || 3780)
 const TOKEN_TTL_MS = 30 * 24 * 60 * 60 * 1000 // 30 天
 const ITER = 150000
+const BACKUP_DAYS = 14            // 每日快照保留天数
+const WEEKLY_KEEP = 8             // 每周深快照保留份数
+const TOMBSTONE_TTL_MS = 14 * 24 * 60 * 60 * 1000 // 墓碑保留 14 天：足够覆盖离线设备，防止无限膨胀与重放
+const LARGE_TOMBSTONE_BACKUP = 20 // 单次同步墓碑达到该数量时，先做数据快照再应用（大批删除的前置保险）
+let lastBackupAt = ''

 const TABLES = ['project', 'work_log', 'log_item', 'attendance', 'weekly_note', 'setting', 'memo', 'credential', 'goal', 'transaction', 'plan']

@@ -35,7 +40,13 @@ function readJson(file, fallback) {
 }
 function writeJson(file, data) {
  const tmp = path.join(DATA_DIR, file + '.tmp')
-  fs.writeFileSync(tmp, JSON.stringify(data))
+  const fd = fs.openSync(tmp, 'w')
+  try {
+    fs.writeSync(fd, JSON.stringify(data))
+    fs.fsyncSync(fd) // 落盘后再原子改名：进程崩溃/断电不会留下半截文件
+  } finally {
+    fs.closeSync(fd)
+  }
  fs.renameSync(tmp, path.join(DATA_DIR, file))
 }

@@ -126,11 +137,37 @@ function isAdminToken(token) {
 function saveRegistry() { writeJson(registryFile, registry); try { doBackup() } catch { /* 忽略 */ } }

 function userFile(userId) { return path.join(DATA_DIR, 'users', userId + '.json') }
+function emptyUserData(userId) {
+  return { userId, cursor: 0, tables: {}, tombstones: {}, created_at: '' }
+}
 function loadUserData(userId) {
-  const data = readJson('users/' + userId + '.json', { userId, cursor: 0, tables: {}, tombstones: {} })
-  // 数据纪元：用户数据文件首次创建时间。目录被误删重建后该值改变，客户端据此识别并全量重推自救
-  if (!data.created_at) data.created_at = new Date().toISOString()
-  return data
+  const full = userFile(userId)
+  if (fs.existsSync(full)) {
+    try {
+      return JSON.parse(fs.readFileSync(full, 'utf-8'))
+    } catch (e) {
+      // 数据文件损坏：隔离保留原件并大声报错，绝不静默当作空数据（否则客户端同步会引发连锁覆盖）
+      const quarantine = full + '.corrupt-' + new Date().toISOString().replace(/[:.]/g, '-')
+      try { fs.renameSync(full, quarantine) } catch { /* 忽略 */ }
+      console.error('[存储] 用户数据文件解析失败，已隔离：' + quarantine + '（' + (e && e.message) + '）')
+      return emptyUserData(userId)
+    }
+  }
+  return emptyUserData(userId)
+}
+// 墓碑回收：超过保留期的墓碑删除（全量快照重建的客户端不依赖过期墓碑也能收敛）
+function gcTombstones(data) {
+  const cutoff = Date.now() - TOMBSTONE_TTL_MS
+  let pruned = 0
+  for (const t of Object.keys(data.tombstones || {})) {
+    const bucket = data.tombstones[t]
+    for (const u of Object.keys(bucket)) {
+      const ts = Date.parse(bucket[u])
+      if (Number.isFinite(ts) && ts < cutoff) { delete bucket[u]; pruned++ }
+    }
+    if (!Object.keys(bucket).length) delete data.tombstones[t]
+  }
+  return pruned
 }
 function saveUserData(userId, data) {
  fs.mkdirSync(path.join(DATA_DIR, 'users'), { recursive: true })
@@ -269,7 +306,15 @@ async function handle(req, res) {
    return
  }

-  if (req.method === 'GET' && route === '/api/health') { json(res, 200, { ok: true, backup: lastOffsiteBackup.state === 'disabled' ? 'local-only' : lastOffsiteBackup }); return }
+  if (req.method === 'GET' && route === '/api/health') {
+    json(res, 200, {
+      ok: true,
+      backup: lastOffsiteBackup.state === 'disabled' ? 'local-only' : lastOffsiteBackup,
+      users: registry.users.length,
+      lastBackup: lastBackupAt
+    })
+    return
+  }

  // SSE 事件流：token 经查询参数传递（EventSource 无法自定义请求头）
  if (req.method === 'GET' && route === '/api/events') {
@@ -359,9 +404,11 @@ async function handle(req, res) {
    if (!auth || !tokenUsable(auth)) { json(res, 401, { error: '未认证' }); return }
    const body = await readBody(req)
    const userData = loadUserData(auth.userId)
-    if (!userData.cursor) userData.cursor = 0
-    if (!userData.tables) userData.tables = {}
-    if (!userData.tombstones) userData.tombstones = {}
+    let dirty = false
+    if (userData.cursor == null) { userData.cursor = 0; dirty = true }
+    if (!userData.tables) { userData.tables = {}; dirty = true }
+    if (!userData.tombstones) { userData.tombstones = {}; dirty = true }
+    if (!userData.created_at) { userData.created_at = new Date().toISOString(); dirty = true } // 数据纪元（客户端灾难自检依据）

    // 应用推送的变更（新者胜 + 自然键去重）
    const incomingTables = body.tables || {}
@@ -411,12 +458,18 @@ async function handle(req, res) {
          userData.tombstones[t][conflictUuid] = now
          changed++
        }
+        // 内容与已存行完全一致：不计变更、不触发保存（空推送不落盘）
+        if (!conflictUuid && existing && JSON.stringify(existing) === JSON.stringify(clean)) continue
        userData.tables[t][row.uuid] = clean
        changed++
      }
    }
    // 应用墓碑
    const incomingTombs = body.tombstones || []
+    if (incomingTombs.length >= LARGE_TOMBSTONE_BACKUP) {
+      // 大批删除的前置保险：先把当前数据快照留存，再应用删除（可追溯、可回滚）
+      try { doBackup(); console.log('[存储] 检测到大批删除（' + incomingTombs.length + ' 条墓碑），已先行快照') } catch { /* 忽略 */ }
+    }
    for (const tb of incomingTombs) {
      if (!TABLES.includes(tb.table) || !tb.uuid) continue
      userData.tombstones[tb.table] = userData.tombstones[tb.table] || {}
@@ -434,7 +487,8 @@ async function handle(req, res) {
        if (row && String(row.updated_at || '') <= String(deletedAt)) delete userData.tables[t][u]
      }
    }
-    saveUserData(auth.userId, userData)
+    if (gcTombstones(userData) > 0) dirty = true
+    if (dirty || changed > 0) saveUserData(auth.userId, userData)
    if (changed) broadcast(auth.userId, 'change')

    // 返回权威全量
@@ -511,27 +565,53 @@ async function handle(req, res) {
 }

 // ---------- 自动备份：服务器是唯一数据源，必须自带兜底 ----------
+function snapshotTo(dir) {
+  fs.mkdirSync(path.join(dir, 'users'), { recursive: true })
+  try { fs.copyFileSync(path.join(DATA_DIR, registryFile), path.join(dir, 'registry.json')) } catch { /* 忽略 */ }
+  try { fs.copyFileSync(path.join(DATA_DIR, 'secret.key'), path.join(dir, 'secret.key')) } catch { /* 忽略 */ }
+  try {
+    const usersDir = path.join(DATA_DIR, 'users')
+    if (fs.existsSync(usersDir)) {
+      for (const f of fs.readdirSync(usersDir)) {
+        if (f.endsWith('.json') && !f.includes('.corrupt-')) fs.copyFileSync(path.join(usersDir, f), path.join(dir, 'users', f))
+      }
+    }
+  } catch { /* 忽略 */ }
+}
+function pruneBackups() {
+  try {
+    const root = path.join(DATA_DIR, 'backups')
+    const entries = fs.readdirSync(root).filter(d => /^\d{4}-\d{2}-\d{2}/.test(d))
+    const days = Array.from(new Set(entries.map(d => d.slice(0, 10)))).sort()
+    while (days.length > BACKUP_DAYS) {
+      const dead = days.shift()
+      for (const d of entries) {
+        if (d.startsWith(dead)) { try { fs.rmSync(path.join(root, d), { recursive: true, force: true }) } catch { /* 忽略 */ } }
+      }
+    }
+  } catch { /* 忽略 */ }
+}
 function doBackup() {
  try {
-    const day = new Date().toISOString().slice(0, 10)
-    const dir = path.join(DATA_DIR, 'backups', day)
-    fs.mkdirSync(path.join(dir, 'users'), { recursive: true })
-    try { fs.copyFileSync(path.join(DATA_DIR, registryFile), path.join(dir, 'registry.json')) } catch { /* 忽略 */ }
-    try { fs.copyFileSync(path.join(DATA_DIR, 'secret.key'), path.join(dir, 'secret.key')) } catch { /* 忽略 */ }
-    try {
-      const usersDir = path.join(DATA_DIR, 'users')
-      if (fs.existsSync(usersDir)) {
-        for (const f of fs.readdirSync(usersDir)) {
-          if (f.endsWith('.json')) fs.copyFileSync(path.join(usersDir, f), path.join(dir, 'users', f))
-        }
-      }
-    } catch { /* 忽略 */ }
-    // 保留最近 14 天
-    try {
-      const backupsRoot = path.join(DATA_DIR, 'backups')
-      const days = fs.readdirSync(backupsRoot).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort()
-      while (days.length > 14) { const old = days.shift(); try { fs.rmSync(path.join(backupsRoot, old), { recursive: true, force: true }) } catch { /* 忽略 */ } }
-    } catch { /* 忽略 */ }
+    const now = new Date()
+    const day = now.toISOString().slice(0, 10)
+    const root = path.join(DATA_DIR, 'backups')
+    let dir = path.join(root, day)
+    // 当日首份快照不可变（当日完整证据链）；之后的同日备份落时间戳副本
+    if (fs.existsSync(dir)) dir = path.join(root, day + '_t' + now.toTimeString().slice(0, 8).replace(/:/g, ''))
+    snapshotTo(dir)
+    // 每周一额外做周快照（更深恢复点），保留 8 份
+    if (now.getDay() === 1) {
+      const wdir = path.join(root, 'weekly', day)
+      if (!fs.existsSync(wdir)) snapshotTo(wdir)
+      try {
+        const wroot = path.join(root, 'weekly')
+        const ws = fs.readdirSync(wroot).filter(d => /^\d{4}-\d{2}-\d{2}$/.test(d)).sort()
+        while (ws.length > WEEKLY_KEEP) { const old = ws.shift(); try { fs.rmSync(path.join(wroot, old), { recursive: true, force: true }) } catch { /* 忽略 */ } }
+      } catch { /* 忽略 */ }
+    }
+    pruneBackups()
+    lastBackupAt = new Date().toISOString()
  } catch (e) { console.error('backup failed:', e && e.message) }
  void backupOffsite()
 }
@@ -0,0 +1,61 @@
+// 服务端存储加固冒烟：空同步不落盘 / 墓碑GC / 大批删除前置快照 / 损坏隔离
+const BASE = 'http://127.0.0.1:3780'
+let pass = 0, fail = 0
+function check(name, ok, detail) {
+  if (ok) { pass++; console.log('  PASS ' + name) } else { fail++; console.log('  FAIL ' + name + (detail ? ' -> ' + detail : '')) }
+}
+const sleep = ms => new Promise(r => setTimeout(r, ms))
+async function j(path, body, token) {
+  const r = await fetch(BASE + path, { method: 'POST', headers: { 'Content-Type': 'application/json', ...(token ? { Authorization: 'Bearer ' + token } : {}) }, body: body ? JSON.stringify(body) : undefined })
+  return { status: r.status, json: await r.json().catch(() => ({})) }
+}
+async function main() {
+  const fsp = await import('fs')
+  let reg = await j('/api/auth/register', { username: 'storagetest', password: 'test123456' })
+  if (reg.status === 409) reg = await j('/api/auth/login', { username: 'storagetest', password: 'test123456' })
+  console.log('auth status:', reg.status)
+  const tok = reg.json.token
+  const file = 'server/data/users/' + reg.json.userId + '.json'
+  const row = { uuid: 'm1', title: 't', content: 'c', created_at: '2026-10-26T00:00:00', updated_at: '2026-10-26T00:00:00' }
+  const push = tables => j('/api/sync', { device: 't', tables, tombstones: [] }, tok)
+
+  const p1 = await push({ memo: [row] })
+  console.log('push1 status:', p1.status)
+  check('首次同步落盘(200+文件存在)', p1.status === 200 && fsp.existsSync(file))
+  const m1 = fsp.statSync(file).mtimeMs
+
+  await sleep(30)
+  const p2 = await push({ memo: [row] })
+  await sleep(30)
+  const m2 = fsp.statSync(file).mtimeMs
+  check('空推送不重写文件(脏检查)', p2.status === 200 && m2 === m1, 'mtime ' + m1 + ' -> ' + m2)
+
+  await j('/api/sync', { device: 't', tables: {}, tombstones: [{ table: 'memo', uuid: 'old1', deleted_at: '2026-09-01T00:00:00' }] }, tok)
+  const d = JSON.parse(fsp.readFileSync(file, 'utf8'))
+  check('过期墓碑(>14天)自动回收', !d.tombstones.memo || !d.tombstones.memo.old1, JSON.stringify(d.tombstones))
+
+  const dirBefore = fsp.readdirSync('server/data/backups').filter(x => /^\d{4}-\d{2}-\d{2}/.test(x))
+  const tombs = Array.from({ length: 20 }, (_, i) => ({ table: 'memo', uuid: 'ghost' + i, deleted_at: '2026-10-26T23:59:00' }))
+  tombs.push({ table: 'memo', uuid: 'm1', deleted_at: '2026-10-26T23:59:00' }) // m1 本尊：应被删除
+  await sleep(1100) // 错开秒级时间戳，避免 _t 目录同名覆盖
+  const bigResp = await j('/api/sync', { device: 't', tables: {}, tombstones: tombs }, tok)
+  console.log('big-tombstone status:', bigResp.status)
+  const dirAfter = fsp.readdirSync('server/data/backups').filter(x => /^\d{4}-\d{2}-\d{2}/.test(x))
+  console.log('backup dirs:', dirAfter.join(' | '))
+  check('大批删除触发前置快照', dirAfter.length > dirBefore.length, dirBefore.length + '->' + dirAfter.length)
+  const d2 = JSON.parse(fsp.readFileSync(file, 'utf8'))
+  check('墓碑应用后行已删除', !d2.tables.memo || !d2.tables.memo.m1, JSON.stringify(Object.keys(d2.tables.memo || {})))
+  check('墓碑已记录(待客户端收敛)', Object.keys(d2.tombstones.memo || {}).length >= 21, JSON.stringify(d2.tombstones.memo || {}))
+
+  fsp.writeFileSync(file, '{ this is not json !!!')
+  const r5 = await push({ memo: [{ uuid: 'm2', title: 'x', content: '', created_at: '2026-10-26T00:00:00', updated_at: '2026-10-26T00:00:00' }] })
+  check('损坏文件不再静默当空(同步仍200)', r5.status === 200, 'status=' + r5.status)
+  const quarantined = fsp.readdirSync('server/data/users').some(f => f.includes('.corrupt-'))
+  check('损坏文件已隔离保留原件', quarantined, fsp.readdirSync('server/data/users').join(','))
+  const d3 = JSON.parse(fsp.readFileSync('server/data/users/' + reg.json.userId + '.json', 'utf8'))
+  check('新纪元已写入(created_at 非空)', !!d3.created_at)
+
+  console.log('RESULT: ' + (fail === 0 ? 'ALL PASS ' + pass : 'FAIL ' + fail + '/' + (pass + fail)))
+  if (fail) process.exit(1)
+}
+main().catch(e => { console.error('SMOKE ERROR', e); process.exit(1) })