audio_codebuddy/deploy-package/scripts/clean-oss-orphans.cjs

/**
 * OSS audio/ 孤儿文件清理
 *
 * 策略：OSS audio/ 下的文件，跟 DB 里的 AudioRecord.audioUrl + BookChapter.audioUrl 做差集。
 * DB 没引用的 = 孤儿，可选删除。
 *
 * 用法:
 *   node deploy-package/scripts/clean-oss-orphans.cjs --dry-run   # 默认，列出孤儿不删
 *   node deploy-package/scripts/clean-oss-orphans.cjs --apply     # 真删
 *   node deploy-package/scripts/clean-oss-orphans.cjs --keep-recent-days 7 --dry-run
 *   node deploy-package/scripts/clean-oss-orphans.cjs --max-delete 1000 --apply  # 单次上限
 *   node deploy-package/scripts/clean-oss-orphans.cjs --out orphans.json --apply  # 留档
 *
 * 保护: merged/ 目录(章节合并稳定路径)默认不删; BookChapter.audioUrl 引用的绝不删;
 *       AudioRecord.status='completed' 引用的绝不删; --keep-recent-days 内的不删。
 */

const fs = require('fs');
const path = require('path');

// ---- 读 .env.production ----
const envTxt = fs.readFileSync(path.join(__dirname, '..', '..', 'server', '.env.production'), 'utf8');
const env = {};
for (const line of envTxt.split(/\r?\n/)) {
  const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/);
  if (m) env[m[1]] = m[2].replace(/^["']|["']$/g, '');
}
const AK = env.OSS_ACCESS_KEY_ID;
const SK = env.OSS_ACCESS_KEY_SECRET;
if (!AK || !SK) {
  console.error('❌ 缺少 OSS AK/SK');
  process.exit(1);
}
const serverNM = path.join(__dirname, '..', '..', 'server', 'node_modules');
const OSS = require(path.join(serverNM, 'ali-oss'));
let mysql = null;
try { mysql = require(path.join(serverNM, 'mysql2', 'promise')); } catch {}

// ---- 解析参数 ----
const args = process.argv.slice(2);
const opts = {
  apply: args.includes('--apply'),
  dryRun: !args.includes('--apply'),
  keepRecentDays: 0,
  maxDelete: Infinity,
  bucket: 'rrbrr-book-audio',
  out: null,
  limitPrefix: null, // 调试用: 只扫某个 prefix
  liveUrlsFile: null, // 外部提供的"活 URL"清单 (TSV: src<TAB>url)
  skipDb: false,
};
for (let i = 0; i < args.length; i++) {
  if (args[i] === '--keep-recent-days') opts.keepRecentDays = parseInt(args[++i], 10);
  if (args[i] === '--max-delete') opts.maxDelete = parseInt(args[++i], 10);
  if (args[i] === '--out') opts.out = args[++i];
  if (args[i] === '--bucket') opts.bucket = args[++i];
  if (args[i] === '--prefix') opts.limitPrefix = args[++i];
  if (args[i] === '--live-urls-file') opts.liveUrlsFile = args[++i];
  if (args[i] === '--skip-db') opts.skipDb = true;
}

function fmtBytes(b) {
  if (b < 1024) return b + ' B';
  if (b < 1024 ** 2) return (b / 1024).toFixed(1) + ' KB';
  if (b < 1024 ** 3) return (b / 1024 ** 2).toFixed(1) + ' MB';
  return (b / 1024 ** 3).toFixed(2) + ' GB';
}

async function listAll(client, prefix) {
  const all = [];
  let marker = null;
  do {
    const res = await client.list({ prefix, 'max-keys': 1000, marker });
    all.push(...(res.objects || []));
    marker = res.nextMarker || null;
  } while (marker);
  return all;
}

// 从 URL 提取 OSS object key。兼容本地路径 / 其它域。
// 不管 host 是哪个 bucket / 哪个 region，只取 path 部分。OSS key 是全局唯一的，bucket 不影响匹配。
function urlToKey(url) {
  if (!url) return null;
  const s = String(url).trim();
  if (!s) return null;
  if (s.startsWith('/')) return null; // 本地路径，不算 OSS key
  try {
    const u = new URL(s);
    return decodeURIComponent(u.pathname.replace(/^\/+/, ''));
  } catch {
    return null;
  }
}

(async () => {
  console.log(`\n=== OSS 孤儿清理 (bucket=${opts.bucket}) ===`);
  console.log(`模式: ${opts.apply ? '🗑  真实删除' : '👀 dry-run（不删）'}`);
  if (opts.keepRecentDays) console.log(`保留: 最近 ${opts.keepRecentDays} 天内的文件不删`);
  if (isFinite(opts.maxDelete)) console.log(`单次上限: ${opts.maxDelete} 个`);

  // ---- 1. 连 OSS ----
  const epCandidates = ['oss-cn-shanghai.aliyuncs.com', 'oss-cn-shanghai-internal.aliyuncs.com', 'oss-cn-hangzhou.aliyuncs.com'];
  let client = null, usedEp = null;
  for (const e of epCandidates) {
    try {
      const c = new OSS({ accessKeyId: AK, accessKeySecret: SK, bucket: opts.bucket, endpoint: e, secure: true });
      await c.getBucketInfo(opts.bucket);
      client = c; usedEp = e; break;
    } catch {}
  }
  if (!client) { console.error('❌ OSS 连不上'); process.exit(1); }
  console.log(`endpoint: ${usedEp}`);

  // ---- 2. 列 OSS audio/ ----
  const t0 = Date.now();
  const prefix = opts.limitPrefix || 'audio/';
  console.log(`\n[1] 扫描 OSS ${prefix} ...`);
  const ossObjs = await listAll(client, prefix);
  console.log(`    完成,文件数=${ossObjs.length} (${((Date.now()-t0)/1000).toFixed(1)}s)`);

  // ---- 3. 拿所有活引用 ----
  console.log(`\n[2] 查活引用...`);
  const liveKeys = new Set();
  const liveKeysBySource = { AudioRecord_completed: 0, AudioRecord_failed: 0, AudioRecord_processing: 0, BookChapter: 0 };

  if (opts.liveUrlsFile) {
    // 外部提供的清单（SSH 到线上 dump 下来的）
    const lines = fs.readFileSync(opts.liveUrlsFile, 'utf8').split(/\r?\n/);
    for (const line of lines) {
      if (!line.trim()) continue;
      const [src, url] = line.split(/\t/);
      const k = urlToKey(url);
      if (k) {
        liveKeys.add(k);
        const tag = src === 'BC' ? 'BookChapter' : `AudioRecord_${src === 'AR' ? 'completed' : src}`;
        liveKeysBySource[tag] = (liveKeysBySource[tag] || 0) + 1;
      }
    }
  } else if (mysql && !opts.skipDb) {
    // 用本地 .env.production 里的 DATABASE_URL 直接连（用户机器上的 DB）
    const conn = await mysql.createConnection({ uri: env.DATABASE_URL, multipleStatements: false });
    const [arRows] = await conn.query(`SELECT audioId, audioUrl, status, createdAt FROM AudioRecord`);
    const [bcRows] = await conn.query(`SELECT id, audioUrl, bookId FROM BookChapter WHERE audioUrl IS NOT NULL AND audioUrl <> ''`);
    await conn.end();
    for (const r of arRows) {
      const k = urlToKey(r.audioUrl);
      if (k) { liveKeys.add(k); liveKeysBySource[`AudioRecord_${r.status || 'unknown'}`] = (liveKeysBySource[`AudioRecord_${r.status || 'unknown'}`] || 0) + 1; }
    }
    for (const r of bcRows) {
      const k = urlToKey(r.audioUrl);
      if (k) { liveKeys.add(k); liveKeysBySource.BookChapter++; }
    }
  } else {
    console.log('    ⚠️  未提供 --live-urls-file 且未配置 mysql2 / --skip-db，跳过活引用收集');
  }

  console.log(`    活引用 keys 总数: ${liveKeys.size}`);
  for (const [k, v] of Object.entries(liveKeysBySource)) console.log(`      ${k}: ${v}`);

  // ---- 4. 找孤儿 ----
  console.log(`\n[3] 匹配并找孤儿...`);
  const orphanList = [];
  const keptList = { live: 0, merged: 0, recent: 0, malformed: 0 };
  const nowMs = Date.now();
  for (const o of ossObjs) {
    const key = o.name;
    // 保护 merged/ 章节合并稳定路径
    if (key.startsWith('audio/merged/')) { keptList.merged++; continue; }
    // 保护有 DB 引用
    if (liveKeys.has(key)) { keptList.live++; continue; }
    // 保护最近 N 天的
    if (opts.keepRecentDays > 0 && o.lastModified) {
      const ageMs = nowMs - new Date(o.lastModified).getTime();
      if (ageMs < opts.keepRecentDays * 86400_000) { keptList.recent++; continue; }
    }
    orphanList.push({ key, size: o.size, lastModified: o.lastModified });
  }

  // 按 size 降序
  orphanList.sort((a, b) => b.size - a.size);
  const totalOrphanBytes = orphanList.reduce((s, x) => s + x.size, 0);
  const totalOssBytes = ossObjs.reduce((s, x) => s + x.size, 0);

  console.log(`\n[4] 结果:`);
  console.log(`  OSS  audio/ 总文件:       ${ossObjs.length.toLocaleString()}  (${fmtBytes(totalOssBytes)})`);
  console.log(`  ✅ 保留(DB 活引用):       ${keptList.live.toLocaleString()}`);
  console.log(`  ✅ 保留(merged/):          ${keptList.merged.toLocaleString()}`);
  if (opts.keepRecentDays) console.log(`  ✅ 保留(最近${opts.keepRecentDays}天):     ${keptList.recent.toLocaleString()}`);
  console.log(`  ❌ 孤儿(将被删除):        ${orphanList.length.toLocaleString()}  (${fmtBytes(totalOrphanBytes)})`);
  console.log(`     占 OSS 总占用:         ${(totalOrphanBytes / totalOssBytes * 100).toFixed(1)}%`);

  // 孤儿按月分布
  const byMonth = new Map();
  for (const x of orphanList) {
    const m = x.lastModified ? x.lastModified.slice(0, 7) : '(未知)';
    const c = byMonth.get(m) || { count: 0, bytes: 0 };
    c.count++; c.bytes += x.size; byMonth.set(m, c);
  }
  console.log(`\n  孤儿按月分布:`);
  for (const [m, c] of [...byMonth.entries()].sort()) {
    console.log(`    ${m}  ${String(c.count).padStart(6)}  ${fmtBytes(c.bytes)}`);
  }

  console.log(`\n  孤儿 Top 20 最大:`);
  for (const x of orphanList.slice(0, 20)) {
    console.log(`    ${fmtBytes(x.size).padStart(10)}  ${x.key}  (${x.lastModified?.slice(0,10) || '?'})`);
  }

  // 落档
  if (opts.out) {
    fs.writeFileSync(opts.out, JSON.stringify({ scannedAt: new Date().toISOString(), bucket: opts.bucket, totalOssBytes, orphanCount: orphanList.length, orphanBytes: totalOrphanBytes, orphans: orphanList }, null, 2));
    console.log(`\n  📄 孤儿清单已写到 ${opts.out}`);
  }

  if (opts.dryRun) {
    console.log(`\n👀 DRY-RUN 模式，没动任何文件。`);
    console.log(`   确认无误后用 --apply 真实删除:`);
    console.log(`   node deploy-package/scripts/clean-oss-orphans.cjs --apply${opts.keepRecentDays ? ' --keep-recent-days ' + opts.keepRecentDays : ''}${isFinite(opts.maxDelete) ? ' --max-delete ' + opts.maxDelete : ''}`);
    return;
  }

  // ---- 5. 真实删除 ----
  const toDelete = orphanList.slice(0, opts.maxDelete);
  console.log(`\n[5] 开始删除 ${toDelete.length} 个文件...`);
  const batchSize = 1000; // aliyun deleteMulti 一次最多 1000
  let deleted = 0, deletedBytes = 0, fail = 0;
  for (let i = 0; i < toDelete.length; i += batchSize) {
    const batch = toDelete.slice(i, i + batchSize);
    try {
      await client.deleteMulti(batch.map(x => x.key));
      deleted += batch.length;
      deletedBytes += batch.reduce((s, x) => s + x.size, 0);
      process.stdout.write(`\r  进度 ${deleted}/${toDelete.length} (${fmtBytes(deletedBytes)})  `);
    } catch (e) {
      fail++;
      console.error(`\n  ❌ batch ${i}-${i+batch.length} 删除失败:`, e.code || e.message);
    }
  }
  console.log(`\n\n✅ 完成: 删了 ${deleted} 个文件,释放 ${fmtBytes(deletedBytes)}。失败 batch: ${fail}`);
})().catch(e => { console.error('异常:', e); process.exit(1); });