/** * OSS audio/ 孤儿文件清理 * * 策略:OSS audio/ 下的文件,跟 DB 里的 AudioRecord.audioUrl + BookChapter.audioUrl 做差集。 * DB 没引用的 = 孤儿,可选删除。 * * 用法: * node deploy-package/scripts/clean-oss-orphans.cjs --dry-run # 默认,列出孤儿不删 * node deploy-package/scripts/clean-oss-orphans.cjs --apply # 真删 * node deploy-package/scripts/clean-oss-orphans.cjs --keep-recent-days 7 --dry-run * node deploy-package/scripts/clean-oss-orphans.cjs --max-delete 1000 --apply # 单次上限 * node deploy-package/scripts/clean-oss-orphans.cjs --out orphans.json --apply # 留档 * * 保护: merged/ 目录(章节合并稳定路径)默认不删; BookChapter.audioUrl 引用的绝不删; * AudioRecord.status='completed' 引用的绝不删; --keep-recent-days 内的不删。 */ const fs = require('fs'); const path = require('path'); // ---- 读 .env.production ---- const envTxt = fs.readFileSync(path.join(__dirname, '..', '..', 'server', '.env.production'), 'utf8'); const env = {}; for (const line of envTxt.split(/\r?\n/)) { const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/); if (m) env[m[1]] = m[2].replace(/^["']|["']$/g, ''); } const AK = env.OSS_ACCESS_KEY_ID; const SK = env.OSS_ACCESS_KEY_SECRET; if (!AK || !SK) { console.error('❌ 缺少 OSS AK/SK'); process.exit(1); } const serverNM = path.join(__dirname, '..', '..', 'server', 'node_modules'); const OSS = require(path.join(serverNM, 'ali-oss')); let mysql = null; try { mysql = require(path.join(serverNM, 'mysql2', 'promise')); } catch {} // ---- 解析参数 ---- const args = process.argv.slice(2); const opts = { apply: args.includes('--apply'), dryRun: !args.includes('--apply'), keepRecentDays: 0, maxDelete: Infinity, bucket: 'rrbrr-book-audio', out: null, limitPrefix: null, // 调试用: 只扫某个 prefix liveUrlsFile: null, // 外部提供的"活 URL"清单 (TSV: srcurl) skipDb: false, }; for (let i = 0; i < args.length; i++) { if (args[i] === '--keep-recent-days') opts.keepRecentDays = parseInt(args[++i], 10); if (args[i] === '--max-delete') opts.maxDelete = parseInt(args[++i], 10); if (args[i] === '--out') opts.out = args[++i]; if (args[i] === '--bucket') opts.bucket = args[++i]; if (args[i] === '--prefix') opts.limitPrefix = args[++i]; if (args[i] === '--live-urls-file') opts.liveUrlsFile = args[++i]; if (args[i] === '--skip-db') opts.skipDb = true; } function fmtBytes(b) { if (b < 1024) return b + ' B'; if (b < 1024 ** 2) return (b / 1024).toFixed(1) + ' KB'; if (b < 1024 ** 3) return (b / 1024 ** 2).toFixed(1) + ' MB'; return (b / 1024 ** 3).toFixed(2) + ' GB'; } async function listAll(client, prefix) { const all = []; let marker = null; do { const res = await client.list({ prefix, 'max-keys': 1000, marker }); all.push(...(res.objects || [])); marker = res.nextMarker || null; } while (marker); return all; } // 从 URL 提取 OSS object key。兼容本地路径 / 其它域。 // 不管 host 是哪个 bucket / 哪个 region,只取 path 部分。OSS key 是全局唯一的,bucket 不影响匹配。 function urlToKey(url) { if (!url) return null; const s = String(url).trim(); if (!s) return null; if (s.startsWith('/')) return null; // 本地路径,不算 OSS key try { const u = new URL(s); return decodeURIComponent(u.pathname.replace(/^\/+/, '')); } catch { return null; } } (async () => { console.log(`\n=== OSS 孤儿清理 (bucket=${opts.bucket}) ===`); console.log(`模式: ${opts.apply ? '🗑 真实删除' : '👀 dry-run(不删)'}`); if (opts.keepRecentDays) console.log(`保留: 最近 ${opts.keepRecentDays} 天内的文件不删`); if (isFinite(opts.maxDelete)) console.log(`单次上限: ${opts.maxDelete} 个`); // ---- 1. 连 OSS ---- const epCandidates = ['oss-cn-shanghai.aliyuncs.com', 'oss-cn-shanghai-internal.aliyuncs.com', 'oss-cn-hangzhou.aliyuncs.com']; let client = null, usedEp = null; for (const e of epCandidates) { try { const c = new OSS({ accessKeyId: AK, accessKeySecret: SK, bucket: opts.bucket, endpoint: e, secure: true }); await c.getBucketInfo(opts.bucket); client = c; usedEp = e; break; } catch {} } if (!client) { console.error('❌ OSS 连不上'); process.exit(1); } console.log(`endpoint: ${usedEp}`); // ---- 2. 列 OSS audio/ ---- const t0 = Date.now(); const prefix = opts.limitPrefix || 'audio/'; console.log(`\n[1] 扫描 OSS ${prefix} ...`); const ossObjs = await listAll(client, prefix); console.log(` 完成,文件数=${ossObjs.length} (${((Date.now()-t0)/1000).toFixed(1)}s)`); // ---- 3. 拿所有活引用 ---- console.log(`\n[2] 查活引用...`); const liveKeys = new Set(); const liveKeysBySource = { AudioRecord_completed: 0, AudioRecord_failed: 0, AudioRecord_processing: 0, BookChapter: 0 }; if (opts.liveUrlsFile) { // 外部提供的清单(SSH 到线上 dump 下来的) const lines = fs.readFileSync(opts.liveUrlsFile, 'utf8').split(/\r?\n/); for (const line of lines) { if (!line.trim()) continue; const [src, url] = line.split(/\t/); const k = urlToKey(url); if (k) { liveKeys.add(k); const tag = src === 'BC' ? 'BookChapter' : `AudioRecord_${src === 'AR' ? 'completed' : src}`; liveKeysBySource[tag] = (liveKeysBySource[tag] || 0) + 1; } } } else if (mysql && !opts.skipDb) { // 用本地 .env.production 里的 DATABASE_URL 直接连(用户机器上的 DB) const conn = await mysql.createConnection({ uri: env.DATABASE_URL, multipleStatements: false }); const [arRows] = await conn.query(`SELECT audioId, audioUrl, status, createdAt FROM AudioRecord`); const [bcRows] = await conn.query(`SELECT id, audioUrl, bookId FROM BookChapter WHERE audioUrl IS NOT NULL AND audioUrl <> ''`); await conn.end(); for (const r of arRows) { const k = urlToKey(r.audioUrl); if (k) { liveKeys.add(k); liveKeysBySource[`AudioRecord_${r.status || 'unknown'}`] = (liveKeysBySource[`AudioRecord_${r.status || 'unknown'}`] || 0) + 1; } } for (const r of bcRows) { const k = urlToKey(r.audioUrl); if (k) { liveKeys.add(k); liveKeysBySource.BookChapter++; } } } else { console.log(' ⚠️ 未提供 --live-urls-file 且未配置 mysql2 / --skip-db,跳过活引用收集'); } console.log(` 活引用 keys 总数: ${liveKeys.size}`); for (const [k, v] of Object.entries(liveKeysBySource)) console.log(` ${k}: ${v}`); // ---- 4. 找孤儿 ---- console.log(`\n[3] 匹配并找孤儿...`); const orphanList = []; const keptList = { live: 0, merged: 0, recent: 0, malformed: 0 }; const nowMs = Date.now(); for (const o of ossObjs) { const key = o.name; // 保护 merged/ 章节合并稳定路径 if (key.startsWith('audio/merged/')) { keptList.merged++; continue; } // 保护有 DB 引用 if (liveKeys.has(key)) { keptList.live++; continue; } // 保护最近 N 天的 if (opts.keepRecentDays > 0 && o.lastModified) { const ageMs = nowMs - new Date(o.lastModified).getTime(); if (ageMs < opts.keepRecentDays * 86400_000) { keptList.recent++; continue; } } orphanList.push({ key, size: o.size, lastModified: o.lastModified }); } // 按 size 降序 orphanList.sort((a, b) => b.size - a.size); const totalOrphanBytes = orphanList.reduce((s, x) => s + x.size, 0); const totalOssBytes = ossObjs.reduce((s, x) => s + x.size, 0); console.log(`\n[4] 结果:`); console.log(` OSS audio/ 总文件: ${ossObjs.length.toLocaleString()} (${fmtBytes(totalOssBytes)})`); console.log(` ✅ 保留(DB 活引用): ${keptList.live.toLocaleString()}`); console.log(` ✅ 保留(merged/): ${keptList.merged.toLocaleString()}`); if (opts.keepRecentDays) console.log(` ✅ 保留(最近${opts.keepRecentDays}天): ${keptList.recent.toLocaleString()}`); console.log(` ❌ 孤儿(将被删除): ${orphanList.length.toLocaleString()} (${fmtBytes(totalOrphanBytes)})`); console.log(` 占 OSS 总占用: ${(totalOrphanBytes / totalOssBytes * 100).toFixed(1)}%`); // 孤儿按月分布 const byMonth = new Map(); for (const x of orphanList) { const m = x.lastModified ? x.lastModified.slice(0, 7) : '(未知)'; const c = byMonth.get(m) || { count: 0, bytes: 0 }; c.count++; c.bytes += x.size; byMonth.set(m, c); } console.log(`\n 孤儿按月分布:`); for (const [m, c] of [...byMonth.entries()].sort()) { console.log(` ${m} ${String(c.count).padStart(6)} ${fmtBytes(c.bytes)}`); } console.log(`\n 孤儿 Top 20 最大:`); for (const x of orphanList.slice(0, 20)) { console.log(` ${fmtBytes(x.size).padStart(10)} ${x.key} (${x.lastModified?.slice(0,10) || '?'})`); } // 落档 if (opts.out) { fs.writeFileSync(opts.out, JSON.stringify({ scannedAt: new Date().toISOString(), bucket: opts.bucket, totalOssBytes, orphanCount: orphanList.length, orphanBytes: totalOrphanBytes, orphans: orphanList }, null, 2)); console.log(`\n 📄 孤儿清单已写到 ${opts.out}`); } if (opts.dryRun) { console.log(`\n👀 DRY-RUN 模式,没动任何文件。`); console.log(` 确认无误后用 --apply 真实删除:`); console.log(` node deploy-package/scripts/clean-oss-orphans.cjs --apply${opts.keepRecentDays ? ' --keep-recent-days ' + opts.keepRecentDays : ''}${isFinite(opts.maxDelete) ? ' --max-delete ' + opts.maxDelete : ''}`); return; } // ---- 5. 真实删除 ---- const toDelete = orphanList.slice(0, opts.maxDelete); console.log(`\n[5] 开始删除 ${toDelete.length} 个文件...`); const batchSize = 1000; // aliyun deleteMulti 一次最多 1000 let deleted = 0, deletedBytes = 0, fail = 0; for (let i = 0; i < toDelete.length; i += batchSize) { const batch = toDelete.slice(i, i + batchSize); try { await client.deleteMulti(batch.map(x => x.key)); deleted += batch.length; deletedBytes += batch.reduce((s, x) => s + x.size, 0); process.stdout.write(`\r 进度 ${deleted}/${toDelete.length} (${fmtBytes(deletedBytes)}) `); } catch (e) { fail++; console.error(`\n ❌ batch ${i}-${i+batch.length} 删除失败:`, e.code || e.message); } } console.log(`\n\n✅ 完成: 删了 ${deleted} 个文件,释放 ${fmtBytes(deletedBytes)}。失败 batch: ${fail}`); })().catch(e => { console.error('异常:', e); process.exit(1); });