From 4b25231c01e90d21bb704f9670ce24f49256d79a Mon Sep 17 00:00:00 2001 From: lou Date: Tue, 11 Aug 2026 14:22:28 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A4=E5=84=BF=E6=B8=85?= =?UTF-8?q?=E7=90=86=E5=B7=A5=E5=85=B7=20+=20=E7=9C=8B=E9=97=A8=E7=8B=97?= =?UTF-8?q?=E6=AF=8F=E5=B0=8F=E6=97=B6=E8=87=AA=E5=8A=A8=E6=B8=85=E7=90=86?= =?UTF-8?q?=20+=20start.sh=20=E6=97=A0=E6=9D=A1=E4=BB=B6=E8=A1=A5=E6=8B=89?= =?UTF-8?q?=E7=9C=8B=E9=97=A8=E7=8B=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - clean_orphans.py: 只删数据库无记录的孤儿 (tmp 终态/僵尸任务卷, storage 无记录目录) - 目录级判断: files 记录指向的目录整体跳过, 内部文件永不单独删除 (防误删) - 路径 resolve 统一, 调用方 cwd 无关 - 边界: tmp 终态超 1h 删/进行中保留/超 7 天判僵尸; storage 孤儿超 24h 删 - watchdog.sh: 每 30s 保活服务 + 每小时自动跑清理 (失败 30s 后重试) - start.sh: 看门狗拉起移到无条件位置 (原 main.py 在跑时 exit 短路, 看门狗死亡无人接管) - tests: 新增 test_clean_orphans 5 项 (含有效目录内 chunk 不删的误删回归断言) - README: 数据目录补 tmp, 新增运维节, 测试数 13->17 --- README.md | 22 +++- clean_orphans.py | 221 ++++++++++++++++++++++++++++++++++++ start.sh | 44 +++---- tests/test_clean_orphans.py | 129 +++++++++++++++++++++ watchdog.sh | 12 ++ 5 files changed, 405 insertions(+), 23 deletions(-) create mode 100644 clean_orphans.py create mode 100644 tests/test_clean_orphans.py diff --git a/README.md b/README.md index 4c65ca4..77bdbfc 100644 --- a/README.md +++ b/README.md @@ -53,11 +53,31 @@ python3 -m venv .venv && .venv/bin/pip install fastapi "uvicorn[standard]" crypt ``` data/ ├── app.db # SQLite: users / files / transfers / dirs +├── tmp/{task_id}/ # 传输中临时卷 (aria2 落盘, complete 后移走) └── storage/{user}/{date}/{task_id}/chunk_NNNN # 密文卷原样 ``` +## 运维: 孤儿清理 + 看门狗 + +中断传输/失败任务会在 `data/tmp/` 留下残留卷, 旧架构迁移也可能在 `data/storage/` +留下无记录目录。`clean_orphans.py` 只删**数据库无记录**的孤儿, 绝不碰有效数据 +(files 记录指向的目录整体跳过, 内部文件永不单独判断): + +```bash +./clean_orphans.py --dry-run # 预览不删 +./clean_orphans.py # 实际清理 +./clean_orphans.py --quiet # 静默 (日志仍写 /tmp/sz-clean.log) +``` + +清理边界: tmp 终态任务 (done/failed) 超 1 小时删, 进行中保留, 超 7 天无动静 +判僵尸删; storage 孤儿超 24 小时删 (给客户端 complete 重试窗口)。 + +`watchdog.sh` (由 start.sh 拉起) 每 30s 检查服务进程, 挂了自动拉起, 并每小时 +自动跑一次孤儿清理。`start.sh` 幂等: main.py 已在跑则跳过启动, 但**无条件补拉 +看门狗** (防看门狗死亡后无人接管)。 + ## 测试 ```bash -.venv/bin/pytest -q # 13 tests (含账户隔离 / 目录 / 完整链路上传下载) +.venv/bin/pytest -q # 17 tests (含账户隔离 / 目录 / 完整链路上传下载 / 孤儿清理边界) ``` diff --git a/clean_orphans.py b/clean_orphans.py new file mode 100644 index 0000000..82c2b73 --- /dev/null +++ b/clean_orphans.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""孤儿文件清理脚本: 删除数据库无记录的传输临时目录与存储残留。 + +清理目标: + 1. data/tmp// 中断/失败/僵尸任务的临时卷目录 + 2. data/storage/ 下 files 表无记录的孤儿目录与单文件 (含旧架构残留) + +安全边界 (只删确认孤儿, 绝不碰有效数据): + - tmp: 状态 uploading/storing 且 mtime 新 (< 7 天) = 进行中, 保留; + 终态 (done/failed) 超 1 小时 = 任务已结束, tmp 卷无价值, 删; + 无任务记录超 1 小时 = 孤儿, 删; uploading/storing 超 7 天 = 僵尸任务 + (客户端早已不在), 删 + - storage: 只删 files.path 集合中不存在的目录/文件; 目录删除前检查其下 + 没有任何 files.path 前缀 (防父目录误删); mtime 超 24 小时才动 + (给客户端 complete 重试窗口) + - 删除一律 shutil.rmtree, 失败跳过并记录 + +用法: + clean_orphans.py # 实际清理 + clean_orphans.py --dry-run # 预览不删 + clean_orphans.py --quiet # 静默 (日志仍写 /tmp/sz-clean.log, watchdog 调用用) +""" + +import argparse +import os +import shutil +import sqlite3 +import sys +from datetime import datetime +from pathlib import Path + +DB_PATH = Path("data/app.db") +TMP_ROOT = Path("data/tmp") +STORAGE_ROOT = Path("data/storage") +CLEAN_LOG = Path("/tmp/sz-clean.log") + +# 终态任务 tmp 保留窗口 (秒): 任务结束 1 小时后卷可删 +TMP_FINAL_AGE = 3600 +# 进行中任务僵尸判定 (秒): 7 天无动静 = 客户端早死, 卷可删 +TMP_STALE_AGE = 7 * 86400 +# storage 孤儿 mtime 保留窗口 (秒): 24 小时, 给 complete 重试窗口 +STORAGE_MIN_AGE = 86400 +# 进行中的任务状态 (tmp 目录保留) +ACTIVE_STATUSES = {"uploading", "storing"} + + +def log(msg: str, quiet: bool = False) -> None: + """写日志到 stdout 与 /tmp/sz-clean.log, 中文业务日志。""" + line = f"[{datetime.now():%Y-%m-%d %H:%M:%S}] {msg}" + if not quiet: + print(line) + try: + with CLEAN_LOG.open("a", encoding="utf-8") as f: + f.write(line + "\n") + except OSError: + pass + + +def load_db(db_path: Path) -> tuple[dict[str, str], set[str]]: + """读数据库, 返回 (transfer_id->status 映射, files.path 集合)。 + + files.path 可能存相对路径 (data/storage/...) 或绝对路径, 统一 resolve + 成绝对路径, 与磁盘遍历的 resolve 结果可比。 + """ + conn = sqlite3.connect(db_path) + conn.row_factory = sqlite3.Row + cur = conn.cursor() + status_map: dict[str, str] = {} + for r in cur.execute("SELECT transfer_id, status FROM transfers"): + status_map[str(r["transfer_id"])] = str(r["status"]) + paths: set[str] = set() + for r in cur.execute("SELECT path FROM files"): + paths.add(str(Path(str(r["path"])).resolve())) + conn.close() + return status_map, paths + + +def dir_age_ok(p: Path, min_age: int) -> bool: + """路径 mtime 距今超过 min_age 秒才返回 True (过新的一律跳过)。""" + try: + return (datetime.now().timestamp() - p.stat().st_mtime) >= min_age + except OSError: + return False + + +def dir_size(p: Path) -> int: + """目录内文件总字节数 (删除失败兜底统计用)。""" + return sum(f.stat().st_size for f in p.rglob("*") if f.is_file()) + + +def clean_tmp( + tmp_root: Path, status_map: dict[str, str], dry: bool, quiet: bool +) -> tuple[int, int]: + """清理 tmp 下的孤儿任务目录, 返回 (删除数, 释放字节)。""" + removed = 0 + freed = 0 + if not tmp_root.is_dir(): + return removed, freed + for tid_dir in sorted(tmp_root.iterdir()): + name = tid_dir.name + if not tid_dir.is_dir(): + continue # 散文件不动 + status = status_map.get(name) # None = 无任务记录 + if status in ACTIVE_STATUSES: + # 进行中: 超 7 天没动静 = 僵尸任务, 删; 否则保留 + if not dir_age_ok(tid_dir, TMP_STALE_AGE): + log(f"[tmp] {name} 任务进行中, 保留", quiet) + continue + log(f"[tmp] {name} 僵尸任务 (进行中超 7 天), 删除", quiet) + else: + # 无记录或终态 (done/failed): 超 1 小时可删 + if not dir_age_ok(tid_dir, TMP_FINAL_AGE): + log(f"[tmp] {name} 目录过新, 跳过", quiet) + continue + reason = "无任务记录" if status is None else f"任务已{status}" + log(f"[tmp] {reason}, 删除孤儿任务目录 {name} ({dir_size(tid_dir) / 1024 / 1024:.1f} MB)", quiet) + if not dry: + size = dir_size(tid_dir) + try: + shutil.rmtree(tid_dir) + removed += 1 + freed += size + except OSError as e: + log(f"[tmp] 删除失败 {name}: {e}", quiet) + return removed, freed + + +def clean_storage( + storage_root: Path, file_paths: set[str], dry: bool, quiet: bool +) -> tuple[int, int]: + """清理 storage 下 files 表无记录的孤儿目录, 返回 (删除数, 释放字节)。 + + 安全边界 (防误删有效数据): + - files.path 记录的是**目录路径**, 目录有效 => 内部所有文件有效, + **文件永不单独判断删除** (曾因逐文件判断把有效目录内的 chunk 误删) + - 目录 rel 在 file_paths 中, 或其下有任意 files.path 前缀 => 有效, + 整个子树跳过 (剪枝不深入) + - 目录无记录且无子记录 => 孤儿候选, 整个目录删除 (含旧架构单文件 + 所在的日期目录, 一并覆盖) + """ + removed = 0 + freed = 0 + if not storage_root.is_dir(): + return removed, freed + # 收集候选 (仅目录), 统一删除, 避免遍历中 rmtree + candidates: list[Path] = [] + for dirpath, dirnames, filenames in os.walk(storage_root): + d = Path(dirpath) + if d == storage_root: + continue + rel = str(d.resolve()) + if rel in file_paths: + # 有效叶子目录: files 记录精确指向它, 整个子树跳过 + dirnames[:] = [] + continue + if not any(fp.startswith(rel + "/") for fp in file_paths): + # 无记录且无子记录 => 孤儿候选, 整个目录删除 + if dir_age_ok(d, STORAGE_MIN_AGE): + candidates.append(d) + dirnames[:] = [] # 目录将删, 剪枝不再深入 + continue + # 有子记录 (父目录层如 user/日期): 不是孤儿, 深入遍历子目录 + for p in candidates: + size = dir_size(p) + log(f"[storage] 删除孤儿目录 {p} ({size / 1024 / 1024:.1f} MB)", quiet) + if not dry: + try: + shutil.rmtree(p) + removed += 1 + freed += size + except OSError as e: + log(f"[storage] 删除失败 {p}: {e}", quiet) + return removed, freed + + +def main() -> int: + parser = argparse.ArgumentParser(description="7z-encrypt 服务端孤儿文件清理") + parser.add_argument("--dry-run", action="store_true", help="预览模式, 不实际删除") + parser.add_argument("--quiet", action="store_true", help="静默模式, 只写日志文件") + parser.add_argument("--db", default=str(DB_PATH), help="SQLite 数据库路径") + parser.add_argument("--tmp", default=str(TMP_ROOT), help="临时目录根") + parser.add_argument("--storage", default=str(STORAGE_ROOT), help="存储目录根") + args = parser.parse_args() + + # 默认路径 (data/app.db 等) 相对脚本所在目录解析, 调用方 cwd 无关 + script_dir = Path(__file__).resolve().parent + db_path = Path(args.db) + if not db_path.is_absolute(): + db_path = script_dir / db_path + tmp_root = Path(args.tmp) + if not tmp_root.is_absolute(): + tmp_root = script_dir / tmp_root + storage_root = Path(args.storage) + if not storage_root.is_absolute(): + storage_root = script_dir / storage_root + dry = args.dry_run + quiet = args.quiet + + mode = "预览" if dry else "清理" + log(f"=== {mode}开始 (db={db_path}) ===", quiet) + if not db_path.is_file(): + log(f"[错误] 数据库不存在: {db_path}", quiet) + return 1 + + status_map, file_paths = load_db(db_path) + log(f"[db] 任务 {len(status_map)} 个, 文件记录 {len(file_paths)} 条", quiet) + + t_removed, t_freed = clean_tmp(tmp_root, status_map, dry, quiet) + s_removed, s_freed = clean_storage(storage_root, file_paths, dry, quiet) + + total_removed = t_removed + s_removed + total_freed = t_freed + s_freed + log( + f"=== {mode}完成: 删除 {total_removed} 项, 释放 {total_freed / 1024 / 1024:.1f} MB ===", + quiet, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/start.sh b/start.sh index 81d9c80..35321a3 100755 --- a/start.sh +++ b/start.sh @@ -7,31 +7,31 @@ cd /root/server if pgrep -f 'python3 [m]ain\.py' > /dev/null 2>&1 && [ "$1" != "--force" ]; then echo "[start] 服务已在运行, 跳过" - exit 0 -fi - -# 杀掉残留进程 (防重复拉起), 再启动 -pkill -9 -f 'python3 [m]ain\.py' 2>/dev/null || true -sleep 1 - -# 后台常驻: setsid + nohup 防 SSH 断连杀进程; SZ_TOKEN 与客户端预共享 -(setsid nohup env SZ_TOKEN=sz-token-95197 python3 main.py > /tmp/sz-server.log 2>&1 &) - -# 等启动完成 -for i in 1 2 3 4 5 6; do - sleep 1 - grep -q 'startup complete' /tmp/sz-server.log 2>/dev/null && break -done - -if grep -q 'startup complete' /tmp/sz-server.log 2>/dev/null; then - echo "[start] 服务已启动 (PID $(pgrep -f 'python3 [m]ain\.py' | head -1))" else - echo "[start] 启动失败, 日志尾部:" - tail -5 /tmp/sz-server.log - exit 1 + # 杀掉残留进程 (防重复拉起), 再启动 + pkill -9 -f 'python3 [m]ain\.py' 2>/dev/null || true + sleep 1 + + # 后台常驻: setsid + nohup 防 SSH 断连杀进程; SZ_TOKEN 与客户端预共享 + (setsid nohup env SZ_TOKEN=sz-token-95197 python3 main.py > /tmp/sz-server.log 2>&1 &) + + # 等启动完成 + for i in 1 2 3 4 5 6; do + sleep 1 + grep -q 'startup complete' /tmp/sz-server.log 2>/dev/null && break + done + + if grep -q 'startup complete' /tmp/sz-server.log 2>/dev/null; then + echo "[start] 服务已启动 (PID $(pgrep -f 'python3 [m]ain\.py' | head -1))" + else + echo "[start] 启动失败, 日志尾部:" + tail -5 /tmp/sz-server.log + exit 1 + fi fi -# 拉起看门狗 (防重复, watchdog.sh 自身检查) +# 拉起看门狗 (无条件检查: main.py 在跑也会补拉, 防看门狗死亡后无人接管) +# pgrep 用 [.] 正则防自匹配 (远程命令字符串含字面量也会被 -f 匹配) if ! pgrep -f 'watchdog[.]sh' > /dev/null 2>&1; then (setsid nohup /root/server/watchdog.sh > /dev/null 2>&1 &) echo "[start] 看门狗已启动" diff --git a/tests/test_clean_orphans.py b/tests/test_clean_orphans.py new file mode 100644 index 0000000..bfd7201 --- /dev/null +++ b/tests/test_clean_orphans.py @@ -0,0 +1,129 @@ +"""clean_orphans 孤儿清理逻辑测试: 临时 db + 临时目录, 验证删除/保留边界。 + +覆盖: tmp 终态/无记录删除、进行中保留、僵尸任务删除; storage 有记录保留、 +无记录目录/单文件删除、父目录含记录不删; dry-run 不实际删除。 +""" + +import os +import sqlite3 +import time +from pathlib import Path + +from clean_orphans import clean_storage, clean_tmp, load_db + +# 测试时间基准: 全部相对 now 偏移, 避免真实时间波动 +NOW = time.time() +HOUR = 3600 +DAY = 86400 + + +def _set_mtime(p: Path, age_sec: float) -> None: + """把路径 mtime 调到 age_sec 秒前。""" + t = NOW - age_sec + os.utime(p, (t, t)) + + +def make_env(tmp_path: Path) -> tuple[Path, Path, Path]: + """造测试库与目录结构, 返回 (db, tmp_root, storage_root)。""" + db = tmp_path / "app.db" + conn = sqlite3.connect(db) + cur = conn.cursor() + cur.execute("CREATE TABLE transfers (transfer_id TEXT, status TEXT)") + cur.execute("CREATE TABLE files (path TEXT)") + cur.executemany( + "INSERT INTO transfers VALUES (?,?)", + [ + ("done_tid", "done"), + ("failed_tid", "failed"), + ("uploading_tid", "uploading"), + ("zombie_tid", "uploading"), + ], + ) + storage_root = tmp_path / "data" / "storage" + recorded = [ + storage_root / "user1" / "2026-08-11" / "valid_tid", + storage_root / "user1" / "2026-08-11" / "clone_tid_clone", + ] + cur.executemany( + "INSERT INTO files VALUES (?)", [(str(p),) for p in recorded] + ) + conn.commit() + conn.close() + + # tmp 目录 + for tid in ("done_tid", "failed_tid", "uploading_tid", "zombie_tid", "orphan_tid"): + d = tmp_path / "tmp" / tid + d.mkdir(parents=True) + (d / "chunk_0001").write_bytes(b"data") + _set_mtime(tmp_path / "tmp" / "done_tid", 2 * DAY) + _set_mtime(tmp_path / "tmp" / "failed_tid", 2 * DAY) + _set_mtime(tmp_path / "tmp" / "uploading_tid", 60) # 新, 进行中 + _set_mtime(tmp_path / "tmp" / "zombie_tid", 8 * DAY) # 僵尸 + _set_mtime(tmp_path / "tmp" / "orphan_tid", 2 * DAY) + + # storage 目录 + for d in recorded: + d.mkdir(parents=True) + (d / "chunk_0001").write_bytes(b"x") + _set_mtime(d, 2 * DAY) + orphan_dir = storage_root / "user1" / "2026-08-11" / "orphan_storage_tid" + orphan_dir.mkdir(parents=True) + (orphan_dir / "chunk_0001").write_bytes(b"x") + _set_mtime(orphan_dir, 2 * DAY) + orphan_file = storage_root / "user1" / "2026-08-10" / "old_arch_single.enc" + orphan_file.parent.mkdir(parents=True) + orphan_file.write_bytes(b"y") + _set_mtime(orphan_file, 2 * DAY) + _set_mtime(orphan_file.parent, 2 * DAY) + + return db, tmp_path / "tmp", storage_root + + +def test_load_db_resolves_paths(tmp_path: Path) -> None: + """files.path 相对/绝对统一 resolve, 与磁盘绝对路径可比。""" + db, _, _ = make_env(tmp_path) + status_map, paths = load_db(db) + assert status_map["done_tid"] == "done" + assert status_map["zombie_tid"] == "uploading" + assert str((tmp_path / "data" / "storage" / "user1" / "2026-08-11" / "valid_tid").resolve()) in paths + + +def test_clean_tmp_removes_final_orphan_zombie_keeps_active(tmp_path: Path) -> None: + """终态/无记录/僵尸 tmp 删, 进行中(新 mtime)保留。""" + db, tmp_root, _ = make_env(tmp_path) + status_map, _ = load_db(db) + removed, _ = clean_tmp(tmp_root, status_map, dry=False, quiet=True) + assert removed == 4 # done + failed + orphan + zombie + assert not (tmp_root / "done_tid").exists() + assert not (tmp_root / "failed_tid").exists() + assert not (tmp_root / "orphan_tid").exists() + assert not (tmp_root / "zombie_tid").exists() + assert (tmp_root / "uploading_tid").exists() # 进行中保留 + + +def test_clean_storage_keeps_recorded_removes_orphan(tmp_path: Path) -> None: + """files 有记录保留 (含目录内 chunk), 无记录目录删, 含记录父目录不删。""" + db, _, storage_root = make_env(tmp_path) + _, paths = load_db(db) + removed, _ = clean_storage(storage_root, paths, dry=False, quiet=True) + assert removed == 2 # orphan_storage_tid + user1/2026-08-10 (含 old_arch_single.enc) + # 有效目录整体保留, 内部 chunk 文件绝不单独判断删除 (防误删回归) + assert (storage_root / "user1" / "2026-08-11" / "valid_tid").exists() + assert (storage_root / "user1" / "2026-08-11" / "valid_tid" / "chunk_0001").exists() + assert (storage_root / "user1" / "2026-08-11" / "clone_tid_clone").exists() + assert (storage_root / "user1" / "2026-08-11" / "clone_tid_clone" / "chunk_0001").exists() + assert not (storage_root / "user1" / "2026-08-11" / "orphan_storage_tid").exists() + assert not (storage_root / "user1" / "2026-08-10" / "old_arch_single.enc").exists() + # 有记录的子目录存在 => 父目录 user1/2026-08-11 不被删 + assert (storage_root / "user1" / "2026-08-11").exists() + + +def test_dry_run_does_not_delete(tmp_path: Path) -> None: + """dry-run 预览不实际删除。""" + db, tmp_root, storage_root = make_env(tmp_path) + status_map, paths = load_db(db) + t_removed, _ = clean_tmp(tmp_root, status_map, dry=True, quiet=True) + s_removed, _ = clean_storage(storage_root, paths, dry=True, quiet=True) + assert t_removed == 0 and s_removed == 0 + assert (tmp_root / "done_tid").exists() + assert (storage_root / "user1" / "2026-08-11" / "orphan_storage_tid").exists() diff --git a/watchdog.sh b/watchdog.sh index 81dee6b..73052ec 100755 --- a/watchdog.sh +++ b/watchdog.sh @@ -1,6 +1,9 @@ #!/bin/bash # 7z-encrypt 服务看门狗: 每 30s 检查服务进程, 挂了自动拉起 +# 每小时清理一次孤儿文件 (中断传输残留等) # 由 start.sh 启动 (容器唤醒后自动恢复); 日志 /tmp/sz-watchdog.log +# 注意: 本脚本不依赖调用方 cwd, 显式 cd 到服务端目录 +cd /root/server || exit 1 LOGFILE=/tmp/sz-watchdog.log log() { @@ -16,6 +19,15 @@ fi log "[watchdog] 启动, PID $$" while true; do + # 每小时清理一次孤儿文件 (启动时 last 文件不存在, 立即清一次) + if [ ! -f /tmp/sz-clean.last ] || [ $(( $(date +%s) - $(cat /tmp/sz-clean.last) )) -ge 3600 ]; then + log "[watchdog] 清理孤儿文件..." + if /root/server/clean_orphans.py --quiet >> /tmp/sz-clean.log 2>&1; then + date +%s > /tmp/sz-clean.last + else + log "[watchdog] 清理失败, 稍后重试" + fi + fi if ! pgrep -f 'python3 [m]ain\.py' > /dev/null 2>&1; then log "[watchdog] 服务挂了, 拉起..." /root/server/start.sh >> "$LOGFILE" 2>&1 || log "[watchdog] 拉起失败"