零合并重构: complete 只校验卷齐+卷目录 move 入库 (秒回); 下载改逐卷 GET chunk/{n}; 删 assembler; 状态机 uploading->storing->done; 测试 7 项

This commit is contained in:
lou
2026-08-10 01:30:19 +08:00
parent 3c4e683210
commit 34496b4bf7
5 changed files with 78 additions and 125 deletions
+37 -38
View File
@@ -4,26 +4,24 @@ API 层 (api)
输入: HTTP 请求 (带认证)
输出: 响应 / 路由到各模块
零知识设计: 服务端只传输+存储密文, 不持有密钥, 不解密
零知识 + 零合并设计: 服务端只存卷/传卷, 不合并不解密, 不持有密钥。
端点:
POST /api/transfer/init {init json} -> {transfer_id}
GET /api/transfer/{id}/chunks -> {received: [n...]}
PUT /api/transfer/{id}/chunk/{n} 卷密文 -> {ok} / 409
POST /api/transfer/{id}/complete -> {status, file_id} / 409
POST /api/transfer/{id}/complete -> {status, file_id} / 409 (秒回, 不合并)
GET /api/files -> {files: [...]}
GET /api/files/{file_id} -> 密文文件 (头 X-Enc-Params: base64(enc_params))
GET /api/files/{id}/chunk/{n} -> 单卷密文 (头 X-Enc-Params / X-Chunk-Count)
全部端点需 Authorization: Bearer <token> (与客户端预共享)
"""
import base64
import hashlib
import json
from pathlib import Path
from fastapi import Depends, FastAPI, Header, HTTPException, Request
from fastapi.responses import FileResponse, JSONResponse
from assembler import Assembler, AssemblerError
from db import ServerDB
from receiver import Receiver, ReceiverError
from settings import STORAGE_ROOT, TMP_ROOT, TOKEN
@@ -34,10 +32,9 @@ from task_manager import TaskManager
db = ServerDB()
tasks = TaskManager(db)
receiver = Receiver(tasks, TMP_ROOT)
assembler = Assembler(tasks, TMP_ROOT)
storage = Storage(tasks, STORAGE_ROOT)
app = FastAPI(title="7z-encrypt 服务端 (零知识)")
app = FastAPI(title="7z-encrypt 服务端 (零知识 + 零合并)")
# ---------- 认证 ----------
@@ -48,15 +45,16 @@ def verify_token(authorization: str | None = Header(default=None)) -> None:
raise HTTPException(status_code=401, detail="未授权: token 无效或缺失")
def _sha256(path: str | Path) -> str:
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def _enc_headers(rec: dict) -> dict[str, str]:
"""下载响应头: 解密参数 + 卷数"""
ep = json.dumps(rec["enc_params"], ensure_ascii=False)
return {
"X-Enc-Params": base64.b64encode(ep.encode()).decode(),
"X-Chunk-Count": str(rec["chunk_count"]),
}
# ---------- 文件: ls / 下载 (密文 + 解密参数) ----------
# ---------- 文件: ls / 逐卷下载 ----------
@app.get("/api/files", dependencies=[Depends(verify_token)])
async def list_files():
@@ -64,26 +62,22 @@ async def list_files():
return {"files": db.list_files()}
@app.get("/api/files/{file_id}", dependencies=[Depends(verify_token)])
async def download_file(file_id: str):
"""下载密文文件。enc_params 放响应头 X-Enc-Params (客户端本地解密用)"""
@app.get("/api/files/{file_id}/chunk/{idx}", dependencies=[Depends(verify_token)])
async def download_chunk(file_id: str, idx: int):
"""下载单卷密文 (零合并: 服务端不拼接, 客户端逐卷拉取本地合并)"""
rec = db.get_file(file_id)
if rec is None:
return JSONResponse(status_code=404, content={"error": "文件不存在"})
path = Path(rec["path"])
if not path.exists():
return JSONResponse(status_code=404, content={"error": f"存储文件缺失: {path}"})
# 解密参数: 任务 init 时存的 enc {alg, key_id, context}
enc_params = json.dumps(rec["enc_params"], ensure_ascii=False)
headers = {
"X-Enc-Params": base64.b64encode(enc_params.encode()).decode(),
"X-Chunk-Count": str(rec["chunk_count"]), # 客户端按卷粒度显示进度
}
if not 1 <= idx <= rec["chunk_count"]:
return JSONResponse(status_code=404, content={"error": f"卷号越界: {idx}"})
chunk_path = Path(rec["path"]) / f"chunk_{idx:04d}"
if not chunk_path.exists():
return JSONResponse(status_code=404, content={"error": f"卷文件缺失: {chunk_path}"})
return FileResponse(
path,
chunk_path,
filename=rec["file_name"],
media_type="application/octet-stream",
headers=headers,
headers=_enc_headers(rec),
)
@@ -122,27 +116,32 @@ async def put_chunk(transfer_id: str, idx: int, request: Request):
async def complete(transfer_id: str):
if tasks.get(transfer_id) is None:
return JSONResponse(status_code=404, content={"error": "任务不存在"})
# 幂等: 已完成的任务直接返回已有 file_id (客户端超时重试/断线重连场景)
# 幂等: 已完成的任务直接返回已有 file_id
done_rec = db.get_file_by_transfer(transfer_id)
if done_rec is not None:
return {"status": "done", "file_id": done_rec["file_id"]}
try:
tasks.set_status(transfer_id, "assembling")
merged = assembler.assemble(transfer_id)
transfer = tasks.get(transfer_id)
assert transfer is not None
# 零知识: 合并后的密文直接入库, 服务端不碰解密
final_path = storage.store(merged, transfer["file_name"])
# 零合并: 只校验卷齐, 卷目录直接 move 入存储区, 秒回
got = tasks.received(transfer_id)
if len(got) != transfer["chunk_count"]:
return JSONResponse(status_code=409, content={
"status": "incomplete",
"received": sorted(got),
"error": f"缺卷: {transfer['chunk_count'] - len(got)} 卷未上传",
})
tasks.set_status(transfer_id, "storing")
final_dir = storage.store_chunks(transfer_id, TMP_ROOT / transfer_id)
file_id = db.insert_file(
transfer_id,
transfer["file_name"],
str(final_path),
final_path.stat().st_size,
_sha256(final_path),
str(final_dir),
transfer["file_size"],
transfer["total_sha256"],
)
tasks.set_status(transfer_id, "done")
return {"status": "done", "file_id": file_id}
except AssemblerError as e:
except OSError as e:
tasks.set_status(transfer_id, "failed")
return JSONResponse(status_code=409, content={"error": str(e), "status": "failed"})
-55
View File
@@ -1,55 +0,0 @@
"""
汇聚器模块 (assembler)
模块: 服务端 / 汇聚器
输入: 已收卷集合
输出: 完整密文文件
两道门: 卷齐才合并 (缺卷返回列表让客户端补); 合并后总哈希必须匹配才进解包。
"""
import hashlib
import shutil
from pathlib import Path
from task_manager import TaskManager
class AssemblerError(RuntimeError):
"""汇聚失败 (卷未齐/总哈希不一致)"""
class Assembler:
def __init__(self, tasks: TaskManager, tmp_root: str | Path) -> None:
self.tasks = tasks
self.tmp_root = Path(tmp_root)
def assemble(self, transfer_id: str) -> Path:
"""按 idx 顺序合并全部卷 -> 返回合并后密文路径"""
transfer = self.tasks.get(transfer_id)
if transfer is None:
raise AssemblerError(f"任务不存在: {transfer_id}")
total = transfer["chunk_count"]
received = self.tasks.received(transfer_id)
if len(received) < total:
missing = sorted(set(range(1, total + 1)) - received)
raise AssemblerError(f"缺卷 {missing}, 等齐再合并")
chunk_dir = self.tmp_root / transfer_id
merged_path = chunk_dir / "merged.bin"
with open(merged_path, "wb") as out:
for idx in range(1, total + 1):
chunk_path = chunk_dir / f"chunk_{idx:04d}"
with open(chunk_path, "rb") as f:
shutil.copyfileobj(f, out)
# 总哈希校验
h = hashlib.sha256()
with open(merged_path, "rb") as f:
for chunk in iter(lambda: f.read(1 << 20), b""):
h.update(chunk)
if h.hexdigest() != transfer["total_sha256"]:
merged_path.unlink(missing_ok=True)
raise AssemblerError("合并后总 SHA-256 不一致, 任务 failed (需整体重传)")
return merged_path
+14 -18
View File
@@ -1,11 +1,11 @@
"""
存储层模块 (storage)
模块: 服务端 / 存储层
输入: 合并后的密文文件 + 元数据
输出: 最终文件路径
输入: 已校验的密文卷目录 + 元数据
输出: 最终存储目录
零知识: 只存密文, 服务端无密钥。
按日期目录落盘, 同名冲突加后缀去重, 流式移动不整块进内存
零知识 + 零合并: 只存卷, 不做任何拼接/合并, 服务端无密钥。
卷目录整体 move 到日期/transfer_id 目录 (transfer_id 唯一, 同名冲突)
"""
import shutil
@@ -20,19 +20,15 @@ class Storage:
self.tasks = tasks
self.storage_root = Path(storage_root)
def store(self, enc_path: Path, file_name: str) -> Path:
"""密文文件 -> 最终存储路径 (返回路径)"""
def store_chunks(self, transfer_id: str, chunk_dir: Path) -> Path:
"""卷目录 -> 最终存储目录 (整体 move, 返回目标目录路径)
注意: 目标目录不能预创建, 否则 shutil.move 会嵌套成 dst/dst/。
"""
day_dir = self.storage_root / datetime.now().strftime("%Y-%m-%d")
final_dir = day_dir / transfer_id
if final_dir.exists():
raise OSError(f"存储目录已存在: {final_dir}")
day_dir.mkdir(parents=True, exist_ok=True)
# 同名冲突: 加 -1/-2 后缀
target = day_dir / file_name
counter = 1
while target.exists():
stem, suffix = file_name.rsplit(".", 1) if "." in file_name else (file_name, "")
name = f"{stem}-{counter}" + (f".{suffix}" if suffix else "")
target = day_dir / name
counter += 1
shutil.move(str(enc_path), str(target))
return target
shutil.move(str(chunk_dir), str(final_dir))
return final_dir
+3 -3
View File
@@ -4,7 +4,8 @@
输入: init 请求 / 卷到达事件
输出: transfer 状态机
状态机: uploading -> assembling -> decrypting -> done / failed
状态机: uploading -> storing -> done / failed
(零合并: 无 assembling/decrypting 阶段, 卷齐即入库)
transfer_id 生命周期贯穿接收->入库; 孤儿任务由清理逻辑回收 (见 server/data/tmp)。
"""
@@ -14,8 +15,7 @@ from typing import Any
from db import ServerDB
STATUS_UPLOADING = "uploading"
STATUS_ASSEMBLING = "assembling"
STATUS_DECRYPTING = "decrypting"
STATUS_STORING = "storing"
STATUS_DONE = "done"
STATUS_FAILED = "failed"
+24 -11
View File
@@ -95,7 +95,7 @@ class ServerPipelineTest(unittest.TestCase):
bad = {"Authorization": "Bearer wrong"}
self.assertEqual(self.client.get("/api/files", headers=bad).status_code, 401)
# ---------- 全链路 (零知识: 服务端只存密文) ----------
# ---------- 全链路 (零知识 + 零合并: 服务端只存) ----------
def test_full_pipeline_roundtrip(self):
data = os.urandom(3 << 20) # 3MB
@@ -108,17 +108,20 @@ class ServerPipelineTest(unittest.TestCase):
file_id = r.json()["file_id"]
self.assertTrue(file_id)
# 存储的是密文 (不是明文 data), 零知识验证
# 零合并: 存储的是卷目录 (chunk_XXXX 文件), 不是合并单文件, 没有明文
db = ServerDB()
with db._conn() as conn:
row = conn.execute(
"SELECT path, size FROM files WHERE file_id = ?", (file_id,)
"SELECT path FROM files WHERE file_id = ?", (file_id,)
).fetchone()
self.assertIsNotNone(row)
with open(row["path"], "rb") as f:
stored = f.read()
chunk_dir = Path(row["path"])
self.assertTrue(chunk_dir.is_dir())
chunk_files_on_disk = sorted(chunk_dir.glob("chunk_*"))
self.assertEqual(len(chunk_files_on_disk), len(chunk_files))
# 卷内容与客户端上传的卷一致
stored = b"".join(p.read_bytes() for p in chunk_files_on_disk)
self.assertEqual(stored, enc_bytes)
self.assertEqual(row["size"], len(enc_bytes))
self.assertNotEqual(stored, data) # 服务端手里不是明文
# transfer 状态 done
@@ -200,19 +203,29 @@ class ServerPipelineTest(unittest.TestCase):
files = r.json()["files"]
self.assertTrue(any(f["file_id"] == file_id for f in files))
# 下载返回密文 + 解密参数头 (客户端本地解密用)
r = self.client.get(f"/api/files/{file_id}", headers=AUTH)
# 逐卷下载: 每卷内容与上传卷一致, 拼接 == 完整密文
r = self.client.get(f"/api/files/{file_id}/chunk/1", headers=AUTH)
self.assertEqual(r.status_code, 200, r.text)
self.assertEqual(r.content, enc_bytes)
enc_params = json.loads(
__import__("base64").b64decode(r.headers["X-Enc-Params"]).decode()
)
self.assertEqual(enc_params["key_id"], TEST_KEY_ID)
self.assertEqual(enc_params["alg"], "cobblestone-aes256gcm")
self.assertEqual(r.headers["X-Chunk-Count"], str(len(chunk_files)))
# 逐卷拉取拼接 == 完整密文
pieces = []
for idx in range(1, len(chunk_files) + 1):
r = self.client.get(f"/api/files/{file_id}/chunk/{idx}", headers=AUTH)
self.assertEqual(r.status_code, 200, r.text)
pieces.append(r.content)
self.assertEqual(b"".join(pieces), enc_bytes)
# 越界卷 -> 404
self.assertEqual(
self.client.get(f"/api/files/{file_id}/chunk/999", headers=AUTH).status_code, 404
)
# 未知 file_id -> 404
self.assertEqual(
self.client.get("/api/files/nope", headers=AUTH).status_code, 404
self.client.get("/api/files/nope/chunk/1", headers=AUTH).status_code, 404
)
# complete 幂等: 重复 complete 返回同一个 file_id, 不新增入库记录