transfer: 新增 list_files/download (流式+进度+Content-Disposition 文件名); CLI --list/--download/--out; 测试 57 项

This commit is contained in:
lou
2026-08-10 00:46:08 +08:00
parent 29f885fd48
commit c972948d94
3 changed files with 154 additions and 5 deletions
+15
View File
@@ -143,6 +143,7 @@ def main() -> int:
ap = argparse.ArgumentParser(description="配置: 加载校验 / 生成模板")
ap.add_argument("--init", action="store_true", help="生成默认 config.json 模板")
ap.add_argument("--path", default=str(DEFAULT_CONFIG), help="配置文件路径")
ap.add_argument("--set-server", metavar="URL", help="更新 server.url 并保存")
args = ap.parse_args()
if args.init:
@@ -173,6 +174,20 @@ def main() -> int:
print(f"[config] 模板已生成: {path} (改 server.url 后即可用)")
return 0
if args.set_server:
path = Path(args.path)
if not path.exists():
print(f"[错误] 配置文件不存在: {path}, 先 --init 生成")
return 1
data = json.loads(path.read_text(encoding="utf-8"))
data.setdefault("server", {})["url"] = args.set_server
path.write_text(
json.dumps(data, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
print(f"[config] server.url -> {args.set_server} ({path})")
return 0
try:
cfg = AppConfig(args.path)
except (ConfigError, json.JSONDecodeError) as e:
+63 -1
View File
@@ -11,6 +11,7 @@ import sys
import tempfile
import threading
import unittest
import urllib.parse
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
@@ -33,6 +34,7 @@ class ProtocolServer:
self.transfer_id = "T-1"
self.completed = False
self.file_id = "F-42"
self.files: list[dict] = [] # ls/下载测试: [{"file_id","file_name","size","data"}]
def make_handler(self):
srv = self
@@ -74,6 +76,33 @@ class ProtocolServer:
if m:
self._json(200, {"received": sorted(srv.received)})
return
if self.path == "/api/files":
# 列表只含元信息, 不含 data (bytes 不可 JSON 序列化)
self._json(200, {
"files": [
{k: v for k, v in f.items() if k != "data"}
for f in srv.files
]
})
return
m = re.fullmatch(r"/api/files/([^/]+)", self.path)
if m:
fid = m.group(1)
rec = next((f for f in srv.files if f["file_id"] == fid), None)
if rec is None:
self._json(404, {"error": "文件不存在"})
return
body = rec["data"]
self.send_response(200)
self.send_header("Content-Type", "application/octet-stream")
self.send_header("Content-Length", str(len(body)))
self.send_header(
"Content-Disposition",
f"attachment; filename*=utf-8''{urllib.parse.quote(rec['file_name'])}",
)
self.end_headers()
self.wfile.write(body)
return
self._json(404, {"error": "not found"})
def do_PUT(self):
@@ -109,6 +138,7 @@ class TransferTestBase(unittest.TestCase):
)
self._td = tempfile.TemporaryDirectory(prefix='hermes-test-')
self.addCleanup(self._td.cleanup)
self.tmp = Path(self._td.name)
self.chunk_files: dict[int, Path] = {}
for i in range(1, CHUNK_COUNT + 1):
p = Path(self._td.name) / f"chunk{i}.bin"
@@ -174,5 +204,37 @@ class TestProtocolErrors(TransferTestBase):
self.assertEqual(payload["status"], "incomplete")
if __name__ == '__main__':
class TestListDownload(TransferTestBase):
"""ls 列表 + 下载 (真实 HTTP 服务器)"""
def setUp(self):
super().setUp()
self.srv.files = [
{"file_id": "F-1", "file_name": "报告.pdf", "size": 6, "data": b"hello"},
]
def test_list_files(self):
files = self.client.list_files()
self.assertEqual(len(files), 1)
self.assertEqual(files[0]["file_id"], "F-1")
self.assertEqual(files[0]["file_name"], "报告.pdf")
def test_download_to_path(self):
dest = self.tmp / "out.bin"
got = self.client.download("F-1", dest)
self.assertEqual(got, dest)
self.assertEqual(dest.read_bytes(), b"hello")
def test_download_uses_server_filename(self):
got = self.client.download("F-1")
self.assertEqual(got.name, "报告.pdf")
self.assertEqual(got.read_bytes(), b"hello")
got.unlink()
def test_download_unknown_404(self):
with self.assertRaises(TransferError):
self.client.download("NOPE")
if __name__ == "__main__":
unittest.main()
+76 -4
View File
@@ -20,6 +20,7 @@ import time
from pathlib import Path
from typing import Any
from urllib import error as urlerror
from urllib import parse as urlparse
from urllib import request as urlrequest
DEFAULT_MAX_RETRIES = 3
@@ -110,6 +111,53 @@ class TransferClient:
raise TransferError(f"complete 失败 (HTTP {code}): {payload}")
return payload
def list_files(self) -> list[dict[str, Any]]:
"""GET /api/files: 服务端文件列表 (ls)"""
code, payload = self._request("GET", "/api/files")
if code != 200:
raise TransferError(f"文件列表失败 (HTTP {code}): {payload}")
return payload.get("files", [])
def download(self, file_id: str, dest: Path | None = None) -> Path:
"""GET /api/files/{id}: 流式下载, 实时进度。dest 缺省用服务端文件名。"""
req = urlrequest.Request(self.base_url + f"/api/files/{file_id}", method="GET")
try:
with urlrequest.urlopen(req, timeout=self.timeout) as resp:
if dest is None:
# 从 Content-Disposition 取文件名 (filename*=utf-8''... 或 filename=...)
cd = resp.headers.get("Content-Disposition") or ""
name = "download.bin"
if "filename*=utf-8''" in cd:
name = urlparse.unquote(cd.split("filename*=utf-8''")[1].split(";")[0])
elif "filename=" in cd:
name = cd.split("filename=")[1].split(";")[0].strip('"')
dest = Path(name)
total = int(resp.headers.get("Content-Length") or 0)
done = 0
with open(dest, "wb") as f:
while True:
chunk = resp.read(1 << 20)
if not chunk:
break
f.write(chunk)
done += len(chunk)
if total:
pct = done * 100 // total
sys.stdout.write(
f"\r[下载] {done / 1048576:.1f}/{total / 1048576:.1f} MB ({pct}%)"
)
sys.stdout.flush()
if total:
sys.stdout.write("\n")
sys.stdout.flush()
return dest
except urlerror.HTTPError as e:
if e.code == 404:
raise TransferError("文件不存在 (HTTP 404)") from e
raise TransferError(f"下载失败 (HTTP {e.code})") from e
except urlerror.URLError as e:
raise TransferError(f"网络错误: {e.reason}") from e
# ---------- 全流程 ----------
def transfer(
@@ -153,12 +201,37 @@ class TransferClient:
def main() -> int:
ap = argparse.ArgumentParser(description="传输客户端: 上传 init json + 卷文件")
ap = argparse.ArgumentParser(description="传输客户端: 上传 / 文件列表 / 下载")
ap.add_argument("--server", required=True, help="服务端地址, 如 http://127.0.0.1:8000")
ap.add_argument("--init", required=True, help="init json 路径 (metadata 产物)")
ap.add_argument("--manifest", required=True, help="卷清单路径 (splitter 产物)")
ap.add_argument("--init", help="init json 路径 (上传模式)")
ap.add_argument("--manifest", help="卷清单路径 (上传模式)")
ap.add_argument("--list", action="store_true", help="列出服务端文件")
ap.add_argument("--download", metavar="FILE_ID", help="下载文件")
ap.add_argument("--out", help="下载保存路径 (默认当前目录, 用服务端文件名)")
args = ap.parse_args()
client = TransferClient(args.server)
if args.list:
files = client.list_files()
print(f"[文件] 共 {len(files)} 个:")
for f in files:
size_mb = f["size"] / 1048576
print(f" {f['file_id']} {f['file_name']} ({size_mb:.1f} MB) {f['created_at']}")
return 0
if args.download:
try:
dest = client.download(args.download, Path(args.out) if args.out else None)
except TransferError as e:
print(f"[错误] {e}")
return 1
print(f"[下载] 完成 -> {dest} ({dest.stat().st_size / 1048576:.1f} MB)")
return 0
if not (args.init and args.manifest):
ap.error("需指定 --list / --download / (--init + --manifest) 之一")
init_json = json.loads(Path(args.init).read_text(encoding="utf-8"))
manifest = json.loads(Path(args.manifest).read_text(encoding="utf-8"))
chunk_files = {
@@ -170,7 +243,6 @@ def main() -> int:
print(f"[错误] 卷文件缺失: {missing}")
return 1
client = TransferClient(args.server)
try:
client.transfer(init_json, chunk_files)
except TransferError as e: