lou
efa634b574
真凶(两次"阿里云黑洞"实为系统卡死): server 每连接新建 SSLContext (每次重读证书, 内存+盘 IO 双爆点) + ThreadPoolExecutor 无界队列 (4MB 帧在内存堆积) -> 1.6G 云服 OOM -> swap 抖动 -> 系统盘 IOPS/BPS 打满 -> 整机假死(ping 丢+应用全断, 形似黑洞)。 - server: SSLContext 启动建一次复用; TLS 握手移入连接线程(不再 阻塞 accept 主循环); 在飞请求/并发连接双上限(64); 响应上限 8MB; 内存自愈(RSS 达阈值主动退出, 交由 systemd 重启释放) - proxy_dns: 通道不可用 -> 直连 fallback_dns 降级(全屋 DNS 不断), 再失败回 SERVFAIL(客户端立即切备用 DNS, 不再干等超时); 重连改 后台限频, 不再阻塞查询路径 - client: DNS 请求超时 15s -> 3.5s(让降级快速触发) - watchdog: 判据由端口/进程探测改为直连云端完成 TLS+AUTH(端口在听、 本地 DNS 探针都会给假健康信号 - 降级路径照样能答); 通道不健康即 撤 L2 劫持规则, DNS 回落 smartdns 直连; HTTP 规则排除内网段 - 配置外置: fallback_dns + 服务端防护参数 - 新增 scripts/test_dns_fallback.py(降级链单测, 2/2 PASS) QEMU OpenWrt lab 全链路验证: 降级 / 判据 / 故障演练(云挂->自动撤 规则、云回->自动重挂) 全过。
407 lines
15 KiB
Python
407 lines
15 KiB
Python
"""L3 云服务器套壳转发器(部署在云端中转服务器)。
|
|
|
|
职责:
|
|
1. TLS 监听, 首帧 AUTH 校验 token(防公网滥用)
|
|
2. HTTP_REQ -> 解析明文请求 -> 以 HTTPS 重新发起(统一 UA, 剔 hop-by-hop
|
|
头) -> HTTP_RESP 回传原始响应字节
|
|
3. DNS_REQ -> UDP 递归解析(dns_upstream) -> DNS_RESP 原样回传
|
|
4. 边界: forward_scheme=https 默认; deny_suffixes 可配(阿里云批量外连
|
|
有黑洞前科, 国外/高风险目标按需拉黑)
|
|
|
|
指纹说明: Python 标准库 TLS 的 JA3 不是 Chrome 的, 但"统一"目标已达成
|
|
(所有走通道的流量同一指纹同一 UA), 检测方要的是"多设备多指纹"而不是
|
|
"精确 OS"; 完美 JA3 伪装需 Go uTLS 重写, 留作后续。
|
|
|
|
2026-09-10 内存加固(此前两次把 1.6G 云服撑到 OOM, swap 抖动打满系统盘 IO,
|
|
整机卡死被误判为"黑洞"):
|
|
- SSLContext 改为启动时构建一次复用(原: 每连接 new + 每连接重读证书,
|
|
是内存碎片与系统盘 IO 的双爆点)。
|
|
- TLS 握手移入连接线程(原: 在 accept 主线程同步握手, 慢速/扫描连接可
|
|
挂死整个 accept 循环)。
|
|
- 请求队列有界(max_inflight): 超限直接回 ERROR 帧, 不再无界堆积 4MB 帧。
|
|
- 并发连接有界(max_connections): 超限立即关闭新连接。
|
|
- 响应体上限(max_response_bytes): 超限回 502, 不再整体读入内存。
|
|
- 内存守卫线程(memory_limit_mb): 自身 RSS 达阈值即主动退出, 由 systemd
|
|
Restart=on-failure 拉起清空(阈值释放, 而不是等内核 OOM 砍)。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import http.client
|
|
import os
|
|
import socket
|
|
import socketserver
|
|
import ssl
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from typing import Any, Callable, cast
|
|
|
|
from .httpmsg import HttpRequest, parse_request
|
|
from .proto import (
|
|
FrameReader,
|
|
TYPE_AUTH,
|
|
TYPE_AUTH_OK,
|
|
TYPE_DNS_REQ,
|
|
TYPE_DNS_RESP,
|
|
TYPE_ERROR,
|
|
TYPE_HTTP_REQ,
|
|
TYPE_HTTP_RESP,
|
|
TYPE_PING,
|
|
TYPE_PONG,
|
|
pack_frame,
|
|
)
|
|
|
|
_DEFAULT_UA = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
|
|
)
|
|
|
|
_DEFAULT_MAX_INFLIGHT = 64
|
|
_DEFAULT_MAX_CONNECTIONS = 64
|
|
_DEFAULT_HANDSHAKE_TIMEOUT = 8.0
|
|
_DEFAULT_MAX_RESPONSE = 8 * 1024 * 1024
|
|
_DEFAULT_MEMORY_LIMIT_MB = 220
|
|
_DEFAULT_MEMORY_INTERVAL = 30.0
|
|
|
|
|
|
def _rss_mb() -> float:
|
|
"""当前进程 RSS(MB), 读 /proc/self/statm 第二字段。"""
|
|
with open("/proc/self/statm", encoding="ascii") as fh:
|
|
fields = fh.read().split()
|
|
pages = int(fields[1]) if len(fields) > 1 else 0
|
|
return pages * os.sysconf("SC_PAGE_SIZE") / (1024.0 * 1024.0)
|
|
|
|
|
|
class CloudConfig:
|
|
def __init__(self, cfg: dict[str, Any]) -> None:
|
|
side: dict[str, Any] = cfg["cloud_side"]
|
|
self.listen_port = int(side.get("listen_port", 8443))
|
|
self.token = str(side["token"])
|
|
self.dns_upstream = str(side.get("dns_upstream", "223.5.5.5"))
|
|
self.forward_scheme = str(side.get("forward_scheme", "https"))
|
|
self.ua = str(side.get("ua", _DEFAULT_UA))
|
|
self.deny_suffixes = [str(s) for s in side.get("deny_suffixes", [])]
|
|
cert: dict[str, Any] = side.get("tls", {})
|
|
self.cert_file = str(cert.get("cert_file", ""))
|
|
self.key_file = str(cert.get("key_file", ""))
|
|
self.max_inflight = int(side.get("max_inflight", _DEFAULT_MAX_INFLIGHT))
|
|
self.max_connections = int(
|
|
side.get("max_connections", _DEFAULT_MAX_CONNECTIONS)
|
|
)
|
|
self.tls_handshake_timeout = float(
|
|
side.get("tls_handshake_timeout", _DEFAULT_HANDSHAKE_TIMEOUT)
|
|
)
|
|
self.max_response_bytes = int(
|
|
side.get("max_response_bytes", _DEFAULT_MAX_RESPONSE)
|
|
)
|
|
self.memory_limit_mb = int(
|
|
side.get("memory_limit_mb", _DEFAULT_MEMORY_LIMIT_MB)
|
|
)
|
|
self.memory_check_interval = float(
|
|
side.get("memory_check_interval", _DEFAULT_MEMORY_INTERVAL)
|
|
)
|
|
|
|
|
|
def _upstream_addr(dns_upstream: str) -> tuple[str, int]:
|
|
"""解析 dns_upstream 的 host:port(缺省 53)。"""
|
|
h = dns_upstream.strip()
|
|
if h.count(":") == 1:
|
|
name, _, port_s = h.rpartition(":")
|
|
if port_s.isdigit():
|
|
return name, int(port_s)
|
|
return h, 53
|
|
|
|
|
|
def _denied(host: str, suffixes: list[str]) -> bool:
|
|
low = host.lower()
|
|
for s in suffixes:
|
|
s = s.strip().lower()
|
|
if s and (low == s or low.endswith("." + s) or low.endswith(s)):
|
|
return True
|
|
return False
|
|
|
|
|
|
def _split_host_port(host: str, default_port: int) -> tuple[str, int]:
|
|
"""拆分 Host 头里的 host:port(支持 [v6]:port)。"""
|
|
h = host.strip()
|
|
if h.startswith("["):
|
|
idx = h.find("]")
|
|
if idx > 0:
|
|
core = h[1:idx]
|
|
rest = h[idx + 1 :]
|
|
if rest.startswith(":") and rest[1:].isdigit():
|
|
return core, int(rest[1:])
|
|
return core, default_port
|
|
if h.count(":") == 1:
|
|
name, _, port_s = h.rpartition(":")
|
|
if port_s.isdigit():
|
|
return name, int(port_s)
|
|
return h, default_port
|
|
|
|
|
|
def _forward_http(req: HttpRequest, conf: CloudConfig) -> bytes:
|
|
"""套壳转发: 组 HTTPS 请求到目标, 返回重组后的原始 HTTP 响应。"""
|
|
if not req.host:
|
|
return _error_resp(400, "missing host")
|
|
if _denied(req.host, conf.deny_suffixes):
|
|
return _error_resp(403, "host denied")
|
|
scheme = conf.forward_scheme
|
|
default_port = 443 if scheme == "https" else 80
|
|
target_host, target_port = _split_host_port(req.host, default_port)
|
|
headers = req.without_hop_by_hop()
|
|
headers = [(k, v) for k, v in headers if k.lower() != "content-length"]
|
|
out_headers: list[tuple[str, str]] = [("Host", req.host)]
|
|
ua_replaced = False
|
|
for key, value in headers:
|
|
if key.lower() == "user-agent":
|
|
out_headers.append(("User-Agent", conf.ua))
|
|
ua_replaced = True
|
|
else:
|
|
out_headers.append((key, value))
|
|
if not ua_replaced:
|
|
out_headers.append(("User-Agent", conf.ua))
|
|
try:
|
|
if scheme == "https":
|
|
conn: http.client.HTTPConnection = http.client.HTTPSConnection(
|
|
target_host, target_port, timeout=30
|
|
)
|
|
else:
|
|
conn = http.client.HTTPConnection(target_host, target_port, timeout=30)
|
|
conn.request(req.method, req.path, body=req.body, headers=dict(out_headers))
|
|
resp = conn.getresponse()
|
|
# 有界读取: 超过上限直接判定过大, 不再整体读进内存
|
|
body = resp.read(conf.max_response_bytes + 1)
|
|
if len(body) > conf.max_response_bytes:
|
|
conn.close()
|
|
return _error_resp(502, "response too large")
|
|
status = resp.status
|
|
reason = resp.reason or ""
|
|
resp_headers: list[str] = []
|
|
for key, value in resp.getheaders():
|
|
low = key.lower()
|
|
if low in ("transfer-encoding", "connection", "keep-alive"):
|
|
continue
|
|
resp_headers.append(f"{key}: {value}")
|
|
conn.close()
|
|
head = (
|
|
f"HTTP/1.1 {status} {reason}\r\n".encode("latin-1")
|
|
+ "\r\n".join(resp_headers).encode("latin-1")
|
|
+ f"\r\nContent-Length: {len(body)}\r\n\r\n".encode("ascii")
|
|
)
|
|
return head + body
|
|
except Exception as exc: # noqa: BLE001 - 任何转发失败都回 502
|
|
return _error_resp(502, f"forward failed: {exc}")
|
|
|
|
|
|
def _error_resp(status: int, text: str) -> bytes:
|
|
body = f"<html><body><h1>{status}</h1><p>{text}</p></body></html>".encode(
|
|
"utf-8", "replace"
|
|
)
|
|
head = (
|
|
f"HTTP/1.1 {status} {text}\r\n".encode("latin-1", "replace")
|
|
+ b"Content-Type: text/html; charset=utf-8\r\n"
|
|
+ f"Content-Length: {len(body)}\r\n".encode("ascii")
|
|
+ b"Connection: close\r\n\r\n"
|
|
)
|
|
return head + body
|
|
|
|
|
|
class _ConnThread:
|
|
"""单连接读循环(在连接线程里同步跑)。"""
|
|
|
|
def __init__(self, sock: socket.socket, server: "CloudServer") -> None:
|
|
self.sock = sock
|
|
self.server = server
|
|
self.conf = server.conf
|
|
self._write_lock = threading.Lock()
|
|
|
|
def _send_frame(self, frame_type: int, req_id: int, payload: bytes) -> None:
|
|
with self._write_lock:
|
|
self.sock.sendall(pack_frame(frame_type, req_id, payload))
|
|
|
|
def run(self) -> None:
|
|
reader = FrameReader()
|
|
authed = False
|
|
try:
|
|
self.sock.settimeout(15)
|
|
while True:
|
|
data = self.sock.recv(65536)
|
|
if not data:
|
|
return
|
|
reader.feed(data)
|
|
while True:
|
|
frame = reader.poll()
|
|
if frame is None:
|
|
break
|
|
frame_type, req_id, payload = frame
|
|
if frame_type == TYPE_PING:
|
|
self._send_frame(TYPE_PONG, req_id, b"")
|
|
continue
|
|
if frame_type == TYPE_AUTH:
|
|
ok = payload.decode("utf-8", "replace") == self.conf.token
|
|
if not ok:
|
|
return
|
|
authed = True
|
|
self._send_frame(TYPE_AUTH_OK, req_id, b"")
|
|
continue
|
|
if not authed:
|
|
return
|
|
if frame_type == TYPE_HTTP_REQ:
|
|
if not self.server.submit(self._handle_http, req_id, payload):
|
|
self._send_frame(TYPE_ERROR, req_id, b"server busy")
|
|
elif frame_type == TYPE_DNS_REQ:
|
|
if not self.server.submit(self._handle_dns, req_id, payload):
|
|
self._send_frame(TYPE_ERROR, req_id, b"server busy")
|
|
except (OSError, ValueError):
|
|
return
|
|
finally:
|
|
try:
|
|
self.sock.close()
|
|
except OSError:
|
|
pass
|
|
|
|
def _handle_http(self, req_id: int, payload: bytes) -> None:
|
|
try:
|
|
req = parse_request(payload)
|
|
if req is None:
|
|
resp = _error_resp(400, "bad http request")
|
|
else:
|
|
resp = _forward_http(req, self.conf)
|
|
self._send_frame(TYPE_HTTP_RESP, req_id, resp)
|
|
except Exception: # noqa: BLE001 - 单请求失败不杀连接
|
|
self._send_frame(TYPE_ERROR, req_id, b"http forward internal error")
|
|
|
|
def _handle_dns(self, req_id: int, payload: bytes) -> None:
|
|
try:
|
|
up = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
try:
|
|
up.settimeout(5)
|
|
up.sendto(payload, _upstream_addr(self.conf.dns_upstream))
|
|
resp, _ = up.recvfrom(4096)
|
|
finally:
|
|
up.close()
|
|
self._send_frame(TYPE_DNS_RESP, req_id, resp)
|
|
except OSError:
|
|
self._send_frame(TYPE_ERROR, req_id, b"dns upstream timeout")
|
|
|
|
|
|
class CloudServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
|
allow_reuse_address = True
|
|
daemon_threads = True
|
|
|
|
def __init__(self, addr: tuple[str, int], conf: CloudConfig) -> None:
|
|
self.conf = conf
|
|
self.pool = ThreadPoolExecutor(max_workers=16)
|
|
self._inflight = threading.BoundedSemaphore(conf.max_inflight)
|
|
self._conns = threading.BoundedSemaphore(conf.max_connections)
|
|
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
|
ctx.load_cert_chain(conf.cert_file, conf.key_file)
|
|
self.ssl_ctx = ctx
|
|
super().__init__(addr, _ConnHandler)
|
|
self._guard = threading.Thread(
|
|
target=self._mem_guard, name="cloud-memguard", daemon=True
|
|
)
|
|
self._guard.start()
|
|
|
|
def submit(self, fn: Callable[..., None], *args: Any) -> bool:
|
|
"""有界提交: 在飞请求达上限直接拒绝, 防止队列无界堆积吃内存。"""
|
|
if not self._inflight.acquire(blocking=False):
|
|
return False
|
|
|
|
def _run() -> None:
|
|
try:
|
|
fn(*args)
|
|
finally:
|
|
self._inflight.release()
|
|
|
|
self.pool.submit(_run)
|
|
return True
|
|
|
|
def process_request(
|
|
self, request: socket.socket, client_address: Any
|
|
) -> None:
|
|
"""并发连接上限: 超限立即关闭, 防止线程/缓冲堆积。"""
|
|
if not self._conns.acquire(blocking=False):
|
|
try:
|
|
request.close()
|
|
except OSError:
|
|
pass
|
|
return
|
|
super().process_request(request, client_address)
|
|
|
|
def process_request_thread(
|
|
self, request: socket.socket, client_address: Any
|
|
) -> None:
|
|
try:
|
|
super().process_request_thread(request, client_address)
|
|
finally:
|
|
self._conns.release()
|
|
|
|
def _mem_guard(self) -> None:
|
|
"""阈值自愈: RSS 达 memory_limit_mb 主动退出, 由 systemd 重启清空。"""
|
|
limit = self.conf.memory_limit_mb
|
|
while True:
|
|
time.sleep(self.conf.memory_check_interval)
|
|
try:
|
|
rss = _rss_mb()
|
|
except OSError:
|
|
continue
|
|
if rss >= limit:
|
|
print(
|
|
f"[cloud] 内存 {rss:.0f}MB 达阈值 {limit}MB, "
|
|
"主动退出由 systemd 重启释放",
|
|
flush=True,
|
|
)
|
|
os._exit(1)
|
|
|
|
|
|
class _ConnHandler(socketserver.BaseRequestHandler):
|
|
def handle(self) -> None:
|
|
server = cast(CloudServer, self.server)
|
|
sock = self.request
|
|
# TLS 握手放在连接线程里做(不在 accept 主线程阻塞)
|
|
try:
|
|
sock.settimeout(server.conf.tls_handshake_timeout)
|
|
tls = server.ssl_ctx.wrap_socket(sock, server_side=True)
|
|
except (ssl.SSLError, OSError):
|
|
try:
|
|
sock.close()
|
|
except OSError:
|
|
pass
|
|
return
|
|
_ConnThread(tls, server).run()
|
|
|
|
|
|
def main(cfg_path: str) -> int:
|
|
import json
|
|
from pathlib import Path
|
|
|
|
cfg = json.loads(Path(cfg_path).read_text(encoding="utf-8"))
|
|
conf = CloudConfig(cfg)
|
|
if not conf.cert_file or not conf.key_file:
|
|
print(
|
|
"cloud_side.tls.cert_file/key_file required "
|
|
"(自签: openssl req -x509 -newkey rsa:2048 -nodes "
|
|
"-keyout key.pem -out cert.pem -days 365)",
|
|
file=__import__("sys").stderr,
|
|
)
|
|
return 2
|
|
srv = CloudServer(("0.0.0.0", conf.listen_port), conf)
|
|
print(
|
|
f"[cloud] listen 0.0.0.0:{conf.listen_port} "
|
|
f"memlimit={conf.memory_limit_mb}MB inflight<={conf.max_inflight} "
|
|
f"conns<={conf.max_connections}",
|
|
flush=True,
|
|
)
|
|
srv.serve_forever()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
|
|
if len(sys.argv) < 2:
|
|
print("usage: python -m channel.server <config.json>", file=sys.stderr)
|
|
raise SystemExit(2)
|
|
raise SystemExit(main(sys.argv[1]))
|