fix(download): 补齐 download_bytes 与等许可两处无界 await,预算耗尽不再重试(M289 r1)
r1 复审 p2-1:`stall.rs` 那句「全仓只有两处无界 await」是假话。真实清单五处, M289 之前只修了两处;这次补齐: * `bytes.rs::fetch_bytes` 的 `send()` 改走 `http::send_bounded`(与文件下载同一个构造点); 整包 `response.bytes()` 换成 `bytes_stream` + `http::read_step` 的空闲判定,并边读边卡 `MAX_IN_MEMORY_BYTES`(`Content-Length` 可缺失/可撒谎)。生产调用点:版本清单、 版本 JSON、Modrinth 元数据。 * `engine::acquire` 的 `acquire_owned()` 加上界(`EngineConfig::permit_timeout`,默认 120min ≥ 单个作业总期限),到点报新阶段 `StallStage::AcquirePermit`(「等许可」), 不冒充「服务器不给数据」。这就是「卡死作业攥着许可把队列堵死」的连坐形态。 * `stall.rs` 模块头改成一张「哪些入口有界、谁托管给谁」的表,并写清五处 await 的真实清单。 r1 复审 p2-2:总期限用完是链路级判决,不再按满预算重试/换源: * `DownloadError::job_budget_exhausted()` + `DownloadError::is_retryable()`(覆盖 `kind.is_retryable()`),引擎与 `download_bytes` 都改用它; * 引擎在预算耗尽时直接结束候选循环,不再换源(同一份字节、同一条链路,换源只会再 烧满一整个预算)。188MB/20 KiB/s 的用户不再从「1.6 小时慢慢下完」变成「几小时反复重下」。 * `stall.rs` 写明「慢链路照样活」的前提 = **平均吞吐 ≥ 32 KiB/s**,以及它挡掉哪类链路; `download.*` schema 接线留给另一个 mission(`setting/keys.rs` 不在 scope)。 顺带(复审 p3): * `transfer.rs` 的读空闲统一走 `EngineConfig::deadlines()`(不再各读字段); * `verify_part` 补一条代价说明:`spawn_blocking` 到点不可取消,可能让同名重建拿到 `Access is denied`(**该假设未验**,注释里已标); * `engine.rs` 因这轮新增又贴到 410 行 ⇒ `finish` / `materialize` / `millis` 逐字节搬到 `engine/finish.rs`(`engine::millis` 路径用 `pub(crate) use` 保持可用)。 新增用例(全离线、<2s):`download_bytes` 静默服务端 / 正文中途卡住、等许可超时、 预算耗尽不重试(`max_attempts_per_source = 3` 下仍只发一轮)、预算耗尽不换源 (官方源 `hits == 0`)。
This commit is contained in:
1 parent
cc0def3415
commit
d6f862a43c
13 files changed
+571
-146
No files matched your search
@@ -10,6 +10,7 @@ use super::engine::{millis, DownloadEngine};
|
||||
use super::error::{leaf_of, DownloadError, DownloadErrorKind};
|
||||
use super::http;
|
||||
use super::progress::ProgressEvent;
|
||||
use super::stall::StallReport;
|
||||
use super::types::{Candidate, DownloadId, DownloadRequest};
|
||||
use super::verify;
|
||||
|
||||
@@ -31,10 +32,12 @@ impl DownloadEngine {
|
||||
));
|
||||
}
|
||||
|
||||
let _permit = self.acquire().await?;
|
||||
let _permit = self.acquire(req.expected_size).await?;
|
||||
let mut suppressed: Vec<String> = Vec::new();
|
||||
// 逐源的叶子分类(出去时汇总成一条,见 `DownloadError::leaf`)。
|
||||
let mut kinds: Vec<DownloadErrorKind> = Vec::new();
|
||||
// 与 `download()` 一致:最后一次尝试的失速现场带出去(含阶段与三个数)。
|
||||
let mut last_stall: Option<StallReport> = None;
|
||||
|
||||
for candidate in &plan {
|
||||
// 与文件下载同一套规则:冷却中/刚出冷却的源只给一次探测机会。
|
||||
@@ -59,13 +62,14 @@ impl DownloadEngine {
|
||||
return Ok(bytes);
|
||||
}
|
||||
Err(err) => {
|
||||
last_stall = err.stall.as_deref().copied();
|
||||
self.health.record_failure(
|
||||
&candidate.source,
|
||||
req.kind,
|
||||
&err.message,
|
||||
now_ms(),
|
||||
);
|
||||
let retry = err.kind.is_retryable() && attempt + 1 < max_attempts;
|
||||
let retry = err.is_retryable() && attempt + 1 < max_attempts;
|
||||
let delay = self.next_delay(attempt);
|
||||
self.emit(ProgressEvent::SourceFailed {
|
||||
id,
|
||||
@@ -90,12 +94,17 @@ impl DownloadEngine {
|
||||
self.flush_health();
|
||||
// 叶子成因要跟着出去:上层(在线内容源的 `Transport`)靠它区分「这个 URL 不存在」
|
||||
// 与「没联网」,只留 `AllSourcesFailed` 的话那套分类在生产路径上永远不可达。
|
||||
let detail = match &last_stall {
|
||||
Some(stall) => format!("(最后一次失速:{stall})"),
|
||||
None => String::new(),
|
||||
};
|
||||
Err(DownloadError::new(
|
||||
DownloadErrorKind::AllSourcesFailed,
|
||||
format!("所有候选源都取不到文本资源:{}", req.url),
|
||||
format!("所有候选源都取不到文本资源:{}{detail}", req.url),
|
||||
)
|
||||
.with_suppressed(suppressed)
|
||||
.with_leaf(leaf_of(kinds)))
|
||||
.with_leaf(leaf_of(kinds))
|
||||
.with_last_stall(last_stall))
|
||||
}
|
||||
|
||||
async fn fetch_bytes(
|
||||
@@ -103,18 +112,25 @@ impl DownloadEngine {
|
||||
req: &DownloadRequest,
|
||||
candidate: &Candidate,
|
||||
) -> Result<Vec<u8>, DownloadError> {
|
||||
let response = self
|
||||
.http
|
||||
.get(&candidate.url)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|err| http::classify_reqwest(&err, &candidate.url))?;
|
||||
// 两处 await 都有界(复审 r1 p2-1):等响应头走 `send_bounded`(与文件下载同一构造点),
|
||||
// 整包读改成流式累加 + `read_step` 的空闲判定。不能再退回 `response.bytes()`:
|
||||
// 它是一次无界的整包读,`MAX_IN_MEMORY_BYTES` 只管大小不管时间。
|
||||
let deadlines = self.cfg.deadlines();
|
||||
let response = http::send_bounded(
|
||||
self.http.get(&candidate.url),
|
||||
&candidate.url,
|
||||
deadlines,
|
||||
req.expected_size,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| err.with_source(candidate.source.as_str()))?;
|
||||
let status = response.status();
|
||||
if !status.is_success() {
|
||||
return Err(http::classify_status(status.as_u16(), &candidate.url)
|
||||
.with_source(candidate.source.as_str()));
|
||||
}
|
||||
if let Some(len) = http::content_length(response.headers()) {
|
||||
let declared = http::content_length(response.headers());
|
||||
if let Some(len) = declared {
|
||||
if len > MAX_IN_MEMORY_BYTES {
|
||||
return Err(DownloadError::new(
|
||||
DownloadErrorKind::InvalidRequest,
|
||||
@@ -126,10 +142,42 @@ impl DownloadEngine {
|
||||
.with_source(candidate.source.as_str()));
|
||||
}
|
||||
}
|
||||
let body = response
|
||||
.bytes()
|
||||
|
||||
let mut stream = response.bytes_stream();
|
||||
let mut body: Vec<u8> = Vec::with_capacity(
|
||||
declared
|
||||
.and_then(|len| usize::try_from(len).ok())
|
||||
.unwrap_or(0),
|
||||
);
|
||||
loop {
|
||||
let next = http::read_step(
|
||||
futures_util::StreamExt::next(&mut stream),
|
||||
deadlines.read_idle,
|
||||
body.len() as u64,
|
||||
declared,
|
||||
&candidate.url,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| http::classify_reqwest(&err, &candidate.url))?;
|
||||
.map_err(|err| err.with_source(candidate.source.as_str()))?;
|
||||
let Some(chunk) = next else { break };
|
||||
let chunk = chunk.map_err(|err| {
|
||||
http::classify_reqwest(&err, &candidate.url).with_source(candidate.source.as_str())
|
||||
})?;
|
||||
body.extend_from_slice(&chunk);
|
||||
// `Content-Length` 可以撒谎、也可以缺失(靠关连接分帧):边读边卡,不能只看头。
|
||||
if body.len() as u64 > MAX_IN_MEMORY_BYTES {
|
||||
return Err(DownloadError::new(
|
||||
DownloadErrorKind::InvalidRequest,
|
||||
format!(
|
||||
"文本资源超过内存上限({} 字节),拒绝继续载入:{}",
|
||||
body.len(),
|
||||
candidate.url
|
||||
),
|
||||
)
|
||||
.with_source(candidate.source.as_str()));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(integrity) = &req.integrity {
|
||||
let actual = verify::digest_bytes(integrity.algo, &body);
|
||||
if !verify::matches(&integrity.hex, &actual) {
|
||||
@@ -140,6 +188,6 @@ impl DownloadEngine {
|
||||
.with_source(candidate.source.as_str()));
|
||||
}
|
||||
}
|
||||
Ok(body.to_vec())
|
||||
Ok(body)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user