Fix site downloading
All checks were successful
Build / build (push) Successful in 1m56s

This commit is contained in:
Moritz Ruth 2025-03-01 00:20:43 +01:00
parent 05d3d5bf16
commit abbaaf8381
Signed by: moritzruth
GPG key ID: C9BBAB79405EE56D
3 changed files with 68 additions and 371 deletions

387
Cargo.lock generated
View file

@ -282,7 +282,7 @@ dependencies = [
"polling 2.8.0",
"rustix 0.37.28",
"slab",
"socket2 0.4.10",
"socket2",
"waker-fn",
]
@ -605,12 +605,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.40"
@ -1198,10 +1192,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"wasm-bindgen",
]
[[package]]
@ -1297,29 +1289,6 @@ dependencies = [
"itoa",
]
[[package]]
name = "http-body"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http",
]
[[package]]
name = "http-body-util"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
dependencies = [
"bytes",
"futures-util",
"http",
"http-body",
"pin-project-lite 0.2.16",
]
[[package]]
name = "http-client"
version = "6.5.3"
@ -1371,62 +1340,6 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"httparse",
"itoa",
"pin-project-lite 0.2.16",
"smallvec",
"tokio",
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.27.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2"
dependencies = [
"futures-util",
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tower-service",
"webpki-roots",
]
[[package]]
name = "hyper-util"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http",
"http-body",
"hyper",
"pin-project-lite 0.2.16",
"socket2 0.5.8",
"tokio",
"tower-service",
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.61"
@ -1637,12 +1550,6 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "ipnet"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.1"
@ -1749,12 +1656,6 @@ version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
[[package]]
name = "mime"
version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
[[package]]
name = "miniz_oxide"
version = "0.7.4"
@ -1773,17 +1674,6 @@ dependencies = [
"adler2",
]
[[package]]
name = "mio"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd"
dependencies = [
"libc",
"wasi 0.11.0+wasi-snapshot-preview1",
"windows-sys 0.52.0",
]
[[package]]
name = "nom"
version = "5.1.3"
@ -2003,58 +1893,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "quinn"
version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e96808277ec6f97351a2380e6c25114bc9e67037775464979f3037c92d05ef"
dependencies = [
"bytes",
"pin-project-lite 0.2.16",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls",
"socket2 0.5.8",
"thiserror 2.0.11",
"tokio",
"tracing",
]
[[package]]
name = "quinn-proto"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2fe5ef3495d7d2e377ff17b1a8ce2ee2ec2a18cde8b6ad6619d65d0701c135d"
dependencies = [
"bytes",
"getrandom 0.2.15",
"rand 0.8.5",
"ring",
"rustc-hash",
"rustls",
"rustls-pki-types",
"slab",
"thiserror 2.0.11",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e46f3055866785f6b92bc6164b76be02ca8f2eb4b002c0354b28cf4c119e5944"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.5.8",
"tracing",
"windows-sys 0.59.0",
]
[[package]]
name = "quote"
version = "1.0.38"
@ -2173,50 +2011,6 @@ version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "reqwest"
version = "0.12.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43e734407157c3c2034e0258f5e4473ddb361b1e85f95a66690d67264d7cd1da"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"once_cell",
"percent-encoding",
"pin-project-lite 0.2.16",
"quinn",
"rustls",
"rustls-pemfile",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tower",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
"windows-registry",
]
[[package]]
name = "ring"
version = "0.17.11"
@ -2247,12 +2041,6 @@ version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustc_version"
version = "0.2.3"
@ -2295,6 +2083,7 @@ version = "0.23.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47796c98c480fce5406ef69d1c76378375492c3b0a0de587be0c1d9feb12f395"
dependencies = [
"log",
"once_cell",
"ring",
"rustls-pki-types",
@ -2317,9 +2106,6 @@ name = "rustls-pki-types"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "917ce264624a4b4db1c364dcc35bfca9ded014d0a958cd47ad3e960e988ea51c"
dependencies = [
"web-time",
]
[[package]]
name = "rustls-webpki"
@ -2549,16 +2335,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "socket2"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c970269d99b64e60ec3bd6ad27270092a5394c4e309314b18ae3fe575695fbe8"
dependencies = [
"libc",
"windows-sys 0.52.0",
]
[[package]]
name = "sscdc"
version = "0.1.0"
@ -2573,12 +2349,12 @@ dependencies = [
"log",
"once_cell",
"regex",
"reqwest",
"serde",
"serde_json",
"serde_regex",
"tide",
"toml",
"ureq",
"validator",
"zip",
]
@ -2765,15 +2541,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
version = "0.13.1"
@ -2906,21 +2673,6 @@ dependencies = [
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.43.0"
@ -2928,22 +2680,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d61fa4ffa3de412bfea335c6ecff681de2b609ba3c77ef3e00e521813a9ed9e"
dependencies = [
"backtrace",
"bytes",
"libc",
"mio",
"pin-project-lite 0.2.16",
"socket2 0.5.8",
"windows-sys 0.52.0",
]
[[package]]
name = "tokio-rustls"
version = "0.26.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37"
dependencies = [
"rustls",
"tokio",
]
[[package]]
@ -2980,33 +2717,6 @@ dependencies = [
"winnow",
]
[[package]]
name = "tower"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite 0.2.16",
"sync_wrapper",
"tokio",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
[[package]]
name = "tower-service"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
[[package]]
name = "tracing"
version = "0.1.41"
@ -3048,12 +2758,6 @@ dependencies = [
"tracing-core",
]
[[package]]
name = "try-lock"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "typeid"
version = "1.0.2"
@ -3097,6 +2801,36 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "ureq"
version = "3.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06f78313c985f2fba11100dd06d60dd402d0cabb458af4d94791b8e09c025323"
dependencies = [
"base64 0.22.1",
"flate2",
"log",
"percent-encoding",
"rustls",
"rustls-pemfile",
"rustls-pki-types",
"ureq-proto",
"utf-8",
"webpki-roots",
]
[[package]]
name = "ureq-proto"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64adb55464bad1ab1aa9229133d0d59d2f679180f4d15f0d9debe616f541f25e"
dependencies = [
"base64 0.22.1",
"http",
"httparse",
"log",
]
[[package]]
name = "url"
version = "2.5.4"
@ -3109,6 +2843,12 @@ dependencies = [
"serde",
]
[[package]]
name = "utf-8"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
[[package]]
name = "utf16_iter"
version = "1.0.5"
@ -3211,15 +2951,6 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
[[package]]
name = "want"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
dependencies = [
"try-lock",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
@ -3315,16 +3046,6 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "0.26.8"
@ -3371,36 +3092,6 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dccfd733ce2b1753b03b6d3c65edf020262ea35e20ccdf3e288043e6dd620e3"
[[package]]
name = "windows-registry"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result",
"windows-strings",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-result"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-strings"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.48.0"

View file

@ -14,11 +14,11 @@ http-client = { version = "6.5.3", default-features = false, features = ["async-
log = "0.4.26"
once_cell = "1.20.3"
regex = "1.11.1"
reqwest = { version = "0.12.12", default-features = false, features = ["blocking", "rustls-tls"] }
serde = "1.0.217"
serde_json = "1.0.139"
serde_regex = "1.1.0"
tide = "0.17.0-beta.1"
toml = "0.8.19"
ureq = "3.0.8"
validator = { version = "0.20.0", features = ["derive"] }
zip = { version = "2.2.3", default-features = false, features = ["deflate"] }

View file

@ -126,7 +126,7 @@ pub async fn load_sites(sites_directory_path: &Utf8Path) -> Result<Sites> {
Ok(RwLock::new(sites))
}
async fn handle_download(reqwest_client: &mut reqwest::Client, sites: &Sites, caddy_controller: &mut CaddyController, domain: String) -> Result<()> {
async fn handle_download(ureq_agent: Arc<ureq::Agent>, sites: &Sites, caddy_controller: &mut CaddyController, domain: String) -> Result<()> {
let site = {
let sites = sites.read().await;
match sites.get(&domain) {
@ -151,26 +151,31 @@ async fn handle_download(reqwest_client: &mut reqwest::Client, sites: &Sites, ca
// Download
let archive_file_path = site.path.join(format!("{}.zip", current_version.id));
let mut archive_file_for_writing = fs::OpenOptions::new()
.truncate(true)
.create(true)
.write(true)
.open(archive_file_path.as_std_path())
.await?;
let mut file_writer = BufWriter::new(&mut archive_file_for_writing);
let mut response = reqwest_client.get(&current_version.download_url).send().await?;
let mut response = task::spawn_blocking({
let url = current_version.download_url.clone();
move || ureq_agent.get(url).call() }).await?;
let status = response.status();
if !status.is_success() {
return Err(eyre!("Download request failed with status code {status}"));
}
while let Some(chunk) = response.chunk().await? {
file_writer.write_all(&chunk).await?;
}
let archive_file = task::spawn_blocking({
let archive_file_path = archive_file_path.to_owned().into_std_path_buf();
file_writer.flush().await?;
move || -> std::io::Result<std::fs::File> {
let mut writer = std::io::BufWriter::new(std::fs::OpenOptions::new()
.truncate(true)
.create(true)
.write(true)
.read(true)
.open(archive_file_path)?);
std::io::copy(&mut response.body_mut().as_reader(), &mut writer)?;
Ok(writer.into_inner()?)
}
}).await?;
let extraction_directory_path = site.path.join(&current_version.id);
let _ = fs::remove_dir_all(extraction_directory_path.as_std_path()).await;
@ -181,10 +186,9 @@ async fn handle_download(reqwest_client: &mut reqwest::Client, sites: &Sites, ca
// Unpack to temp dir
task::spawn_blocking({
let extraction_directory_path = extraction_directory_path.clone();
let archive_file_path = archive_file_path.clone().into_std_path_buf();
move || -> Result<()> {
let mut archive = zip::ZipArchive::new(std::io::BufReader::new(std::fs::File::open(archive_file_path)?))?;
let mut archive = zip::ZipArchive::new(std::io::BufReader::new(archive_file))?;
archive.extract(&extraction_directory_path)?;
Ok(())
}
@ -212,14 +216,16 @@ async fn handle_download(reqwest_client: &mut reqwest::Client, sites: &Sites, ca
}
async fn sites_worker(mut download_tasks_receiver: async_std::channel::Receiver<String>, sites: Arc<Sites>, mut caddy_controller: CaddyController) -> Result<!> {
let mut reqwest_client = reqwest::Client::builder()
.read_timeout(Duration::from_mins(1))
.timeout(Duration::from_hours(1))
.build()?;
let mut ureq_agent = Arc::new(
ureq::Agent::new_with_config(ureq::Agent::config_builder()
.timeout_connect(Some(Duration::from_secs(30)))
.timeout_global(Some(Duration::from_mins(15)))
.build())
);
loop {
let domain = download_tasks_receiver.recv().await.unwrap();
handle_download(&mut reqwest_client, &sites, &mut caddy_controller, domain).await?;
handle_download(Arc::clone(&ureq_agent), &sites, &mut caddy_controller, domain).await?;
}
}