Skip to content

Commit e4d8aa3

Browse files
committed
check nightlies/pr features with etag avalibility
Users in China often don't have a stable connection to AWS S3, so mirror sites like https://mirrors.tuna.tsinghua.edu.cn exist; they are typically simple static-file HTTP servers and don't have advanced S3-like features such as etags. Previously, we have provided the JULIAUP_SERVER feature to change the download baseurl, but at that time, I didn't realize that the server mirror should support ETag. This commit introduces a server feature check operation for custom JULIAUP_SERVER: * if JULIAUP_SERVER equals the default official ones, it works as usual * Otherwise, send a HEAD check request and set the NIGHTLY_SERVER_SUPPORTS_ETAG feature flag If this feature flag is false, then the PR/nightlies download will be disabled. This doesn't affect the mirror sites because they don't support mirroring Julia nightlies deliberately; too few people use it, and it consumes too much unnecessary sync bandwidth. Known JuliaUp mirror sites in China that can be used to test: https://mirrors.ustc.edu.cn/juliaup-releases https://mirrors.tuna.tsinghua.edu.cn/juliaup-releases https://mirrors.bfsu.edu.cn/juliaup-releases This commit makes it work for the above mirror sites and thus fixes Disclaimer: This commit is primarily generated by AI (cursor + claude 4.5 opus). I've tested it manually on both Windows and Linux, and it works as expected. Example: PR/nightlies features are disabled with a proper error message: ``` ❯ target/debug/juliaup add nightly Error: The configured nightly server does not support etag headers, which are required for nightly and PR channels. Nightly and PR channels cannot be installed from this server. ❯ target/debug/julia +nightly Question: The Juliaup channel 'nightly' is not installed. Would you like to install it?: Yes (install this time only) Installing Julia nightly as requested Error: The configured nightly server does not support etag headers, which are required for nightly and PR channels. Nightly and PR channels cannot be installed from this server. Error: The Julia launcher failed to determine the command for the `nightly` channel. Caused by: Failed to install channel 'nightly'. juliaup add command failed with exit code: Some(1) ``` Example: normal release download just works ``` ❯ target/debug/juliaup add 1.10 Checking for new Julia versions Installing Julia 1.10.10+0.x64.linux.gnu DEBUG: Starting download from URL: https://mirrors.ustc.edu.cn/julia-releases/bin/linux/x64/1.10/julia-1.10.10-linux-x86_64.tar.gz Downloading ━━━━━━━━━━━━━━━━━━━━━━━━╸ 165.61 MiB/165.79 MiB eta: 0s Add Installed Julia channel '1.10' ``` closes #917
1 parent 5944e00 commit e4d8aa3

File tree

3 files changed

+152
-28
lines changed

3 files changed

+152
-28
lines changed

README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ the `JULIAUP_DEPOT_PATH` environment variable. Caution: Previous versions of Jul
173173
Juliaup by default downloads julia binary tarballs from the official server "https://julialang-s3.julialang.org".
174174
If requested, the environment variable `JULIAUP_SERVER` can be used to tell Juliaup to use a third-party mirror server.
175175

176+
**Note:** Nightly and PR channels (e.g., `nightly`, `pr123`) require the server to provide `etag` headers in HTTP responses for version tracking.
177+
If your custom mirror server does not support `etag` headers, these channels will not be available. Regular versioned Julia releases will still work normally.
178+
176179
## Development guides
177180

178181
For juliaup developers, information on how to build juliaup locally, update julia versions, and release updates

src/operations.rs

Lines changed: 64 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use crate::get_bundled_julia_version;
1010
use crate::get_juliaup_target;
1111
use crate::global_paths::GlobalPaths;
1212
use crate::jsonstructs_versionsdb::JuliaupVersionDB;
13+
use crate::utils::check_server_supports_nightlies;
1314
use crate::utils::get_bin_dir;
1415
use crate::utils::get_julianightlies_base_url;
1516
use crate::utils::get_juliaserver_base_url;
@@ -168,13 +169,13 @@ pub fn download_extract_sans_parent(
168169
pb.set_prefix(DOWNLOADING_PREFIX);
169170
pb.set_style(bar_style());
170171

171-
let last_modified = match response
172+
// Extract etag if present, otherwise return empty string
173+
// Empty etag is valid for regular version installs from servers without etag support
174+
let last_modified = response
172175
.headers()
173-
.get("etag") {
174-
Some(etag) => Ok(etag.to_str().unwrap().to_string()),
175-
None => Err(anyhow!(format!("Failed to get etag from `{}`.\n\
176-
This is likely due to requesting a pull request that does not have a cached build available. You may have to build locally.", url))),
177-
}?;
176+
.get("etag")
177+
.map(|etag| etag.to_str().unwrap_or("").to_string())
178+
.unwrap_or_default();
178179

179180
let response_with_pb = pb.wrap_read(response);
180181

@@ -226,15 +227,16 @@ pub fn download_extract_sans_parent(
226227

227228
http_response
228229
.EnsureSuccessStatusCode()
229-
.with_context(|| format!("Failed to get etag from `{}`.\n\
230-
This is likely due to requesting a pull request that does not have a cached build available. You may have to build locally.", url))?;
230+
.with_context(|| format!("Failed to download from `{}`.", url))?;
231231

232+
// Extract etag if present, otherwise return empty string
233+
// Empty etag is valid for regular version installs from servers without etag support
232234
let last_modified = http_response
233235
.Headers()
234-
.unwrap()
235-
.Lookup(&HSTRING::from("etag"))
236-
.unwrap()
237-
.to_string();
236+
.ok()
237+
.and_then(|headers| headers.Lookup(&HSTRING::from("etag")).ok())
238+
.map(|etag| etag.to_string())
239+
.unwrap_or_default();
238240

239241
let http_response_content = http_response
240242
.Content()
@@ -653,6 +655,17 @@ pub fn install_from_url(
653655
}
654656
};
655657

658+
// Nightly and PR channels require etag for version tracking
659+
// If etag is empty, the server doesn't support this functionality
660+
if server_etag.is_empty() {
661+
std::fs::remove_dir_all(temp_dir.path())?;
662+
bail!(
663+
"The server did not provide an etag header, which is required for nightly and PR channels.\n\
664+
This is likely due to requesting a pull request that does not have a cached build available, \
665+
or using a server that does not support etag headers. You may have to build locally."
666+
);
667+
}
668+
656669
// Query the actual version
657670
let julia_path = temp_dir
658671
.path()
@@ -692,6 +705,14 @@ pub fn install_non_db_version(
692705
name: &String,
693706
paths: &GlobalPaths,
694707
) -> Result<crate::config_file::JuliaupConfigChannel> {
708+
// Check if the nightly server supports etag headers (required for nightly/PR channels)
709+
if !check_server_supports_nightlies()? {
710+
bail!(
711+
"The configured nightly server does not support etag headers, which are required for nightly and PR channels.\n\
712+
Nightly and PR channels cannot be installed from this server."
713+
);
714+
}
715+
695716
// Determine the download URL
696717
let download_url_base = get_julianightlies_base_url()?;
697718

@@ -1639,6 +1660,9 @@ fn download_direct_download_etags(
16391660
use windows::Web::Http::HttpMethod;
16401661
use windows::Web::Http::HttpRequestMessage;
16411662

1663+
// Check if the server supports etag headers (required for nightly/PR updates)
1664+
let server_supports_etag = check_server_supports_nightlies().unwrap_or(false);
1665+
16421666
let http_client = HttpClient::new().with_context(|| "Failed to create HttpClient.")?;
16431667

16441668
let mut requests = Vec::new();
@@ -1652,6 +1676,13 @@ fn download_direct_download_etags(
16521676
}
16531677

16541678
if let JuliaupConfigChannel::DirectDownloadChannel { url, .. } = installed_channel {
1679+
// If server doesn't support etag, we can't check for updates on nightly/PR channels
1680+
// Return None gracefully so the update process can continue with other channels
1681+
if !server_supports_etag {
1682+
requests.push((channel_name.clone(), None));
1683+
continue;
1684+
}
1685+
16551686
let http_client = http_client.clone();
16561687
let url_clone = url.clone();
16571688
let channel_name_clone = channel_name.clone();
@@ -1678,16 +1709,14 @@ fn download_direct_download_etags(
16781709
.map_err(|e| anyhow!("Failed to get response: {:?}", e))?;
16791710

16801711
if response.IsSuccessStatusCode()? {
1681-
let headers = response
1712+
// Gracefully handle missing etag - return None instead of error
1713+
let etag = response
16821714
.Headers()
1683-
.map_err(|e| anyhow!("Failed to get headers: {:?}", e))?;
1684-
1685-
let etag = headers
1686-
.Lookup(&HSTRING::from("ETag"))
1687-
.map_err(|e| anyhow!("ETag header not found: {:?}", e))?
1688-
.to_string();
1715+
.ok()
1716+
.and_then(|headers| headers.Lookup(&HSTRING::from("ETag")).ok())
1717+
.map(|s| s.to_string());
16891718

1690-
Ok::<Option<String>, anyhow::Error>(Some(etag))
1719+
Ok::<Option<String>, anyhow::Error>(etag)
16911720
} else {
16921721
Ok::<Option<String>, anyhow::Error>(None)
16931722
}
@@ -1710,6 +1739,9 @@ fn download_direct_download_etags(
17101739
) -> Result<Vec<(String, Option<String>)>> {
17111740
use std::sync::Arc;
17121741

1742+
// Check if the server supports etag headers (required for nightly/PR updates)
1743+
let server_supports_etag = check_server_supports_nightlies().unwrap_or(false);
1744+
17131745
let client = Arc::new(reqwest::blocking::Client::new());
17141746

17151747
let mut requests = Vec::new();
@@ -1723,6 +1755,13 @@ fn download_direct_download_etags(
17231755
}
17241756

17251757
if let JuliaupConfigChannel::DirectDownloadChannel { url, .. } = installed_channel {
1758+
// If server doesn't support etag, we can't check for updates on nightly/PR channels
1759+
// Return None gracefully so the update process can continue with other channels
1760+
if !server_supports_etag {
1761+
requests.push((channel_name.clone(), None));
1762+
continue;
1763+
}
1764+
17261765
let client = Arc::clone(&client);
17271766
let url_clone = url.clone();
17281767
let channel_name_clone = channel_name.clone();
@@ -1740,17 +1779,14 @@ fn download_direct_download_etags(
17401779
})?;
17411780

17421781
if response.status().is_success() {
1782+
// Gracefully handle missing etag - return None instead of error
17431783
let etag = response
17441784
.headers()
17451785
.get("etag")
1746-
.ok_or_else(|| {
1747-
anyhow!("ETag header not found in response from {}", &url_clone)
1748-
})?
1749-
.to_str()
1750-
.map_err(|e| anyhow!("Failed to parse ETag header: {}", e))?
1751-
.to_string();
1752-
1753-
Ok::<Option<String>, anyhow::Error>(Some(etag))
1786+
.and_then(|h| h.to_str().ok())
1787+
.map(|s| s.to_string());
1788+
1789+
Ok::<Option<String>, anyhow::Error>(etag)
17541790
} else {
17551791
Ok::<Option<String>, anyhow::Error>(None)
17561792
}

src/utils.rs

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,93 @@ use anyhow::{anyhow, bail, Context, Result};
22
use console::style;
33
use semver::{BuildMetadata, Version};
44
use std::path::PathBuf;
5+
use std::sync::OnceLock;
56
use url::Url;
67

8+
/// Cached result of whether the nightly server supports etag headers.
9+
/// This is used to avoid repeated HTTP requests to check server capabilities.
10+
static NIGHTLY_SERVER_SUPPORTS_ETAG: OnceLock<bool> = OnceLock::new();
11+
12+
/// Checks if the nightly server supports etag headers.
13+
/// This is required for nightly and PR channel support because we use etags
14+
/// to track versions of these builds.
15+
///
16+
/// The result is cached after the first check.
17+
#[cfg(not(windows))]
18+
pub fn check_server_supports_nightlies() -> Result<bool> {
19+
Ok(*NIGHTLY_SERVER_SUPPORTS_ETAG.get_or_init(|| {
20+
let base_url = match get_julianightlies_base_url() {
21+
Ok(url) => url,
22+
Err(_) => return false,
23+
};
24+
25+
let test_url = match base_url.join("bin/") {
26+
Ok(url) => url,
27+
Err(_) => return false,
28+
};
29+
30+
let client = reqwest::blocking::Client::new();
31+
match client.head(test_url.as_str()).send() {
32+
Ok(response) => response.headers().get("etag").is_some(),
33+
Err(_) => false,
34+
}
35+
}))
36+
}
37+
38+
/// Checks if the nightly server supports etag headers.
39+
/// This is required for nightly and PR channel support because we use etags
40+
/// to track versions of these builds.
41+
///
42+
/// The result is cached after the first check.
43+
#[cfg(windows)]
44+
pub fn check_server_supports_nightlies() -> Result<bool> {
45+
use windows::core::HSTRING;
46+
use windows::Foundation::Uri;
47+
use windows::Web::Http::HttpClient;
48+
use windows::Web::Http::HttpMethod;
49+
use windows::Web::Http::HttpRequestMessage;
50+
51+
Ok(*NIGHTLY_SERVER_SUPPORTS_ETAG.get_or_init(|| {
52+
let base_url = match get_julianightlies_base_url() {
53+
Ok(url) => url,
54+
Err(_) => return false,
55+
};
56+
57+
let test_url = match base_url.join("bin/") {
58+
Ok(url) => url,
59+
Err(_) => return false,
60+
};
61+
62+
let http_client = match HttpClient::new() {
63+
Ok(client) => client,
64+
Err(_) => return false,
65+
};
66+
67+
let request_uri = match Uri::CreateUri(&HSTRING::from(test_url.as_str())) {
68+
Ok(uri) => uri,
69+
Err(_) => return false,
70+
};
71+
72+
let request = match HttpRequestMessage::Create(&HttpMethod::Head().unwrap(), &request_uri) {
73+
Ok(req) => req,
74+
Err(_) => return false,
75+
};
76+
77+
let response = match http_client.SendRequestAsync(&request) {
78+
Ok(async_op) => match async_op.get() {
79+
Ok(resp) => resp,
80+
Err(_) => return false,
81+
},
82+
Err(_) => return false,
83+
};
84+
85+
match response.Headers() {
86+
Ok(headers) => headers.Lookup(&HSTRING::from("ETag")).is_ok(),
87+
Err(_) => false,
88+
}
89+
}))
90+
}
91+
792
pub fn get_juliaserver_base_url() -> Result<Url> {
893
let base_url = if let Ok(val) = std::env::var("JULIAUP_SERVER") {
994
if val.ends_with('/') {

0 commit comments

Comments
 (0)