xtask/drift/
fetch.rs

1//! Fetching the upstream source of truth.
2//!
3//! Only `--refresh` and `--upstream` come here. The offline check that CI gates
4//! on never touches the network.
5
6use std::time::{SystemTime, UNIX_EPOCH};
7
8use crate::drift::baseline::{Source, UPSTREAM_PATH, UPSTREAM_REPO};
9
10const USER_AGENT: &str = "ag-ui-rust-xtask-drift-check";
11
12/// The upstream file, pinned to the commit it was read at.
13pub struct Fetched {
14    pub text: String,
15    pub source: Source,
16}
17
18/// Resolves the newest commit touching `events.ts`, then reads the file at
19/// exactly that commit, so the recorded SHA always describes the recorded text.
20pub fn events_ts() -> Result<Fetched, String> {
21    let commits = get(&format!(
22        "https://api.github.com/repos/{UPSTREAM_REPO}/commits?path={UPSTREAM_PATH}&per_page=1"
23    ))?;
24    let commits: serde_json::Value = serde_json::from_str(&commits)
25        .map_err(|e| format!("GitHub returned a commit list that is not JSON: {e}"))?;
26    let head = commits
27        .get(0)
28        .ok_or_else(|| format!("no commits found for {UPSTREAM_PATH} in {UPSTREAM_REPO}"))?;
29    let commit = head
30        .get("sha")
31        .and_then(|v| v.as_str())
32        .ok_or("GitHub commit list has no `sha`")?
33        .to_string();
34    let commit_date = head
35        .pointer("/commit/committer/date")
36        .or_else(|| head.pointer("/commit/author/date"))
37        .and_then(|v| v.as_str())
38        .unwrap_or("")
39        .split('T')
40        .next()
41        .unwrap_or("")
42        .to_string();
43
44    let url = format!("https://raw.githubusercontent.com/{UPSTREAM_REPO}/{commit}/{UPSTREAM_PATH}");
45    let text = get(&url)?;
46    looks_like_events_ts(&text)
47        .map_err(|why| format!("{url} did not return {UPSTREAM_PATH}: {why}"))?;
48
49    Ok(Fetched {
50        text,
51        source: Source {
52            repo: UPSTREAM_REPO.to_string(),
53            path: UPSTREAM_PATH.to_string(),
54            commit,
55            commit_date,
56            fetched_at: today_utc(),
57        },
58    })
59}
60
61/// Rejects a response body that is not the upstream module.
62///
63/// ureq already turns a 4xx/5xx into an error, so this is about the responses
64/// that arrive with status 200 and are still not the file: a captive portal or
65/// corporate proxy interstitial, a GitHub maintenance page, an HTML 404 from a
66/// CDN in front of raw.githubusercontent.com, or a body that was cut short
67/// before the interesting part. Left alone, all of those reach `extract()`,
68/// which reports the far less helpful "`enum EventType` not found".
69fn looks_like_events_ts(text: &str) -> Result<(), String> {
70    let head = text.trim_start();
71    if head.is_empty() {
72        return Err("the response body was empty".to_string());
73    }
74    if head.starts_with('<') {
75        return Err(format!(
76            "the response is markup, not TypeScript — probably an error page or a proxy \
77             interstitial. It starts: {}",
78            snippet(head)
79        ));
80    }
81    if !text.contains("EventType") {
82        return Err(format!(
83            "the response never mentions `EventType`, so it is not the events module (or it was \
84             truncated before reaching it). {} bytes, starting: {}",
85            text.len(),
86            snippet(head)
87        ));
88    }
89    Ok(())
90}
91
92/// The first line or so of a body, for an error message.
93fn snippet(text: &str) -> String {
94    let line = text.lines().next().unwrap_or("").trim();
95    let cut: String = line.chars().take(80).collect();
96    if cut.chars().count() < line.chars().count() {
97        format!("`{cut}...`")
98    } else {
99        format!("`{cut}`")
100    }
101}
102
103fn get(url: &str) -> Result<String, String> {
104    let mut request = ureq::get(url)
105        .header("User-Agent", USER_AGENT)
106        .header("Accept", "application/vnd.github+json");
107    // A token is not required, but an unauthenticated runner shares GitHub's
108    // 60-requests-per-hour-per-IP budget with every other job on the machine.
109    if let Some(token) = github_token() {
110        request = request.header("Authorization", &format!("Bearer {token}"));
111    }
112    let mut response = request.call().map_err(|e| explain(url, &e))?;
113    response
114        .body_mut()
115        .read_to_string()
116        .map_err(|e| format!("cannot read the response from {url}: {e}"))
117}
118
119fn github_token() -> Option<String> {
120    ["GITHUB_TOKEN", "GH_TOKEN"]
121        .iter()
122        .find_map(|key| std::env::var(key).ok())
123        .filter(|token| !token.is_empty())
124}
125
126fn explain(url: &str, error: &ureq::Error) -> String {
127    match error {
128        ureq::Error::StatusCode(403 | 429) => format!(
129            "{url} returned {error}.\n\
130             That is usually GitHub's unauthenticated rate limit (60 requests/hour/IP).\n\
131             Set GITHUB_TOKEN and retry, or run the offline check without --upstream/--refresh."
132        ),
133        ureq::Error::StatusCode(404) => format!(
134            "{url} returned 404 — upstream may have moved the file.\n\
135             Check {UPSTREAM_REPO} and update UPSTREAM_PATH in xtask/src/drift/baseline.rs."
136        ),
137        _ => format!("cannot fetch {url}: {error}"),
138    }
139}
140
141/// Today's date in UTC as `YYYY-MM-DD`.
142///
143/// Civil-from-days, so the baseline carries a readable capture date without a
144/// calendar dependency.
145pub fn today_utc() -> String {
146    let secs = SystemTime::now()
147        .duration_since(UNIX_EPOCH)
148        .map(|d| d.as_secs())
149        .unwrap_or(0);
150    let (y, m, d) = civil_from_days((secs / 86_400) as i64);
151    format!("{y:04}-{m:02}-{d:02}")
152}
153
154/// Howard Hinnant's `civil_from_days`, with 1970-01-01 as day 0.
155fn civil_from_days(days: i64) -> (i64, u32, u32) {
156    let z = days + 719_468;
157    let era = z.div_euclid(146_097);
158    let doe = z.rem_euclid(146_097);
159    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
160    let y = yoe + era * 400;
161    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
162    let mp = (5 * doy + 2) / 153;
163    let d = (doy - (153 * mp + 2) / 5 + 1) as u32;
164    let m = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
165    (if m <= 2 { y + 1 } else { y }, m, d)
166}
167
168#[cfg(test)]
169mod tests {
170    use super::{civil_from_days, looks_like_events_ts};
171
172    #[test]
173    fn accepts_the_real_module() {
174        let source = "import { z } from \"zod\";\n\nexport enum EventType {\n  RAW = \"RAW\",\n}\n";
175        assert!(looks_like_events_ts(source).is_ok());
176    }
177
178    #[test]
179    fn rejects_an_html_error_page() {
180        let error =
181            looks_like_events_ts("<!DOCTYPE html>\n<title>404 Not Found</title>\n").unwrap_err();
182        assert!(error.contains("markup, not TypeScript"), "{error}");
183        assert!(error.contains("<!DOCTYPE html>"), "{error}");
184    }
185
186    #[test]
187    fn rejects_a_proxy_interstitial_with_leading_whitespace() {
188        let error = looks_like_events_ts("\n\n  <html><body>Sign in to continue</body></html>")
189            .unwrap_err();
190        assert!(error.contains("markup, not TypeScript"), "{error}");
191    }
192
193    #[test]
194    fn rejects_an_empty_body() {
195        assert!(
196            looks_like_events_ts("   \n  ")
197                .unwrap_err()
198                .contains("empty")
199        );
200    }
201
202    /// A body cut short before the enum is TypeScript and still useless.
203    #[test]
204    fn rejects_a_response_truncated_before_the_enum() {
205        let error = looks_like_events_ts("import { z } from \"zod\";\n\nexport const Role = z.")
206            .unwrap_err();
207        assert!(error.contains("never mentions `EventType`"), "{error}");
208    }
209
210    #[test]
211    fn converts_known_days() {
212        assert_eq!(civil_from_days(0), (1970, 1, 1));
213        assert_eq!(civil_from_days(19_723), (2024, 1, 1));
214        assert_eq!(civil_from_days(19_784), (2024, 3, 2)); // leap year
215        assert_eq!(civil_from_days(20_682), (2026, 8, 17));
216    }
217}
xtask/drift/fetch.rs

xtask/drift/
fetch.rs
