diff --git a/docs/architecture.md b/docs/architecture.md index c1d5a84..036e310 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,8 +63,17 @@ EOF, connection resets/timeouts, temporary DNS failures, and HTTP 408, 429, 500, 502, 503, and 504. Authentication, missing repositories, certificate validation, local filesystem errors, and unknown failures are not retried. Classification uses Git's English stderr diagnostics; the child locale is fixed -to `C`. Retry logs contain the operation, attempt, delay, and error category, -without raw upstream stderr or credentials. +to `C`. Retry logs contain the operation, attempt, delay, and error category. + +An operation that still fails is logged once, as `upstream operation failed`, +with its operation, repo, `error_kind`, elapsed time, and a summary of git's +stderr. The summary is the last 8 lines, capped at 1 KiB, because `remote:` +lines are upstream-controlled text. Git already strips credentials from URLs in +its own diagnostics; the proxy repeats that, redacts `Authorization` and +`Cookie` values and the credential in use, and removes inherited `GIT_TRACE*` +and `GIT_CURL_VERBOSE` so trace output never reaches stderr. The stderr stays +out of the returned error, which callers may surface. A cancelled operation (the +client went away, or shutdown) is logged at debug, not as a failure. Retries stay inside the existing per-repository lock. Each failed clone's staging directory is removed before another attempt; an existing fetch mirror is kept. diff --git a/docs/metrics.md b/docs/metrics.md index 5007cb2..c3636d6 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -27,7 +27,9 @@ Label values: `serve_duration_seconds`: `info_refs` (the buffered ref advertisement) or `upload_pack` (the packfile stream, timed to EOF). - `result` on `requests_total`: `ok`, `error`, `upstream_error`, `unauthorized`, - `rejected`. On `upstream_ops_total`: `ok` or `error`. On `lfs_objects_total`: + `rejected`. On `upstream_ops_total`: `ok` or `error`, once per operation after + retries; an operation cancelled because the client went away or the proxy is + shutting down is not counted. On `lfs_objects_total`: `hit` (served from disk), `miss` (fetched from upstream, then cached), `error`. On `auth_validations_total`: `allow`, `deny`, `unavailable` (the endpoint failed, throttled, or answered something else), `saturated` (no call made: no diff --git a/src/git.rs b/src/git.rs index 0a38d25..51b1c57 100644 --- a/src/git.rs +++ b/src/git.rs @@ -21,10 +21,11 @@ //! `PROXY_WANTS_NS` so the mirror can serve them. use std::collections::{HashMap, HashSet}; +use std::ffi::{OsStr, OsString}; use std::future::Future; use std::path::Path; use std::pin::Pin; -use std::process::Stdio; +use std::process::{ExitStatus, Stdio}; use std::sync::Arc; use std::task::Poll; use std::time::Duration; @@ -52,6 +53,9 @@ const UPSTREAM_RETRY_DELAYS: [Duration; 3] = [ ]; const MAX_UPSTREAM_RETRIES: usize = UPSTREAM_RETRY_DELAYS.len(); const MAX_UPSTREAM_STDERR_BYTES: usize = 64 * 1024; +const MAX_STDERR_SUMMARY_LINES: usize = 8; +const MAX_STDERR_SUMMARY_BYTES: usize = 1024; +const REDACTED: &str = ""; // Negotiation is bounded independently of pack size. Once input is written, // only inactivity is bounded: active downloads have no total duration limit. @@ -275,7 +279,7 @@ impl GitCache { } let mut last = tokio::select! { biased; - _ = self.shutdown.cancelled() => bail!("upstream operation cancelled"), + _ = self.shutdown.cancelled() => return Err(UpstreamStopped::Cancelled.into()), guard = slot.fetch_lock.lock() => guard, }; @@ -305,7 +309,10 @@ impl GitCache { if is_auth_failure(&e) || self.shutdown.is_cancelled() || e.is::() { return Err(e); } - tracing::warn!(repo = %repo.name, error = %e, "ensure wanted oids failed"); + if !upstream_failure_logged(&e) { + let (repo, error) = (repo.name.as_str(), format!("{e:#}")); + tracing::warn!(repo, error, "ensure wanted oids failed"); + } } Ok(outcome) } @@ -464,11 +471,7 @@ impl GitCache { ) .await { - // `-` not the repo name: a failed clone must not mint a per-repo series - // for an arbitrary client-supplied path (see `metrics`). The failing - // repo is still named in the returned error, which the caller logs. - self.metrics - .record_upstream(UpstreamOp::Clone, Status::Error, "-"); + self.upstream_failed(UpstreamOp::Clone, repo, cred, started, &error); return Err(error); } let elapsed = started.elapsed().as_secs_f64(); @@ -507,8 +510,7 @@ impl GitCache { ) .await { - self.metrics - .record_upstream(UpstreamOp::Fetch, Status::Error, "-"); + self.upstream_failed(UpstreamOp::Fetch, repo, cred, started, &error); return Err(error); } self.metrics @@ -579,8 +581,7 @@ impl GitCache { ) .await { - self.metrics - .record_upstream(UpstreamOp::WantFetch, Status::Error, "-"); + self.upstream_failed(UpstreamOp::WantFetch, repo, cred, started, &error); return Err(error); } self.metrics @@ -598,6 +599,51 @@ impl GitCache { Ok(()) } + /// Count and log an upstream operation's final failure. This is the one place + /// that sees the operation, the repo, and the credential needed to redact + /// stderr, so callers skip errors for which `upstream_failure_logged` holds. + fn upstream_failed( + &self, + op: UpstreamOp, + repo: &RepoRef, + cred: Option<&str>, + started: Instant, + error: &anyhow::Error, + ) { + let kind = failure_kind(error); + // A client that went away, or shutdown, is not an upstream failure. + if kind != Some(FailureKind::Cancelled) { + // `-` not the repo name: a failure must not mint a per-repo series for + // an arbitrary client-supplied path (see `metrics`). The log names it. + self.metrics.record_upstream(op, Status::Error, "-"); + } + let Some(kind) = kind else { + return; + }; + let elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX); + let (op, repo) = (op.as_str(), repo.name.as_str()); + if kind == FailureKind::Cancelled { + tracing::debug!(op, repo, elapsed_ms, "upstream operation cancelled"); + return; + } + let secret = cred.or(self.cfg.upstream_auth_header.as_deref()); + let stderr = error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .map(|failed| redact_stderr(&failed.stderr, secret)) + .unwrap_or_default(); + let (error_kind, error) = (kind.as_str(), format!("{error:#}")); + tracing::warn!( + op, + repo, + error_kind, + elapsed_ms, + stderr, + error, + "upstream operation failed" + ); + } + /// Cap how many want-by-SHA pins a mirror keeps, deleting the oldest by creation /// date beyond `max_wants`. Pins are excluded from `fetch --prune` and `git gc`, /// so this is what bounds their growth - the cache-size LRU scoped to a mirror's @@ -712,6 +758,8 @@ impl GitCache { let mut c = Command::new(&self.cfg.git_binary); c.env("GIT_TERMINAL_PROMPT", "0"); // fail instead of hanging on a prompt c.env("LC_ALL", "C"); // Retry classification relies on Git's English diagnostics. + // Trace output would bypass redaction and can carry request headers. + remove_trace_vars(&mut c, std::env::vars_os().map(|(key, _)| key)); for (k, v) in git_config_env( &self.cfg.big_file_threshold, cred.or(self.cfg.upstream_auth_header.as_deref()), @@ -761,17 +809,139 @@ pub fn validate_upstream_timeout(timeout: Duration) -> Result<()> { Ok(()) } -#[derive(Debug)] -struct UpstreamStopped; +/// Whether `err` is an upstream git failure already logged where the operation +/// failed, so a caller need not log it again. +pub fn upstream_failure_logged(err: &anyhow::Error) -> bool { + failure_kind(err).is_some() +} + +/// The `error_kind` of an upstream failure; `None` for a local error (spawn, +/// staging cleanup), which is left to the caller to log. +fn failure_kind(err: &anyhow::Error) -> Option { + err.chain().find_map(|cause| { + if let Some(failed) = cause.downcast_ref::() { + return Some(failed.kind); + } + cause + .downcast_ref::() + .map(|stopped| match stopped { + UpstreamStopped::Cancelled => FailureKind::Cancelled, + UpstreamStopped::TimedOut => FailureKind::Timeout, + }) + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FailureKind { + Auth, + TransientHttp, + Tls, + Connection, + Dns, + Unknown, + Timeout, + Cancelled, +} + +impl FailureKind { + fn as_str(self) -> &'static str { + match self { + Self::Auth => "auth", + Self::TransientHttp => "transient_http", + Self::Tls => "tls", + Self::Connection => "connection", + Self::Dns => "dns", + Self::Unknown => "unknown", + Self::Timeout => "timeout", + Self::Cancelled => "cancelled", + } + } + + fn reason(self) -> &'static str { + match self { + Self::Auth => "upstream rejected the credential", + Self::TransientHttp => "transient HTTP status", + Self::Tls => "TLS connection interrupted", + Self::Connection => "connection interrupted or timed out", + Self::Dns => "DNS resolution failed", + Self::Unknown => "permanent or unrecognized error", + Self::Timeout => "total budget exhausted", + Self::Cancelled => "cancelled", + } + } +} + +#[derive(Debug, Clone, Copy)] +enum UpstreamStopped { + Cancelled, + TimedOut, +} impl std::fmt::Display for UpstreamStopped { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("upstream operation cancelled or total budget exhausted") + f.write_str(match self { + Self::Cancelled => "upstream operation cancelled", + Self::TimedOut => "upstream operation exhausted its total budget", + }) } } impl std::error::Error for UpstreamStopped {} +/// A git process that exited unsuccessfully. The stderr tail is kept for the +/// failure log only: it is upstream-controlled text, so it stays out of +/// `Display` and `Debug` and is redacted before it is logged. +struct UpstreamFailed { + operation: String, + repo: String, + attempts: usize, + status: ExitStatus, + kind: FailureKind, + stderr: Vec, +} + +impl std::fmt::Debug for UpstreamFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UpstreamFailed") + .field("operation", &self.operation) + .field("repo", &self.repo) + .field("attempts", &self.attempts) + .field("status", &self.status) + .field("kind", &self.kind) + .finish_non_exhaustive() + } +} + +impl std::fmt::Display for UpstreamFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if self.kind == FailureKind::Auth { + // The cause, `UpstreamAuthFailed`, follows in the error chain. + return write!( + f, + "git {} failed for {} (status {})", + self.operation, self.repo, self.status + ); + } + write!( + f, + "git {} failed for {} after {} attempt(s) ({}, status {})", + self.operation, + self.repo, + self.attempts, + self.kind.reason(), + self.status + ) + } +} + +impl std::error::Error for UpstreamFailed { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + // Keeps `is_auth_failure` a plain chain walk. + static AUTH: UpstreamAuthFailed = UpstreamAuthFailed; + (self.kind == FailureKind::Auth).then_some(&AUTH as _) + } +} + // The caller holds the repository lock throughout retries. Never retry local // upload-pack: once its response starts streaming it cannot be replayed safely. async fn run_upstream( @@ -790,8 +960,11 @@ async fn run_upstream( #[cfg(unix)] cmd.process_group(0); for attempt in 0..=MAX_UPSTREAM_RETRIES { - if cancellation.is_cancelled() || Instant::now() >= deadline { - return Err(UpstreamStopped.into()); + if cancellation.is_cancelled() { + return Err(UpstreamStopped::Cancelled.into()); + } + if Instant::now() >= deadline { + return Err(UpstreamStopped::TimedOut.into()); } if let Some(path) = staging { clean_staging(path).await?; @@ -820,8 +993,8 @@ async fn run_upstream( }; let result = tokio::select! { biased; - _ = cancellation.cancelled() => Err(UpstreamStopped.into()), - _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped.into()), + _ = cancellation.cancelled() => Err(UpstreamStopped::Cancelled.into()), + _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped::TimedOut.into()), result = operation_result => result, }; let (status, tail) = match result { @@ -845,6 +1018,7 @@ async fn run_upstream( && status.code().is_some() && let Some(delay) = UPSTREAM_RETRY_DELAYS.get(attempt) { + let reason = reason.reason(); tracing::warn!( repo, operation, @@ -859,16 +1033,20 @@ async fn run_upstream( } // An upstream credential rejection is carried out typed, so a `client`-mode // caller can map it to a client 401 rather than a blanket 502. - if reason.is_none() && upstream_auth_diagnostic(&tail) { - return Err(anyhow::Error::new(UpstreamAuthFailed).context(format!( - "git {operation} failed for {repo}: upstream rejected the credential" - ))); - } - bail!( - "git {operation} failed for {repo} after {} attempt(s) ({}, status {status})", - attempt + 1, - reason.unwrap_or("permanent or unrecognized error") - ); + let kind = match reason { + Some(kind) => kind, + None if upstream_auth_diagnostic(&tail) => FailureKind::Auth, + None => FailureKind::Unknown, + }; + return Err(UpstreamFailed { + operation: operation.to_owned(), + repo: repo.to_owned(), + attempts: attempt + 1, + status, + kind, + stderr: tail, + } + .into()); } unreachable!("final attempt returns an error or succeeds") } @@ -887,8 +1065,8 @@ async fn wait_upstream_retry( ) -> Result<()> { tokio::select! { biased; - _ = cancellation.cancelled() => Err(UpstreamStopped.into()), - _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped.into()), + _ = cancellation.cancelled() => Err(UpstreamStopped::Cancelled.into()), + _ = tokio::time::sleep_until(deadline) => Err(UpstreamStopped::TimedOut.into()), _ = tokio::time::sleep(delay) => Ok(()), } } @@ -920,7 +1098,7 @@ fn upstream_auth_diagnostic(stderr: &[u8]) -> bool { .any(|message| text.contains(message)) } -fn transient_upstream_error(stderr: &[u8]) -> Option<&'static str> { +fn transient_upstream_error(stderr: &[u8]) -> Option { let text = String::from_utf8_lossy(stderr).to_ascii_lowercase(); // Permanent errors take precedence if Git reports more than one diagnostic. if [ @@ -950,7 +1128,7 @@ fn transient_upstream_error(stderr: &[u8]) -> Option<&'static str> { .ends_with(&format!("returned error: {code}")) }) }) { - return Some("transient HTTP status"); + return Some(FailureKind::TransientHttp); } if [ "unexpected eof while reading", @@ -961,7 +1139,7 @@ fn transient_upstream_error(stderr: &[u8]) -> Option<&'static str> { .iter() .any(|message| text.contains(message)) { - return Some("TLS connection interrupted"); + return Some(FailureKind::Tls); } if [ "connection reset", @@ -983,7 +1161,7 @@ fn transient_upstream_error(stderr: &[u8]) -> Option<&'static str> { .iter() .any(|message| text.contains(message)) { - return Some("connection interrupted or timed out"); + return Some(FailureKind::Connection); } if [ "temporary failure in name resolution", @@ -993,11 +1171,111 @@ fn transient_upstream_error(stderr: &[u8]) -> Option<&'static str> { .iter() .any(|message| text.contains(message)) { - return Some("DNS resolution failed"); + return Some(FailureKind::Dns); } None } +fn remove_trace_vars(cmd: &mut Command, keys: impl IntoIterator) { + for key in keys { + if is_trace_var(&key) { + cmd.env_remove(key); + } + } +} + +/// Git environment variables that send trace output to stderr. +fn is_trace_var(key: &OsStr) -> bool { + key.to_str() + .is_some_and(|key| key.starts_with("GIT_TRACE") || key == "GIT_CURL_VERBOSE") +} + +/// A bounded, single-line summary of upstream git stderr for the failure log. +/// Git already strips credentials from URLs in its own diagnostics, but +/// `remote:` lines are upstream-controlled text, so the same rules are applied +/// again and the credential in use is removed wherever it appears verbatim. +fn redact_stderr(stderr: &[u8], secret: Option<&str>) -> String { + let text = String::from_utf8_lossy(stderr); + let lines: Vec<&str> = text + .lines() + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect(); + let recent = &lines[lines.len().saturating_sub(MAX_STDERR_SUMMARY_LINES)..]; + let mut summary = recent + .iter() + .map(|line| redact_line(line, secret)) + .collect::>() + .join(" | "); + if summary.len() > MAX_STDERR_SUMMARY_BYTES { + let mut cut = summary.len() - MAX_STDERR_SUMMARY_BYTES; + while !summary.is_char_boundary(cut) { + cut += 1; + } + summary.replace_range(..cut, "..."); + } + summary +} + +fn redact_line(line: &str, secret: Option<&str>) -> String { + let replaced: String = line + .chars() + .map(|c| if c.is_control() { ' ' } else { c }) + .collect(); + let mut line = replaced.trim().to_owned(); + if let Some(secret) = secret { + // The whole header first, then its credential alone. A short value is + // skipped: replacing it would mangle unrelated text. + let credential = secret.rsplit(char::is_whitespace).next().unwrap_or(""); + for value in [secret.trim(), credential] { + if value.len() >= 8 { + line = line.replace(value, REDACTED); + } + } + } + redact_userinfo(&redact_header(&line)) +} + +/// Keep an `Authorization` header's scheme and drop the rest of the line, as +/// git's curl trace does. Git keeps cookie names and redacts their values; +/// this drops everything after `Cookie:`, which is stricter. +fn redact_header(line: &str) -> String { + let lower = line.to_ascii_lowercase(); + let auth = lower.find("authorization:").map(|at| { + let value = &line[at + "authorization:".len()..]; + let scheme = value.trim_start(); + let scheme_len = scheme.find(char::is_whitespace).unwrap_or(scheme.len()); + line.len() - scheme.len() + scheme_len + }); + let cookie = lower.find("cookie:").map(|at| at + "cookie:".len()); + match auth.into_iter().chain(cookie).min() { + Some(end) => format!("{} {REDACTED}", &line[..end]), + None => line.to_owned(), + } +} + +/// Replace the userinfo of every `scheme://user:pass@host` URL in `line`. +fn redact_userinfo(line: &str) -> String { + let mut out = String::with_capacity(line.len()); + let mut rest = line; + while let Some(at) = rest.find("://") { + let (head, tail) = rest.split_at(at + 3); + out.push_str(head); + let authority_end = tail + .find(|c: char| c == '/' || c.is_whitespace() || matches!(c, '\'' | '"' | '<' | '>')) + .unwrap_or(tail.len()); + match tail[..authority_end].rfind('@') { + Some(userinfo_end) => { + out.push_str(REDACTED); + rest = &tail[userinfo_end..]; + } + None => rest = tail, + } + } + out.push_str(rest); + out +} + async fn write_upload_pack_request( stdin: &mut (impl tokio::io::AsyncWrite + Unpin), body: &[u8], @@ -1102,7 +1380,7 @@ const UPLOAD_PACK_CAN_OBSERVE_EXIT_BEFORE_EOF: bool = cfg!(all( target_os = "redox", )) ))] -async fn wait_upload_pack(child: &mut Child) -> std::io::Result { +async fn wait_upload_pack(child: &mut Child) -> std::io::Result { use rustix::process::{Pid, WaitId, WaitIdOptions, waitid}; use tokio::signal::unix::{SignalKind, signal}; @@ -1136,7 +1414,7 @@ async fn wait_upload_pack(child: &mut Child) -> std::io::Result std::io::Result { +async fn wait_upload_pack(child: &mut Child) -> std::io::Result { child.wait().await } @@ -1390,7 +1668,10 @@ mod tests { .await .unwrap_err(); assert!( - error.is::(), + matches!( + error.downcast_ref::(), + Some(UpstreamStopped::Cancelled) + ), "cancellation must precede spawn: {error}" ); } @@ -1538,7 +1819,7 @@ mod tests { ) .await .unwrap_err(); - assert!(error.is::()); + assert_eq!(failure_kind(&error), Some(FailureKind::Timeout)); assert!(!error.to_string().contains("private-diagnostic")); } } @@ -1581,7 +1862,10 @@ mod tests { }) .await .unwrap(); - assert!(result.unwrap_err().is::()); + assert_eq!( + failure_kind(&result.unwrap_err()), + Some(FailureKind::Cancelled) + ); let mut pids = pids.split_whitespace(); let leader = pids.next().unwrap(); assert!( @@ -1686,7 +1970,7 @@ mod tests { "fatal: unable to access 'https://github.com/a/b/': The requested URL returned error: {code}\n" ); assert_eq!( - Some("transient HTTP status"), + Some(FailureKind::TransientHttp), transient_upstream_error(message.as_bytes()) ); } @@ -1769,6 +2053,177 @@ mod tests { } } + #[cfg(unix)] + #[tokio::test] + async fn upstream_failure_keeps_stderr_out_of_the_error() { + let token = CancellationToken::new(); + let mut cmd = Command::new("sh"); + cmd.args(["-c", "echo 'remote: private-diagnostic' >&2; exit 128"]); + let error = run_upstream( + &mut cmd, + "fetch", + "repo.git", + None, + Duration::from_secs(5), + &token, + ) + .await + .unwrap_err(); + assert_eq!(failure_kind(&error), Some(FailureKind::Unknown)); + assert!(upstream_failure_logged(&error)); + let failed = error.downcast_ref::().unwrap(); + assert_eq!(failed.attempts, 1); + let debug = format!("{failed:?}"); + assert!(debug.contains("Unknown"), "{debug}"); + assert!(!debug.contains("private-diagnostic"), "{debug}"); + assert!(String::from_utf8_lossy(&failed.stderr).contains("private-diagnostic")); + for rendered in [format!("{error:#}"), format!("{error:?}")] { + assert!(!rendered.contains("private-diagnostic"), "{rendered}"); + assert!( + rendered.contains("permanent or unrecognized error"), + "{rendered}" + ); + } + + let mut cmd = Command::new("sh"); + cmd.args([ + "-c", + "echo 'fatal: The requested URL returned error: 404' >&2; exit 128", + ]); + let error = run_upstream( + &mut cmd, + "fetch", + "repo.git", + None, + Duration::from_secs(5), + &token, + ) + .await + .unwrap_err(); + assert!(is_auth_failure(&error)); + assert_eq!(failure_kind(&error), Some(FailureKind::Auth)); + assert!(format!("{error:#}").ends_with("upstream rejected the credential")); + } + + #[test] + fn local_errors_are_left_to_the_caller_to_log() { + let error = anyhow::anyhow!("disk full").context("spawn upstream git"); + assert_eq!(failure_kind(&error), None); + assert!(!upstream_failure_logged(&error)); + let stopped = anyhow::Error::new(UpstreamStopped::TimedOut).context("outer"); + assert_eq!(failure_kind(&stopped), Some(FailureKind::Timeout)); + } + + #[test] + fn redact_stderr_removes_credentials() { + let secret = "Authorization: Bearer s3cret-token-value"; + let stderr = "\ +remote: token s3cret-token-value is not allowed +fatal: unable to access 'https://user:hunter2@git.example/a/b.git/': error +> Authorization: Basic dXNlcjpwYXNz +> proxy-authorization: Bearer abc.def +> Cookie: session=abc; other=def +plain https://git.example/a/b.git and ssh://git@git.example/a/b.git +"; + let summary = redact_stderr(stderr.as_bytes(), Some(secret)); + for leaked in [ + "s3cret-token-value", + "hunter2", + "user:", + "dXNlcjpwYXNz", + "abc.def", + "session=", + ] { + assert!(!summary.contains(leaked), "{leaked}: {summary}"); + } + for kept in [ + "remote: token is not allowed", + "https://@git.example/a/b.git/", + "Authorization: Basic ", + "proxy-authorization: Bearer ", + "Cookie: ", + "plain https://git.example/a/b.git and ssh://@git.example/a/b.git", + ] { + assert!(summary.contains(kept), "{kept}: {summary}"); + } + // A short credential is not replaced; it would mangle unrelated text. + assert_eq!(redact_stderr(b"a b c", Some("Bearer b")), "a b c"); + } + + #[test] + fn redact_stderr_is_bounded_and_single_line() { + let stderr: String = (0..20).map(|n| format!("line {n}\x07\r\x1b\n\n")).collect(); + let summary = redact_stderr(stderr.as_bytes(), None); + assert!(!summary.contains("line 11 "), "{summary}"); + assert!(summary.starts_with("line 12"), "{summary}"); + assert!(summary.ends_with("line 19"), "{summary}"); + assert!(!summary.chars().any(char::is_control), "{summary:?}"); + + // Odd length, so the cut first lands inside a two-byte character. + let long = format!("{}z", "\u{e9}".repeat(MAX_STDERR_SUMMARY_BYTES)); + let summary = redact_stderr(long.as_bytes(), None); + assert!(summary.starts_with("...")); + assert!(summary.len() <= MAX_STDERR_SUMMARY_BYTES + "...".len()); + } + + #[test] + fn trace_variables_are_recognized() { + for key in [ + "GIT_TRACE", + "GIT_TRACE_CURL", + "GIT_TRACE_PACKET", + "GIT_TRACE2_EVENT", + "GIT_CURL_VERBOSE", + ] { + assert!(is_trace_var(OsStr::new(key)), "{key}"); + } + for key in ["GIT_TERMINAL_PROMPT", "GIT_CONFIG_COUNT", "TRACE"] { + assert!(!is_trace_var(OsStr::new(key)), "{key}"); + } + let mut cmd = Command::new("git"); + remove_trace_vars(&mut cmd, ["GIT_TRACE_CURL".into(), "HOME".into()]); + let envs: Vec<_> = cmd.as_std().get_envs().collect(); + assert_eq!(envs, [(OsStr::new("GIT_TRACE_CURL"), None)]); + } + + #[test] + fn failure_kinds_have_distinct_labels_and_reasons() { + let kinds = [ + FailureKind::Auth, + FailureKind::TransientHttp, + FailureKind::Tls, + FailureKind::Connection, + FailureKind::Dns, + FailureKind::Unknown, + FailureKind::Timeout, + FailureKind::Cancelled, + ]; + let labels: HashSet<_> = kinds.iter().map(|kind| kind.as_str()).collect(); + let reasons: HashSet<_> = kinds.iter().map(|kind| kind.reason()).collect(); + assert_eq!(labels.len(), kinds.len()); + assert_eq!(reasons.len(), kinds.len()); + assert_eq!( + UpstreamStopped::Cancelled.to_string(), + "upstream operation cancelled" + ); + } + + #[tokio::test] + async fn exhausted_budget_stops_before_spawning() { + let mut cmd = Command::new("missing-git"); + let error = run_upstream( + &mut cmd, + "test", + "test", + None, + Duration::ZERO, + &CancellationToken::new(), + ) + .await + .unwrap_err(); + assert_eq!(failure_kind(&error), Some(FailureKind::Timeout)); + } + #[test] fn is_auth_failure_sees_the_typed_error_under_context() { let typed = anyhow::Error::new(UpstreamAuthFailed).context("git clone failed for r"); diff --git a/src/metrics.rs b/src/metrics.rs index fbbdfd4..4b1e493 100644 --- a/src/metrics.rs +++ b/src/metrics.rs @@ -106,7 +106,7 @@ impl Status { } impl UpstreamOp { - fn as_str(self) -> &'static str { + pub(crate) fn as_str(self) -> &'static str { match self { Self::Clone => "clone", Self::Fetch => "fetch", @@ -154,6 +154,8 @@ pub struct Metrics { requests: IntCounterVec, /// `upstream_ops_total{op, result, repo}` - op = clone | fetch | want_fetch; /// result = ok | error; repo = the repo path when result = ok, else `-`. + /// One sample per operation, after retries; a cancelled operation (client + /// gone, or shutdown) is not counted. upstream: IntCounterVec, /// `cache_bytes` - total size of the on-disk mirror cache, maintained /// incrementally as mirrors are added, refreshed, and evicted. Populated only diff --git a/src/server.rs b/src/server.rs index 7707f91..1d5a4b4 100644 --- a/src/server.rs +++ b/src/server.rs @@ -22,7 +22,7 @@ use tower::limit::GlobalConcurrencyLimitLayer; use crate::auth; use crate::config::UpstreamCredential; -use crate::git::{GitCache, is_auth_failure}; +use crate::git::{GitCache, is_auth_failure, upstream_failure_logged}; use crate::lfs::{Lfs, Outcome}; use crate::metrics::{LfsResult, Metrics, RequestKind, Status}; use crate::repo; @@ -503,7 +503,10 @@ async fn info_refs( } st.metrics .record_request(RequestKind::InfoRefs, Status::UpstreamError, "-"); - tracing::warn!(repo = %name, error = %e, "ensure_fresh failed"); + if !upstream_failure_logged(&e) { + let (repo, error) = (name, format!("{e:#}")); + tracing::warn!(repo, error, "ensure_fresh failed"); + } return err(StatusCode::BAD_GATEWAY, "upstream fetch failed"); } } @@ -559,7 +562,10 @@ async fn upload_pack( } st.metrics .record_request(RequestKind::UploadPack, Status::UpstreamError, "-"); - tracing::warn!(repo = %name, error = %e, "ensure mirror exists failed"); + if !upstream_failure_logged(&e) { + let (repo, error) = (name, format!("{e:#}")); + tracing::warn!(repo, error, "ensure mirror exists failed"); + } return err(StatusCode::BAD_GATEWAY, "upstream unavailable"); } diff --git a/tests/http.rs b/tests/http.rs index 783da58..d918469 100644 --- a/tests/http.rs +++ b/tests/http.rs @@ -182,6 +182,17 @@ async fn upload_pack_rejects_undecodable_body() { async fn upstream_failure_returns_bad_gateway_and_records_error() { // A real (but doomed) clone: the upstream points at a path that does not // exist, so `git clone --mirror` fails and the request surfaces a 502. + // A global subscriber that wants every callsite: without one, a test thread + // with no subscriber can cache a callsite as disabled for all threads. + let _ = tracing::subscriber::set_global_default(tracing_subscriber::registry()); + let logs = tempfile::NamedTempFile::new().unwrap(); + let _logs = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(logs.reopen().unwrap()) + .finish(), + ); let cache = tempfile::tempdir().unwrap(); let metrics = Arc::new(Metrics::new()); let cfg = GitConfig { @@ -217,6 +228,12 @@ async fn upstream_failure_returns_bad_gateway_and_records_error() { let resp = get(st, "/missing.git/info/refs?service=git-upload-pack").await; assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + // Logged once, where the clone failed; the handler does not repeat it. + let logs = std::fs::read_to_string(logs.path()).unwrap(); + assert_eq!(1, logs.matches(" WARN ").count(), "{logs}"); + assert!(logs.contains("upstream operation failed"), "{logs}"); + assert!(logs.contains("repo=\"missing.git\""), "{logs}"); + let scraped = metrics.gather(); // A failed clone records under the `-` sentinel, not the client-supplied path, // so a flood of doomed repo names cannot inflate label cardinality. @@ -284,6 +301,125 @@ async fn upload_pack_upstream_failure_returns_bad_gateway() { assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); } +#[tokio::test] +async fn upload_pack_local_failure_is_logged_by_the_handler() { + // The cache root is a file, so creating the mirror's parent fails before any + // upstream git runs: a local error the handler logs, not the git layer. + let _ = tracing::subscriber::set_global_default(tracing_subscriber::registry()); + let logs = tempfile::NamedTempFile::new().unwrap(); + let _logs = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(logs.reopen().unwrap()) + .finish(), + ); + let root = tempfile::NamedTempFile::new().unwrap(); + let metrics = Arc::new(Metrics::new()); + let upstream = "file:///nonexistent/git-cache-proxy-upstream"; + let cfg = GitConfig { + git_binary: "git".into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }; + let lfs = Arc::new(Lfs::new( + LfsConfig { + upstream_base: upstream.into(), + cache_root: root.path().to_path_buf(), + upstream_auth_header: None, + }, + None, + )); + let st = AppState { + cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + lfs, + upstream_base: upstream.into(), + cache_root: root.path().to_path_buf(), + auth: anon_auth(), + upstream_credential: UpstreamCredential::Static, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 0, + metrics, + }; + + let resp = router(st) + .oneshot( + Request::post("/missing.git/git-upload-pack") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let logs = std::fs::read_to_string(logs.path()).unwrap(); + assert_eq!(1, logs.matches(" WARN ").count(), "{logs}"); + assert!(logs.contains("ensure mirror exists failed"), "{logs}"); + assert!(logs.contains("create cache parent"), "{logs}"); +} + +#[tokio::test] +async fn unspawnable_git_is_logged_by_the_handler() { + // git cannot start, so the clone fails with no upstream diagnostic to + // classify: it still counts as an upstream error, and the handler logs it. + let _ = tracing::subscriber::set_global_default(tracing_subscriber::registry()); + let logs = tempfile::NamedTempFile::new().unwrap(); + let _logs = tracing::subscriber::set_default( + tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_writer(logs.reopen().unwrap()) + .finish(), + ); + let cache = tempfile::tempdir().unwrap(); + let metrics = Arc::new(Metrics::new()); + let upstream = "file:///nonexistent/git-cache-proxy-upstream"; + let cfg = GitConfig { + git_binary: cache.path().join("missing-git").to_str().unwrap().into(), + upstream_auth_header: None, + big_file_threshold: "8m".into(), + fetch_ttl: Duration::from_secs(10), + max_wants: 100, + max_upload_packs: 0, + pack_threads: None, + pack_window_memory: None, + }; + let lfs = Arc::new(Lfs::new( + LfsConfig { + upstream_base: upstream.into(), + cache_root: cache.path().to_path_buf(), + upstream_auth_header: None, + }, + None, + )); + let st = AppState { + cache: Arc::new(GitCache::new(cfg, metrics.clone(), None)), + lfs, + upstream_base: upstream.into(), + cache_root: cache.path().to_path_buf(), + auth: anon_auth(), + upstream_credential: UpstreamCredential::Static, + max_decoded_body: 512 * 1024 * 1024, + max_concurrent: 0, + metrics: metrics.clone(), + }; + + let resp = get(st, "/missing.git/info/refs?service=git-upload-pack").await; + assert_eq!(resp.status(), StatusCode::BAD_GATEWAY); + let logs = std::fs::read_to_string(logs.path()).unwrap(); + assert_eq!(1, logs.matches(" WARN ").count(), "{logs}"); + assert!(logs.contains("ensure_fresh failed"), "{logs}"); + let scraped = metrics.gather(); + assert!( + scraped.contains(r#"op="clone",repo="-",result="error""#), + "{scraped}" + ); +} + #[tokio::test] async fn concurrency_limit_serializes_without_deadlock() { // The limit wraps the git-serving fallback (health/readiness/metrics are diff --git a/tests/retry.rs b/tests/retry.rs index b04d57b..d150701 100644 --- a/tests/retry.rs +++ b/tests/retry.rs @@ -242,10 +242,198 @@ async fn dropped_request_cancels_owner_and_releases_repository_lock() { f.cache.wait_for_shutdown().await; } +#[tokio::test] +async fn final_failure_is_logged_once_with_redacted_stderr() { + let logs = Logs::capture(); + let f = Fixture::new(); + f.seed_mirror(); + f.fail( + "fetch", + 10, + "remote: token s3cret-token-value rejected\n\ + fatal: unable to access 'https://user:hunter2@git.example/repo.git/': odd failure", + ); + let cred = "Authorization: Bearer s3cret-token-value"; + assert!( + f.cache + .ensure_fresh(&f.repo, true, b"", Some(cred)) + .await + .is_err() + ); + let failures = logs.failures(); + assert_eq!(1, failures.len(), "{}", logs.read()); + let line = &failures[0]; + assert!(line.starts_with(" WARN"), "{line}"); + for expected in [ + "op=\"fetch\"", + "repo=\"repo.git\"", + "error_kind=\"unknown\"", + "elapsed_ms=", + "remote: token rejected", + "https://@git.example/repo.git/': odd failure", + "after 1 attempt(s) (permanent or unrecognized error, status exit status: 128)", + ] { + assert!(line.contains(expected), "{expected}: {line}"); + } + let all = logs.read(); + assert!(!all.contains("s3cret-token-value"), "{all}"); + assert!(!all.contains("hunter2"), "{all}"); + assert_eq!(1, f.upstream_errors("fetch")); +} + +#[tokio::test] +async fn static_auth_header_is_redacted_from_logs() { + let logs = Logs::capture(); + let f = Fixture::build( + Duration::from_secs(60), + Some("Authorization: Bearer static-token-value"), + ); + f.seed_mirror(); + f.fail("fetch", 10, "remote: static-token-value is revoked"); + assert!( + f.cache + .ensure_fresh(&f.repo, true, b"", None) + .await + .is_err() + ); + let all = logs.read(); + assert!(!all.contains("static-token-value"), "{all}"); + assert!(all.contains("remote: is revoked"), "{all}"); +} + +#[tokio::test] +async fn success_after_retry_logs_no_failure() { + let logs = Logs::capture(); + let f = Fixture::new(); + f.seed_mirror(); + f.fail("fetch", 1, "The requested URL returned error: 503"); + f.cache + .ensure_fresh(&f.repo, true, b"", None) + .await + .unwrap(); + assert!(logs.read().contains("retrying upstream git")); + assert!(logs.failures().is_empty(), "{}", logs.read()); + assert_eq!(0, f.upstream_errors("fetch")); +} + +#[tokio::test] +async fn auth_rejection_is_logged_with_its_diagnostic() { + let logs = Logs::capture(); + let f = Fixture::new(); + f.fail( + "clone", + 10, + "fatal: unable to access 'https://git.example/repo.git/': The requested URL returned error: 404", + ); + let error = f + .cache + .ensure_fresh(&f.repo, true, b"", None) + .await + .unwrap_err(); + assert!(git_cache_proxy::git::is_auth_failure(&error)); + let failures = logs.failures(); + assert_eq!(1, failures.len(), "{}", logs.read()); + assert!( + failures[0].contains("error_kind=\"auth\""), + "{}", + failures[0] + ); + assert!( + failures[0].contains("returned error: 404"), + "{}", + failures[0] + ); + assert!( + failures[0].contains("upstream rejected the credential"), + "{}", + failures[0] + ); +} + +#[tokio::test] +async fn budget_timeout_is_logged_as_timeout() { + let logs = Logs::capture(); + let mut f = Fixture::new(); + f.cache = f + .cache + .clone() + .with_upstream_timeout(Duration::from_millis(300)) + .unwrap(); + let _ready = f.stall("clone"); + assert!( + f.cache + .ensure_fresh(&f.repo, true, b"", None) + .await + .is_err() + ); + let failures = logs.failures(); + assert_eq!(1, failures.len(), "{}", logs.read()); + assert!( + failures[0].contains("error_kind=\"timeout\""), + "{}", + failures[0] + ); + assert!( + failures[0].contains("exhausted its total budget"), + "{}", + failures[0] + ); + assert_eq!(1, f.upstream_errors("clone")); +} + +#[tokio::test] +async fn request_cancellation_is_not_logged_as_a_failure() { + let logs = Logs::capture(); + let f = Fixture::new(); + let ready = f.stall("clone"); + let cache = f.cache.clone(); + let repo = f.repo.clone(); + let request = tokio::spawn(async move { cache.ensure_fresh(&repo, true, b"", None).await }); + tokio::time::timeout(Duration::from_secs(5), tokio::fs::read_to_string(ready)) + .await + .unwrap() + .unwrap(); + request.abort(); + assert!(request.await.unwrap_err().is_cancelled()); + std::fs::remove_file(f.root.path().join("stall-clone")).unwrap(); + // Queues behind the cancelled owner, so its outcome is recorded by now. + tokio::time::timeout( + Duration::from_secs(5), + f.cache.ensure_fresh(&f.repo, true, b"", None), + ) + .await + .unwrap() + .unwrap(); + assert!(logs.failures().is_empty(), "{}", logs.read()); + assert!(logs.read().contains("upstream operation cancelled")); + assert_eq!(0, f.upstream_errors("clone")); +} + +#[tokio::test] +async fn shutdown_is_not_logged_as_a_failure() { + let logs = Logs::capture(); + let f = Fixture::new(); + let ready = f.stall("clone"); + let cache = f.cache.clone(); + let repo = f.repo.clone(); + let request = tokio::spawn(async move { cache.ensure_fresh(&repo, true, b"", None).await }); + tokio::time::timeout(Duration::from_secs(5), tokio::fs::read_to_string(ready)) + .await + .unwrap() + .unwrap(); + f.cache.shutdown(); + assert!(request.await.unwrap().is_err()); + f.cache.wait_for_shutdown().await; + assert!(logs.failures().is_empty(), "{}", logs.read()); + assert!(logs.read().contains("upstream operation cancelled")); + assert_eq!(0, f.upstream_errors("clone")); +} + struct Fixture { root: tempfile::TempDir, repo: RepoRef, cache: GitCache, + metrics: Arc, } impl Fixture { @@ -254,6 +442,10 @@ impl Fixture { } fn with_ttl(fetch_ttl: Duration) -> Self { + Self::build(fetch_ttl, None) + } + + fn build(fetch_ttl: Duration, upstream_auth_header: Option<&str>) -> Self { let root = tempfile::tempdir().unwrap(); let upstream = root.path().join("upstream"); std::fs::create_dir(&upstream).unwrap(); @@ -296,10 +488,11 @@ exec git "$@" &root.path().join("cache"), ) .unwrap(); + let metrics = Arc::new(Metrics::new()); let cache = GitCache::new( GitConfig { git_binary: binary.to_str().unwrap().into(), - upstream_auth_header: None, + upstream_auth_header: upstream_auth_header.map(str::to_owned), big_file_threshold: "8m".into(), fetch_ttl, max_wants: 0, @@ -307,10 +500,15 @@ exec git "$@" pack_threads: None, pack_window_memory: None, }, - Arc::new(Metrics::new()), + metrics.clone(), None, ); - Self { root, repo, cache } + Self { + root, + repo, + cache, + metrics, + } } fn seed_mirror(&self) { @@ -335,6 +533,29 @@ exec git "$@" std::fs::write(self.root.path().join(format!("error-{op}")), message).unwrap(); } + fn stall(&self, op: &str) -> std::path::PathBuf { + std::fs::write(self.root.path().join(format!("stall-{op}")), "").unwrap(); + let ready = self.root.path().join(format!("ready-{op}")); + assert!( + Command::new("mkfifo") + .arg(&ready) + .status() + .unwrap() + .success() + ); + ready + } + + fn upstream_errors(&self, op: &str) -> u64 { + let prefix = + format!(r#"gitcacheproxy_upstream_ops_total{{op="{op}",repo="-",result="error"}} "#); + self.metrics + .gather() + .lines() + .find_map(|line| line.strip_prefix(&prefix)) + .map_or(0, |count| count.parse().unwrap()) + } + fn attempts(&self, op: &str) -> u32 { std::fs::read_to_string(self.root.path().join(format!("count-{op}"))) .unwrap() @@ -344,6 +565,41 @@ exec git "$@" } } +/// Capture this thread's log events; `#[tokio::test]` runs spawned tasks on it. +struct Logs { + file: tempfile::NamedTempFile, + _guard: tracing::subscriber::DefaultGuard, +} + +impl Logs { + fn capture() -> Self { + // A global subscriber that wants every callsite: without one, a test thread + // with no subscriber can cache a callsite as disabled for all threads. + let _ = tracing::subscriber::set_global_default(tracing_subscriber::registry()); + let file = tempfile::NamedTempFile::new().unwrap(); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .without_time() + .with_max_level(tracing::Level::DEBUG) + .with_writer(file.reopen().unwrap()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + Self { file, _guard } + } + + fn read(&self) -> String { + std::fs::read_to_string(self.file.path()).unwrap() + } + + fn failures(&self) -> Vec { + self.read() + .lines() + .filter(|line| line.contains("upstream operation failed")) + .map(str::to_owned) + .collect() + } +} + fn git(cwd: &Path, args: &[&str]) { let output = Command::new("git") .current_dir(cwd)