fix old log loading
This commit is contained in:
+8
-2
@@ -82,8 +82,14 @@ async fn logs_response(
|
||||
match state.history.page(before, limit).await {
|
||||
Ok(page) => Json(page).into_response(),
|
||||
Err(code) if code == "invalid_cursor" => error(StatusCode::BAD_REQUEST, "invalid_cursor"),
|
||||
Err(code) if code == "stale_cursor" => error(StatusCode::CONFLICT, "stale_cursor"),
|
||||
Err(_) => error(StatusCode::UNPROCESSABLE_ENTITY, "history_unavailable"),
|
||||
Err(code) if code == "stale_cursor" => {
|
||||
tracing::debug!(event = "history_page_failed", reason = "stale_cursor");
|
||||
error(StatusCode::CONFLICT, "stale_cursor")
|
||||
}
|
||||
Err(code) => {
|
||||
tracing::warn!(event = "history_page_failed", reason = %code);
|
||||
error(StatusCode::UNPROCESSABLE_ENTITY, "history_unavailable")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+134
-9
@@ -35,7 +35,10 @@ impl HistoryIndex {
|
||||
for entry in fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if !entry.file_type()?.is_file() || archive::detect(&path).is_none() {
|
||||
if !entry.file_type()?.is_file()
|
||||
|| is_debug_log(&path)
|
||||
|| archive::detect(&path).is_none()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let meta = entry.metadata()?;
|
||||
@@ -75,6 +78,38 @@ impl HistoryIndex {
|
||||
generation,
|
||||
})
|
||||
}
|
||||
|
||||
// A cursor remains valid while latest.log only grows: existing source and
|
||||
// line positions do not move. Archive changes, rotation, and truncation do
|
||||
// change pagination and must invalidate outstanding cursors.
|
||||
fn is_append_compatible_with(&self, previous: &Self) -> bool {
|
||||
self.sources.len() == previous.sources.len()
|
||||
&& self
|
||||
.sources
|
||||
.iter()
|
||||
.zip(&previous.sources)
|
||||
.all(|(new, old)| {
|
||||
new.label == old.label
|
||||
&& new.path == old.path
|
||||
&& new.device == old.device
|
||||
&& new.inode == old.inode
|
||||
&& if new.label == "latest.log" {
|
||||
new.compressed_size >= old.compressed_size
|
||||
} else {
|
||||
new.modified_ms == old.modified_ms
|
||||
&& new.compressed_size == old.compressed_size
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
fn is_debug_log(path: &Path) -> bool {
|
||||
let name = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase();
|
||||
name == "debug.log"
|
||||
|| (name.starts_with("debug-") && (name.ends_with(".log") || name.ends_with(".log.gz")))
|
||||
}
|
||||
fn filename_key(name: &str) -> String {
|
||||
name.chars().filter(|c| c.is_ascii_digit()).collect()
|
||||
@@ -105,7 +140,12 @@ impl ArchiveCache {
|
||||
}
|
||||
}
|
||||
fn get(&mut self, p: &Path) -> Option<Arc<Vec<String>>> {
|
||||
self.items.get(p).cloned()
|
||||
let value = self.items.get(p).cloned()?;
|
||||
if let Some(position) = self.order.iter().position(|item| item == p) {
|
||||
self.order.remove(position);
|
||||
}
|
||||
self.order.push_back(p.to_path_buf());
|
||||
Some(value)
|
||||
}
|
||||
fn put(&mut self, p: PathBuf, v: Arc<Vec<String>>) {
|
||||
if self.max_files == 0 || self.max_bytes == 0 {
|
||||
@@ -148,7 +188,12 @@ impl HistoryStore {
|
||||
}
|
||||
}
|
||||
pub async fn refresh(&self, dir: &Path) -> std::io::Result<()> {
|
||||
*self.index.write().await = HistoryIndex::scan(dir)?;
|
||||
let mut next = HistoryIndex::scan(dir)?;
|
||||
let mut index = self.index.write().await;
|
||||
if next.is_append_compatible_with(&index) {
|
||||
next.generation = index.generation;
|
||||
}
|
||||
*index = next;
|
||||
Ok(())
|
||||
}
|
||||
pub async fn page(&self, before: Option<&str>, limit: usize) -> Result<HistoryPage, String> {
|
||||
@@ -207,14 +252,22 @@ impl HistoryStore {
|
||||
})
|
||||
}
|
||||
fn load(&self, source: &Source) -> Result<Arc<Vec<String>>, archive::ArchiveError> {
|
||||
if let Some(v) = self.cache.lock().unwrap().get(&source.path) {
|
||||
return Ok(v);
|
||||
let cacheable = matches!(
|
||||
archive::detect(&source.path),
|
||||
Some(archive::ArchiveKind::Gzip | archive::ArchiveKind::TarGzip)
|
||||
);
|
||||
if cacheable {
|
||||
if let Some(v) = self.cache.lock().unwrap().get(&source.path) {
|
||||
return Ok(v);
|
||||
}
|
||||
}
|
||||
let v = Arc::new(archive::read_lines(&source.path, self.max_archive_bytes)?);
|
||||
self.cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.put(source.path.clone(), v.clone());
|
||||
if cacheable {
|
||||
self.cache
|
||||
.lock()
|
||||
.unwrap()
|
||||
.put(source.path.clone(), v.clone());
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
}
|
||||
@@ -236,6 +289,14 @@ fn make_line(source: &Source, index: usize, text: &str, redact: bool) -> LogLine
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use flate2::{write::GzEncoder, Compression};
|
||||
use std::io::Write;
|
||||
|
||||
fn write_gzip(path: &Path, contents: &[u8]) {
|
||||
let mut gzip = GzEncoder::new(fs::File::create(path).unwrap(), Compression::default());
|
||||
gzip.write_all(contents).unwrap();
|
||||
gzip.finish().unwrap();
|
||||
}
|
||||
#[test]
|
||||
fn deterministic_filename_order() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
@@ -252,6 +313,23 @@ mod tests {
|
||||
let i = HistoryIndex::scan(d.path()).unwrap();
|
||||
assert_eq!(i.sources.last().unwrap().label, "latest.log");
|
||||
}
|
||||
#[test]
|
||||
fn excludes_minecraft_debug_logs() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
fs::write(d.path().join("latest.log"), "normal\n").unwrap();
|
||||
fs::write(d.path().join("debug.log"), "debug\n").unwrap();
|
||||
write_gzip(&d.path().join("debug-1.log.gz"), b"rotated debug\n");
|
||||
|
||||
let index = HistoryIndex::scan(d.path()).unwrap();
|
||||
assert_eq!(
|
||||
index
|
||||
.sources
|
||||
.iter()
|
||||
.map(|source| source.label.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
vec!["latest.log"]
|
||||
);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn paginates_without_exposing_paths() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
@@ -278,4 +356,51 @@ mod tests {
|
||||
vec!["old", "new"]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cursor_survives_latest_log_append_but_not_truncation() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let latest = d.path().join("latest.log");
|
||||
fs::write(&latest, "one\ntwo\n").unwrap();
|
||||
let store = HistoryStore::new(HistoryIndex::scan(d.path()).unwrap(), 1024, 2, 1024, false);
|
||||
let cursor = store.page(None, 1).await.unwrap().next_before.unwrap();
|
||||
|
||||
fs::OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&latest)
|
||||
.unwrap()
|
||||
.write_all(b"three\n")
|
||||
.unwrap();
|
||||
store.refresh(d.path()).await.unwrap();
|
||||
assert!(store.page(Some(&cursor), 1).await.is_ok());
|
||||
|
||||
fs::write(&latest, "replacement\n").unwrap();
|
||||
store.refresh(d.path()).await.unwrap();
|
||||
assert_eq!(
|
||||
store.page(Some(&cursor), 1).await.unwrap_err(),
|
||||
"stale_cursor"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn evicted_archive_is_reloaded_on_demand() {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let first_path = d.path().join("1.log.gz");
|
||||
let second_path = d.path().join("2.log.gz");
|
||||
write_gzip(&first_path, b"old\n");
|
||||
write_gzip(&second_path, b"second\n");
|
||||
let index = HistoryIndex::scan(d.path()).unwrap();
|
||||
let first = index.sources.iter().find(|s| s.path == first_path).unwrap();
|
||||
let second = index
|
||||
.sources
|
||||
.iter()
|
||||
.find(|s| s.path == second_path)
|
||||
.unwrap();
|
||||
let store = HistoryStore::new(index.clone(), 1024, 1, 1024, false);
|
||||
|
||||
assert_eq!(store.load(first).unwrap()[0], "old");
|
||||
assert_eq!(store.load(second).unwrap()[0], "second"); // evicts first
|
||||
write_gzip(&first_path, b"reloaded\n");
|
||||
assert_eq!(store.load(first).unwrap()[0], "reloaded");
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user