Skip to content

Commit

Permalink
fix: amplification of "couldn't batch lines - file not found" error l…
Browse files Browse the repository at this point in the history 
…og flood (#511)

- removed amplification of error "couldn't batch lines" + "file not found" - disabled retrying
- fixed rare panic in common/notify_stream/src/lib.rs in unwrap() when inotify event does not have path
- resolved clippy issue (choking on macro expansion) by pinning rust toolchain to previous version 1.69 using rust-toolchain.toml

Ref: LOG-14646
  • Loading branch information
@dkhokhlov
dkhokhlov authored Jun 13, 2023
1 parent 00bd5d5 commit d2ecf88
Show file tree
Hide file tree
Showing 3 changed files with 81 additions and 66 deletions.
26 changes: 15 additions & 11 deletions common/fs/src/cache/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -500,9 +500,8 @@ impl FileSystem {
watcher.receive()
};

let mfs = fs.clone(); // clone fs for missing files
let (missing_dirs, missing_dir_watcher, missing_dir_event_stream, retry_event_sender) = {
let mut _mfs = mfs.try_lock().expect("could not lock filesystem cache");
let mut _mfs = fs.try_lock().expect("could not lock filesystem cache");

let missing_dirs = _mfs.missing_dirs.clone();
let missing_dir_watcher = _mfs.missing_dir_watcher.take().unwrap_or_else(Watcher::new);
Expand Down Expand Up @@ -599,15 +598,20 @@ impl FileSystem {
Ok(_) => Some((e, event_time)),
Err(e) => {
match e {
Error::File(_) | Error::DirectoryListNotValid(_, _) => {
retry_event_sender
.send((
event.unwrap(),
event_time,
retries.unwrap_or(0),
))
.await
.unwrap();
Error::File(io_err)
| Error::DirectoryListNotValid(io_err, _) => {
if io_err.kind() == io::ErrorKind::NotFound {
{}
} else {
retry_event_sender
.send((
event.unwrap(),
event_time,
retries.unwrap_or(0),
))
.await
.unwrap();
}
}
_ => {}
}
Expand Down
116 changes: 61 additions & 55 deletions common/notify_stream/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use notify::{Config, ErrorKind, EventKind, Watcher as NotifyWatcher};
use std::path::Path;
use std::rc::Rc;
use time::OffsetDateTime;
use tracing::{debug, info, instrument, trace};
use tracing::{info, instrument, trace, warn};

type PathId = std::path::PathBuf;

Expand Down Expand Up @@ -163,63 +163,69 @@ impl Watcher {
.expect("channel closed unexpectedly")
.unwrap();
trace!("received raw notify event: {:?}", received);
if let Some(mapped_event) = match received.kind {
EventKind::Remove(RemoveKind::File) => {
Some(Event::Remove(received.paths.first().unwrap().clone()))
let notify::Event { kind, paths, attrs } = received;
let mut paths = paths.into_iter();
if let Some(mapped_event) = match ((paths.next(), paths.last()), kind) {
// looks like something removed a file
(
(Some(from_path), None),
EventKind::Remove(
RemoveKind::File
| RemoveKind::Folder
| RemoveKind::Other
| RemoveKind::Any,
),
) => Some(Event::Remove(from_path)),
// looks like something created a file
(
(Some(from_path), None),
EventKind::Create(
CreateKind::File
| CreateKind::Folder
| CreateKind::Other
| CreateKind::Any,
)
| EventKind::Modify(ModifyKind::Name(RenameMode::To)),
) => Some(Event::Create(from_path)),
// looks like something modified the contents of the file
(
(Some(from_path), None),
EventKind::Modify(
ModifyKind::Data(DataChange::Any | DataChange::Content)
| ModifyKind::Other
| ModifyKind::Any,
),
) => Some(Event::Write(from_path)), // looks like a rename
(
(Some(from_path), Some(to_path)),
EventKind::Modify(ModifyKind::Name(RenameMode::Both)),
) => Some(Event::Rename(from_path, to_path)),
// looks like something moved a file but we don't know where to
// if we know where it was moved from we can treat it as a delete
// if we don't even know that we need to rescan
(
(from_path, None),
kind @ EventKind::Modify(ModifyKind::Name(RenameMode::From)),
) => {
from_path
.map(Event::Remove)
.or_else(|| {
warn!("raw notify event with None path Event {{ kind: {:?}, attrs: {:?} }}", kind, attrs);
Some(Event::Rescan) // trigger FS rescan
})
}
EventKind::Remove(RemoveKind::Folder) => {
Some(Event::Remove(received.paths.first().unwrap().clone()))
// looks like the file metadata was modified by the contents weren't changed
(_, EventKind::Modify(ModifyKind::Metadata(_)) | EventKind::Access(_)) => None,
// something we didn't expect happened, log and move on
((from_path, to_path), kind) => {
warn!(
"unexpected event, kind: {:?}, from_path: {:?}, to_path: {:?}, attrs: {:?}",
kind, from_path, to_path, attrs
);
None
}
EventKind::Remove(RemoveKind::Other) => {
Some(Event::Remove(received.paths.first().unwrap().clone()))
}
EventKind::Remove(RemoveKind::Any) => {
Some(Event::Remove(received.paths.first().unwrap().clone()))
}
EventKind::Create(CreateKind::File) => {
Some(Event::Create(received.paths.first().unwrap().clone()))
}
EventKind::Create(CreateKind::Folder) => {
Some(Event::Create(received.paths.first().unwrap().clone()))
}
EventKind::Create(CreateKind::Other) => {
Some(Event::Create(received.paths.first().unwrap().clone()))
}
EventKind::Create(CreateKind::Any) => {
Some(Event::Create(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Data(DataChange::Any)) => {
Some(Event::Write(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Data(DataChange::Content)) => {
Some(Event::Write(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Name(RenameMode::From)) => received
.paths
.first()
.map(|path| Event::Remove(path.clone()))
.or_else(|| {
debug!("raw notify event with None path: {:?}", received);
Some(Event::Rescan) // trigger FS rescan
}),
EventKind::Modify(ModifyKind::Name(RenameMode::To)) => {
Some(Event::Create(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => Some(Event::Rename(
received.paths.first().unwrap().clone(),
received.paths.last().unwrap().clone(),
)),
EventKind::Modify(ModifyKind::Other) => {
Some(Event::Write(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Any) => {
Some(Event::Write(received.paths.first().unwrap().clone()))
}
EventKind::Modify(ModifyKind::Metadata(_)) => None,
EventKind::Access(_) => None,
_ => None,
} {
trace!("mapped event: {:?}\n", mapped_event);
trace!("mapped event: {:?}", mapped_event);
return Some(((mapped_event, OffsetDateTime::now_utc()), rx));
}
}
Expand Down
5 changes: 5 additions & 0 deletions rust-toolchain.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
[toolchain]
channel = "1.69"
components = [ "rustfmt", "cargo", "clippy", "rustc", "llvm-tools-preview" ]
targets = ["x86_64-apple-darwin", "x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu", "aarch64-unknown-linux-musl", "x86_64-unknown-linux-musl"]
profile = "default"

0 comments on commit d2ecf88

Please sign in to comment.