diff --git a/include/clang/DirectoryWatcher/DirectoryWatcher.h b/include/clang/DirectoryWatcher/DirectoryWatcher.h index 2739955a113..0bf966bb832 100644 --- a/include/clang/DirectoryWatcher/DirectoryWatcher.h +++ b/include/clang/DirectoryWatcher/DirectoryWatcher.h @@ -1,67 +1,123 @@ //===- DirectoryWatcher.h - Listens for directory file changes --*- C++ -*-===// // -// The LLVM Compiler Infrastructure +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// \file -/// \brief Utility class for listening for file system changes in a directory. //===----------------------------------------------------------------------===// #ifndef LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H #define LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H -#include "clang/Basic/LLVM.h" -#include "llvm/Support/Chrono.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/StringRef.h" #include #include #include namespace clang { - -/// Provides notifications for file system changes in a directory. +/// Provides notifications for file changes in a directory. +/// +/// Invokes client-provided function on every filesystem event in the watched +/// directory. Initially the the watched directory is scanned and for every file +/// found, an event is synthesized as if the file was added. +/// +/// This is not a general purpose directory monitoring tool - list of +/// limitations follows. +/// +/// Only flat directories with no subdirectories are supported. In case +/// subdirectories are present the behavior is unspecified - events *might* be +/// passed to Receiver on macOS (due to FSEvents being used) while they +/// *probably* won't be passed on Linux (due to inotify being used). +/// +/// Known potential inconsistencies +/// - For files that are deleted befor the initial scan processed them, clients +/// might receive Removed notification without any prior Added notification. +/// - Multiple notifications might be produced when a file is added to the +/// watched directory during the initial scan. We are choosing the lesser evil +/// here as the only known alternative strategy would be to invalidate the +/// watcher instance and force user to create a new one whenever filesystem +/// event occurs during the initial scan but that would introduce continuous +/// restarting failure mode (watched directory is not always "owned" by the same +/// process that is consuming it). Since existing clients can handle duplicate +/// events well, we decided for simplicity. +/// +/// Notifications are provided only for changes done through local user-space +/// filesystem interface. Specifically, it's unspecified if notification would +/// be provided in case of a: +/// - a file mmap-ed and changed +/// - a file changed via remote (NFS) or virtual (/proc) FS access to monitored +/// directory +/// - another filesystem mounted to the watched directory +/// +/// No support for LLVM VFS. /// -/// Guarantees that the first time the directory is processed, the receiver will -/// be invoked even if the directory is empty. +/// It is unspecified whether notifications for files being deleted are sent in +/// case the whole watched directory is sent. +/// +/// Directories containing "too many" files and/or receiving events "too +/// frequently" are not supported - if the initial scan can't be finished before +/// the watcher instance gets invalidated (see WatcherGotInvalidated) there's no +/// good error handling strategy - the only option for client is to destroy the +/// watcher, restart watching with new instance and hope it won't repeat. class DirectoryWatcher { public: - enum class EventKind { - /// A file was added. - Added, - /// A file was removed. - Removed, - /// A file was modified. - Modified, - /// The watched directory got deleted. No more events will follow. - DirectoryDeleted, - }; - struct Event { + enum class EventKind { + Removed, + /// Content of a file was modified. + Modified, + /// The watched directory got deleted. + WatchedDirRemoved, + /// The DirectoryWatcher that originated this event is no longer valid and + /// its behavior is unspecified. + /// + /// The prime case is kernel signalling to OS-specific implementation of + /// DirectoryWatcher some resource limit being hit. + /// *Usually* kernel starts dropping or squashing events together after + /// that and so would DirectoryWatcher. This means that *some* events + /// might still be passed to Receiver but this behavior is unspecified. + /// + /// Another case is after the watched directory itself is deleted. + /// WatcherGotInvalidated will be received at least once during + /// DirectoryWatcher instance lifetime - when handling errors this is done + /// on best effort basis, when an instance is being destroyed then this is + /// guaranteed. + /// + /// The only proper response to this kind of event is to destruct the + /// originating DirectoryWatcher instance and create a new one. + WatcherGotInvalidated + }; + EventKind Kind; + /// Filename that this event is related to or an empty string in + /// case this event is related to the watched directory itself. std::string Filename; - llvm::sys::TimePoint<> ModTime; - }; - - typedef std::function Events, bool isInitial)> EventReceiver; - ~DirectoryWatcher(); + Event(EventKind Kind, llvm::StringRef Filename) + : Kind(Kind), Filename(Filename) {} + }; + /// Returns nullptr if \param Path doesn't exist. + /// Returns nullptr if \param Path isn't a directory. + /// Returns nullptr if OS kernel API told us we can't start watching. In such + /// case it's unclear whether just retrying has any chance to succeeed. static std::unique_ptr - create(StringRef Path, EventReceiver Receiver, bool waitInitialSync, - std::string &Error); - -private: - struct Implementation; - Implementation &Impl; + create(llvm::StringRef Path, + std::function Events, + bool IsInitial)> + Receiver, + bool WaitForInitialSync); - DirectoryWatcher(); + virtual ~DirectoryWatcher() = default; + DirectoryWatcher(const DirectoryWatcher &) = delete; + DirectoryWatcher &operator=(const DirectoryWatcher &) = delete; + DirectoryWatcher(DirectoryWatcher &&) = default; - DirectoryWatcher(const DirectoryWatcher&) = delete; - DirectoryWatcher &operator =(const DirectoryWatcher&) = delete; +protected: + DirectoryWatcher() = default; }; } // namespace clang -#endif +#endif // LLVM_CLANG_DIRECTORYWATCHER_DIRECTORYWATCHER_H diff --git a/include/clang/Index/IndexDataStore.h b/include/clang/Index/IndexDataStore.h index 145a83dbcb7..8e438fd7dc8 100644 --- a/include/clang/Index/IndexDataStore.h +++ b/include/clang/Index/IndexDataStore.h @@ -36,16 +36,15 @@ class IndexDataStore { static unsigned getFormatVersion(); enum class UnitEventKind { - Added, Removed, Modified, /// The directory got deleted. No more events will follow. DirectoryDeleted, + Failure }; struct UnitEvent { UnitEventKind Kind; StringRef UnitName; - llvm::sys::TimePoint<> ModTime; }; struct UnitEventNotification { bool IsInitial; diff --git a/include/indexstore/IndexStoreCXX.h b/include/indexstore/IndexStoreCXX.h index 5b9fd896e95..6765ffa9e82 100644 --- a/include/indexstore/IndexStoreCXX.h +++ b/include/indexstore/IndexStoreCXX.h @@ -19,7 +19,6 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallString.h" -#include namespace indexstore { using llvm::ArrayRef; @@ -152,19 +151,19 @@ class IndexStore { UnitEvent(indexstore_unit_event_t obj) : obj(obj) {} enum class Kind { - Added, Removed, Modified, DirectoryDeleted, + Failure }; Kind getKind() const { indexstore_unit_event_kind_t c_k = indexstore_unit_event_get_kind(obj); Kind K; switch (c_k) { - case INDEXSTORE_UNIT_EVENT_ADDED: K = Kind::Added; break; case INDEXSTORE_UNIT_EVENT_REMOVED: K = Kind::Removed; break; case INDEXSTORE_UNIT_EVENT_MODIFIED: K = Kind::Modified; break; case INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED: K = Kind::DirectoryDeleted; break; + case INDEXSTORE_UNIT_EVENT_FAILURE: K = Kind::Failure; break; } return K; } @@ -172,8 +171,6 @@ class IndexStore { StringRef getUnitName() const { return stringFromIndexStoreStringRef(indexstore_unit_event_get_unit_name(obj)); } - - timespec getModificationTime() const { return indexstore_unit_event_get_modification_time(obj); } }; class UnitEventNotification { @@ -258,24 +255,6 @@ class IndexStore { nameBuf.append(unitName.begin(), unitName.begin()+nameLen); } - llvm::Optional - getUnitModificationTime(StringRef unitName, std::string &error) { - llvm::SmallString<64> buf = unitName; - int64_t seconds, nanoseconds; - indexstore_error_t c_err = nullptr; - bool err = indexstore_store_get_unit_modification_time(obj, buf.c_str(), - &seconds, &nanoseconds, &c_err); - if (err && c_err) { - error = indexstore_error_get_description(c_err); - indexstore_error_dispose(c_err); - return llvm::None; - } - timespec ts; - ts.tv_sec = seconds; - ts.tv_nsec = nanoseconds; - return ts; - } - void purgeStaleData() { indexstore_store_purge_stale_data(obj); } diff --git a/include/indexstore/indexstore.h b/include/indexstore/indexstore.h index e5154694eb2..fb32d0f355f 100644 --- a/include/indexstore/indexstore.h +++ b/include/indexstore/indexstore.h @@ -17,7 +17,6 @@ #include #include #include -#include /** * \brief The version constants for the Index Store C API. @@ -134,10 +133,10 @@ INDEXSTORE_PUBLIC bool indexstore_unit_event_notification_is_initial(indexstore_unit_event_notification_t); typedef enum { - INDEXSTORE_UNIT_EVENT_ADDED = 1, - INDEXSTORE_UNIT_EVENT_REMOVED = 2, - INDEXSTORE_UNIT_EVENT_MODIFIED = 3, - INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED = 4, + INDEXSTORE_UNIT_EVENT_REMOVED = 1, + INDEXSTORE_UNIT_EVENT_MODIFIED = 2, + INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED = 3, + INDEXSTORE_UNIT_EVENT_FAILURE = 4, } indexstore_unit_event_kind_t; INDEXSTORE_PUBLIC indexstore_unit_event_kind_t @@ -146,9 +145,6 @@ indexstore_unit_event_get_kind(indexstore_unit_event_t); INDEXSTORE_PUBLIC indexstore_string_ref_t indexstore_unit_event_get_unit_name(indexstore_unit_event_t); -INDEXSTORE_PUBLIC struct timespec -indexstore_unit_event_get_modification_time(indexstore_unit_event_t); - #if INDEXSTORE_HAS_BLOCKS typedef void (^indexstore_unit_event_handler_t)(indexstore_unit_event_notification_t); @@ -197,14 +193,6 @@ indexstore_store_get_unit_name_from_output_path(indexstore_t store, char *name_buf, size_t buf_size); -/// \returns true if an error occurred, false otherwise. -INDEXSTORE_PUBLIC bool -indexstore_store_get_unit_modification_time(indexstore_t store, - const char *unit_name, - int64_t *seconds, - int64_t *nanoseconds, - indexstore_error_t *error); - typedef void *indexstore_symbol_t; typedef enum { diff --git a/lib/DirectoryWatcher/CMakeLists.txt b/lib/DirectoryWatcher/CMakeLists.txt index b1f35aab3ff..b0bbb936933 100644 --- a/lib/DirectoryWatcher/CMakeLists.txt +++ b/lib/DirectoryWatcher/CMakeLists.txt @@ -2,13 +2,26 @@ include(CheckIncludeFiles) set(LLVM_LINK_COMPONENTS support) -add_clang_library(clangDirectoryWatcher - DirectoryWatcher.cpp - ) +set(DIRECTORY_WATCHER_SOURCES DirectoryScanner.cpp) +set(DIRECTORY_WATCHER_LINK_LIBS "") if(APPLE) - check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES_H) - if(HAVE_CORESERVICES_H) - target_link_libraries(clangDirectoryWatcher PRIVATE "-framework CoreServices") + check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES) + if(HAVE_CORESERVICES) + list(APPEND DIRECTORY_WATCHER_SOURCES mac/DirectoryWatcher-mac.cpp) + set(DIRECTORY_WATCHER_LINK_LIBS "-framework CoreServices") + endif() +elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") + check_include_files("sys/inotify.h" HAVE_INOTIFY) + if(HAVE_INOTIFY) + list(APPEND DIRECTORY_WATCHER_SOURCES linux/DirectoryWatcher-linux.cpp) + find_package(Threads REQUIRED) + set(DIRECTORY_WATCHER_LINK_LIBS ${CMAKE_THREAD_LIBS_INIT}) endif() endif() + +add_clang_library(clangDirectoryWatcher + ${DIRECTORY_WATCHER_SOURCES} + ) + +target_link_libraries(clangDirectoryWatcher PRIVATE ${DIRECTORY_WATCHER_LINK_LIBS}) diff --git a/lib/DirectoryWatcher/DirectoryScanner.cpp b/lib/DirectoryWatcher/DirectoryScanner.cpp new file mode 100644 index 00000000000..ecfec52f459 --- /dev/null +++ b/lib/DirectoryWatcher/DirectoryScanner.cpp @@ -0,0 +1,54 @@ +//===- DirectoryScanner.cpp - Utility functions for DirectoryWatcher ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" + +#include "llvm/Support/Path.h" + +namespace clang { + +using namespace llvm; + +Optional getFileStatus(StringRef Path) { + sys::fs::file_status Status; + std::error_code EC = status(Path, Status); + if (EC) + return None; + return Status; +} + +std::vector scanDirectory(StringRef Path) { + using namespace llvm::sys; + std::vector Result; + + std::error_code EC; + for (auto It = fs::directory_iterator(Path, EC), + End = fs::directory_iterator(); + !EC && It != End; It.increment(EC)) { + auto status = getFileStatus(It->path()); + if (!status.hasValue()) + continue; + Result.emplace_back(sys::path::filename(It->path())); + } + + return Result; +} + +std::vector +getAsFileEvents(const std::vector &Scan) { + std::vector Events; + Events.reserve(Scan.size()); + + for (const auto &File : Scan) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::Modified, File}); + } + return Events; +} + +} // namespace clang \ No newline at end of file diff --git a/lib/DirectoryWatcher/DirectoryScanner.h b/lib/DirectoryWatcher/DirectoryScanner.h new file mode 100644 index 00000000000..55731225e25 --- /dev/null +++ b/lib/DirectoryWatcher/DirectoryScanner.h @@ -0,0 +1,29 @@ +//===- DirectoryScanner.h - Utility functions for DirectoryWatcher --------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "clang/DirectoryWatcher/DirectoryWatcher.h" +#include "llvm/Support/FileSystem.h" +#include +#include + +namespace clang { + +/// Gets names (filenames) of items in directory at \p Path. +/// \returns empty vector if \p Path is not a directory, doesn't exist or can't +/// be read from. +std::vector scanDirectory(llvm::StringRef Path); + +/// Create event with EventKind::Added for every element in \p Scan. +std::vector +getAsFileEvents(const std::vector &Scan); + +/// Gets status of file (or directory) at \p Path. +/// \returns llvm::None if \p Path doesn't exist or can't get the status. +llvm::Optional getFileStatus(llvm::StringRef Path); + +} // namespace clang \ No newline at end of file diff --git a/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h b/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h deleted file mode 100644 index ddb492ab2ff..00000000000 --- a/lib/DirectoryWatcher/DirectoryWatcher-linux.inc.h +++ /dev/null @@ -1,196 +0,0 @@ -//===- DirectoryWatcher-linux.inc.h - Linux-platform directory listening --===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Support/Errno.h" -#include "llvm/Support/Path.h" -#include "llvm/Support/Mutex.h" -#include -#include -#include - -namespace { - -struct INotifyEvent { - DirectoryWatcher::EventKind K; - std::string Filename; - Optional Status; -}; - -class EventQueue { - DirectoryWatcher::EventReceiver Receiver; - sys::Mutex Mtx; - bool gotInitialScan = false; - std::vector PendingEvents; - - DirectoryWatcher::Event toDirEvent(const INotifyEvent &evt) { - llvm::sys::TimePoint<> modTime{}; - if (evt.Status.hasValue()) modTime = evt.Status->getLastModificationTime(); - return DirectoryWatcher::Event{evt.K, evt.Filename, modTime}; - } - -public: - explicit EventQueue(DirectoryWatcher::EventReceiver receiver) - : Receiver(receiver) {} - - void onDirectoryEvents(ArrayRef evts) { - sys::ScopedLock L(Mtx); - - if (!gotInitialScan) { - PendingEvents.insert(PendingEvents.end(), evts.begin(), evts.end()); - return; - } - - SmallVector dirEvents; - for (const auto &evt : evts) { - dirEvents.push_back(toDirEvent(evt)); - } - Receiver(dirEvents, /*isInitial=*/false); - } - - void onInitialScan(std::shared_ptr dirScan) { - sys::ScopedLock L(Mtx); - - std::vector events = dirScan->getAsFileEvents(); - Receiver(events, /*isInitial=*/true); - - events.clear(); - for (const auto &evt : PendingEvents) { - if (evt.K == DirectoryWatcher::EventKind::Added && - dirScan->FileIDSet.count(evt.Status->getUniqueID())) { - // Already reported this event at the initial directory scan. - continue; - } - events.push_back(toDirEvent(evt)); - } - if (!events.empty()) { - Receiver(events, /*isInitial=*/false); - } - - gotInitialScan = true; - PendingEvents.clear(); - } -}; -} // namespace - -struct DirectoryWatcher::Implementation { - bool initialize(StringRef Path, EventReceiver Receiver, - bool waitInitialSync, std::string &Error); - ~Implementation() { - stopListening(); - }; - -private: - int inotifyFD = -1; - - void stopListening(); -}; - -static void runWatcher(std::string pathToWatch, int inotifyFD, - std::shared_ptr evtQueue) { - #define EVT_BUF_LEN (30 * (sizeof(struct inotify_event) + NAME_MAX + 1)) - char buf[EVT_BUF_LEN] __attribute__ ((aligned(8))); - - while (1) { - ssize_t numRead = read(inotifyFD, buf, EVT_BUF_LEN); - if (numRead == -1) { - return; // watcher is stopped. - } - - SmallVector iEvents; - for (char *p = buf; p < buf + numRead;) { - struct inotify_event *ievt = (struct inotify_event *)p; - p += sizeof(struct inotify_event) + ievt->len; - - if (ievt->mask & IN_DELETE_SELF) { - INotifyEvent iEvt{DirectoryWatcher::EventKind::DirectoryDeleted, - pathToWatch, None}; - iEvents.push_back(iEvt); - break; - } - - DirectoryWatcher::EventKind K = DirectoryWatcher::EventKind::Added; - if (ievt->mask & IN_MODIFY) { - K = DirectoryWatcher::EventKind::Modified; - } - if (ievt->mask & IN_MOVED_TO) { - K = DirectoryWatcher::EventKind::Added; - } - if (ievt->mask & IN_DELETE) { - K = DirectoryWatcher::EventKind::Removed; - } - - assert(ievt->len > 0 && "expected a filename from inotify"); - SmallString<256> fullPath{pathToWatch}; - sys::path::append(fullPath, ievt->name); - - Optional statusOpt; - if (K != DirectoryWatcher::EventKind::Removed) { - statusOpt = getFileStatus(fullPath); - if (!statusOpt.hasValue()) - K = DirectoryWatcher::EventKind::Removed; - } - INotifyEvent iEvt{K, fullPath.str(), statusOpt}; - iEvents.push_back(iEvt); - } - - if (!iEvents.empty()) - evtQueue->onDirectoryEvents(iEvents); - } -} - -bool DirectoryWatcher::Implementation::initialize(StringRef Path, - EventReceiver Receiver, - bool waitInitialSync, - std::string &errorMsg) { - auto error = [&](StringRef msg) -> bool { - errorMsg = msg; - errorMsg += ": "; - errorMsg += llvm::sys::StrError(); - return true; - }; - - auto evtQueue = std::make_shared(std::move(Receiver)); - - inotifyFD = inotify_init(); - if (inotifyFD == -1) - return error("inotify_init failed"); - - std::string pathToWatch = Path; - int wd = inotify_add_watch( - inotifyFD, pathToWatch.c_str(), - IN_MOVED_TO | IN_DELETE | IN_MODIFY | IN_DELETE_SELF | IN_ONLYDIR); - if (wd == -1) - return error("inotify_add_watch failed"); - - std::thread watchThread( - std::bind(runWatcher, pathToWatch, inotifyFD, evtQueue)); - watchThread.detach(); - - auto initialScan = std::make_shared(); - auto runScan = [pathToWatch, initialScan, evtQueue]() { - initialScan->scanDirectory(pathToWatch); - evtQueue->onInitialScan(std::move(initialScan)); - }; - - if (waitInitialSync) { - runScan(); - } else { - std::thread scanThread(runScan); - scanThread.detach(); - } - - return false; -} - -void DirectoryWatcher::Implementation::stopListening() { - if (inotifyFD == -1) - return; - close(inotifyFD); - inotifyFD = -1; -} diff --git a/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h b/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h deleted file mode 100644 index 826e9bea4fd..00000000000 --- a/lib/DirectoryWatcher/DirectoryWatcher-mac.inc.h +++ /dev/null @@ -1,214 +0,0 @@ -//===- DirectoryWatcher-mac.inc.h - Mac-platform directory listening ------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include - -struct DirectoryWatcher::Implementation { - bool initialize(StringRef Path, EventReceiver Receiver, - bool waitInitialSync, std::string &Error); - ~Implementation() { - stopFSEventStream(); - }; -private: - FSEventStreamRef EventStream = nullptr; - - bool setupFSEventStream(StringRef path, EventReceiver receiver, - dispatch_queue_t queue, - std::shared_ptr initialScanPtr); - void stopFSEventStream(); -}; - -namespace { -struct EventStreamContextData { - std::string WatchedPath; - DirectoryWatcher::EventReceiver Receiver; - std::shared_ptr InitialScan; - - EventStreamContextData(std::string watchedPath, DirectoryWatcher::EventReceiver receiver, - std::shared_ptr initialScanPtr) - : WatchedPath(std::move(watchedPath)), - Receiver(std::move(receiver)), - InitialScan(std::move(initialScanPtr)) { - } - - static void dispose(const void *ctx) { - delete static_cast(ctx); - } -}; -} - -static void eventStreamCallback( - ConstFSEventStreamRef stream, - void *clientCallBackInfo, - size_t numEvents, - void *eventPaths, - const FSEventStreamEventFlags eventFlags[], - const FSEventStreamEventId eventIds[]) { - auto *ctx = static_cast(clientCallBackInfo); - - std::vector Events; - for (size_t i = 0; i < numEvents; ++i) { - StringRef path = ((const char **)eventPaths)[i]; - const FSEventStreamEventFlags flags = eventFlags[i]; - if (!(flags & kFSEventStreamEventFlagItemIsFile)) { - if ((flags & kFSEventStreamEventFlagItemRemoved) && path == ctx->WatchedPath) { - DirectoryWatcher::Event Evt{DirectoryWatcher::EventKind::DirectoryDeleted, path, llvm::sys::TimePoint<>{} }; - Events.push_back(Evt); - break; - } - continue; - } - DirectoryWatcher::EventKind K = DirectoryWatcher::EventKind::Modified; - bool hasAddedFlag = flags & (kFSEventStreamEventFlagItemCreated | - kFSEventStreamEventFlagItemRenamed); - bool hasRemovedFlag = flags & kFSEventStreamEventFlagItemRemoved; - Optional statusOpt; - // NOTE: With low latency sometimes for a file that is moved inside the - // directory, or for a file that is removed from the directory, the flags - // have both 'renamed' and 'removed'. We use getting the file status as a - // way to distinguish between the two. - if (hasAddedFlag) { - statusOpt = getFileStatus(path); - if (statusOpt.hasValue()) { - K = DirectoryWatcher::EventKind::Added; - } else { - K = DirectoryWatcher::EventKind::Removed; - } - } else if (hasRemovedFlag) { - K = DirectoryWatcher::EventKind::Removed; - } else { - statusOpt = getFileStatus(path); - if (!statusOpt.hasValue()) { - K = DirectoryWatcher::EventKind::Removed; - } - } - - if (ctx->InitialScan && K == DirectoryWatcher::EventKind::Added) { - // For the first time we get the events, check that we haven't already - // sent the 'added' event at the initial scan. - if (ctx->InitialScan->FileIDSet.count(statusOpt->getUniqueID())) { - // Already reported this event at the initial directory scan. - continue; - } - } - - llvm::sys::TimePoint<> modTime{}; - if (statusOpt.hasValue()) - modTime = statusOpt->getLastModificationTime(); - DirectoryWatcher::Event Evt{K, path, modTime}; - Events.push_back(Evt); - } - - // We won't need to check again later on. - ctx->InitialScan.reset(); - - if (!Events.empty()) { - ctx->Receiver(Events, /*isInitial=*/false); - } -} - -bool DirectoryWatcher::Implementation::setupFSEventStream(StringRef path, - EventReceiver receiver, - dispatch_queue_t queue, - std::shared_ptr initialScanPtr) { - if (path.empty()) - return true; - - CFMutableArrayRef pathsToWatch = CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); - CFStringRef cfPathStr = CFStringCreateWithBytes(nullptr, (const UInt8 *)path.data(), path.size(), kCFStringEncodingUTF8, false); - CFArrayAppendValue(pathsToWatch, cfPathStr); - CFRelease(cfPathStr); - CFAbsoluteTime latency = 0.0; // Latency in seconds. - - std::string realPath; - { - SmallString<128> Storage; - StringRef P = llvm::Twine(path).toNullTerminatedStringRef(Storage); - char Buffer[PATH_MAX]; - // Use ::realpath to get the real path name - if (::realpath(P.begin(), Buffer) != nullptr) - realPath = Buffer; - else - realPath = path; - } - - EventStreamContextData *ctxData = - new EventStreamContextData(std::move(realPath), std::move(receiver), - std::move(initialScanPtr)); - FSEventStreamContext context; - context.version = 0; - context.info = ctxData; - context.retain = nullptr; - context.release = EventStreamContextData::dispose; - context.copyDescription = nullptr; - - EventStream = FSEventStreamCreate(nullptr, - eventStreamCallback, - &context, - pathsToWatch, - kFSEventStreamEventIdSinceNow, - latency, - kFSEventStreamCreateFlagFileEvents | - kFSEventStreamCreateFlagNoDefer); - CFRelease(pathsToWatch); - if (!EventStream) { - return true; - } - FSEventStreamSetDispatchQueue(EventStream, queue); - FSEventStreamStart(EventStream); - return false; -} - -void DirectoryWatcher::Implementation::stopFSEventStream() { - if (!EventStream) - return; - FSEventStreamStop(EventStream); - FSEventStreamInvalidate(EventStream); - FSEventStreamRelease(EventStream); - EventStream = nullptr; -} - -bool DirectoryWatcher::Implementation::initialize(StringRef Path, - EventReceiver Receiver, bool waitInitialSync, std::string &Error) { - auto initialScan = std::make_shared(); - - dispatch_queue_t queue = dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); - dispatch_semaphore_t initScanSema = dispatch_semaphore_create(0); - dispatch_semaphore_t setupFSEventsSema = dispatch_semaphore_create(0); - - std::string copiedPath = Path; - dispatch_retain(initScanSema); - dispatch_retain(setupFSEventsSema); - dispatch_async(queue, ^{ - // Wait for the event stream to be setup before doing the initial scan, - // to make sure we won't miss any events. - dispatch_semaphore_wait(setupFSEventsSema, DISPATCH_TIME_FOREVER); - initialScan->scanDirectory(copiedPath); - Receiver(initialScan->getAsFileEvents(), /*isInitial=*/true); - dispatch_semaphore_signal(initScanSema); - dispatch_release(setupFSEventsSema); - dispatch_release(initScanSema); - }); - bool fsErr = setupFSEventStream(Path, Receiver, queue, initialScan); - dispatch_semaphore_signal(setupFSEventsSema); - - if (waitInitialSync) { - dispatch_semaphore_wait(initScanSema, DISPATCH_TIME_FOREVER); - } - dispatch_release(setupFSEventsSema); - dispatch_release(initScanSema); - dispatch_release(queue); - - if (fsErr) { - raw_string_ostream(Error) << "failed to setup FSEvents stream for path: " << Path; - return true; - } - - return false; -} diff --git a/lib/DirectoryWatcher/DirectoryWatcher.cpp b/lib/DirectoryWatcher/DirectoryWatcher.cpp deleted file mode 100644 index 0feaacefe7a..00000000000 --- a/lib/DirectoryWatcher/DirectoryWatcher.cpp +++ /dev/null @@ -1,151 +0,0 @@ -//===- DirectoryWatcher.cpp - Listens for directory file changes ----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// \file -/// \brief Utility class for listening for file system changes in a directory. -//===----------------------------------------------------------------------===// - -#include "clang/DirectoryWatcher/DirectoryWatcher.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/Support/FileSystem.h" -#include "llvm/Support/raw_ostream.h" - -using namespace clang; -using namespace llvm; - -static Optional getFileStatus(StringRef path) { - sys::fs::file_status Status; - std::error_code EC = status(path, Status); - if (EC) - return None; - return Status; -} - -namespace llvm { -// Specialize DenseMapInfo for sys::fs::UniqueID. -template <> struct DenseMapInfo { - static sys::fs::UniqueID getEmptyKey() { - return sys::fs::UniqueID{DenseMapInfo::getEmptyKey(), - DenseMapInfo::getEmptyKey()}; - } - - static sys::fs::UniqueID getTombstoneKey() { - return sys::fs::UniqueID{DenseMapInfo::getTombstoneKey(), - DenseMapInfo::getEmptyKey()}; - } - - static unsigned getHashValue(const sys::fs::UniqueID &val) { - return DenseMapInfo>::getHashValue( - std::make_pair(val.getDevice(), val.getFile())); - } - - static bool isEqual(const sys::fs::UniqueID &LHS, const sys::fs::UniqueID &RHS) { - return LHS == RHS; - } -}; -} - -namespace { -/// Used for initial directory scan. -/// -/// Note that the caller must ensure serial access to it. It is not thread safe -/// to access it without additional protection. -struct DirectoryScan { - DenseSet FileIDSet; - std::vector>> Files; - - void scanDirectory(StringRef Path) { - using namespace llvm::sys; - - std::error_code EC; - for (auto It = fs::directory_iterator(Path, EC), End = fs::directory_iterator(); - !EC && It != End; It.increment(EC)) { - auto status = getFileStatus(It->path()); - if (!status.hasValue()) - continue; - Files.push_back(std::make_tuple(It->path(), status->getLastModificationTime())); - FileIDSet.insert(status->getUniqueID()); - } - } - - std::vector getAsFileEvents() const { - std::vector Events; - for (const auto &info : Files) { - DirectoryWatcher::Event Event{DirectoryWatcher::EventKind::Added, std::get<0>(info), std::get<1>(info)}; - Events.push_back(std::move(Event)); - } - return Events; - } -}; -} - -// Add platform-specific functionality. - -#if !defined(__has_include) -# define __has_include(x) 0 -#endif - -#if __has_include() -# include "DirectoryWatcher-mac.inc.h" -#elif __has_include() -# include "DirectoryWatcher-linux.inc.h" -#else - -struct DirectoryWatcher::Implementation { - bool initialize(StringRef Path, EventReceiver Receiver, - bool waitInitialSync, std::string &Error) { - Error = "directory listening not supported for this platform"; - return true; - } -}; - -#endif - - -DirectoryWatcher::DirectoryWatcher() - : Impl(*new Implementation()) {} - -DirectoryWatcher::~DirectoryWatcher() { - delete &Impl; -} - -std::unique_ptr DirectoryWatcher::create(StringRef Path, - EventReceiver Receiver, bool waitInitialSync, std::string &Error) { - using namespace llvm::sys; - - if (!fs::exists(Path)) { - std::error_code EC = fs::create_directories(Path); - if (EC) { - Error = EC.message(); - return nullptr; - } - } - - bool IsDir; - std::error_code EC = fs::is_directory(Path, IsDir); - if (EC) { - Error = EC.message(); - return nullptr; - } - if (!IsDir) { - Error = "path is not a directory: "; - Error += Path; - return nullptr; - } - - std::unique_ptr DirWatch; - DirWatch.reset(new DirectoryWatcher()); - auto &Impl = DirWatch->Impl; - bool hasError = Impl.initialize(Path, std::move(Receiver), waitInitialSync, Error); - if (hasError) - return nullptr; - - return DirWatch; -} diff --git a/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp new file mode 100644 index 00000000000..986ebc5d95f --- /dev/null +++ b/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp @@ -0,0 +1,345 @@ +//===- DirectoryWatcher-linux.cpp - Linux-platform directory watching -----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" +#include "llvm/Support/AlignOf.h" +#include "llvm/Support/Errno.h" +#include "llvm/Support/Mutex.h" +#include "llvm/Support/Path.h" +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace { + +using namespace llvm; +using namespace clang; + +/// Pipe for inter-thread synchronization - for epoll-ing on multiple +/// conditions. It is meant for uni-directional 1:1 signalling - specifically: +/// no multiple consumers, no data passing. Thread waiting for signal should +/// poll the FDRead. Signalling thread should call signal() which writes single +/// character to FDRead. +struct SemaphorePipe { + // Expects two file-descriptors opened as a pipe in the canonical POSIX + // order: pipefd[0] refers to the read end of the pipe. pipefd[1] refers to + // the write end of the pipe. + SemaphorePipe(int pipefd[2]) + : FDRead(pipefd[0]), FDWrite(pipefd[1]), OwnsFDs(true) {} + SemaphorePipe(const SemaphorePipe &) = delete; + void operator=(const SemaphorePipe &) = delete; + SemaphorePipe(SemaphorePipe &&other) + : FDRead(other.FDRead), FDWrite(other.FDWrite), + OwnsFDs(other.OwnsFDs) // Someone could have moved from the other + // instance before. + { + other.OwnsFDs = false; + }; + + void signal() { + ssize_t Result = llvm::sys::RetryAfterSignal(-1, write, FDWrite, "A", 1); + assert(Result != -1); + } + ~SemaphorePipe() { + if (OwnsFDs) { + close(FDWrite); + close(FDRead); + } + } + const int FDRead; + const int FDWrite; + bool OwnsFDs; + + static llvm::Optional create() { + int InotifyPollingStopperFDs[2]; + if (pipe2(InotifyPollingStopperFDs, O_CLOEXEC) == -1) + return llvm::None; + return SemaphorePipe(InotifyPollingStopperFDs); + } +}; + +/// Mutex-protected queue of Events. +class EventQueue { + std::mutex Mtx; + std::condition_variable NonEmpty; + std::queue Events; + +public: + void push_back(const DirectoryWatcher::Event::EventKind K, + StringRef Filename) { + { + std::unique_lock L(Mtx); + Events.emplace(K, Filename); + } + NonEmpty.notify_one(); + } + + // Blocks on caller thread and uses codition_variable to wait until there's an + // event to return. + DirectoryWatcher::Event pop_front_blocking() { + std::unique_lock L(Mtx); + while (true) { + // Since we might have missed all the prior notifications on NonEmpty we + // have to check the queue first (under lock). + if (!Events.empty()) { + DirectoryWatcher::Event Front = Events.front(); + Events.pop(); + return Front; + } + NonEmpty.wait(L, [this]() { return !Events.empty(); }); + } + } +}; + +class DirectoryWatcherLinux : public clang::DirectoryWatcher { +public: + DirectoryWatcherLinux( + llvm::StringRef WatchedDirPath, + std::function, bool)> Receiver, + bool WaitForInitialSync, int InotifyFD, int InotifyWD, + SemaphorePipe &&InotifyPollingStopSignal); + + ~DirectoryWatcherLinux() override { + StopWork(); + InotifyPollingThread.join(); + EventsReceivingThread.join(); + inotify_rm_watch(InotifyFD, InotifyWD); + llvm::sys::RetryAfterSignal(-1, close, InotifyFD); + } + +private: + const std::string WatchedDirPath; + // inotify file descriptor + int InotifyFD = -1; + // inotify watch descriptor + int InotifyWD = -1; + + EventQueue Queue; + + // Make sure lifetime of Receiver fully contains lifetime of + // EventsReceivingThread. + std::function, bool)> Receiver; + + // Consumes inotify events and pushes directory watcher events to the Queue. + void InotifyPollingLoop(); + std::thread InotifyPollingThread; + // Using pipe so we can epoll two file descriptors at once - inotify and + // stopping condition. + SemaphorePipe InotifyPollingStopSignal; + + // Does the initial scan of the directory - directly calling Receiver, + // bypassing the Queue. Both InitialScan and EventReceivingLoop use Receiver + // which isn't necessarily thread-safe. + void InitialScan(); + + // Processing events from the Queue. + // In case client doesn't want to do the initial scan synchronously + // (WaitForInitialSync=false in ctor) we do the initial scan at the beginning + // of this thread. + std::thread EventsReceivingThread; + // Push event of WatcherGotInvalidated kind to the Queue to stop the loop. + // Both InitialScan and EventReceivingLoop use Receiver which isn't + // necessarily thread-safe. + void EventReceivingLoop(); + + // Stops all the async work. Reentrant. + void StopWork() { + Queue.push_back(DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, + ""); + InotifyPollingStopSignal.signal(); + } +}; + +void DirectoryWatcherLinux::InotifyPollingLoop() { + // We want to be able to read ~30 events at once even in the worst case + // (obscenely long filenames). + constexpr size_t EventBufferLength = + 30 * (sizeof(struct inotify_event) + NAME_MAX + 1); + // http://man7.org/linux/man-pages/man7/inotify.7.html + // Some systems cannot read integer variables if they are not + // properly aligned. On other systems, incorrect alignment may + // decrease performance. Hence, the buffer used for reading from + // the inotify file descriptor should have the same alignment as + // struct inotify_event. + + auto ManagedBuffer = + llvm::make_unique>(); + char *const Buf = ManagedBuffer->buffer; + + const int EpollFD = epoll_create1(EPOLL_CLOEXEC); + if (EpollFD == -1) { + StopWork(); + return; + } + auto EpollFDGuard = llvm::make_scope_exit([EpollFD]() { close(EpollFD); }); + + struct epoll_event EventSpec; + EventSpec.events = EPOLLIN; + EventSpec.data.fd = InotifyFD; + if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyFD, &EventSpec) == -1) { + StopWork(); + return; + } + + EventSpec.data.fd = InotifyPollingStopSignal.FDRead; + if (epoll_ctl(EpollFD, EPOLL_CTL_ADD, InotifyPollingStopSignal.FDRead, + &EventSpec) == -1) { + StopWork(); + return; + } + + std::array EpollEventBuffer; + + while (true) { + const int EpollWaitResult = llvm::sys::RetryAfterSignal( + -1, epoll_wait, EpollFD, EpollEventBuffer.data(), + EpollEventBuffer.size(), /*timeout=*/-1 /*== infinity*/); + if (EpollWaitResult == -1) { + StopWork(); + return; + } + + // Multiple epoll_events can be received for a single file descriptor per + // epoll_wait call. + for (const auto &EpollEvent : EpollEventBuffer) { + if (EpollEvent.data.fd == InotifyPollingStopSignal.FDRead) { + StopWork(); + return; + } + } + + // epoll_wait() always return either error or >0 events. Since there was no + // event for stopping, it must be an inotify event ready for reading. + ssize_t NumRead = llvm::sys::RetryAfterSignal(-1, read, InotifyFD, Buf, + EventBufferLength); + for (char *P = Buf; P < Buf + NumRead;) { + if (P + sizeof(struct inotify_event) > Buf + NumRead) { + StopWork(); + llvm_unreachable("an incomplete inotify_event was read"); + return; + } + + struct inotify_event *Event = reinterpret_cast(P); + P += sizeof(struct inotify_event) + Event->len; + + if (Event->mask & (IN_CREATE | IN_MODIFY | IN_MOVED_TO | IN_DELETE) && + Event->len <= 0) { + StopWork(); + llvm_unreachable("expected a filename from inotify"); + return; + } + + if (Event->mask & (IN_CREATE | IN_MOVED_TO | IN_MODIFY)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Modified, + Event->name); + } else if (Event->mask & (IN_DELETE | IN_MOVED_FROM)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::Removed, + Event->name); + } else if (Event->mask & (IN_DELETE_SELF | IN_MOVE_SELF)) { + Queue.push_back(DirectoryWatcher::Event::EventKind::WatchedDirRemoved, + ""); + StopWork(); + return; + } else if (Event->mask & IN_IGNORED) { + StopWork(); + return; + } else { + StopWork(); + llvm_unreachable("Unknown event type."); + return; + } + } + } +} + +void DirectoryWatcherLinux::InitialScan() { + this->Receiver(getAsFileEvents(scanDirectory(WatchedDirPath)), + /*IsInitial=*/true); +} + +void DirectoryWatcherLinux::EventReceivingLoop() { + while (true) { + DirectoryWatcher::Event Event = this->Queue.pop_front_blocking(); + this->Receiver(Event, false); + if (Event.Kind == + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated) { + StopWork(); + return; + } + } +} + +DirectoryWatcherLinux::DirectoryWatcherLinux( + StringRef WatchedDirPath, + std::function, bool)> Receiver, + bool WaitForInitialSync, int InotifyFD, int InotifyWD, + SemaphorePipe &&InotifyPollingStopSignal) + : WatchedDirPath(WatchedDirPath), InotifyFD(InotifyFD), + InotifyWD(InotifyWD), Receiver(Receiver), + InotifyPollingStopSignal(std::move(InotifyPollingStopSignal)) { + + InotifyPollingThread = std::thread([this]() { InotifyPollingLoop(); }); + // We have no guarantees about thread safety of the Receiver which is being + // used in both InitialScan and EventReceivingLoop. We shouldn't run these + // only synchronously. + if (WaitForInitialSync) { + InitialScan(); + EventsReceivingThread = std::thread([this]() { EventReceivingLoop(); }); + } else { + EventsReceivingThread = std::thread([this]() { + // FIXME: We might want to terminate an async initial scan early in case + // of a failure in EventsReceivingThread. + InitialScan(); + EventReceivingLoop(); + }); + } +} + +} // namespace + +std::unique_ptr clang::DirectoryWatcher::create( + StringRef Path, + std::function, bool)> Receiver, + bool WaitForInitialSync) { + if (Path.empty()) + return nullptr; + + const int InotifyFD = inotify_init1(IN_CLOEXEC); + if (InotifyFD == -1) + return nullptr; + + const int InotifyWD = inotify_add_watch( + InotifyFD, Path.str().c_str(), + IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_EXCL_UNLINK | IN_MODIFY | + IN_MOVED_FROM | IN_MOVE_SELF | IN_MOVED_TO | IN_ONLYDIR | IN_IGNORED); + if (InotifyWD == -1) + return nullptr; + + auto InotifyPollingStopper = SemaphorePipe::create(); + + if (!InotifyPollingStopper) + return nullptr; + + return llvm::make_unique( + Path, Receiver, WaitForInitialSync, InotifyFD, InotifyWD, + std::move(*InotifyPollingStopper)); +} \ No newline at end of file diff --git a/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp b/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp new file mode 100644 index 00000000000..3df79ac48a4 --- /dev/null +++ b/lib/DirectoryWatcher/mac/DirectoryWatcher-mac.cpp @@ -0,0 +1,233 @@ +//===- DirectoryWatcher-mac.cpp - Mac-platform directory watching ---------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "DirectoryScanner.h" +#include "clang/DirectoryWatcher/DirectoryWatcher.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Path.h" +#include + +using namespace llvm; +using namespace clang; + +static FSEventStreamRef createFSEventStream( + StringRef Path, + std::function, bool)>, + dispatch_queue_t); +static void stopFSEventStream(FSEventStreamRef); + +namespace { + +class DirectoryWatcherMac : public clang::DirectoryWatcher { +public: + DirectoryWatcherMac( + FSEventStreamRef EventStream, + std::function, bool)> + Receiver, + llvm::StringRef WatchedDirPath) + : EventStream(EventStream), Receiver(Receiver), + WatchedDirPath(WatchedDirPath) {} + + ~DirectoryWatcherMac() override { + stopFSEventStream(EventStream); + EventStream = nullptr; + // Now it's safe to use Receiver as the only other concurrent use would have + // been in EventStream processing. + Receiver(DirectoryWatcher::Event( + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""), + false); + } + +private: + FSEventStreamRef EventStream; + std::function, bool)> Receiver; + const std::string WatchedDirPath; +}; + +struct EventStreamContextData { + std::string WatchedPath; + std::function, bool)> Receiver; + + EventStreamContextData( + std::string &&WatchedPath, + std::function, bool)> + Receiver) + : WatchedPath(std::move(WatchedPath)), Receiver(Receiver) {} + + // Needed for FSEvents + static void dispose(const void *ctx) { + delete static_cast(ctx); + } +}; +} // namespace + +constexpr const FSEventStreamEventFlags StreamInvalidatingFlags = + kFSEventStreamEventFlagUserDropped | kFSEventStreamEventFlagKernelDropped | + kFSEventStreamEventFlagMustScanSubDirs; + +constexpr const FSEventStreamEventFlags ModifyingFileEvents = + kFSEventStreamEventFlagItemCreated | kFSEventStreamEventFlagItemRenamed | + kFSEventStreamEventFlagItemModified; + +static void eventStreamCallback(ConstFSEventStreamRef Stream, + void *ClientCallBackInfo, size_t NumEvents, + void *EventPaths, + const FSEventStreamEventFlags EventFlags[], + const FSEventStreamEventId EventIds[]) { + auto *ctx = static_cast(ClientCallBackInfo); + + std::vector Events; + for (size_t i = 0; i < NumEvents; ++i) { + StringRef Path = ((const char **)EventPaths)[i]; + const FSEventStreamEventFlags Flags = EventFlags[i]; + + if (Flags & StreamInvalidatingFlags) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } else if (!(Flags & kFSEventStreamEventFlagItemIsFile)) { + // Subdirectories aren't supported - if some directory got removed it + // must've been the watched directory itself. + if ((Flags & kFSEventStreamEventFlagItemRemoved) && + Path == ctx->WatchedPath) { + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}); + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + break; + } + // No support for subdirectories - just ignore everything. + continue; + } else if (Flags & kFSEventStreamEventFlagItemRemoved) { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + continue; + } else if (Flags & ModifyingFileEvents) { + if (!getFileStatus(Path).hasValue()) { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Removed, + llvm::sys::path::filename(Path)); + } else { + Events.emplace_back(DirectoryWatcher::Event::EventKind::Modified, + llvm::sys::path::filename(Path)); + } + continue; + } + + // default + Events.emplace_back(DirectoryWatcher::Event{ + DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}); + llvm_unreachable("Unknown FSEvent type."); + } + + if (!Events.empty()) { + ctx->Receiver(Events, /*IsInitial=*/false); + } +} + +FSEventStreamRef createFSEventStream( + StringRef Path, + std::function, bool)> Receiver, + dispatch_queue_t Queue) { + if (Path.empty()) + return nullptr; + + CFMutableArrayRef PathsToWatch = [&]() { + CFMutableArrayRef PathsToWatch = + CFArrayCreateMutable(nullptr, 0, &kCFTypeArrayCallBacks); + CFStringRef CfPathStr = + CFStringCreateWithBytes(nullptr, (const UInt8 *)Path.data(), + Path.size(), kCFStringEncodingUTF8, false); + CFArrayAppendValue(PathsToWatch, CfPathStr); + CFRelease(CfPathStr); + return PathsToWatch; + }(); + + FSEventStreamContext Context = [&]() { + std::string RealPath; + { + SmallString<128> Storage; + StringRef P = llvm::Twine(Path).toNullTerminatedStringRef(Storage); + char Buffer[PATH_MAX]; + if (::realpath(P.begin(), Buffer) != nullptr) + RealPath = Buffer; + else + RealPath = Path; + } + + FSEventStreamContext Context; + Context.version = 0; + Context.info = new EventStreamContextData(std::move(RealPath), Receiver); + Context.retain = nullptr; + Context.release = EventStreamContextData::dispose; + Context.copyDescription = nullptr; + return Context; + }(); + + FSEventStreamRef Result = FSEventStreamCreate( + nullptr, eventStreamCallback, &Context, PathsToWatch, + kFSEventStreamEventIdSinceNow, /* latency in seconds */ 0.0, + kFSEventStreamCreateFlagFileEvents | kFSEventStreamCreateFlagNoDefer); + CFRelease(PathsToWatch); + + return Result; +} + +void stopFSEventStream(FSEventStreamRef EventStream) { + if (!EventStream) + return; + FSEventStreamStop(EventStream); + FSEventStreamInvalidate(EventStream); + FSEventStreamRelease(EventStream); +} + +std::unique_ptr clang::DirectoryWatcher::create( + StringRef Path, + std::function, bool)> Receiver, + bool WaitForInitialSync) { + dispatch_queue_t Queue = + dispatch_queue_create("DirectoryWatcher", DISPATCH_QUEUE_SERIAL); + + if (Path.empty()) + return nullptr; + + auto EventStream = createFSEventStream(Path, Receiver, Queue); + if (!EventStream) { + return nullptr; + } + + std::unique_ptr Result = + llvm::make_unique(EventStream, Receiver, Path); + + // We need to copy the data so the lifetime is ok after a const copy is made + // for the block. + const std::string CopiedPath = Path; + + auto InitWork = ^{ + // We need to start watching the directory before we start scanning in order + // to not miss any event. By dispatching this on the same serial Queue as + // the FSEvents will be handled we manage to start watching BEFORE the + // inital scan and handling events ONLY AFTER the scan finishes. + FSEventStreamSetDispatchQueue(EventStream, Queue); + FSEventStreamStart(EventStream); + // We need to decrement the ref count for Queue as initialize() will return + // and FSEvents has incremented it. Since we have to wait for FSEvents to + // take ownership it's the easiest to do it here rather than main thread. + dispatch_release(Queue); + Receiver(getAsFileEvents(scanDirectory(CopiedPath)), /*IsInitial=*/true); + }; + + if (WaitForInitialSync) { + dispatch_sync(Queue, InitWork); + } else { + dispatch_async(Queue, InitWork); + } + + return Result; +} diff --git a/lib/IndexDataStore/IndexDataStore.cpp b/lib/IndexDataStore/IndexDataStore.cpp index 6f207b4d95e..863cd1d38ac 100644 --- a/lib/IndexDataStore/IndexDataStore.cpp +++ b/lib/IndexDataStore/IndexDataStore.cpp @@ -115,18 +115,20 @@ bool IndexDataStoreImpl::startEventListening(bool waitInitialSync, std::string & IndexDataStore::UnitEventKind K; StringRef UnitName = sys::path::filename(evt.Filename); switch (evt.Kind) { - case DirectoryWatcher::EventKind::Added: - K = IndexDataStore::UnitEventKind::Added; break; - case DirectoryWatcher::EventKind::Removed: + case DirectoryWatcher::Event::EventKind::Removed: K = IndexDataStore::UnitEventKind::Removed; break; - case DirectoryWatcher::EventKind::Modified: + case DirectoryWatcher::Event::EventKind::Modified: K = IndexDataStore::UnitEventKind::Modified; break; - case DirectoryWatcher::EventKind::DirectoryDeleted: + case DirectoryWatcher::Event::EventKind::WatchedDirRemoved: K = IndexDataStore::UnitEventKind::DirectoryDeleted; UnitName = StringRef(); break; + case DirectoryWatcher::Event::EventKind::WatcherGotInvalidated: + K = IndexDataStore::UnitEventKind::Failure; + UnitName = StringRef(); + break; } - UnitEvents.push_back(IndexDataStore::UnitEvent{K, UnitName, evt.ModTime}); + UnitEvents.push_back(IndexDataStore::UnitEvent{K, UnitName}); } if (auto handler = localUnitEventHandlerData->getHandler()) { @@ -136,9 +138,11 @@ bool IndexDataStoreImpl::startEventListening(bool waitInitialSync, std::string & }; DirWatcher = DirectoryWatcher::create(UnitPath.str(), OnUnitsChange, - waitInitialSync, Error); - if (!DirWatcher) + waitInitialSync); + if (!DirWatcher) { + Error = "failed to create directory watcher"; return true; + } return false; } diff --git a/tools/IndexStore/IndexStore.cpp b/tools/IndexStore/IndexStore.cpp index c761b5cbea0..1ff05cfcdef 100644 --- a/tools/IndexStore/IndexStore.cpp +++ b/tools/IndexStore/IndexStore.cpp @@ -36,18 +36,6 @@ static indexstore_string_ref_t toIndexStoreString(StringRef str) { return indexstore_string_ref_t{ str.data(), str.size() }; } -static timespec toTimeSpec(sys::TimePoint<> tp) { - std::chrono::seconds sec = std::chrono::time_point_cast( - tp).time_since_epoch(); - std::chrono::nanoseconds nsec = - std::chrono::time_point_cast(tp - sec) - .time_since_epoch(); - timespec ts; - ts.tv_sec = sec.count(); - ts.tv_nsec = nsec.count(); - return ts; -} - //===----------------------------------------------------------------------===// // Fatal error handling //===----------------------------------------------------------------------===// @@ -164,14 +152,14 @@ indexstore_unit_event_get_kind(indexstore_unit_event_t c_evt) { auto *evt = static_cast(c_evt); indexstore_unit_event_kind_t k; switch (evt->Kind) { - case IndexDataStore::UnitEventKind::Added: - k = INDEXSTORE_UNIT_EVENT_ADDED; break; case IndexDataStore::UnitEventKind::Removed: k = INDEXSTORE_UNIT_EVENT_REMOVED; break; case IndexDataStore::UnitEventKind::Modified: k = INDEXSTORE_UNIT_EVENT_MODIFIED; break; case IndexDataStore::UnitEventKind::DirectoryDeleted: k = INDEXSTORE_UNIT_EVENT_DIRECTORY_DELETED; break; + case IndexDataStore::UnitEventKind::Failure: + k = INDEXSTORE_UNIT_EVENT_FAILURE; break; } return k; } @@ -182,12 +170,6 @@ indexstore_unit_event_get_unit_name(indexstore_unit_event_t c_evt) { return toIndexStoreString(evt->UnitName); } -timespec -indexstore_unit_event_get_modification_time(indexstore_unit_event_t c_evt) { - auto *evt = static_cast(c_evt); - return toTimeSpec(evt->ModTime); -} - #if INDEXSTORE_HAS_BLOCKS void indexstore_store_set_unit_event_handler(indexstore_t c_store, @@ -596,32 +578,6 @@ indexstore_store_get_unit_name_from_output_path(indexstore_t store, return nameLen; } -bool -indexstore_store_get_unit_modification_time(indexstore_t c_store, - const char *unit_name, - int64_t *seconds, - int64_t *nanoseconds, - indexstore_error_t *c_error) { - IndexDataStore *store = static_cast(c_store); - std::string error; - // FIXME: This provides mod time with second-only accuracy. - auto optModTime = IndexUnitReader::getModificationTimeForUnit(unit_name, - store->getFilePath(), error); - if (!optModTime) { - if (c_error) - *c_error = new IndexStoreError{ error }; - return true; - } - - timespec ts = toTimeSpec(*optModTime); - if (seconds) - *seconds = ts.tv_sec; - if (nanoseconds) - *nanoseconds = ts.tv_nsec; - - return false; -} - indexstore_unit_reader_t indexstore_unit_reader_create(indexstore_t c_store, const char *unit_name, indexstore_error_t *c_error) { @@ -656,20 +612,6 @@ indexstore_unit_reader_get_provider_version(indexstore_unit_reader_t rdr) { return toIndexStoreString(reader->getProviderVersion()); } -void -indexstore_unit_reader_get_modification_time(indexstore_unit_reader_t rdr, - int64_t *seconds, - int64_t *nanoseconds) { - auto reader = static_cast(rdr); - // FIXME: This provides mod time with second-only accuracy. - sys::TimePoint<> timeVal = reader->getModificationTime(); - timespec ts = toTimeSpec(timeVal); - if (seconds) - *seconds = ts.tv_sec; - if (nanoseconds) - *nanoseconds = ts.tv_nsec; -} - bool indexstore_unit_reader_is_system_unit(indexstore_unit_reader_t rdr) { auto reader = static_cast(rdr); diff --git a/tools/IndexStore/IndexStore.exports b/tools/IndexStore/IndexStore.exports index 9196ad60c38..3cd8a6513ef 100644 --- a/tools/IndexStore/IndexStore.exports +++ b/tools/IndexStore/IndexStore.exports @@ -3,7 +3,6 @@ indexstore_error_dispose indexstore_format_version indexstore_store_create indexstore_store_dispose -indexstore_store_get_unit_modification_time indexstore_store_get_unit_name_from_output_path indexstore_store_units_apply indexstore_store_units_apply_f @@ -48,15 +47,12 @@ indexstore_unit_dependency_get_modulename indexstore_unit_dependency_get_name indexstore_unit_dependency_is_system indexstore_unit_event_get_kind -indexstore_unit_event_get_modification_time -indexstore_unit_event_get_unit_name indexstore_unit_event_notification_get_event indexstore_unit_event_notification_get_events_count indexstore_unit_event_notification_is_initial indexstore_unit_reader_create indexstore_unit_reader_dispose indexstore_unit_reader_get_main_file -indexstore_unit_reader_get_modification_time indexstore_unit_reader_get_module_name indexstore_unit_reader_get_provider_identifier indexstore_unit_reader_get_provider_version diff --git a/tools/c-index-test/core_main.cpp b/tools/c-index-test/core_main.cpp index 69261eb2a7f..23f3851e864 100644 --- a/tools/c-index-test/core_main.cpp +++ b/tools/c-index-test/core_main.cpp @@ -765,24 +765,23 @@ static int watchDirectory(StringRef dirPath) { OS << "-- " << Events.size() << " :\n"; for (auto evt : Events) { switch (evt.Kind) { - case DirectoryWatcher::EventKind::Added: - OS << "added: "; break; - case DirectoryWatcher::EventKind::Modified: + case DirectoryWatcher::Event::EventKind::Modified: OS << "modified: "; break; - case DirectoryWatcher::EventKind::Removed: + case DirectoryWatcher::Event::EventKind::Removed: OS << "removed: "; break; - case DirectoryWatcher::EventKind::DirectoryDeleted: + case DirectoryWatcher::Event::EventKind::WatchedDirRemoved: OS << "dir deleted: "; break; + case DirectoryWatcher::Event::EventKind::WatcherGotInvalidated: + OS << "watcher got invalidated: "; break; } OS << evt.Filename << '\n'; } }; - std::string Error; auto watcher = DirectoryWatcher::create(dirPath, receiver, - /*waitInitialSync=*/true, Error); + /*waitInitialSync=*/true); if (!watcher) { - errs() << "failed creating directory watcher: " << Error << '\n'; + errs() << "failed creating directory watcher" << '\n'; return 1; } diff --git a/unittests/DirectoryWatcher/CMakeLists.txt b/unittests/DirectoryWatcher/CMakeLists.txt index c96424e2986..ab66eab4779 100644 --- a/unittests/DirectoryWatcher/CMakeLists.txt +++ b/unittests/DirectoryWatcher/CMakeLists.txt @@ -10,13 +10,4 @@ target_link_libraries(DirectoryWatcherTests PRIVATE clangDirectoryWatcher clangBasic - ) - -if(APPLE) - check_include_files("CoreServices/CoreServices.h" HAVE_CORESERVICES_H) - if(HAVE_CORESERVICES_H) - set(DWT_LINK_FLAGS "${DWT_LINK_FLAGS} -framework CoreServices") - set_property(TARGET DirectoryWatcherTests APPEND_STRING PROPERTY - LINK_FLAGS ${DWT_LINK_FLAGS}) - endif() -endif() + ) \ No newline at end of file diff --git a/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp b/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp index 5fec06a64d6..a2c50fc7d00 100644 --- a/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp +++ b/unittests/DirectoryWatcher/DirectoryWatcherTest.cpp @@ -1,17 +1,19 @@ //===- unittests/DirectoryWatcher/DirectoryWatcherTest.cpp ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "clang/DirectoryWatcher/DirectoryWatcher.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/Mutex.h" #include "llvm/Support/Path.h" +#include "llvm/Support/raw_ostream.h" #include "gtest/gtest.h" #include +#include #include #include @@ -20,309 +22,405 @@ using namespace llvm::sys; using namespace llvm::sys::fs; using namespace clang; +namespace clang { +static bool operator==(const DirectoryWatcher::Event &lhs, + const DirectoryWatcher::Event &rhs) { + return lhs.Filename == rhs.Filename && + static_cast(lhs.Kind) == static_cast(rhs.Kind); +} +} // namespace clang + namespace { -class EventCollection { - SmallVector Events; -public: - EventCollection() = default; - explicit EventCollection(ArrayRef events) { - append(events); - } +struct DirectoryWatcherTestFixture { + std::string TestRootDir; + std::string TestWatchedDir; - void append(ArrayRef events) { - Events.append(events.begin(), events.end()); + DirectoryWatcherTestFixture() { + SmallString<128> pathBuf; + std::error_code UniqDirRes = createUniqueDirectory("dirwatcher", pathBuf); + assert(!UniqDirRes); + TestRootDir = pathBuf.str(); + path::append(pathBuf, "watch"); + TestWatchedDir = pathBuf.str(); + std::error_code CreateDirRes = create_directory(TestWatchedDir, false); + assert(!CreateDirRes); } - bool empty() const { return Events.empty(); } - size_t size() const { return Events.size(); } - void clear() { Events.clear(); } - - bool hasEvents(ArrayRef filenames, - ArrayRef kinds, - ArrayRef stats) const { - assert(filenames.size() == kinds.size()); - assert(filenames.size() == stats.size()); - SmallVector evts = Events; - bool hadError = false; - for (unsigned i = 0, e = filenames.size(); i < e; ++i) { - StringRef fname = filenames[i]; - DirectoryWatcher::EventKind kind = kinds[i]; - file_status stat = stats[i]; - auto it = std::find_if(evts.begin(), evts.end(), [&](const DirectoryWatcher::Event &evt)->bool { - return path::filename(evt.Filename) == fname; - }); - if (it == evts.end()) { - hadError = err(Twine("expected filename '"+fname+"' not found")); - continue; - } - if (it->Kind != kind) { - hadError = err(Twine("filename '" + fname + "' has event kind " + - std::to_string((int)it->Kind) + ", expected ") + - std::to_string((int)kind)); - } - if (it->Kind != DirectoryWatcher::EventKind::Removed && - it->ModTime != stat.getLastModificationTime()) - hadError = err(Twine("filename '"+fname+"' has different mod time")); - evts.erase(it); - } - for (const auto &evt : evts) { - hadError = err(Twine("unexpected filename '"+path::filename(evt.Filename)+"' found")); - } - return !hadError; - } + ~DirectoryWatcherTestFixture() { remove_directories(TestRootDir); } - bool hasAdded(ArrayRef filenames, ArrayRef stats) const { - std::vector kinds{ - filenames.size(), DirectoryWatcher::EventKind::Added }; - return hasEvents(filenames, kinds, stats); + SmallString<128> getPathInWatched(const std::string &testFile) { + SmallString<128> pathBuf; + pathBuf = TestWatchedDir; + path::append(pathBuf, testFile); + return pathBuf; } - bool hasRemoved(ArrayRef filenames) const { - std::vector kinds{ - filenames.size(), DirectoryWatcher::EventKind::Removed }; - std::vector stats{ filenames.size(), file_status{} }; - return hasEvents(filenames, kinds, stats); + void addFile(const std::string &testFile) { + Expected ft = openNativeFileForWrite(getPathInWatched(testFile), + CD_CreateNew, OF_None); + if (ft) { + closeFile(*ft); + } else { + llvm::errs() << llvm::toString(ft.takeError()) << "\n"; + llvm::errs() << getPathInWatched(testFile) << "\n"; + llvm_unreachable("Couldn't create test file."); + } } -private: - bool err(Twine msg) const { - SmallString<128> buf; - llvm::errs() << msg.toStringRef(buf) << '\n'; - return true; + void deleteFile(const std::string &testFile) { + std::error_code EC = + remove(getPathInWatched(testFile), /*IgnoreNonExisting=*/false); + ASSERT_FALSE(EC); } }; -struct EventOccurrence { - std::vector Events; - bool IsInitial; -}; +std::string eventKindToString(const DirectoryWatcher::Event::EventKind K) { + switch (K) { + case DirectoryWatcher::Event::EventKind::Removed: + return "Removed"; + case DirectoryWatcher::Event::EventKind::Modified: + return "Modified"; + case DirectoryWatcher::Event::EventKind::WatchedDirRemoved: + return "WatchedDirRemoved"; + case DirectoryWatcher::Event::EventKind::WatcherGotInvalidated: + return "WatcherGotInvalidated"; + } + llvm_unreachable("unknown event kind"); +} -class DirectoryWatcherTest: public std::enable_shared_from_this { - std::string WatchedDir; - std::string TempDir; - std::unique_ptr DirWatcher; +struct VerifyingConsumer { + std::vector ExpectedInitial; + std::vector ExpectedNonInitial; + std::vector OptionalNonInitial; + std::vector UnexpectedInitial; + std::vector UnexpectedNonInitial; + std::mutex Mtx; + std::condition_variable ResultIsReady; + + VerifyingConsumer( + const std::vector &ExpectedInitial, + const std::vector &ExpectedNonInitial, + const std::vector &OptionalNonInitial = {}) + : ExpectedInitial(ExpectedInitial), + ExpectedNonInitial(ExpectedNonInitial), + OptionalNonInitial(OptionalNonInitial) {} + + // This method is used by DirectoryWatcher. + void consume(DirectoryWatcher::Event E, bool IsInitial) { + if (IsInitial) + consumeInitial(E); + else + consumeNonInitial(E); + } - std::condition_variable Condition; - std::mutex Mutex; - std::deque EvtOccurs; + void consumeInitial(DirectoryWatcher::Event E) { + std::unique_lock L(Mtx); + auto It = std::find(ExpectedInitial.begin(), ExpectedInitial.end(), E); + if (It == ExpectedInitial.end()) { + UnexpectedInitial.push_back(E); + } else { + ExpectedInitial.erase(It); + } + if (result()) + ResultIsReady.notify_one(); + } -public: - void init() { - SmallString<128> pathBuf; - std::error_code EC = createUniqueDirectory("dirwatcher", pathBuf); - ASSERT_FALSE(EC); - TempDir = pathBuf.str(); - path::append(pathBuf, "watch"); - WatchedDir = pathBuf.str(); - EC = create_directory(WatchedDir); - ASSERT_FALSE(EC); + void consumeNonInitial(DirectoryWatcher::Event E) { + std::unique_lock L(Mtx); + auto It = + std::find(ExpectedNonInitial.begin(), ExpectedNonInitial.end(), E); + if (It == ExpectedNonInitial.end()) { + auto OptIt = + std::find(OptionalNonInitial.begin(), OptionalNonInitial.end(), E); + if (OptIt != OptionalNonInitial.end()) { + OptionalNonInitial.erase(OptIt); + } else { + UnexpectedNonInitial.push_back(E); + } + } else { + ExpectedNonInitial.erase(It); + } + if (result()) + ResultIsReady.notify_one(); } - ~DirectoryWatcherTest() { - stopWatching(); - remove_directories(TempDir); + // This method is used by DirectoryWatcher. + void consume(llvm::ArrayRef Es, bool IsInitial) { + for (const auto &E : Es) + consume(E, IsInitial); } -public: - StringRef getWatchedDir() const { - return WatchedDir; + // Not locking - caller has to lock Mtx. + llvm::Optional result() const { + if (ExpectedInitial.empty() && ExpectedNonInitial.empty() && + UnexpectedInitial.empty() && UnexpectedNonInitial.empty()) + return true; + if (!UnexpectedInitial.empty() || !UnexpectedNonInitial.empty()) + return false; + return llvm::None; } - void addFile(StringRef filename, file_status &stat) { - SmallString<128> pathBuf; - pathBuf = TempDir; - path::append(pathBuf, filename); - Expected ft = openNativeFileForWrite(pathBuf, CD_CreateNew, OF_None); - ASSERT_TRUE((bool)ft); - closeFile(*ft); - - SmallString<128> newPath; - newPath = WatchedDir; - path::append(newPath, filename); - std::error_code EC = rename(pathBuf, newPath); - ASSERT_FALSE(EC); + // This method is used by tests. + // \returns true on success + bool blockUntilResult() { + std::unique_lock L(Mtx); + while (true) { + if (result()) + return *result(); - EC = status(newPath, stat); - ASSERT_FALSE(EC); + ResultIsReady.wait(L, [this]() { return result().hasValue(); }); + } + return false; // Just to make compiler happy. } - void addFiles(ArrayRef filenames, std::vector &stats) { - for (auto fname : filenames) { - file_status stat; - addFile(fname, stat); - stats.push_back(stat); + void printUnmetExpectations(llvm::raw_ostream &OS) { + if (!ExpectedInitial.empty()) { + OS << "Expected but not seen initial events: \n"; + for (const auto &E : ExpectedInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!ExpectedNonInitial.empty()) { + OS << "Expected but not seen non-initial events: \n"; + for (const auto &E : ExpectedNonInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!UnexpectedInitial.empty()) { + OS << "Unexpected initial events seen: \n"; + for (const auto &E : UnexpectedInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } + } + if (!UnexpectedNonInitial.empty()) { + OS << "Unexpected non-initial events seen: \n"; + for (const auto &E : UnexpectedNonInitial) { + OS << eventKindToString(E.Kind) << " " << E.Filename << "\n"; + } } } +}; - void addFiles(ArrayRef filenames) { - std::vector stats; - addFiles(filenames, stats); +void checkEventualResultWithTimeout(VerifyingConsumer &TestConsumer) { + std::packaged_task task( + [&TestConsumer]() { return TestConsumer.blockUntilResult(); }); + std::future WaitForExpectedStateResult = task.get_future(); + std::thread worker(std::move(task)); + worker.detach(); + + EXPECT_TRUE(WaitForExpectedStateResult.wait_for(std::chrono::seconds(3)) == + std::future_status::ready) + << "The expected result state wasn't reached before the time-out."; + EXPECT_TRUE(TestConsumer.result().hasValue()); + if (TestConsumer.result().hasValue()) { + EXPECT_TRUE(*TestConsumer.result()); } + if ((TestConsumer.result().hasValue() && !TestConsumer.result().getValue()) || + !TestConsumer.result().hasValue()) + TestConsumer.printUnmetExpectations(llvm::outs()); +} - void removeFile(StringRef filename) { - SmallString<128> pathBuf; - pathBuf = WatchedDir; - path::append(pathBuf, filename); - std::error_code EC = remove(pathBuf, /*IgnoreNonExisting=*/false); - ASSERT_FALSE(EC); - } +} // namespace - void removeFiles(ArrayRef filenames) { - for (auto fname : filenames) { - removeFile(fname); - } - } +TEST(DirectoryWatcherTest, InitialScanSync) { + DirectoryWatcherTestFixture fixture; - /// \returns true for error. - bool startWatching(bool waitInitialSync) { - std::weak_ptr weakThis = shared_from_this(); - auto receiver = [weakThis](ArrayRef events, bool isInitial) { - if (auto this_ = weakThis.lock()) - this_->onEvents(events, isInitial); - }; - std::string error; - DirWatcher = DirectoryWatcher::create(getWatchedDir(), receiver, - waitInitialSync, error); - return DirWatcher == nullptr; - } + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); - void stopWatching() { - DirWatcher.reset(); - } + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}, + {}}; - /// \returns None if the timeout is reached before getting an event. - Optional getNextEvent(unsigned timeout_seconds = 5) { - std::unique_lock lck(Mutex); - auto pred = [&]()->bool { return !EvtOccurs.empty(); }; - bool gotEvent = Condition.wait_for(lck, std::chrono::seconds(timeout_seconds), pred); - if (!gotEvent) - return None; - - EventOccurrence occur = EvtOccurs.front(); - EvtOccurs.pop_front(); - return occur; - } + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); - EventOccurrence getNextEventImmediately() { - std::lock_guard LG(Mutex); - assert(!EvtOccurs.empty()); - EventOccurrence occur = EvtOccurs.front(); - EvtOccurs.pop_front(); - return occur; - } + checkEventualResultWithTimeout(TestConsumer); +} -private: - void onEvents(ArrayRef events, bool isInitial) { - std::lock_guard LG(Mutex); - EvtOccurs.push_back({events, isInitial}); - Condition.notify_all(); - } -}; +TEST(DirectoryWatcherTest, InitialScanAsync) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}, + {}}; + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/false); + + checkEventualResultWithTimeout(TestConsumer); } -TEST(DirectoryWatcherTest, initialScan) { - auto t = std::make_shared(); - t->init(); - - std::vector fnames = {"a", "b", "c"}; - std::vector stats; - t->addFiles(fnames, stats); - - bool err = t->startWatching(/*waitInitialSync=*/true); - ASSERT_FALSE(err); - - auto evt = t->getNextEventImmediately(); - EXPECT_TRUE(evt.IsInitial); - EventCollection coll1{evt.Events}; - EXPECT_TRUE(coll1.hasAdded(fnames, stats)); - - StringRef additionalFname = "d"; - file_status additionalStat; - t->addFile(additionalFname, additionalStat); - auto evtOpt = t->getNextEvent(); - ASSERT_TRUE(evtOpt.hasValue()); - EXPECT_FALSE(evtOpt->IsInitial); - EventCollection coll2{evtOpt->Events}; - EXPECT_TRUE(coll2.hasAdded({additionalFname}, {additionalStat})); +TEST(DirectoryWatcherTest, AddFiles) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, + {{DirectoryWatcher::Event::EventKind::Modified, "a"}, + {DirectoryWatcher::Event::EventKind::Modified, "b"}, + {DirectoryWatcher::Event::EventKind::Modified, "c"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + fixture.addFile("a"); + fixture.addFile("b"); + fixture.addFile("c"); + + checkEventualResultWithTimeout(TestConsumer); } -TEST(DirectoryWatcherTest, fileEvents) { - auto t = std::make_shared(); - t->init(); +TEST(DirectoryWatcherTest, ModifyFile) { + DirectoryWatcherTestFixture fixture; - bool err = t->startWatching(/*waitInitialSync=*/false); - ASSERT_FALSE(err); + fixture.addFile("a"); - auto evt = t->getNextEvent(); - ASSERT_TRUE(evt.hasValue()); - EXPECT_TRUE(evt->IsInitial); - EXPECT_TRUE(evt->Events.empty()); - return; + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + // modify the file { - std::vector fnames = {"a", "b"}; - std::vector stats; - t->addFiles(fnames, stats); - - EventCollection coll{}; - while (coll.size() < 2) { - evt = t->getNextEvent(); - ASSERT_TRUE(evt.hasValue()); - coll.append(evt->Events); - } - EXPECT_TRUE(coll.hasAdded(fnames, stats)); - } - { - std::vector fnames = {"b", "c"}; - std::vector stats; - t->addFiles(fnames, stats); - - EventCollection coll{}; - while (coll.size() < 2) { - evt = t->getNextEvent(); - ASSERT_TRUE(evt.hasValue()); - coll.append(evt->Events); - } - EXPECT_TRUE(coll.hasAdded(fnames, stats)); + std::error_code error; + llvm::raw_fd_ostream bStream(fixture.getPathInWatched("a"), error, + CD_OpenExisting); + assert(!error); + bStream << "foo"; } - { - std::vector fnames = {"a", "c"}; - std::vector stats; - t->addFiles(fnames, stats); - t->removeFile("b"); - - EventCollection coll{}; - while (coll.size() < 3) { - evt = t->getNextEvent(); - ASSERT_TRUE(evt.hasValue()); - coll.append(evt->Events); - } - EXPECT_TRUE(coll.hasEvents( - std::vector{"a", "b", "c"}, - std::vector{ - DirectoryWatcher::EventKind::Added, - DirectoryWatcher::EventKind::Removed, - DirectoryWatcher::EventKind::Added, + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, DeleteFile) { + DirectoryWatcherTestFixture fixture; + + fixture.addFile("a"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + {{DirectoryWatcher::Event::EventKind::Removed, "a"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); }, - std::vector{ - stats[0], - file_status{}, - stats[1], - })); - } + /*waitForInitialSync=*/true); + + fixture.deleteFile("a"); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, DeleteWatchedDir) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, + {{DirectoryWatcher::Event::EventKind::WatchedDirRemoved, ""}, + {DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + remove_directories(fixture.TestWatchedDir); + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, InvalidatedWatcher) { + DirectoryWatcherTestFixture fixture; + + VerifyingConsumer TestConsumer{ + {}, {{DirectoryWatcher::Event::EventKind::WatcherGotInvalidated, ""}}}; + { - std::vector fnames = {"a", "c"}; - t->removeFiles(fnames); - - EventCollection coll{}; - while (coll.size() < 2) { - evt = t->getNextEvent(); - ASSERT_TRUE(evt.hasValue()); - coll.append(evt->Events); + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + } // DW is destructed here. + + checkEventualResultWithTimeout(TestConsumer); +} + +TEST(DirectoryWatcherTest, ChangeMetadata) { + DirectoryWatcherTestFixture fixture; + fixture.addFile("a"); + + VerifyingConsumer TestConsumer{ + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}, + // We don't expect any notification for file having access file changed. + {}, + // Given the timing we are ok with receiving the duplicate event. + {{DirectoryWatcher::Event::EventKind::Modified, "a"}}}; + + auto DW = DirectoryWatcher::create( + fixture.TestWatchedDir, + [&TestConsumer](llvm::ArrayRef Events, + bool IsInitial) { + TestConsumer.consume(Events, IsInitial); + }, + /*waitForInitialSync=*/true); + + { // Change access and modification time of file a. + Expected HopefullyTheFD = llvm::sys::fs::openNativeFileForWrite( + fixture.getPathInWatched("a"), CD_OpenExisting, OF_None); + if (!HopefullyTheFD) { + llvm::outs() << HopefullyTheFD.takeError(); } - EXPECT_TRUE(coll.hasRemoved(fnames)); + + const int FD = HopefullyTheFD.get(); + const TimePoint<> NewTimePt = + std::chrono::system_clock::now() - std::chrono::minutes(1); + + std::error_code setTimeRes = + llvm::sys::fs::setLastAccessAndModificationTime(FD, NewTimePt, + NewTimePt); + assert(!setTimeRes); } + + checkEventualResultWithTimeout(TestConsumer); }