test: Add diffs and simplify flows

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ jobs:
           run: make tarball
 
         - name: Run Tests
-          run: make test && make unittest
+          run: make test
 
         - name: Check Wake Formatting
           run: ./bin/wake-format.native-cpp14-release --auto --dry-run
@@ -96,13 +96,13 @@ jobs:
       strategy:
         matrix:
           include:
-            - target: alpine
-              dockerfile: alpine
-              extra_docker_build_args: ''
-              extra_docker_run_args: ''
-              build_cmd: make -C wake-* static
-              test_cmd: tar xvJf wake-*/wake-static_* && cd tests && ../wake-*/bin/wake -x Unit
-              install_src_glob: build/wake-*/wake-static_*
+            # - target: alpine
+            #   dockerfile: alpine
+            #   extra_docker_build_args: ''
+            #   extra_docker_run_args: ''
+            #   build_cmd: make -C wake-* static
+            #   test_cmd: tar xvJf wake-*/wake-static_* && cd tests && ../wake-*/bin/wake -x Unit
+            #   install_src_glob: build/wake-*/wake-static_*
 
             - target: centos_7_6
               dockerfile: centos-7.6

Makefile

@@ -73,13 +73,9 @@ format:
 		bin/wake-format.native-cpp14-release -i $$FILES; \
 	fi || true
 
-test:		wake.db
+test:		all
 	$(WAKE_ENV) ./bin/wake --in test_wake runTests
 
-# We need to bootstrap job-cache to run the tests
-unittest:	all
-	$(WAKE_ENV) ./bin/wake --in test_wake runUnitTests
-
 remoteCacheTests:	all
 	$(WAKE_ENV) ./bin/wake -d -x 'testPostgres Unit'
 

build.wake

@@ -106,13 +106,13 @@ def all variant =
 def doInstall dest kind =
     # kick off in parallel, but rsc and rsc_tool cannot be built at the same time right now.
     # Force them to be serial relative to each other.
-    def install_rsc_bins =
-        require Pass rsc = buildRSC Unit
-        require Pass rsc = installAs "{dest}/bin/wake-remote-cache" rsc
-        require Pass rsc_tool = buildRSCTool Unit
-        require Pass rsc_tool = installAs "{dest}/bin/wake-remote-cache-tool" rsc_tool
-
-        Pass (Pair rsc rsc_tool)
+    #def install_rsc_bins =
+    #    require Pass rsc = buildRSC Unit
+    #    require Pass rsc = installAs "{dest}/bin/wake-remote-cache" rsc
+    #    require Pass rsc_tool = buildRSCTool Unit
+    #    require Pass rsc_tool = installAs "{dest}/bin/wake-remote-cache-tool" rsc_tool
+    #
+    #    Pass (Pair rsc rsc_tool)
 
     def install_log_viewer =
         buildWakeLogViewer Unit
@@ -140,10 +140,10 @@ def doInstall dest kind =
     require Pass binfiles = install_binfiles
 
     # Rust builds are slow, delay blocking on them for as long as possible
-    require Pass (Pair rsc rsc_tool) = install_rsc_bins
+    #require Pass (Pair rsc rsc_tool) = install_rsc_bins
     require Pass log_viewer = install_log_viewer
 
-    (readme, log_viewer, rsc, rsc_tool, binfiles ++ datfiles)
+    (readme, log_viewer, binfiles ++ datfiles)
     | Pass
 
 # Replace @VERSION@ with 'release'
@@ -309,14 +309,10 @@ export def static _: Result Path Error =
     | getJobOutput
 
 from test_wake import topic wakeTestBinary
-from test_wake import topic wakeUnitTestBinary
 
 publish wakeTestBinary =
     defaultWake, Nil
 
-publish wakeUnitTestBinary =
-    buildWakeUnit, Nil
-
 def defaultWake Unit =
     require Pass wakeVisible = doInstall "tmp" "default"
 

src/job_cache/job_cache_impl_common.cpp

@@ -186,7 +186,7 @@ static void copy(int src_fd, int dst_fd) {
   do {
     intptr_t written = sendfile(dst_fd, src_fd, &idx, size);
     if (written < 0) {
-      wcl::log::error("sendfile(src_fd = %d, NULL, dst_fd = %d, size = %d, 0): %s", src_fd, dst_fd,
+      wcl::log::error("sendfile(src_fd = %d, NULL, dst_fd = %d, size = %ld, 0): %s", src_fd, dst_fd,
                       buf.st_size, strerror(errno))
           .urgent()();
       exit(1);
@@ -389,4 +389,4 @@ wcl::optional<wcl::posix_error_t> sync_send_json_message(int fd, const JAST &jso
   return {};
 }
 
-}  // namespace job_cache
+}  // namespace job_cache

tests/job-cache/basic-ttl/pass.sh

@@ -2,6 +2,29 @@
 
 set -e
 WAKE="${1:+$1/wake}"
+
+cleanup () {
+  rm -f wake.db 2> /dev/null || true
+  rm -rf .cache-misses  2> /dev/null || true
+  rm -rf .cache-hit  2> /dev/null || true
+}
+
+expect_hit() {
+  if [ -z "$(ls -A .cache-hit)" ]; then
+    echo "Expected a cache hit but none found"
+    exit 1
+  fi
+}
+
+expect_miss() {
+  if [ -z "$(ls -A .cache-misses)" ]; then
+    echo "Expected a cache miss but none found"
+    exit 1
+  fi
+}
+
+echo "Cleaning up stale artifacts"
+
 rm wake.db 2> /dev/null || true
 rm -rf .cache-hit 2> /dev/null || true
 rm -rf .cache-misses 2> /dev/null || true
@@ -10,99 +33,61 @@ rm one.txt 2> /dev/null || true
 rm two.txt 2> /dev/null || true
 rm three.txt 2> /dev/null || true
 rm four.txt 2> /dev/null || true
+
+echo "Running test one to fill the cache"
 WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
-rm wake.db
-rm -rf .cache-misses
-sleep 1
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
-rm wake.db
-rm -rf .cache-misses
-sleep 1
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
-rm wake.db
-rm -rf .cache-misses
+cleanup
 sleep 1
 
-# We should still be under the limit here. This is to ensure we mark test one as used
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
-rm wake.db
-rm -rf .cache-misses
-if [ -z "$(ls -A .cache-hit)" ]; then
-  echo "No cache hit found"
-  exit 1
-fi
-sleep 4
-
-# Now we're going to go over. Hopefully dropping two and three, but keeping one and four
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
-rm wake.db
-rm -rf .cache-misses
+echo "Running test two to fill the cache"
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
+cleanup
+sleep 2
 
-# Now make sure we still get a hit on 1
-rm -rf .cache-hit  2> /dev/null || true
+echo "Running test three to fill the cache"
 WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
-rm wake.db
-if [ -z "$(ls -A .cache-hit)" ]; then
-  echo "No cache hit found"
-  exit 1
-fi
-if [ -d ".cache-misses" ]; then
-  echo "Found a cache miss"
-  exit 1
-fi
+cleanup
 sleep 1
 
+# Times:
+# one: 4 of 6 old
+# two: 3 of 6 old
+# three: 1 of 6 old
+echo "Running test one again. Should be a cache hit"
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
+expect_hit
+cleanup
+sleep 3
 
-# And check that we get misses on four and three
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
-rm wake.db
-if [ -z "$(ls -A .cache-misses)" ]; then
-  echo "Expected a cache miss!!"
-  exit 1
-fi
-rm -rf .cache-misses
-sleep 1
+# Times:
+# one: 7 of 6 old (evicted)
+# two: 6 of 6 old (evicted)
+# three: 4 of 6 old
 
+echo "Going over. Expecting one and two to be dropped"
 
+echo "Running test one again to refill the cache"
 WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test one
-rm wake.db
-if [ -d ".cache-misses" ]; then
-  echo "Found a cache miss"
-  exit 1
-fi
-rm -rf .cache-hit
+cleanup
 
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
-rm wake.db
-if [ -z "$(ls -A .cache-misses)" ]; then
-  echo "Expected a cache miss!!"
-  exit 1
-fi
-rm -rf .cache-misses
+echo "Running test three again. Should be a cache hit"
+WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test three
+expect_hit
+cleanup
+sleep 1
 
+# Times:
+# one: 1 of 6 old (refilled)
+# two: 7 of 6 old (evicted)
+# three: 5 of 6 old
 
+echo "Running test two again. Should be a cache miss"
 WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test two
 rm wake.db
-if [ -d ".cache-misses" ]; then
-  echo "Found a cache miss"
-  exit 1
-fi
-rm -rf .cache-hit
-
-WAKE_SHARED_CACHE_FAST_CLOSE=1 DEBUG_WAKE_SHARED_CACHE=1 WAKE_LOCAL_JOB_CACHE=.job-cache "${WAKE:-wake}" test four
-rm wake.db
-if [ -z "$(ls -A .cache-hit)" ]; then
-  echo "No cache hit found"
-  exit 1
-fi
-if [ -d ".cache-misses" ]; then
-  echo "Found a cache miss"
-  exit 1
-fi
-
+expect_miss
+cleanup
 
-# Cleanup
+# Cleanup job files 
 rm one.txt
 rm two.txt
 rm three.txt
-rm four.txt

tests/job-cache/basic-ttl/stdout

@@ -0,0 +1,16 @@
+Cleaning up stale artifacts
+Running test one to fill the cache
+Pass ("one.txt", Nil)
+Running test two to fill the cache
+Pass ("two.txt", Nil)
+Running test three to fill the cache
+Pass ("three.txt", Nil)
+Running test one again. Should be a cache hit
+Pass ("one.txt", Nil)
+Going over. Expecting one and two to be dropped
+Running test one again to refill the cache
+Pass ("one.txt", Nil)
+Running test three again. Should be a cache hit
+Pass ("three.txt", Nil)
+Running test two again. Should be a cache miss
+Pass ("two.txt", Nil)