diff --git a/R/test-files.R b/R/test-files.R
index 362ccd85c..6b6fd94ae 100644
--- a/R/test-files.R
+++ b/R/test-files.R
@@ -373,6 +373,10 @@ local_teardown_env <- function(frame = parent.frame()) {
 #' @export
 find_test_scripts <- function(path, filter = NULL, invert = FALSE, ..., full.names = TRUE, start_first = NULL) {
   files <- dir(path, "^test.*\\.[rR]$", full.names = full.names)
+  if (env_var_is_true("CI") || env_var_is_true("NOT_CRAN")) {
+    devfiles <- dir(path, "^dev-test.*\\.[rR]$", full.names = full.names)
+    files <- c(files, devfiles)
+  }
   files <- filter_test_scripts(files, filter, invert, ...)
   order_test_scripts(files, start_first)
 }
diff --git a/tests/testthat/test-test-files.R b/tests/testthat/test-test-files.R
index 42f51e076..1cfcd7710 100644
--- a/tests/testthat/test-test-files.R
+++ b/tests/testthat/test-test-files.R
@@ -8,16 +8,23 @@ test_that("stops on failure", {
 })
 
 test_that("runs all tests and records output", {
-  withr::local_envvar(TESTTHAT_PARALLEL = "FALSE")
+  withr::local_envvar(TESTTHAT_PARALLEL = "FALSE", NOT_CRAN = "false", CI = "false")
   res <- test_dir(test_path("test_dir"), reporter = "silent", stop_on_failure = FALSE)
   df <- as.data.frame(res)
   df$user <- df$system <- df$real <- df$result <- NULL
 
+  expect_equal(nrow(df), 17)
   local_reproducible_output(width = 200)
   local_edition(3) # set to 2 in ./test_dir
   expect_snapshot_output(print(df))
 })
 
+test_that("runs additional dev tests", {
+  withr::local_envvar(TESTTHAT_PARALLEL = "FALSE", NOT_CRAN = "true")
+  res <- test_dir(test_path("test_dir"), reporter = "silent", stop_on_failure = FALSE)
+  expect_equal(nrow(as.data.frame(res)), 18)
+})
+
 test_that("complains if no files", {
   withr::local_envvar(TESTTHAT_PARALLEL = "FALSE")
   path <- withr::local_tempfile()
diff --git a/tests/testthat/test_dir/dev-test-basic.R b/tests/testthat/test_dir/dev-test-basic.R
new file mode 100644
index 000000000..e708c4ded
--- /dev/null
+++ b/tests/testthat/test_dir/dev-test-basic.R
@@ -0,0 +1,3 @@
+test_that("test runs in developer mode only", {
+  expect_true(TRUE)
+})
diff --git a/vignettes/special-files.Rmd b/vignettes/special-files.Rmd
index d73a5ec12..a0628af55 100644
--- a/vignettes/special-files.Rmd
+++ b/vignettes/special-files.Rmd
@@ -24,7 +24,7 @@ This vignette describes the various special files that testthat understands: tes
 ## Test files
 
 These are the bread and butter of testthat.
-Test files live in `tests/testthat/`, start with either `test-` or `test_`, and end with `.r` or `.R`.
+Public test files live in `tests/testthat/`, start with either `test-` or `test_`, and end with `.r` or `.R`. Additional developer tests use the same pattern but prefixed with `dev-`.
 We recommend organising your test files so that there's a one-to-one correspondence between the files in `R/` and the files in `tests/testthat/` so that (e.g.) `R/myfile.R` has a matching `tests/testthat/test-myfile.R`.
 This correspondence is maintained by functions like `usethis::use_r()` and `usethis::use_test()` and is taken advantage of by functions like `devtools::test_active_file()` and `devtools::test_coverage_active_file()`.