From 1c31df3769c7d57765b3a61c50d2d57bef9a4c7c Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Fri, 20 Sep 2024 17:03:46 +0200
Subject: [PATCH 1/9] add require_complete_lastnight_part5 parameter, fixes
 #1196

---
 R/check_params.R                            |  2 +-
 R/g.part5.R                                 |  7 ++-
 R/g.part5.savetimeseries.R                  | 12 ++++-
 R/g.report.part5.R                          | 17 +++++-
 R/load_params.R                             |  3 +-
 man/GGIR.Rd                                 |  7 +++
 tests/testthat/test_recordingEndSleepHour.R | 58 +++++++++++++--------
 7 files changed, 78 insertions(+), 28 deletions(-)

diff --git a/R/check_params.R b/R/check_params.R
index 63ec72270..74ce70145 100644
--- a/R/check_params.R
+++ b/R/check_params.R
@@ -117,7 +117,7 @@ check_params = function(params_sleep = c(), params_metrics = c(),
     boolean_params = c("epochvalues2csv", "save_ms5rawlevels", "save_ms5raw_without_invalid",
                        "storefolderstructure", "dofirstpage", "visualreport", "week_weekend_aggregate.part5",
                        "do.part3.pdf", "outliers.only", "do.visual", "do.sibreport", "visualreport_without_invalid",
-                       "do.part2.pdf")
+                       "do.part2.pdf", "require_complete_lastnight_part5")
     character_params = c("save_ms5raw_format", "timewindow", "sep_reports", "sep_config",
                          "dec_reports", "dec_config")
     check_class("output", params = params_output, parnames = numeric_params, parclass = "numeric")
diff --git a/R/g.part5.R b/R/g.part5.R
index 7d59be15e..82d9383b8 100644
--- a/R/g.part5.R
+++ b/R/g.part5.R
@@ -600,7 +600,8 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
                                            includedaycrit.part5 = params_cleaning[["includedaycrit.part5"]],
                                            ID = ID,
                                            params_output = params_output,
-                                           params_247 = params_247)
+                                           params_247 = params_247,
+                                           timewindow = timewindowi)
                   }
                 }
               }
@@ -662,7 +663,9 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
                 if (length(GGIRversion) != 1) GGIRversion = sessionInfo()$otherPkgs$GGIR$Version
               }
               output$GGIRversion = GGIRversion
-              save(output, tail_expansion_log, GGIRversion,
+              # Capture final timestamp to ease filtering last window in g.report.part5
+              last_timestamp = time_POSIX[length(time_POSIX)] 
+              save(output, tail_expansion_log, GGIRversion, last_timestamp,
                    file = paste(metadatadir, ms5.out, "/", fnames.ms3[i], sep = ""))
             }
           }
diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R
index 60ff60d20..66e986a51 100644
--- a/R/g.part5.savetimeseries.R
+++ b/R/g.part5.savetimeseries.R
@@ -2,7 +2,8 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
                                   DaCleanFile = NULL,
                                   includedaycrit.part5 = 2/3, ID = NULL,
                                   params_output,
-                                  params_247 = NULL) {
+                                  params_247 = NULL,
+                                  timewindow = NULL) {
   
   ms5rawlevels = data.frame(date_time = ts$time, class_id = LEVELS,
                             # class_name = rep("",Nts),
@@ -23,6 +24,15 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   names(mdat)[which(names(mdat) == "nonwear")] = "invalidepoch"
   names(mdat)[which(names(mdat) == "diur")] = "SleepPeriodTime"
   mdat = mdat[,-which(names(mdat) == "date_time")]
+  
+  
+  if (params_output[["require_complete_lastnight_part5"]] == TRUE) {
+    last_timestamp = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H"))
+    if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) {
+      mdat$window[which(mdat$window == max(mdat$window))] = 0 
+    }
+  }
+  
   # Add invalid day indicator
   mdat$invalid_wakinghours = mdat$invalid_sleepperiod =  mdat$invalid_fullwindow = 100
   wakeup = which(diff(c(mdat$SleepPeriodTime,0)) == -1) + 1 # first epoch of each day
diff --git a/R/g.report.part5.R b/R/g.report.part5.R
index 1c512184c..9484750d5 100644
--- a/R/g.report.part5.R
+++ b/R/g.report.part5.R
@@ -47,21 +47,35 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
       
       x$wear_min_day = (1 - (x$nonwear_perc_day / 100)) * x$dur_day_min #valid minute during waking hours
       x$wear_perc_day = 100 - x$nonwear_perc_day #wear percentage during waking hours
+      x$lasttimestamp = as.numeric(x$lasttimestamp)
       
       minimumValidMinutesMM = 0 # default
       if (length(params_cleaning[["includedaycrit"]]) == 2) {
         minimumValidMinutesMM = params_cleaning[["includedaycrit"]][2] * 60
       }
+      if (params_output[["require_complete_lastnight_part5"]] == FALSE) {
+        x$lastnight = FALSE
+      } else {
+        x$lastnight = x$window_number == max(x$window_number)
+      }
       if (window == "WW" | window == "OO") {
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
+                          ((x$lastnight == TRUE & x$lasttimestamp > 15 & window == "WW") |
+                             (x$lastnight == TRUE & x$lasttimestamp > 9 & window == "OO") |
+                             x$lastnight == FALSE) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
       } else if (window == "MM") {
+        # Note: For MM it would also be good to put requirements on the 
+        # timing of the last timestamp to include/exlcude the last window
+        # but not done yet
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
+                          ((x$lastnight == TRUE & x$lasttimestamp > 9) |
+                             x$lastnight == FALSE) &
                           x$dur_day_spt_min >= (params_cleaning[["minimum_MM_length.part5"]] * 60) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
@@ -118,12 +132,13 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                  " take a few minutes\n"))
     }
     myfun = function(x, expectedCols = c()) {
-      tail_expansion_log = NULL
+      tail_expansion_log = last_timestamp = output = NULL
       load(file = x)
       cut = which(output[, 1] == "")
       if (length(cut) > 0 & length(cut) < nrow(output)) {
         output = output[-cut, which(colnames(output) != "")]
       }
+      output$lasttimestamp = as.numeric(format(last_timestamp, "%H"))
       out = as.matrix(output)
       if (length(expectedCols) > 0) {
         tmp = as.data.frame(matrix(0, 0, length(expectedCols)))
diff --git a/R/load_params.R b/R/load_params.R
index cd650a344..298f443aa 100644
--- a/R/load_params.R
+++ b/R/load_params.R
@@ -115,7 +115,8 @@ load_params = function(topic = c("sleep", "metrics", "rawdata",
                          do.sibreport = FALSE, do.part2.pdf = TRUE,
                          sep_reports = ",", sep_config = ",", 
                          dec_reports = ".", dec_config = ".", 
-                         visualreport_without_invalid = TRUE)
+                         visualreport_without_invalid = TRUE,
+                         require_complete_lastnight_part5 = FALSE)
 
   }
   if ("general" %in% topic) {
diff --git a/man/GGIR.Rd b/man/GGIR.Rd
index 5f530ad88..e01bf42e2 100755
--- a/man/GGIR.Rd
+++ b/man/GGIR.Rd
@@ -1539,6 +1539,13 @@ GGIR(mode = 1:5,
           If TRUE, then reports generated with \code{visualreport = TRUE} only show
           the windows with sufficiently valid data according to \code{includedaycrit}
           when viewingwindow = 1 or \code{includenightcrit} when viewingwindow = 2}
+      \item{require_complete_lastnight_part5}{    
+          Boolean (default = FALSE).
+          When set to TRUE: The last WW window is excluded if recording ends 
+          between midnight and 3pm; The last OO and MM window are excluded if 
+          recording ends between midnight and 9am. This to avoid risk that recording
+          end biases the sleep estimates for the last night.
+      }
     }
   }
 }
diff --git a/tests/testthat/test_recordingEndSleepHour.R b/tests/testthat/test_recordingEndSleepHour.R
index 5b6966945..100298d3d 100644
--- a/tests/testthat/test_recordingEndSleepHour.R
+++ b/tests/testthat/test_recordingEndSleepHour.R
@@ -3,7 +3,7 @@ context("recordingEndSleepHour")
 test_that("recordingEndSleepHour works as expected", {
   skip_on_cran()
   #=======================
-  create_test_acc_csv(Nmin = 2.5 * 1440) # ends at 20:45
+  create_test_acc_csv(Nmin = 3.5 * 1440) # ends at 20:45
   fn = "123A_testaccfile.csv"
   tz = "Europe/Amsterdam"
   # this should NOT trigger data expansion
@@ -13,7 +13,7 @@ test_that("recordingEndSleepHour works as expected", {
        visualreport = FALSE, do.report = c(), verbose = FALSE)
   rn = dir("output_test/meta/basic/", full.names = TRUE)
   load(rn[1])
-  expect_true(nrow(M$metashort) == 43020)
+  expect_true(nrow(M$metashort) == 60300)
   # expect_true(M$metashort$timestamp[nrow(M$metashort)] == "2016-06-25T20:44:55+0200")
   
   # errors and warnings work properly
@@ -46,7 +46,7 @@ test_that("recordingEndSleepHour works as expected", {
        minimum_MM_length.part5 = 6, verbose = FALSE)
   rn = dir("output_test/meta/basic/", full.names = TRUE)
   load(rn[1])
-  expect_true(nrow(M$metashort) > 43020) # metashort is expanded
+  expect_true(nrow(M$metashort) > 60300) # metashort is expanded
   
   # expanded time is not reports
   p2 = read.csv("output_test/results/part2_daysummary.csv")
@@ -54,17 +54,17 @@ test_that("recordingEndSleepHour works as expected", {
   
   p4 = read.csv("output_test/results/part4_nightsummary_sleep_cleaned.csv")
   p4full = read.csv("output_test/results/QC/part4_nightsummary_sleep_full.csv")
-  expect_equal(nrow(p4), 2) # Night 3 is not in the part 4 reports
-  expect_equal(nrow(p4full), 2) # Night 3 is not in the part 4 reports
-  expect_equal(sum(p4$sleeponset), 41.848)
-  expect_equal(sum(p4$wakeup), 62.336)
-  expect_equal(sum(p4$guider_onset), 41.851)
-  expect_equal(sum(p4$guider_wakeup), 62.34)
-  expect_equal(sum(p4$number_sib_sleepperiod), 73)
+  expect_equal(nrow(p4), 3) # Night 3 is not in the part 4 reports
+  expect_equal(nrow(p4full), 3) # Night 3 is not in the part 4 reports
+  expect_equal(sum(p4$sleeponset), 62.77)
+  expect_equal(sum(p4$wakeup), 93.505)
+  expect_equal(sum(p4$guider_onset), 62.778)
+  expect_equal(sum(p4$guider_wakeup), 93.511)
+  expect_equal(sum(p4$number_sib_sleepperiod), 114)
   expect_true(all(is.na(p4$longitudinal_axis)))
   
   p5 = read.csv("output_test/results/part5_daysummary_MM_L40M100V400_T5A5.csv")
-  expect_equal(nrow(p5), 3) # expanded day appears in MM report
+  expect_equal(nrow(p5), 4) # expanded day appears in MM report
   expect_true(p5$dur_day_spt_min[nrow(p5)] < 23*60) # but expanded time is not accounted for in estimates
   
   #================================================================
@@ -83,7 +83,7 @@ test_that("recordingEndSleepHour works as expected", {
        minimum_MM_length.part5 = 6, verbose = FALSE)
   rn = dir("output_test/meta/basic/", full.names = TRUE)
   load(rn[1])
-  expect_true(nrow(M$metashort) > 43020) # metashort is expanded
+  expect_true(nrow(M$metashort) > 60300) # metashort is expanded
   
   # expanded time is not reports
   p2 = read.csv("output_test/results/part2_daysummary.csv")
@@ -91,18 +91,18 @@ test_that("recordingEndSleepHour works as expected", {
   
   p4 = read.csv("output_test/results/part4_nightsummary_sleep_cleaned.csv")
   p4full = read.csv("output_test/results/QC/part4_nightsummary_sleep_full.csv")
-  expect_equal(nrow(p4), 2) # Night 3 is not in the part 4 reports
-  expect_equal(nrow(p4full), 2) # Night 3 is not in the part 4 reports
-  expect_equal(sum(p4$sleeponset), 41.581)
-  expect_equal(sum(p4$wakeup), 58.924)
-  expect_equal(sum(p4$guider_inbedStart), 41.17)
-  expect_equal(sum(p4$guider_inbedEnd), 64.427)
-  expect_equal(sum(p4$number_sib_sleepperiod), 18)
-  expect_equal(sum(p4$sleepefficiency), 0.358)
-  expect_equal(sum(p4$longitudinal_axis), 6)
+  expect_equal(nrow(p4), 3) # Night 3 is not in the part 4 reports
+  expect_equal(nrow(p4full), 3) # Night 3 is not in the part 4 reports
+  expect_equal(sum(p4$sleeponset), 62.1)
+  expect_equal(sum(p4$wakeup), 81.096)
+  expect_equal(sum(p4$guider_inbedStart), 53.171)
+  expect_equal(sum(p4$guider_inbedEnd), 86.853)
+  expect_equal(sum(p4$number_sib_sleepperiod), 24)
+  expect_equal(sum(p4$sleepefficiency), 0.501)
+  expect_equal(sum(p4$longitudinal_axis), 9)
   
   p5 = read.csv("output_test/results/part5_daysummary_MM_L40M100V400_T5A5.csv")
-  expect_equal(nrow(p5), 2) # expanded day appears in MM report
+  expect_equal(nrow(p5), 3) # expanded day appears in MM report
   expect_true(p5$dur_day_spt_min[nrow(p5)] < 23*60) # but expanded time is not accounted for in estimates
   
   # test that time series output has all expected columns including the multiple angle columns
@@ -116,6 +116,20 @@ test_that("recordingEndSleepHour works as expected", {
                     "class_id", "invalid_fullwindow", "invalid_sleepperiod",
                     "invalid_wakinghours", "timestamp") %in% names(mdat)))
   
+  #========================================================
+  # Test require_complete_lastnight_part5 = TRUE
+  create_test_acc_csv(Nmin = 3 * 1440) # ends at 8:45
+  # delete config.csv such that GGIR uses its own default parameter values
+  if (file.exists("output_test/config.csv"))  unlink("output_test/config.csv", recursive = TRUE)
+  # No warning, this should work
+  GGIR(datadir = fn, outputdir = getwd(), mode = 1:5,
+       studyname = "test", overwrite = TRUE, desiredtz = tz, 
+       verbose = FALSE, require_complete_lastnight_part5 = TRUE,
+       do.report = 5)
+  p5 = read.csv("output_test/results/part5_daysummary_MM_L40M100V400_T5A5.csv")
+  expect_equal(nrow(p5), 2) # expanded day appears in MM report
+  
+  
   if (file.exists("output_test"))  unlink("output_test", recursive = TRUE)
   if (file.exists(fn)) file.remove(fn)
 })

From 54a615a1ed07fd4090a5a38930c594fb814e5aa7 Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Fri, 20 Sep 2024 17:18:00 +0200
Subject: [PATCH 2/9] fix failing tests and update changelog, #1196

---
 NEWS.md                                 | 4 ++++
 man/g.part5.savetimeseries.Rd           | 6 +++++-
 tests/testthat/test_load_check_params.R | 2 +-
 3 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/NEWS.md b/NEWS.md
index fb827382e..bc5dda744 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,7 @@
+# CHANGES IN GGIR VERSION 3.1-?
+
+- Part 5: Add parameters require_complete_lastnight_part5 to control whether incomplete last nights are included. #1196
+
 # CHANGES IN GGIR VERSION 3.1-4
 
 - Part 3: Update threshold used for HorAngle to 60 degree, and auto-setting HASPT.ignore.invalid to NA when NotWorn guider is used. #1186
diff --git a/man/g.part5.savetimeseries.Rd b/man/g.part5.savetimeseries.Rd
index 820fd090c..194b05ff1 100644
--- a/man/g.part5.savetimeseries.Rd
+++ b/man/g.part5.savetimeseries.Rd
@@ -12,7 +12,8 @@
                               DaCleanFile = NULL,
                               includedaycrit.part5 = 2/3,
                               ID = NULL, params_output,
-                              params_247 = NULL)
+                              params_247 = NULL,
+                              timewindow = NULL)
 }
 \arguments{
   \item{ts}{
@@ -48,6 +49,9 @@
   \item{params_247}{
     See \link{GGIR}
   }
+  \item{timewindow}{
+    See \link{GGIR}
+  }
 }
 \value{
   Function does not provide output, it only prepare data for saving
diff --git a/tests/testthat/test_load_check_params.R b/tests/testthat/test_load_check_params.R
index 35a56b84d..ea8e2df3f 100644
--- a/tests/testthat/test_load_check_params.R
+++ b/tests/testthat/test_load_check_params.R
@@ -17,7 +17,7 @@ test_that("load_params can load parameters", {
   expect_equal(length(params$params_247), 22)
   expect_equal(length(params$params_cleaning), 24)
   expect_equal(length(params$params_phyact), 14)
-  expect_equal(length(params$params_output), 21)
+  expect_equal(length(params$params_output), 22)
   expect_equal(length(params$params_general), 17)
 
   params_sleep = params$params_sleep

From 536b0b1b95a615b693fd78d3e690a302ddae39ed Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Fri, 20 Sep 2024 17:43:27 +0200
Subject: [PATCH 3/9] also apply to WW timeseries and fix > sign #1196

---
 R/g.part5.savetimeseries.R | 3 +++
 R/g.report.part5.R         | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R
index 66e986a51..c3a1182f7 100644
--- a/R/g.part5.savetimeseries.R
+++ b/R/g.part5.savetimeseries.R
@@ -31,6 +31,9 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
     if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) {
       mdat$window[which(mdat$window == max(mdat$window))] = 0 
     }
+    if (timewindow == "WW" && last_timestamp < 15) {
+      mdat$window[which(mdat$window == max(mdat$window))] = 0 
+    }
   }
   
   # Add invalid day indicator
diff --git a/R/g.report.part5.R b/R/g.report.part5.R
index 9484750d5..4018155c3 100644
--- a/R/g.report.part5.R
+++ b/R/g.report.part5.R
@@ -62,8 +62,8 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &
-                          ((x$lastnight == TRUE & x$lasttimestamp > 15 & window == "WW") |
-                             (x$lastnight == TRUE & x$lasttimestamp > 9 & window == "OO") |
+                          ((x$lastnight == TRUE & x$lasttimestamp >= 15 & window == "WW") |
+                             (x$lastnight == TRUE & x$lasttimestamp >= 9 & window == "OO") |
                              x$lastnight == FALSE) &
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)

From 253a2b0a9d8bf1292ce9a5b0d2d55185371b5a1d Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 11:15:46 +0200
Subject: [PATCH 4/9] allow new g.report.part5 to run on older part5 milestone
 data #1196

---
 R/g.part5.savetimeseries.R |  5 ++---
 R/g.report.part5.R         | 11 +++++++++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R
index c3a1182f7..92ee73326 100644
--- a/R/g.part5.savetimeseries.R
+++ b/R/g.part5.savetimeseries.R
@@ -24,9 +24,8 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   names(mdat)[which(names(mdat) == "nonwear")] = "invalidepoch"
   names(mdat)[which(names(mdat) == "diur")] = "SleepPeriodTime"
   mdat = mdat[,-which(names(mdat) == "date_time")]
-  
-  
-  if (params_output[["require_complete_lastnight_part5"]] == TRUE) {
+  if ("require_complete_lastnight_part5" %in% names(params_output) &&
+      params_output[["require_complete_lastnight_part5"]] == TRUE) {
     last_timestamp = as.numeric(format(mdat$timestamp[length(mdat$timestamp)], "%H"))
     if ((timewindow == "MM" || timewindow == "OO") && last_timestamp < 9) {
       mdat$window[which(mdat$window == max(mdat$window))] = 0 
diff --git a/R/g.report.part5.R b/R/g.report.part5.R
index 4018155c3..559f53e14 100644
--- a/R/g.report.part5.R
+++ b/R/g.report.part5.R
@@ -138,7 +138,11 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
       if (length(cut) > 0 & length(cut) < nrow(output)) {
         output = output[-cut, which(colnames(output) != "")]
       }
-      output$lasttimestamp = as.numeric(format(last_timestamp, "%H"))
+      if (exists("last_timestamp") == TRUE) {
+        output$lasttimestamp = as.numeric(format(last_timestamp, "%H"))
+      } else {
+        output$lasttimestamp = Inf # use dummy value
+      }
       out = as.matrix(output)
       if (length(expectedCols) > 0) {
         tmp = as.data.frame(matrix(0, 0, length(expectedCols)))
@@ -284,8 +288,11 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                   sep = params_output[["sep_reports"]],
                   dec = params_output[["dec_reports"]])
                 # store all summaries in csv files with cleaning criteria
-                validdaysi = getValidDayIndices(x = OF3, window = uwi[j],
+                validdaysi = getValidDayIndices(x = OF3_clean, window = uwi[j],
                                                 params_cleaning = params_cleaning)
+                if ("lasttimestamp" %in% colnames(OF3_clean)) {
+                  OF3_clean = OF3_clean[, -which(colnames(OF3_clean) == "lasttimestamp")]
+                }
                 if (length(validdaysi) > 0) {
                   data.table::fwrite(
                     OF3_clean[validdaysi, ],

From 9a938309c63ae346c1e90e67457ca5955a797e38 Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 11:30:34 +0200
Subject: [PATCH 5/9] improve code comment

---
 tests/testthat/test_recordingEndSleepHour.R | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/testthat/test_recordingEndSleepHour.R b/tests/testthat/test_recordingEndSleepHour.R
index 100298d3d..cf322bd89 100644
--- a/tests/testthat/test_recordingEndSleepHour.R
+++ b/tests/testthat/test_recordingEndSleepHour.R
@@ -121,13 +121,12 @@ test_that("recordingEndSleepHour works as expected", {
   create_test_acc_csv(Nmin = 3 * 1440) # ends at 8:45
   # delete config.csv such that GGIR uses its own default parameter values
   if (file.exists("output_test/config.csv"))  unlink("output_test/config.csv", recursive = TRUE)
-  # No warning, this should work
   GGIR(datadir = fn, outputdir = getwd(), mode = 1:5,
        studyname = "test", overwrite = TRUE, desiredtz = tz, 
        verbose = FALSE, require_complete_lastnight_part5 = TRUE,
        do.report = 5)
   p5 = read.csv("output_test/results/part5_daysummary_MM_L40M100V400_T5A5.csv")
-  expect_equal(nrow(p5), 2) # expanded day appears in MM report
+  expect_equal(nrow(p5), 2) # last window is ignored from because the night ends at 8:45
   
   
   if (file.exists("output_test"))  unlink("output_test", recursive = TRUE)

From 6365e52403e851cc133dc2129c536042275cc4b0 Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 11:34:04 +0200
Subject: [PATCH 6/9] add new parameter to parameters vignette

---
 vignettes/GGIRParameters.Rmd | 1 +
 1 file changed, 1 insertion(+)

diff --git a/vignettes/GGIRParameters.Rmd b/vignettes/GGIRParameters.Rmd
index 8aa3d10a9..390c899f0 100644
--- a/vignettes/GGIRParameters.Rmd
+++ b/vignettes/GGIRParameters.Rmd
@@ -215,6 +215,7 @@ find a description and default value for all the arguments.
 | save_ms5raw_format          | 5                 | params_output            |
 | save_ms5raw_without_invalid | 5                 | params_output            |
 | do.sibreport                | 5                 | params_output            |
+| require_complete_lastnight_part5 | 5                | params_output            |
 | visualreport_without_invalid| visualreport      | params_output            |
 | dofirstpage                 | visualreport      | params_output            |
 | visualreport                | visualreport      | params_output            |

From 7d29e85c5129e61c4d12b9e192b465dcb0e2dd5d Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 11:42:06 +0200
Subject: [PATCH 7/9] mention new param require_complete_lastnight_part5 in
 gh-pages chapter 12 #1196

---
 vignettes/chapter12_TimeUseAnalysis.Rmd | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/vignettes/chapter12_TimeUseAnalysis.Rmd b/vignettes/chapter12_TimeUseAnalysis.Rmd
index 1a61d5b2c..8102b6628 100644
--- a/vignettes/chapter12_TimeUseAnalysis.Rmd
+++ b/vignettes/chapter12_TimeUseAnalysis.Rmd
@@ -45,7 +45,9 @@ GGIR part 5 facilitates the following time window definitions, which can be sele
 | “WW”       | waking up to waking up     | Each day is defined as the time from the participant wakes up a given day to the time they wake up the next day.                             |
 | “OO”       | sleep onset to sleep onset | Each day is defined as the time from the sleep onset a given day to the sleep onset of the next day.                                         |
 
-: For "WW" and "OO", the onset and waking times are guided by the estimates from part 4, but if they are missing, part 5 will attempt to retrieve the estimate from the guider method. Note that the parameter `timewindow` can consist of one of the options beforementioned or any combination of them, for example, the default value is `timewindow = c("MM", "WW")`.
+For "WW" and "OO", the onset and waking times are guided by the estimates from part 4, but if they are missing, part 5 will attempt to retrieve the estimate from the guider method. Note that the parameter `timewindow` can consist of one of the options beforementioned or any combination of them, for example, the default value is `timewindow = c("MM", "WW")`.
+
+When recordings end in the night or early morning the sleep estimates for the night are likely affected. For example, if a recording ends at 10am we cannot be sure that the participant did not sleep until after 10am, and if a recording ends at 2am we cannot be sure that the sleep onset time was reliably estimated. To handle this and ignore the final window in the data, set parameter `require_complete_lastnight_part5 = TRUE` (not default).
 
 ### Defining segments within the MM window
 

From 6a22be969df099a93f6b2c5b6c15bfc35d01be57 Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 11:43:04 +0200
Subject: [PATCH 8/9] Update NEWS.md

---
 NEWS.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/NEWS.md b/NEWS.md
index bc5dda744..e09508c7e 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,6 +1,6 @@
 # CHANGES IN GGIR VERSION 3.1-?
 
-- Part 5: Add parameters require_complete_lastnight_part5 to control whether incomplete last nights are included. #1196
+- Part 5: Add parameters require_complete_lastnight_part5 to control whether last window is included if last night is incomplete. #1196
 
 # CHANGES IN GGIR VERSION 3.1-4
 

From 594f361be2738d5eae13bbc4e114cf0d500e6b5b Mon Sep 17 00:00:00 2001
From: Vincent van Hees <v.vanhees@accelting.com>
Date: Mon, 23 Sep 2024 15:30:35 +0200
Subject: [PATCH 9/9] omit code comment that is no longer applicable

---
 R/g.report.part5.R | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/R/g.report.part5.R b/R/g.report.part5.R
index 559f53e14..388f3f60f 100644
--- a/R/g.report.part5.R
+++ b/R/g.report.part5.R
@@ -68,9 +68,6 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                           include_window == TRUE &
                           x$wear_min_day_spt >= minimumValidMinutesMM)
       } else if (window == "MM") {
-        # Note: For MM it would also be good to put requirements on the 
-        # timing of the last timestamp to include/exlcude the last window
-        # but not done yet
         indices = which(x$wear_perc_day >= includeday_wearPercentage &
                           x$wear_min_day >= includeday_absolute &
                           x$dur_spt_min > 0 & x$dur_day_min > 0 &