Handle sequence of json time series for Fitbit

NEWS.md

@@ -1,5 +1,7 @@
 # Changes in version 1.0.4 (release date:X-X-2025)
 
+- Fitbit: Fix bug preventing the loading of a sequence of json files. #76
+
 - Add functions for reading Parmay Tech Matrix sensor (bin/BIN files) with accelerometer, gyroscope, temperature, and heart rate #70.
 
 # Changes in version 1.0.3 (release date:07-03-2025)

R/mergeFitbitData.R

@@ -8,15 +8,46 @@ mergeFitbitData = function(filenames = NULL, desiredtz = "", configtz = NULL) {
     if (cnt == 1) {
       data = D
     } else {
-      if (length(intersect(x = data$dateTime, D$dateTime)) == 0) {
-        warning(paste0("Time series do not intersect for files ",
-                       basename(filenames[cnt]), " and ", basename(filenames[cnt - 1])),
-                call. = FALSE)
+      # double names is possible when recording is split across json files
+      # in that case there may be multiple calories, steps and sleep files
+      doubleNames = colnames(D)[colnames(D) %in% colnames(data)] 
+      new_times = which(D$dateTime %in% data$dateTime == FALSE)
+      double_times = which(D$dateTime %in% data$dateTime == TRUE)
+      if (length(new_times) > 0) {
+        if (all(colnames(data) %in% colnames(D))) {
+          data = rbind(data, D[new_times, ])
+          data = data[order(data$dateTime),]
+        } else {
+          data = merge(data, D[new_times, ], by = doubleNames, all = TRUE)
+        }
+      }
+      if (length(double_times) > 0) {
+        by_names = colnames(D)[colnames(D) %in% c("dateTime", "seconds")]
+        data2 = merge(data, D[double_times,], by = by_names, all.x = TRUE)
+        xcol = grep(pattern = "[.]x", x = colnames(data2))
+        ycol = grep(pattern = "[.]y", x = colnames(data2))
+        if (length(xcol) > 0 && length(ycol) > 0) {
+          # check whether new column has values that were missing before
+          replace_times = which(is.na(data2[, xcol]) & !is.na(data2[, ycol]))
+          if (length(replace_times) > 0) {
+            # replace previously missing values by new values
+            data2[replace_times, xcol] = data2[replace_times, ycol]
+          }
+          colnames(data2)[xcol] = gsub(pattern = "[.]x", replacement = "", x = colnames(data2)[xcol])
+          data = data2[, -ycol]
+        } else {
+          data = data2
+        }
       }
-
-      data = merge(data, D, by = "dateTime", all = TRUE)
     }
     cnt = cnt + 1
   }
+  data = data[order(data$dateTime),]
+
+  # fill gaps
+  timeRange = range(data$dateTime)
+  epochSize = min(diff(as.numeric(data$dateTime[1:pmin(10, nrow(data))])))
+  timeFrame = data.frame(dateTime = seq( timeRange[1], timeRange[2], by = epochSize))
+  data = merge(data, timeFrame, by = c("dateTime"), all.y = TRUE)
   return(data)
 }

R/readActiwatchCount.R

@@ -35,8 +35,8 @@ readActiwatchCount = function(filename = NULL,
     colnames(D)[grep(pattern = "activiteit|activity", x = colnames(D))] = "counts"
     colnames(D)[grep(pattern = "slapen|sleep", x = colnames(D))] = "sleep"
     colnames(D)[grep(pattern = "niet-om|wear|worn", x = colnames(D))] = "nonwear"
-
-    D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker", x = colnames(D))]
+    colnames(D)[grep(pattern = "light", x = colnames(D))] = "light"
+    D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker|light", x = colnames(D))]
     timestamp_POSIX = as.POSIXct(x = paste(D$date[1:4], D$time[1:4], sep = " "),
                                  format = timeformat,
                                  tz = configtz)

R/readFitbit.R

@@ -32,6 +32,7 @@ readFitbit = function(filename = NULL, desiredtz = "",
   if (dataType == "sleep") {
     epochSize = 30
     # Put all data in data.frame
+    first_block_found = FALSE
     for (i in 1:length(D)) {
       tmp = D[[i]][15]$levels
       data = as.data.frame(data.table::rbindlist(tmp$data, fill = TRUE))
@@ -44,37 +45,29 @@ readFitbit = function(filename = NULL, desiredtz = "",
       if ("shortData" %in% names(tmp)) {
         shortData = data.table::rbindlist(tmp$shortData, fill = TRUE)
         shortData$dateTime = as.POSIXct(shortData$dateTime, format = "%Y-%m-%dT%H:%M:%S", tz = configtz)
-        if (i == 1) {
+        if (first_block_found == FALSE) {
           all_shortData = shortData
+          first_block_found = TRUE
         } else {
           all_shortData = rbind(all_shortData, shortData)
         }
       }
     }
     # Expand to full time series
+    all_data = all_data[order(all_data$dateTime),]
     D = as.data.frame(lapply(all_data, rep, all_data$seconds/epochSize))
-    D$dateTime = seq(from = D$dateTime[1], length.out = nrow(D), by = epochSize) 
-    D$seconds = epochSize
-    D = handleTimeGaps(D, epochSize) # Handle time gaps, if any
-
-    S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/30))
-    S$dateTime = seq(from = S$dateTime[1], length.out = nrow(S), by = 30) 
-    S$seconds = epochSize
-
-    # merge in shortData (S)
-    matching_times = which(S$dateTime %in% D$dateTime ==  TRUE)
-    non_matching_times = which(S$dateTime %in% D$dateTime == FALSE)
-    if (length(matching_times) > 0) {
-      times_to_replace = S$dateTime[matching_times]
-      D[which(D$dateTime %in% times_to_replace), ] = S[matching_times,]
-    }
-    if (length(non_matching_times) > 0) {
-      D = rbind(D, S[non_matching_times,])
-    }
-    D = handleTimeGaps(D, epochSize) # Handle new time gaps, if any
-
-    # Order time stamps
-    D = D[order(D$dateTime), ]
+    D$index = unlist(mapply(seq, rep(0, nrow(all_data)), (all_data$seconds/epochSize) - 1))
+    D$dateTime = D$dateTime + D$index * epochSize
+    S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/epochSize))
+    S$index = unlist(mapply(seq, rep(0, nrow(all_shortData)), (all_shortData$seconds/epochSize) - 1))
+    S$dateTime = S$dateTime + S$index * epochSize
+    D = rbind(D, S)
+    D = D[, -which(colnames(D) %in% c("seconds", "index"))]
+    D = D[order(D$dateTime),]
+    dup_times = unique(D$dateTime[duplicated(D$dateTime)])
+    # wake overrules other classifications
+    D = D[-which(D$dateTime %in% dup_times & D$level != "wake"), ]
+    D = D[!duplicated(D),]
     colnames(D)[2] = "sleeplevel"
   } else if (dataType == "steps" || dataType == "calories") {
     data = as.data.frame(data.table::rbindlist(D, fill = TRUE))

tests/testthat/test_mergeFitbitData.R

@@ -15,8 +15,10 @@ test_that("merging of PHB files goes correctly", {
   expect_equal(format(D$dateTime[1]), "1995-06-24 16:00:00")
 
   # apply function to merge the files
-  expect_warning(mergeFitbitData(filenames = c(file1, file2, file3),
-                      desiredtz =  "Europe/Amsterdam"),
-               regexp = "Time series*")
+  D2 = mergeFitbitData(filenames = c(file1, file2, file3),
+                      desiredtz =  "Europe/Amsterdam")
+  expect_true(all(colnames(D2) %in% c("dateTime", "steps", "calories", "sleeplevel")))
+  expect_equal(nrow(D2), 47874)
+  rm(D2, D)
 })
 

tests/testthat/test_readFitbit.R

@@ -5,11 +5,11 @@ test_that("Fitbit json is correctly read", {
   file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread")
   D = readFitbit(filename = file, desiredtz =  "Europe/Amsterdam")
   expect_equal(nrow(D), 695)
-  expect_equal(ncol(D), 3)
-  expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30")
+  expect_equal(ncol(D), 2)
+  expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30")
   TB = table(D$sleeplevel)
   expect_equal(names(TB), c("asleep", "awake", "deep", "light", "rem", "restless", "wake"))
-  expect_equal(as.numeric(TB), c(146, 2, 118, 283, 71, 10, 65))
+  expect_equal(as.numeric(TB), c(146, 2, 118, 285, 69, 10, 65))
 
   # Steps
   file = system.file("testfiles/steps-1995-06-23_Fitbit.json", package = "GGIRread")
@@ -32,14 +32,14 @@ test_that("Timezones are correctly handled", {
   file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread")
   # Configured and worn in same place
   D = readFitbit(filename = file, desiredtz =  "Europe/Amsterdam")
-  expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30")
+  expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30")
 
   # Configured 1 hour earlier than timezone where device was worn
   D = readFitbit(filename = file, desiredtz =  "Europe/London", configtz =  "Europe/Amsterdam")
-  expect_equal(format(D$dateTime[1]), "1995-07-11 01:28:30")
+  expect_equal(format(D$dateTime[1]), "1995-06-24 21:47:30")
 
   # Configured 6 hours later than timezone where device was worn
   D = readFitbit(filename = file, desiredtz =  "Europe/Amsterdam",
                  configtz =  "America/New_York")
-  expect_equal(format(D$dateTime[1]), "1995-07-11 08:28:30")
+  expect_equal(format(D$dateTime[1]), "1995-06-25 04:47:30")
 })