diff --git a/NEWS.md b/NEWS.md index e540a41..64f346d 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,7 @@ # Changes in version 1.0.4 (release date:X-X-2025) +- Fitbit: Fix bug preventing the loading of a sequence of json files. #76 + - Add functions for reading Parmay Tech Matrix sensor (bin/BIN files) with accelerometer, gyroscope, temperature, and heart rate #70. # Changes in version 1.0.3 (release date:07-03-2025) diff --git a/R/mergeFitbitData.R b/R/mergeFitbitData.R index 7163b5e..4fa4dbe 100644 --- a/R/mergeFitbitData.R +++ b/R/mergeFitbitData.R @@ -8,15 +8,46 @@ mergeFitbitData = function(filenames = NULL, desiredtz = "", configtz = NULL) { if (cnt == 1) { data = D } else { - if (length(intersect(x = data$dateTime, D$dateTime)) == 0) { - warning(paste0("Time series do not intersect for files ", - basename(filenames[cnt]), " and ", basename(filenames[cnt - 1])), - call. = FALSE) + # double names is possible when recording is split across json files + # in that case there may be multiple calories, steps and sleep files + doubleNames = colnames(D)[colnames(D) %in% colnames(data)] + new_times = which(D$dateTime %in% data$dateTime == FALSE) + double_times = which(D$dateTime %in% data$dateTime == TRUE) + if (length(new_times) > 0) { + if (all(colnames(data) %in% colnames(D))) { + data = rbind(data, D[new_times, ]) + data = data[order(data$dateTime),] + } else { + data = merge(data, D[new_times, ], by = doubleNames, all = TRUE) + } + } + if (length(double_times) > 0) { + by_names = colnames(D)[colnames(D) %in% c("dateTime", "seconds")] + data2 = merge(data, D[double_times,], by = by_names, all.x = TRUE) + xcol = grep(pattern = "[.]x", x = colnames(data2)) + ycol = grep(pattern = "[.]y", x = colnames(data2)) + if (length(xcol) > 0 && length(ycol) > 0) { + # check whether new column has values that were missing before + replace_times = which(is.na(data2[, xcol]) & !is.na(data2[, ycol])) + if (length(replace_times) > 0) { + # replace previously missing values by new values + data2[replace_times, xcol] = data2[replace_times, ycol] + } + colnames(data2)[xcol] = gsub(pattern = "[.]x", replacement = "", x = colnames(data2)[xcol]) + data = data2[, -ycol] + } else { + data = data2 + } } - - data = merge(data, D, by = "dateTime", all = TRUE) } cnt = cnt + 1 } + data = data[order(data$dateTime),] + + # fill gaps + timeRange = range(data$dateTime) + epochSize = min(diff(as.numeric(data$dateTime[1:pmin(10, nrow(data))]))) + timeFrame = data.frame(dateTime = seq( timeRange[1], timeRange[2], by = epochSize)) + data = merge(data, timeFrame, by = c("dateTime"), all.y = TRUE) return(data) } \ No newline at end of file diff --git a/R/readActiwatchCount.R b/R/readActiwatchCount.R index 41d2ce6..0d47e0b 100644 --- a/R/readActiwatchCount.R +++ b/R/readActiwatchCount.R @@ -35,8 +35,8 @@ readActiwatchCount = function(filename = NULL, colnames(D)[grep(pattern = "activiteit|activity", x = colnames(D))] = "counts" colnames(D)[grep(pattern = "slapen|sleep", x = colnames(D))] = "sleep" colnames(D)[grep(pattern = "niet-om|wear|worn", x = colnames(D))] = "nonwear" - - D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker", x = colnames(D))] + colnames(D)[grep(pattern = "light", x = colnames(D))] = "light" + D = D[, grep(pattern = "time|date|counts|sleep|nonwear|marker|light", x = colnames(D))] timestamp_POSIX = as.POSIXct(x = paste(D$date[1:4], D$time[1:4], sep = " "), format = timeformat, tz = configtz) diff --git a/R/readFitbit.R b/R/readFitbit.R index 5da0114..6a48398 100644 --- a/R/readFitbit.R +++ b/R/readFitbit.R @@ -32,6 +32,7 @@ readFitbit = function(filename = NULL, desiredtz = "", if (dataType == "sleep") { epochSize = 30 # Put all data in data.frame + first_block_found = FALSE for (i in 1:length(D)) { tmp = D[[i]][15]$levels data = as.data.frame(data.table::rbindlist(tmp$data, fill = TRUE)) @@ -44,37 +45,29 @@ readFitbit = function(filename = NULL, desiredtz = "", if ("shortData" %in% names(tmp)) { shortData = data.table::rbindlist(tmp$shortData, fill = TRUE) shortData$dateTime = as.POSIXct(shortData$dateTime, format = "%Y-%m-%dT%H:%M:%S", tz = configtz) - if (i == 1) { + if (first_block_found == FALSE) { all_shortData = shortData + first_block_found = TRUE } else { all_shortData = rbind(all_shortData, shortData) } } } # Expand to full time series + all_data = all_data[order(all_data$dateTime),] D = as.data.frame(lapply(all_data, rep, all_data$seconds/epochSize)) - D$dateTime = seq(from = D$dateTime[1], length.out = nrow(D), by = epochSize) - D$seconds = epochSize - D = handleTimeGaps(D, epochSize) # Handle time gaps, if any - - S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/30)) - S$dateTime = seq(from = S$dateTime[1], length.out = nrow(S), by = 30) - S$seconds = epochSize - - # merge in shortData (S) - matching_times = which(S$dateTime %in% D$dateTime == TRUE) - non_matching_times = which(S$dateTime %in% D$dateTime == FALSE) - if (length(matching_times) > 0) { - times_to_replace = S$dateTime[matching_times] - D[which(D$dateTime %in% times_to_replace), ] = S[matching_times,] - } - if (length(non_matching_times) > 0) { - D = rbind(D, S[non_matching_times,]) - } - D = handleTimeGaps(D, epochSize) # Handle new time gaps, if any - - # Order time stamps - D = D[order(D$dateTime), ] + D$index = unlist(mapply(seq, rep(0, nrow(all_data)), (all_data$seconds/epochSize) - 1)) + D$dateTime = D$dateTime + D$index * epochSize + S = as.data.frame(lapply(all_shortData, rep, all_shortData$seconds/epochSize)) + S$index = unlist(mapply(seq, rep(0, nrow(all_shortData)), (all_shortData$seconds/epochSize) - 1)) + S$dateTime = S$dateTime + S$index * epochSize + D = rbind(D, S) + D = D[, -which(colnames(D) %in% c("seconds", "index"))] + D = D[order(D$dateTime),] + dup_times = unique(D$dateTime[duplicated(D$dateTime)]) + # wake overrules other classifications + D = D[-which(D$dateTime %in% dup_times & D$level != "wake"), ] + D = D[!duplicated(D),] colnames(D)[2] = "sleeplevel" } else if (dataType == "steps" || dataType == "calories") { data = as.data.frame(data.table::rbindlist(D, fill = TRUE)) diff --git a/tests/testthat/test_mergeFitbitData.R b/tests/testthat/test_mergeFitbitData.R index 6c27166..9ce06be 100644 --- a/tests/testthat/test_mergeFitbitData.R +++ b/tests/testthat/test_mergeFitbitData.R @@ -15,8 +15,10 @@ test_that("merging of PHB files goes correctly", { expect_equal(format(D$dateTime[1]), "1995-06-24 16:00:00") # apply function to merge the files - expect_warning(mergeFitbitData(filenames = c(file1, file2, file3), - desiredtz = "Europe/Amsterdam"), - regexp = "Time series*") + D2 = mergeFitbitData(filenames = c(file1, file2, file3), + desiredtz = "Europe/Amsterdam") + expect_true(all(colnames(D2) %in% c("dateTime", "steps", "calories", "sleeplevel"))) + expect_equal(nrow(D2), 47874) + rm(D2, D) }) \ No newline at end of file diff --git a/tests/testthat/test_readFitbit.R b/tests/testthat/test_readFitbit.R index a2fdaec..90a08d6 100644 --- a/tests/testthat/test_readFitbit.R +++ b/tests/testthat/test_readFitbit.R @@ -5,11 +5,11 @@ test_that("Fitbit json is correctly read", { file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam") expect_equal(nrow(D), 695) - expect_equal(ncol(D), 3) - expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30") + expect_equal(ncol(D), 2) + expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30") TB = table(D$sleeplevel) expect_equal(names(TB), c("asleep", "awake", "deep", "light", "rem", "restless", "wake")) - expect_equal(as.numeric(TB), c(146, 2, 118, 283, 71, 10, 65)) + expect_equal(as.numeric(TB), c(146, 2, 118, 285, 69, 10, 65)) # Steps file = system.file("testfiles/steps-1995-06-23_Fitbit.json", package = "GGIRread") @@ -32,14 +32,14 @@ test_that("Timezones are correctly handled", { file = system.file("testfiles/sleep-1995-06-23_Fitbit.json", package = "GGIRread") # Configured and worn in same place D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam") - expect_equal(format(D$dateTime[1]), "1995-07-11 02:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-24 22:47:30") # Configured 1 hour earlier than timezone where device was worn D = readFitbit(filename = file, desiredtz = "Europe/London", configtz = "Europe/Amsterdam") - expect_equal(format(D$dateTime[1]), "1995-07-11 01:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-24 21:47:30") # Configured 6 hours later than timezone where device was worn D = readFitbit(filename = file, desiredtz = "Europe/Amsterdam", configtz = "America/New_York") - expect_equal(format(D$dateTime[1]), "1995-07-11 08:28:30") + expect_equal(format(D$dateTime[1]), "1995-06-25 04:47:30") })