From dac190030c59615a80ae126d5606d4195d4f7d0c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 24 Jan 2022 10:24:00 +0100 Subject: [PATCH 1/5] Started on adding a Fitbit notification endpoint --- .editorconfig | 3 + build.gradle | 2 - gradle.properties | 1 + .../FitbitActivityLogAvroConverter.java | 38 ++-- .../connect/rest/RestSourceTask.java | 2 +- .../connect/rest/request/RestRequest.java | 6 +- radar-fitbit-converter/build.gradle | 15 ++ .../radarbase/convert/fitbit/DateRange.java | 65 ++++++ .../FitbitActivityLogAvroConverter.java | 200 ++++++++++++++++++ .../convert/fitbit/FitbitAvroConverter.java | 183 ++++++++++++++++ .../FitbitIntradayCaloriesAvroConverter.java | 79 +++++++ .../FitbitIntradayHeartRateAvroConverter.java | 84 ++++++++ .../FitbitIntradayStepsAvroConverter.java | 87 ++++++++ .../fitbit/FitbitSleepAvroConverter.java | 143 +++++++++++++ .../fitbit/FitbitTimeZoneAvroConverter.java | 63 ++++++ radar-fitbit-endpoint/build.gradle | 17 ++ .../org/radarbase/fitbit/endpoint/Main.kt | 54 +++++ .../fitbit/endpoint/config/FitbitConfig.kt | 37 ++++ .../endpoint/config/FitbitEndpointConfig.kt | 40 ++++ .../fitbit/endpoint/config/ServerConfig.kt | 36 ++++ .../filter/ClientDomainVerification.kt | 22 ++ .../filter/ClientDomainVerificationFeature.kt | 29 +++ .../filter/ClientDomainVerificationFilter.kt | 53 +++++ .../inject/GatewayResourceEnhancer.kt | 22 ++ .../inject/ManagementPortalEnhancerFactory.kt | 19 ++ .../endpoint/resource/WebhookResource.kt | 59 ++++++ .../service/FitbitVerificationService.kt | 37 ++++ .../endpoint/service/NotificationService.kt | 24 +++ radar-oauth2-user-repository/build.gradle | 15 ++ settings.gradle | 3 +- 30 files changed, 1410 insertions(+), 28 deletions(-) create mode 100644 radar-fitbit-converter/build.gradle create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java create mode 100644 radar-fitbit-endpoint/build.gradle create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/ServerConfig.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerification.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFeature.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFilter.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/GatewayResourceEnhancer.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/ManagementPortalEnhancerFactory.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/FitbitVerificationService.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt create mode 100644 radar-oauth2-user-repository/build.gradle diff --git a/.editorconfig b/.editorconfig index 4801aa6a..3a328cfc 100644 --- a/.editorconfig +++ b/.editorconfig @@ -13,6 +13,9 @@ indent_size = 2 ij_continuation_indent_size = 4 max_line_length=100 +[*.kt] +indent_size = 4 + [{*.gradle, *.py}] indent_size = 4 ij_continuation_indent_size = 8 diff --git a/build.gradle b/build.gradle index d887b8ba..7375f979 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,3 @@ -import com.github.benmanes.gradle.versions.updates.DependencyUpdatesTask - plugins { id("com.github.ben-manes.versions") version "0.39.0" } diff --git a/gradle.properties b/gradle.properties index e69de29b..7fc6f1ff 100644 --- a/gradle.properties +++ b/gradle.properties @@ -0,0 +1 @@ +kotlin.code.style=official diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java index 254bb786..dd7256d1 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java @@ -119,21 +119,21 @@ private FitbitActivityLevels getActivityLevels(JsonNode s) { .map(levels -> { FitbitActivityLevels.Builder activityLevels = FitbitActivityLevels.newBuilder(); for (JsonNode level : levels) { - Integer time = optInt(level, "minutes") + Integer duration = optInt(level, "minutes") .map(t -> t * 60) .orElse(null); switch (optString(level, "name").orElse("")) { case "sedentary": - activityLevels.setDurationSedentary(time); + activityLevels.setDurationSedentary(duration); break; case "lightly": - activityLevels.setDurationLightly(time); + activityLevels.setDurationLightly(duration); break; case "fairly": - activityLevels.setDurationFairly(time); + activityLevels.setDurationFairly(duration); break; case "very": - activityLevels.setDurationVery(time); + activityLevels.setDurationVery(duration); } } @@ -165,31 +165,27 @@ private FitbitActivityHeartRate getHeartRate(JsonNode activity) { zones.ifPresent(z -> { for (JsonNode zone : z) { + Integer minValue = optInt(zone, "min").orElse(null); + Integer duration = optInt(zone, "minutes") + .map(m -> m * 60) + .orElse(null); switch (optString(zone, "name").orElse("")) { case "Out of Range": - heartRate.setMin(optInt(zone, "min").orElse(null)); - heartRate.setDurationOutOfRange(optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null)); + heartRate.setMin(minValue); + heartRate.setDurationOutOfRange(duration); break; case "Fat Burn": - heartRate.setMinFatBurn(optInt(zone, "min").orElse(null)); - heartRate.setDurationFatBurn(optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null)); + heartRate.setMinFatBurn(minValue); + heartRate.setDurationFatBurn(duration); break; case "Cardio": - heartRate.setMinCardio(optInt(zone, "min").orElse(null)); - heartRate.setDurationCardio(optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null)); + heartRate.setMinCardio(minValue); + heartRate.setDurationCardio(duration); break; case "Peak": - heartRate.setMinPeak(optInt(zone, "min").orElse(null)); + heartRate.setMinPeak(minValue); heartRate.setMax(optInt(zone, "max").orElse(null)); - heartRate.setDurationPeak(optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null)); + heartRate.setDurationPeak(duration); break; default: logger.warn("Cannot process unknown heart rate zone {}", zone.get("name").asText()); diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java index a425f16a..ee8d35b4 100644 --- a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/RestSourceTask.java @@ -77,7 +77,7 @@ public List poll() throws InterruptedException { while (requests.isEmpty() && requestIterator.hasNext()) { RestRequest request = requestIterator.next(); - if (!request.isStillValid()) { + if (request.isNoLongerValid()) { continue; } diff --git a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java index 8e7d5ce2..1386a377 100644 --- a/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java +++ b/kafka-connect-rest-source/src/main/java/org/radarbase/connect/rest/request/RestRequest.java @@ -69,8 +69,8 @@ public Map getPartition() { return partition; } - public boolean isStillValid() { - return isValid == null || isValid.test(this); + public boolean isNoLongerValid() { + return isValid != null && !isValid.test(this); } /** @@ -79,7 +79,7 @@ public boolean isStillValid() { * @throws IOException if making or parsing the request failed. */ public Stream handleRequest() throws IOException { - if (!isStillValid()) { + if (isNoLongerValid()) { return Stream.empty(); } diff --git a/radar-fitbit-converter/build.gradle b/radar-fitbit-converter/build.gradle new file mode 100644 index 00000000..7d6ad79c --- /dev/null +++ b/radar-fitbit-converter/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.6.10' +} + +dependencies { + implementation("org.slf4j:slf4j-api:1.7.32") +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { + kotlinOptions { + jvmTarget = "17" + apiVersion = "1.6" + languageVersion = "1.6" + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java new file mode 100644 index 00000000..d5f61565 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java @@ -0,0 +1,65 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.util; + +import java.time.ZonedDateTime; +import java.util.Objects; + +public class DateRange { + private final ZonedDateTime start; + private final ZonedDateTime end; + + public DateRange(ZonedDateTime start, ZonedDateTime end) { + this.start = start; + this.end = end; + } + + public ZonedDateTime start() { + return start; + } + + public ZonedDateTime end() { + return end; + } + + @Override + public String toString() { + return "DateRange{" + + "start=" + start + + ", end=" + end + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DateRange dateRange = (DateRange) o; + return Objects.equals(start, dateRange.start) && + Objects.equals(end, dateRange.end); + } + + @Override + public int hashCode() { + return Objects.hash(start, end); + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java new file mode 100644 index 00000000..dd7256d1 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java @@ -0,0 +1,200 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.OffsetDateTime; +import java.util.Comparator; +import java.util.Optional; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitActivityHeartRate; +import org.radarcns.connector.fitbit.FitbitActivityHeartRate.Builder; +import org.radarcns.connector.fitbit.FitbitActivityLevels; +import org.radarcns.connector.fitbit.FitbitActivityLogRecord; +import org.radarcns.connector.fitbit.FitbitManualDataEntry; +import org.radarcns.connector.fitbit.FitbitSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitActivityLogAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitActivityLogAvroConverter.class); + private static final float FOOD_CAL_TO_KJOULE_FACTOR = 4.1868f; + + private String activityLogTopic; + + public FitbitActivityLogAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + activityLogTopic = ((FitbitRestSourceConnectorConfig)config).getActivityLogTopic(); + + logger.info("Using activity log topic {}", activityLogTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + + JsonNode array = root.get("activities"); + if (array == null || !array.isArray()) { + return Stream.empty(); + } + + return iterableToStream(array) + .sorted(Comparator.comparing(s -> s.get("startTime").textValue())) + .map(tryOrNull(s -> { + OffsetDateTime startTime = OffsetDateTime.parse(s.get("startTime").textValue()); + FitbitActivityLogRecord record = getRecord(s, startTime); + return new TopicData(startTime.toInstant(), activityLogTopic, record); + }, (s, ex) -> logger.warn( + "Failed to convert sleep patterns from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), s, ex))); + } + + private FitbitActivityLogRecord getRecord(JsonNode s, OffsetDateTime startTime) { + return FitbitActivityLogRecord.newBuilder() + .setTime(startTime.toInstant().toEpochMilli() / 1000d) + .setTimeReceived(System.currentTimeMillis() / 1000d) + .setTimeLastModified(Instant.parse(s.get("lastModified").asText()).toEpochMilli() / 1000d) + .setId(optLong(s, "logId").orElseThrow( + () -> new IllegalArgumentException("Activity log ID not specified"))) + .setLogType(optString(s, "logType").orElse(null)) + .setType(optLong(s, "activityType").orElse(null)) + .setSpeed(optDouble(s, "speed").orElse(null)) + .setDistance(optDouble(s, "distance").map(Double::floatValue).orElse(null)) + .setSteps(optInt(s, "steps").orElse(null)) + .setEnergy(optInt(s, "calories").map(e -> e * FOOD_CAL_TO_KJOULE_FACTOR) + .orElse(null)) + .setDuration(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( + () -> new IllegalArgumentException("Activity log duration not specified"))) + .setDurationActive(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( + () -> new IllegalArgumentException("Activity duration active not specified"))) + .setTimeZoneOffset(startTime.getOffset().getTotalSeconds()) + .setName(optString(s, "activityName").orElse(null)) + .setHeartRate(getHeartRate(s)) + .setManualDataEntry(getManualDataEntry(s)) + .setLevels(getActivityLevels(s)) + .setSource(getSource(s)) + .build(); + } + + private FitbitSource getSource(JsonNode s) { + return optObject(s, "source") + .flatMap(source -> optString(source, "id") + .map(id -> FitbitSource.newBuilder() + .setId(id) + .setName(optString(source, "name").orElse(null)) + .setType(optString(source, "type").orElse(null)) + .setUrl(optString(source, "url").orElse(null)) + .build())) + .orElse(null); + } + + private FitbitActivityLevels getActivityLevels(JsonNode s) { + return optArray(s, "activityLevels") + .map(levels -> { + FitbitActivityLevels.Builder activityLevels = FitbitActivityLevels.newBuilder(); + for (JsonNode level : levels) { + Integer duration = optInt(level, "minutes") + .map(t -> t * 60) + .orElse(null); + switch (optString(level, "name").orElse("")) { + case "sedentary": + activityLevels.setDurationSedentary(duration); + break; + case "lightly": + activityLevels.setDurationLightly(duration); + break; + case "fairly": + activityLevels.setDurationFairly(duration); + break; + case "very": + activityLevels.setDurationVery(duration); + } + } + + return activityLevels.build(); + }) + .orElse(null); + } + + private FitbitManualDataEntry getManualDataEntry(JsonNode s) { + return optObject(s, "manualValuesSpecified") + .map(manual -> FitbitManualDataEntry.newBuilder() + .setSteps(optBoolean(manual, "steps").orElse(null)) + .setDistance(optBoolean(manual, "distance").orElse(null)) + .setEnergy(optBoolean(manual, "calorie").orElse(null)) + .build()) + .orElse(null); + } + + private FitbitActivityHeartRate getHeartRate(JsonNode activity) { + Optional mean = optInt(activity, "averageHeartRate"); + Optional> zones = optArray(activity, "heartRateZones"); + + if (mean.isEmpty() && zones.isEmpty()) { + return null; + } + + Builder heartRate = FitbitActivityHeartRate.newBuilder() + .setMean(mean.orElse(null)); + + zones.ifPresent(z -> { + for (JsonNode zone : z) { + Integer minValue = optInt(zone, "min").orElse(null); + Integer duration = optInt(zone, "minutes") + .map(m -> m * 60) + .orElse(null); + switch (optString(zone, "name").orElse("")) { + case "Out of Range": + heartRate.setMin(minValue); + heartRate.setDurationOutOfRange(duration); + break; + case "Fat Burn": + heartRate.setMinFatBurn(minValue); + heartRate.setDurationFatBurn(duration); + break; + case "Cardio": + heartRate.setMinCardio(minValue); + heartRate.setDurationCardio(duration); + break; + case "Peak": + heartRate.setMinPeak(minValue); + heartRate.setMax(optInt(zone, "max").orElse(null)); + heartRate.setDurationPeak(duration); + break; + default: + logger.warn("Cannot process unknown heart rate zone {}", zone.get("name").asText()); + break; + } + } + }); + + return heartRate.build(); + } + +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java new file mode 100644 index 00000000..7b2e248c --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java @@ -0,0 +1,183 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.confluent.connect.avro.AvroData; +import java.io.IOException; +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import okhttp3.Headers; +import org.apache.avro.Schema.Field; +import org.apache.avro.generic.IndexedRecord; +import org.apache.kafka.connect.data.SchemaAndValue; +import org.apache.kafka.connect.source.SourceRecord; +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarbase.connect.rest.fitbit.user.User; +import org.radarbase.connect.rest.request.RestRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Abstract class to help convert Fitbit data to Avro Data. + */ +public abstract class FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitAvroConverter.class); + private static final Map TIME_UNIT_MAP = new HashMap<>(); + + static { + TIME_UNIT_MAP.put("minute", TimeUnit.MINUTES); + TIME_UNIT_MAP.put("second", TimeUnit.SECONDS); + TIME_UNIT_MAP.put("hour", TimeUnit.HOURS); + TIME_UNIT_MAP.put("day", TimeUnit.DAYS); + TIME_UNIT_MAP.put("millisecond", TimeUnit.MILLISECONDS); + TIME_UNIT_MAP.put("nanosecond", TimeUnit.NANOSECONDS); + TIME_UNIT_MAP.put("microsecond", TimeUnit.MICROSECONDS); + } + + private final AvroData avroData; + + public FitbitAvroConverter(AvroData avroData) { + this.avroData = avroData; + } + + public Collection convert( + DateRange userRange, Headers headers, byte[] data) throws IOException { + if (data == null) { + throw new IOException("Failed to read body"); + } + JsonNode activities = JSON_READER.readTree(data); + + User user = ((FitbitRestRequest) restRequest).getUser(); + final SchemaAndValue key = user.getObservationKey(avroData); + double timeReceived = System.currentTimeMillis() / 1000d; + + return processRecords((FitbitRestRequest)restRequest, activities, timeReceived) + .filter(t -> validateRecord((FitbitRestRequest)restRequest, t)) + .map(t -> { + SchemaAndValue avro = avroData.toConnectData(t.value.getSchema(), t.value); + Map offset = Collections.singletonMap( + TIMESTAMP_OFFSET_KEY, t.sourceOffset.toEpochMilli()); + + return new SourceRecord(restRequest.getPartition(), offset, t.topic, + key.schema(), key.value(), avro.schema(), avro.value()); + }) + .collect(Collectors.toList()); + } + + private boolean validateRecord(FitbitRestRequest request, TopicData record) { + if (record == null) { + return false; + } + Instant endDate = request.getUser().getEndDate(); + if (endDate == null) { + return true; + } + Field timeField = record.value.getSchema().getField("time"); + if (timeField != null) { + long time = (long) (((Double)record.value.get(timeField.pos()) * 1000.0)); + return Instant.ofEpochMilli(time).isBefore(endDate); + } + return true; + } + + /** Process the JSON records generated by given request. */ + protected abstract Stream processRecords( + FitbitRestRequest request, + JsonNode root, + double timeReceived); + + /** Get Fitbit dataset interval used in some intraday API calls. */ + protected static int getRecordInterval(JsonNode root, int defaultValue) { + JsonNode type = root.get("datasetType"); + JsonNode interval = root.get("datasetInterval"); + if (type == null || interval == null) { + logger.warn("Failed to get data interval; using {} instead", defaultValue); + return defaultValue; + } + return (int)TIME_UNIT_MAP + .getOrDefault(type.asText(), TimeUnit.SECONDS) + .toSeconds(interval.asLong()); + } + + /** Converts an iterable (like a JsonNode containing an array) to a stream. */ + protected static Stream iterableToStream(Iterable iter) { + return StreamSupport.stream(iter.spliterator(), false); + } + + protected static Optional optLong(JsonNode node, String fieldName) { + JsonNode v = node.get(fieldName); + return v != null && v.canConvertToLong() ? Optional.of(v.longValue()) : Optional.empty(); + } + + protected static Optional optDouble(JsonNode node, String fieldName) { + JsonNode v = node.get(fieldName); + return v != null && v.isNumber() ? Optional.of(v.doubleValue()) : Optional.empty(); + } + + protected static Optional optInt(JsonNode node, String fieldName) { + JsonNode v = node.get(fieldName); + return v != null && v.canConvertToInt() ? Optional.of(v.intValue()) : Optional.empty(); + } + + protected static Optional optString(JsonNode node, String fieldName) { + JsonNode v = node.get(fieldName); + return v != null && v.isTextual() ? Optional.ofNullable(v.textValue()) : Optional.empty(); + } + + protected static Optional optBoolean(JsonNode node, String fieldName) { + JsonNode v = node.get(fieldName); + return v != null && v.isBoolean() ? Optional.of(v.booleanValue()) : Optional.empty(); + } + + protected static Optional optObject(JsonNode parent, String fieldName) { + JsonNode v = parent.get(fieldName); + return v != null && v.isObject() ? Optional.of((ObjectNode) v) : Optional.empty(); + } + + protected static Optional> optArray(JsonNode parent, String fieldName) { + JsonNode v = parent.get(fieldName); + return v != null && v.isArray() && v.size() != 0 ? + Optional.of(v) : Optional.empty(); + } + + /** Single value for a topic. */ + protected static class TopicData { + Instant sourceOffset; + final String topic; + final IndexedRecord value; + + public TopicData(Instant sourceOffset, String topic, IndexedRecord value) { + this.sourceOffset = sourceOffset; + this.topic = topic; + this.value = value; + } + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java new file mode 100644 index 00000000..93fb8847 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java @@ -0,0 +1,79 @@ +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradayCalories; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitIntradayCaloriesAvroConverter extends FitbitAvroConverter { + + private static final Logger logger = + LoggerFactory.getLogger(FitbitIntradayCaloriesAvroConverter.class); + + private String caloriesTopic; + + public FitbitIntradayCaloriesAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode intraday = root.get("activities-calories-intraday"); + if (intraday == null) { + return Stream.empty(); + } + + JsonNode dataset = intraday.get("dataset"); + if (dataset == null) { + return Stream.empty(); + } + + int interval = getRecordInterval(intraday, 60); + + // Used as the date to convert the local times in the dataset to absolute times. + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(dataset) + .map( + tryOrNull( + activity -> { + Instant time = + startDate.with(LocalTime.parse(activity.get("time").asText())).toInstant(); + + FitbitIntradayCalories calories = + new FitbitIntradayCalories( + time.toEpochMilli() / 1000d, + timeReceived, + interval, + activity.get("value").asDouble(), + activity.get("level").asInt(), + activity.get("mets").asDouble()); + + return new TopicData(time, caloriesTopic, calories); + }, + (a, ex) -> + logger.warn( + "Failed to convert calories from request {} of user {}, {}", + request.getRequest().url(), + request.getUser(), + a, + ex))); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + caloriesTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayCaloriesTopic(); + logger.info("Using calories topic {}", caloriesTopic); + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java new file mode 100644 index 00000000..89bc4cf1 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java @@ -0,0 +1,84 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradayHeartRate; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitIntradayHeartRateAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger( + FitbitIntradayHeartRateAvroConverter.class); + private String heartRateTopic; + + public FitbitIntradayHeartRateAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + heartRateTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayHeartRateTopic(); + logger.info("Using heart rate topic {}", heartRateTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode intraday = root.get("activities-heart-intraday"); + if (intraday == null || !intraday.isObject()) { + return Stream.empty(); + } + + JsonNode dataset = intraday.get("dataset"); + if (dataset == null || !dataset.isArray()) { + return Stream.empty(); + } + + int interval = getRecordInterval(intraday, 1); + + // Used as the date to convert the local times in the dataset to absolute times. + ZonedDateTime startDate = request.getDateRange().start(); + + return iterableToStream(dataset) + .map(tryOrNull(activity -> { + Instant time = startDate.with(LocalTime.parse(activity.get("time").asText())) + .toInstant(); + + FitbitIntradayHeartRate heartRate = new FitbitIntradayHeartRate( + time.toEpochMilli() / 1000d, + timeReceived, + interval, + activity.get("value").asInt()); + + return new TopicData(time, heartRateTopic, heartRate); + }, (a, ex) -> logger.warn( + "Failed to convert heart rate from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), a, ex))); + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java new file mode 100644 index 00000000..2381afa4 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java @@ -0,0 +1,87 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Instant; +import java.time.LocalTime; +import java.time.ZonedDateTime; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitIntradaySteps; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitIntradayStepsAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger( + FitbitIntradayStepsAvroConverter.class); + + private String stepTopic; + + public FitbitIntradayStepsAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + stepTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayStepsTopic(); + logger.info("Using step topic {}", stepTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode intraday = root.get("activities-steps-intraday"); + if (intraday == null) { + return Stream.empty(); + } + + JsonNode dataset = intraday.get("dataset"); + if (dataset == null) { + return Stream.empty(); + } + + int interval = getRecordInterval(intraday, 60); + + // Used as the date to convert the local times in the dataset to absolute times. + ZonedDateTime startDate = request.getDateRange().end(); + + return iterableToStream(dataset) + .map(tryOrNull(activity -> { + + Instant time = startDate + .with(LocalTime.parse(activity.get("time").asText())) + .toInstant(); + + FitbitIntradaySteps steps = new FitbitIntradaySteps( + time.toEpochMilli() / 1000d, + timeReceived, + interval, + activity.get("value").asInt()); + + return new TopicData(time, stepTopic, steps); + }, (a, ex) -> logger.warn( + "Failed to convert steps from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), a, ex))); + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java new file mode 100644 index 00000000..976a348b --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java @@ -0,0 +1,143 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import static org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute.DATE_TIME_FORMAT; +import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.time.Duration; +import java.time.Instant; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.avro.generic.IndexedRecord; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitSleepClassic; +import org.radarcns.connector.fitbit.FitbitSleepClassicLevel; +import org.radarcns.connector.fitbit.FitbitSleepStage; +import org.radarcns.connector.fitbit.FitbitSleepStageLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitSleepAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitSleepAvroConverter.class); + + private static final Map CLASSIC_MAP = new HashMap<>(); + private static final Map STAGES_MAP = new HashMap<>(); + static { + CLASSIC_MAP.put("awake", FitbitSleepClassicLevel.AWAKE); + CLASSIC_MAP.put("asleep", FitbitSleepClassicLevel.ASLEEP); + CLASSIC_MAP.put("restless", FitbitSleepClassicLevel.RESTLESS); + + STAGES_MAP.put("wake", FitbitSleepStageLevel.AWAKE); + STAGES_MAP.put("rem", FitbitSleepStageLevel.REM); + STAGES_MAP.put("deep", FitbitSleepStageLevel.DEEP); + STAGES_MAP.put("light", FitbitSleepStageLevel.LIGHT); + } + + private String sleepStagesTopic; + private String sleepClassicTopic; + + public FitbitSleepAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + sleepStagesTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepStagesTopic(); + sleepClassicTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepClassicTopic(); + + logger.info("Using sleep topic {} and {}", sleepStagesTopic, sleepClassicTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, JsonNode root, double timeReceived) { + JsonNode meta = root.get("meta"); + if (meta != null) { + JsonNode state = meta.get("state"); + if (state != null && meta.get("state").asText().equals("pending")) { + return Stream.empty(); + } + } + JsonNode sleepArray = root.get("sleep"); + if (sleepArray == null) { + return Stream.empty(); + } + + return iterableToStream(sleepArray) + .sorted(Comparator.comparing(s -> s.get("startTime").asText())) + .flatMap(tryOrNull(s -> { + Instant startTime = Instant.from(DATE_TIME_FORMAT.parse(s.get("startTime").asText())); + boolean isStages = s.get("type") == null || s.get("type").asText().equals("stages"); + + // use an intermediate offset for all records but the last. Since the query time + // depends only on the start time of a sleep stages group, this will reprocess the entire + // sleep stages group if something goes wrong while processing. + Instant intermediateOffset = startTime.minus(Duration.ofSeconds(1)); + + List allRecords = iterableToStream(s.get("levels").get("data")) + .map(d -> { + IndexedRecord sleep; + String topic; + + String dateTime = d.get("dateTime").asText(); + int duration = d.get("seconds").asInt(); + String level = d.get("level").asText(); + + if (isStages) { + sleep = new FitbitSleepStage( + dateTime, + timeReceived, + duration, + STAGES_MAP.getOrDefault(level, FitbitSleepStageLevel.UNKNOWN)); + topic = sleepStagesTopic; + } else { + sleep = new FitbitSleepClassic( + dateTime, + timeReceived, + duration, + CLASSIC_MAP.getOrDefault(level, FitbitSleepClassicLevel.UNKNOWN)); + topic = sleepClassicTopic; + } + + return new TopicData(intermediateOffset, topic, sleep); + }) + .collect(Collectors.toList()); + + if (allRecords.isEmpty()) { + return Stream.empty(); + } + + // The final group gets the actual offset, to ensure that the group does not get queried + // again. + allRecords.get(allRecords.size() - 1).sourceOffset = startTime; + + return allRecords.stream(); + }, (s, ex) -> logger.warn( + "Failed to convert sleep patterns from request {} of user {}, {}", + request.getRequest().url(), request.getUser(), s, ex))); + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java new file mode 100644 index 00000000..01c77d8f --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java @@ -0,0 +1,63 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.connect.rest.fitbit.converter; + +import com.fasterxml.jackson.databind.JsonNode; +import io.confluent.connect.avro.AvroData; +import java.util.stream.Stream; +import org.radarbase.connect.rest.RestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; +import org.radarcns.connector.fitbit.FitbitTimeZone; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class FitbitTimeZoneAvroConverter extends FitbitAvroConverter { + private static final Logger logger = LoggerFactory.getLogger(FitbitTimeZoneAvroConverter.class); + + private String timeZoneTopic; + + public FitbitTimeZoneAvroConverter(AvroData avroData) { + super(avroData); + } + + @Override + public void initialize(RestSourceConnectorConfig config) { + timeZoneTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitTimeZoneTopic(); + logger.info("Using timezone topic {}", timeZoneTopic); + } + + @Override + protected Stream processRecords( + FitbitRestRequest request, + JsonNode root, + double timeReceived) { + JsonNode user = root.get("user"); + if (user == null) { + logger.warn("Failed to get timezone from {}, {}", request.getRequest().url(), root); + return Stream.empty(); + } + JsonNode offsetNode = user.get("offsetFromUTCMillis"); + Integer offset = offsetNode == null ? null : (int) (offsetNode.asLong() / 1000L); + + FitbitTimeZone timeZone = new FitbitTimeZone(timeReceived, offset); + + return Stream.of(new TopicData(request.getDateRange().start().toInstant(), + timeZoneTopic, timeZone)); + } +} diff --git a/radar-fitbit-endpoint/build.gradle b/radar-fitbit-endpoint/build.gradle new file mode 100644 index 00000000..2aa08612 --- /dev/null +++ b/radar-fitbit-endpoint/build.gradle @@ -0,0 +1,17 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.6.10' + id 'application' +} + +dependencies { + implementation("org.radarbase:radar-jersey:0.8.0.1") + implementation("org.slf4j:slf4j-api:1.7.32") +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { + kotlinOptions { + jvmTarget = "17" + apiVersion = "1.6" + languageVersion = "1.6" + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt new file mode 100644 index 00000000..be5755f5 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt @@ -0,0 +1,54 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint + +import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig +import org.radarbase.jersey.GrizzlyServer +import org.radarbase.jersey.config.ConfigLoader +import org.slf4j.LoggerFactory +import kotlin.system.exitProcess + +fun main(args: Array) { + System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager") + val logger = LoggerFactory.getLogger("org.radarbase.fitbit.endpoint.MainKt") + + val config = try { + ConfigLoader.loadConfig( + listOf( + "fitbit.yml", + "/etc/radar-fitbit-endpoint/fitbit.yml" + ), + args, + ).copyFromEnv() + } catch (ex: IllegalArgumentException) { + logger.error("No configuration file was found.") + logger.error("Usage: radar-fitbit-endpoint ") + exitProcess(1) + } + + try { + config.validate() + } catch (ex: IllegalStateException) { + logger.error("Configuration incomplete: {}", ex.message) + exitProcess(1) + } + + val resources = ConfigLoader.loadResources(config.resourceConfig, config) + val server = GrizzlyServer(config.server.baseUri, resources, config.server.isJmxEnabled) + server.listen() +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt new file mode 100644 index 00000000..b60f59ad --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.config + +import org.radarbase.jersey.config.ConfigLoader.copyEnv + +data class FitbitConfig( + val verificationCode: String? = null, + val clientId: String? = null, + val clientSecret: String? = null, +) { + fun copyFromEnv(): FitbitConfig = this + .copyEnv("FITBIT_CLIENT_ID") { copy(clientId = it) } + .copyEnv("FITBIT_CLIENT_SECRET") { copy(clientSecret = it) } + .copyEnv("FITBIT_VERIFICATION_CODE") { copy(verificationCode = it) } + + fun validate() { + requireNotNull(verificationCode) { "fitbit.verificationCode configuration or FITBIT_VERIFICATION_CODE environment variable must be set." } + requireNotNull(clientId) { "fitbit.clientId configuration or FITBIT_CLIENT_ID environment variable must be set." } + requireNotNull(clientSecret) { "fitbit.clientSecret configuration or FITBIT_CLIENT_SECRET environment variable must be set." } + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt new file mode 100644 index 00000000..ce6a0ebf --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt @@ -0,0 +1,40 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.config + +import org.radarbase.fitbit.endpoint.inject.ManagementPortalEnhancerFactory +import org.radarbase.jersey.auth.AuthConfig +import org.radarbase.jersey.config.ConfigLoader.copyOnChange +import org.radarbase.jersey.enhancer.EnhancerFactory + +data class FitbitEndpointConfig( + /** Radar-jersey resource configuration class. */ + val resourceConfig: Class = ManagementPortalEnhancerFactory::class.java, + /** Server configurations. */ + val server: ServerConfig = ServerConfig(), + /** Fitbit configuration. */ + val fitbit: FitbitConfig = FitbitConfig(), + val auth: AuthConfig = AuthConfig(jwtResourceName = "res_fitbitEndpoint"), +) { + fun copyFromEnv(): FitbitEndpointConfig = copyOnChange(fitbit, { it.copyFromEnv() }) { copy(fitbit = it) } + + fun validate() { + fitbit.validate() + } +} + diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/ServerConfig.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/ServerConfig.kt new file mode 100644 index 00000000..3da5ed04 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/ServerConfig.kt @@ -0,0 +1,36 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.config + +import java.net.URI + +data class ServerConfig( + /** Base URL to serve data with. This will determine the base path and the port. */ + val baseUri: URI = URI.create("http://0.0.0.0:8090/fitbit/"), + /** Maximum number of simultaneous requests. */ + val maxRequests: Int = 200, + /** + * Maximum request content length, also when decompressed. + * This protects against memory overflows. + */ + val maxRequestSize: Long = 24 * 1024 * 1024, + /** + * Whether JMX should be enabled. Disable if not needed, for higher performance. + */ + val isJmxEnabled: Boolean = true, +) diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerification.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerification.kt new file mode 100644 index 00000000..9dfb45e9 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerification.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.filter + +@Target(AnnotationTarget.FUNCTION) +@Retention(AnnotationRetention.RUNTIME) +annotation class ClientDomainVerification(val domainName: String) diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFeature.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFeature.kt new file mode 100644 index 00000000..562db0a7 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFeature.kt @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2019. The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * + * See the file LICENSE in the root of this repository. + */ + +package org.radarbase.fitbit.endpoint.filter + +import jakarta.inject.Singleton +import jakarta.ws.rs.Priorities +import jakarta.ws.rs.container.DynamicFeature +import jakarta.ws.rs.container.ResourceInfo +import jakarta.ws.rs.core.FeatureContext +import jakarta.ws.rs.ext.Provider + +/** Authorization for different auth tags. */ +@Provider +@Singleton +class ClientDomainVerificationFeature : DynamicFeature { + override fun configure(resourceInfo: ResourceInfo, context: FeatureContext) { + val resourceMethod = resourceInfo.resourceMethod + if (resourceMethod.isAnnotationPresent(ClientDomainVerification::class.java)) { + context.register(ClientDomainVerificationFilter::class.java, Priorities.AUTHORIZATION) + } + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFilter.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFilter.kt new file mode 100644 index 00000000..66e17c00 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/filter/ClientDomainVerificationFilter.kt @@ -0,0 +1,53 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.filter + +import jakarta.inject.Provider +import jakarta.ws.rs.container.ContainerRequestContext +import jakarta.ws.rs.container.ContainerRequestFilter +import jakarta.ws.rs.container.ResourceInfo +import jakarta.ws.rs.core.Context +import jakarta.ws.rs.core.Response +import org.glassfish.grizzly.http.server.Request +import org.slf4j.LoggerFactory +import java.net.InetAddress + +class ClientDomainVerificationFilter( + /** + * Check that the token has given permissions. + */ + @Context private val resourceInfo: ResourceInfo, + @Context private val req: Provider +) : ContainerRequestFilter { + override fun filter(requestContext: ContainerRequestContext) { + val annotation = resourceInfo.resourceMethod.getAnnotation(ClientDomainVerification::class.java) + + val ipAddress = requestContext.getHeaderString("X-Forwarded-For") + ?: req.get().remoteAddr + + val remoteHostName = InetAddress.getByName(ipAddress).hostName + if (remoteHostName != annotation.domainName && !remoteHostName.endsWith(".${annotation.domainName}")) { + logger.error("Failed to verify that IP address {} belongs to domain name {}. It resolves to {} instead.", ipAddress, annotation.domainName, remoteHostName) + requestContext.abortWith(Response.status(Response.Status.NOT_FOUND).build()) + } + } + + companion object { + private val logger = LoggerFactory.getLogger(ClientDomainVerificationFilter::class.java.name) + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/GatewayResourceEnhancer.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/GatewayResourceEnhancer.kt new file mode 100644 index 00000000..19fd0052 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/GatewayResourceEnhancer.kt @@ -0,0 +1,22 @@ +package org.radarbase.fitbit.endpoint.inject + +import org.glassfish.jersey.internal.inject.AbstractBinder +import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig +import org.radarbase.jersey.enhancer.JerseyResourceEnhancer +import org.radarbase.jersey.filter.Filters + +class GatewayResourceEnhancer(private val config: FitbitEndpointConfig) : JerseyResourceEnhancer { + override val packages: Array = arrayOf( + "org.radarbase.fitbit.endpoint.filter", + "org.radarbase.fitbit.endpoint.resource", + ) + + override val classes: Array> = arrayOf( + Filters.logResponse, + ) + + override fun AbstractBinder.enhance() { + bind(config) + .to(FitbitEndpointConfig::class.java) + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/ManagementPortalEnhancerFactory.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/ManagementPortalEnhancerFactory.kt new file mode 100644 index 00000000..963a4f3a --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/inject/ManagementPortalEnhancerFactory.kt @@ -0,0 +1,19 @@ +package org.radarbase.fitbit.endpoint.inject + +import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig +import org.radarbase.jersey.enhancer.EnhancerFactory +import org.radarbase.jersey.enhancer.Enhancers +import org.radarbase.jersey.enhancer.JerseyResourceEnhancer + +/** This binder needs to register all non-Jersey classes, otherwise initialization fails. */ +class ManagementPortalEnhancerFactory(private val config: FitbitEndpointConfig) : EnhancerFactory { + override fun createEnhancers(): List { + return listOf( + GatewayResourceEnhancer(config), + Enhancers.radar(config.auth), + Enhancers.managementPortal(config.auth), + Enhancers.health, + Enhancers.exception, + ) + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt new file mode 100644 index 00000000..5a172227 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.fitbit.endpoint.resource + +import jakarta.ws.rs.* +import jakarta.ws.rs.core.Context +import jakarta.ws.rs.core.Response +import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig +import org.radarbase.fitbit.endpoint.filter.ClientDomainVerification +import org.radarbase.fitbit.endpoint.service.FitbitVerificationService +import org.radarbase.fitbit.endpoint.service.NotificationService + +@Path("webhook") +class WebhookResource( + @Context private val fitbitVerificationService: FitbitVerificationService, + @Context private val notificationService: NotificationService, +) { + @GET + @ClientDomainVerification("fitbit.com") + fun verifyCode( + @Context config: FitbitEndpointConfig, + @PathParam("verify") verificationCode: String?, + ): Response { + return if (verificationCode == config.fitbit.verificationCode) { + Response.noContent().build() + } else { + Response.status(Response.Status.NOT_FOUND).build() + } + } + + @POST + @ClientDomainVerification("fitbit.com") + fun submitNotification( + @HeaderParam("X-Fitbit-Signature") fitbitSignature: String?, + contents: String, + ): Response { + if (!fitbitVerificationService.isSignatureValid(fitbitSignature, contents)) { + return Response.status(Response.Status.NOT_FOUND).build() + } + + notificationService.add(contents) + + return Response.noContent().build() + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/FitbitVerificationService.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/FitbitVerificationService.kt new file mode 100644 index 00000000..74cdd033 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/FitbitVerificationService.kt @@ -0,0 +1,37 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.service + +import jakarta.ws.rs.core.Context +import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig + +class FitbitVerificationService( + @Context config: FitbitEndpointConfig, +) { + private val clientSecret = requireNotNull(config.fitbit.clientSecret) + + fun isSignatureValid(signature: String?, contents: String): Boolean { + if (signature == null) { + return false + } + + // TODO: fix signature verification + + return true + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt new file mode 100644 index 00000000..6b3f50ea --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.service + +class NotificationService { + fun add(contents: String) { + // TODO add to redis + } +} diff --git a/radar-oauth2-user-repository/build.gradle b/radar-oauth2-user-repository/build.gradle new file mode 100644 index 00000000..7d6ad79c --- /dev/null +++ b/radar-oauth2-user-repository/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.6.10' +} + +dependencies { + implementation("org.slf4j:slf4j-api:1.7.32") +} + +tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { + kotlinOptions { + jvmTarget = "17" + apiVersion = "1.6" + languageVersion = "1.6" + } +} diff --git a/settings.gradle b/settings.gradle index c2ce6f87..9be5d9d9 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1,5 @@ rootProject.name = 'kafka-connect-rest-source' include ':kafka-connect-fitbit-source' include ':kafka-connect-rest-source' - +include ':radar-fitbit-endpoint' +include ':radar-fitbit-converter' From 342fc1cddc1e972efa68c7c8e5762463439142e1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 7 Feb 2022 17:06:35 +0100 Subject: [PATCH 2/5] Add notification endpoint --- .../org/radarbase/fitbit/endpoint/Main.kt | 2 +- .../fitbit/endpoint/api/NotificationFilter.kt | 22 +++++++ .../endpoint/api/NotificationSelection.kt | 31 ++++++++++ .../fitbit/endpoint/config/FitbitConfig.kt | 2 +- .../endpoint/config/FitbitEndpointConfig.kt | 4 +- .../endpoint/resource/NotificationResource.kt | 60 +++++++++++++++++++ .../endpoint/resource/WebhookResource.kt | 3 + .../endpoint/service/NotificationService.kt | 29 ++++++++- 8 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationFilter.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationSelection.kt create mode 100644 radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/NotificationResource.kt diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt index be5755f5..a5c050ad 100644 --- a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/Main.kt @@ -34,7 +34,7 @@ fun main(args: Array) { "/etc/radar-fitbit-endpoint/fitbit.yml" ), args, - ).copyFromEnv() + ).withEnv() } catch (ex: IllegalArgumentException) { logger.error("No configuration file was found.") logger.error("Usage: radar-fitbit-endpoint ") diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationFilter.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationFilter.kt new file mode 100644 index 00000000..eaf39dff --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationFilter.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.api + +data class NotificationFilter( + val users: List, +) diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationSelection.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationSelection.kt new file mode 100644 index 00000000..d5a61017 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/api/NotificationSelection.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.fitbit.endpoint.api + +data class NotificationSelection( + val id: String, + val notifications: List, +) + +data class FitbitNotification( + val collectionType: String, + val date: String, + val ownerId: String, + val ownerType: String, + val subscriptionId: String, +) diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt index b60f59ad..ae2e423e 100644 --- a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitConfig.kt @@ -24,7 +24,7 @@ data class FitbitConfig( val clientId: String? = null, val clientSecret: String? = null, ) { - fun copyFromEnv(): FitbitConfig = this + fun withEnv(): FitbitConfig = this .copyEnv("FITBIT_CLIENT_ID") { copy(clientId = it) } .copyEnv("FITBIT_CLIENT_SECRET") { copy(clientSecret = it) } .copyEnv("FITBIT_VERIFICATION_CODE") { copy(verificationCode = it) } diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt index ce6a0ebf..26072073 100644 --- a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/config/FitbitEndpointConfig.kt @@ -31,7 +31,9 @@ data class FitbitEndpointConfig( val fitbit: FitbitConfig = FitbitConfig(), val auth: AuthConfig = AuthConfig(jwtResourceName = "res_fitbitEndpoint"), ) { - fun copyFromEnv(): FitbitEndpointConfig = copyOnChange(fitbit, { it.copyFromEnv() }) { copy(fitbit = it) } + fun withEnv(): FitbitEndpointConfig = this + .copyOnChange(auth, { it.withEnv() }) { copy(auth = it) } + .copyOnChange(fitbit, { it.withEnv() }) { copy(fitbit = it) } fun validate() { fitbit.validate() diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/NotificationResource.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/NotificationResource.kt new file mode 100644 index 00000000..7a9b6ce7 --- /dev/null +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/NotificationResource.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.fitbit.endpoint.resource + +import jakarta.ws.rs.* +import jakarta.ws.rs.core.Context +import jakarta.ws.rs.core.MediaType.APPLICATION_JSON +import jakarta.ws.rs.core.Response +import org.radarbase.fitbit.endpoint.api.NotificationFilter +import org.radarbase.fitbit.endpoint.api.NotificationSelection +import org.radarbase.fitbit.endpoint.service.NotificationService +import org.radarbase.jersey.auth.Authenticated +import java.net.URI + +@Path("notifications") +@Authenticated +@Consumes(APPLICATION_JSON) +@Produces(APPLICATION_JSON) +class NotificationResource( + @Context private val notificationService: NotificationService, +) { + @POST + fun query( + filter: NotificationFilter, + ): Response { + val selected = notificationService.makeSelection(filter) + return Response.created(URI.create("notifications/${selected.id}")) + .entity(selected) + .build() + } + + @GET + @Path("/{selectionId}") + fun notifications( + @PathParam("selectionId") selectionId: String, + ): NotificationSelection = notificationService.getSelection(selectionId) + + @DELETE + @Path("/{selectionId}") + fun deleteNotifications( + @PathParam("selectionId") selectionId: String, + ): Response { + notificationService.deleteSelection(selectionId) + return Response.noContent().build() + } +} diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt index 5a172227..10244447 100644 --- a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/resource/WebhookResource.kt @@ -18,6 +18,7 @@ package org.radarbase.fitbit.endpoint.resource import jakarta.ws.rs.* import jakarta.ws.rs.core.Context +import jakarta.ws.rs.core.MediaType import jakarta.ws.rs.core.Response import org.radarbase.fitbit.endpoint.config.FitbitEndpointConfig import org.radarbase.fitbit.endpoint.filter.ClientDomainVerification @@ -25,6 +26,8 @@ import org.radarbase.fitbit.endpoint.service.FitbitVerificationService import org.radarbase.fitbit.endpoint.service.NotificationService @Path("webhook") +@Consumes(MediaType.APPLICATION_JSON) +@Produces(MediaType.APPLICATION_JSON) class WebhookResource( @Context private val fitbitVerificationService: FitbitVerificationService, @Context private val notificationService: NotificationService, diff --git a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt index 6b3f50ea..547c6f73 100644 --- a/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt +++ b/radar-fitbit-endpoint/src/main/kotlin/org/radarbase/fitbit/endpoint/service/NotificationService.kt @@ -17,8 +17,33 @@ package org.radarbase.fitbit.endpoint.service -class NotificationService { +import com.fasterxml.jackson.core.type.TypeReference +import com.fasterxml.jackson.databind.ObjectMapper +import jakarta.ws.rs.core.Context +import org.radarbase.fitbit.endpoint.api.FitbitNotification +import org.radarbase.fitbit.endpoint.api.NotificationFilter +import org.radarbase.fitbit.endpoint.api.NotificationSelection + +class NotificationService( + @Context objectMapper: ObjectMapper +) { + private val contentReader = objectMapper.readerFor(object : TypeReference>() {}) + fun add(contents: String) { - // TODO add to redis + val notifications = contentReader.readValue>(contents) + + TODO("Add notifications to redis") + } + + fun makeSelection(filter: NotificationFilter): NotificationSelection { + TODO("Retrieve notifications from redis and mark them for being currently processing") + } + + fun getSelection(selectionId: String): NotificationSelection { + TODO("Retrieve selection by ID") + } + + fun deleteSelection(selectionId: String) { + TODO("Delete selection and associated notifications by ID") } } From 5f2307c133cb5b8966e7eaab4736dc11a773c1ed Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 8 Mar 2022 14:44:17 +0100 Subject: [PATCH 3/5] Move converters to separate package --- kafka-connect-fitbit-source/build.gradle | 6 + .../FitbitActivityLogAvroConverter.java | 200 ------------------ .../fitbit/converter/FitbitAvroConverter.java | 184 ---------------- .../fitbit/converter/FitbitAvroConverter.kt | 97 +++++++++ .../FitbitIntradayCaloriesAvroConverter.java | 79 ------- .../FitbitIntradayHeartRateAvroConverter.java | 84 -------- .../FitbitIntradayStepsAvroConverter.java | 87 -------- .../converter/FitbitSleepAvroConverter.java | 143 ------------- .../FitbitTimeZoneAvroConverter.java | 63 ------ .../fitbit/request/FitbitRestRequest.java | 2 +- .../fitbit/route/FitbitActivityLogRoute.java | 36 ++-- .../route/FitbitIntradayCaloriesRoute.java | 32 ++- .../route/FitbitIntradayHeartRateRoute.java | 34 +-- .../route/FitbitIntradayStepsRoute.java | 30 +-- .../rest/fitbit/route/FitbitPollingRoute.java | 22 +- .../rest/fitbit/route/FitbitSleepRoute.java | 22 +- .../fitbit/route/FitbitTimeZoneRoute.java | 32 +-- .../connect/rest/fitbit/util/DateRange.java | 65 ------ radar-fitbit-converter/build.gradle | 5 +- .../convert/fitbit/ConverterContext.kt | 24 +++ .../radarbase/convert/fitbit/DateRange.java | 65 ------ .../org/radarbase/convert/fitbit/DateRange.kt | 25 +++ .../FitbitActivityLogAvroConverter.java | 200 ------------------ .../fitbit/FitbitActivityLogDataConverter.kt | 145 +++++++++++++ .../convert/fitbit/FitbitAvroConverter.java | 183 ---------------- .../convert/fitbit/FitbitDataConverter.kt | 31 +++ .../FitbitIntradayCaloriesAvroConverter.java | 79 ------- .../FitbitIntradayCaloriesDataConverter.kt | 41 ++++ .../FitbitIntradayHeartRateAvroConverter.java | 84 -------- .../FitbitIntradayHeartRateDataConverter.kt | 60 ++++++ .../FitbitIntradayStepsAvroConverter.java | 87 -------- .../FitbitIntradayStepsDataConverter.kt | 58 +++++ .../fitbit/FitbitSleepAvroConverter.java | 143 ------------- .../fitbit/FitbitSleepDataConverter.kt | 120 +++++++++++ .../fitbit/FitbitTimeZoneAvroConverter.java | 63 ------ .../fitbit/FitbitTimeZoneDataConverter.kt | 51 +++++ .../convert/fitbit/JsonNodeExtensions.kt | 81 +++++++ .../org/radarbase/convert/fitbit/TopicData.kt | 28 +++ 38 files changed, 882 insertions(+), 1909 deletions(-) delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java create mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayCaloriesAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java delete mode 100644 kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt create mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/TopicData.kt diff --git a/kafka-connect-fitbit-source/build.gradle b/kafka-connect-fitbit-source/build.gradle index be07a1c0..e74f5348 100644 --- a/kafka-connect-fitbit-source/build.gradle +++ b/kafka-connect-fitbit-source/build.gradle @@ -1,8 +1,14 @@ +plugins { + id 'org.jetbrains.kotlin.jvm' version '1.6.10' +} + dependencies { api project(':kafka-connect-rest-source') api group: 'io.confluent', name: 'kafka-connect-avro-converter', version: confluentVersion api group: 'org.radarbase', name: 'radar-schemas-commons', version: '0.7.3' + implementation(project(":radar-fitbit-converter")) + implementation group: 'org.radarbase', name: 'oauth-client-util', version: '0.8.0' implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-yaml', version: jacksonVersion diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java deleted file mode 100644 index dd7256d1..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitActivityLogAvroConverter.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.util.Comparator; -import java.util.Optional; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitActivityHeartRate; -import org.radarcns.connector.fitbit.FitbitActivityHeartRate.Builder; -import org.radarcns.connector.fitbit.FitbitActivityLevels; -import org.radarcns.connector.fitbit.FitbitActivityLogRecord; -import org.radarcns.connector.fitbit.FitbitManualDataEntry; -import org.radarcns.connector.fitbit.FitbitSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitActivityLogAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitActivityLogAvroConverter.class); - private static final float FOOD_CAL_TO_KJOULE_FACTOR = 4.1868f; - - private String activityLogTopic; - - public FitbitActivityLogAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - activityLogTopic = ((FitbitRestSourceConnectorConfig)config).getActivityLogTopic(); - - logger.info("Using activity log topic {}", activityLogTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - - JsonNode array = root.get("activities"); - if (array == null || !array.isArray()) { - return Stream.empty(); - } - - return iterableToStream(array) - .sorted(Comparator.comparing(s -> s.get("startTime").textValue())) - .map(tryOrNull(s -> { - OffsetDateTime startTime = OffsetDateTime.parse(s.get("startTime").textValue()); - FitbitActivityLogRecord record = getRecord(s, startTime); - return new TopicData(startTime.toInstant(), activityLogTopic, record); - }, (s, ex) -> logger.warn( - "Failed to convert sleep patterns from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), s, ex))); - } - - private FitbitActivityLogRecord getRecord(JsonNode s, OffsetDateTime startTime) { - return FitbitActivityLogRecord.newBuilder() - .setTime(startTime.toInstant().toEpochMilli() / 1000d) - .setTimeReceived(System.currentTimeMillis() / 1000d) - .setTimeLastModified(Instant.parse(s.get("lastModified").asText()).toEpochMilli() / 1000d) - .setId(optLong(s, "logId").orElseThrow( - () -> new IllegalArgumentException("Activity log ID not specified"))) - .setLogType(optString(s, "logType").orElse(null)) - .setType(optLong(s, "activityType").orElse(null)) - .setSpeed(optDouble(s, "speed").orElse(null)) - .setDistance(optDouble(s, "distance").map(Double::floatValue).orElse(null)) - .setSteps(optInt(s, "steps").orElse(null)) - .setEnergy(optInt(s, "calories").map(e -> e * FOOD_CAL_TO_KJOULE_FACTOR) - .orElse(null)) - .setDuration(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( - () -> new IllegalArgumentException("Activity log duration not specified"))) - .setDurationActive(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( - () -> new IllegalArgumentException("Activity duration active not specified"))) - .setTimeZoneOffset(startTime.getOffset().getTotalSeconds()) - .setName(optString(s, "activityName").orElse(null)) - .setHeartRate(getHeartRate(s)) - .setManualDataEntry(getManualDataEntry(s)) - .setLevels(getActivityLevels(s)) - .setSource(getSource(s)) - .build(); - } - - private FitbitSource getSource(JsonNode s) { - return optObject(s, "source") - .flatMap(source -> optString(source, "id") - .map(id -> FitbitSource.newBuilder() - .setId(id) - .setName(optString(source, "name").orElse(null)) - .setType(optString(source, "type").orElse(null)) - .setUrl(optString(source, "url").orElse(null)) - .build())) - .orElse(null); - } - - private FitbitActivityLevels getActivityLevels(JsonNode s) { - return optArray(s, "activityLevels") - .map(levels -> { - FitbitActivityLevels.Builder activityLevels = FitbitActivityLevels.newBuilder(); - for (JsonNode level : levels) { - Integer duration = optInt(level, "minutes") - .map(t -> t * 60) - .orElse(null); - switch (optString(level, "name").orElse("")) { - case "sedentary": - activityLevels.setDurationSedentary(duration); - break; - case "lightly": - activityLevels.setDurationLightly(duration); - break; - case "fairly": - activityLevels.setDurationFairly(duration); - break; - case "very": - activityLevels.setDurationVery(duration); - } - } - - return activityLevels.build(); - }) - .orElse(null); - } - - private FitbitManualDataEntry getManualDataEntry(JsonNode s) { - return optObject(s, "manualValuesSpecified") - .map(manual -> FitbitManualDataEntry.newBuilder() - .setSteps(optBoolean(manual, "steps").orElse(null)) - .setDistance(optBoolean(manual, "distance").orElse(null)) - .setEnergy(optBoolean(manual, "calorie").orElse(null)) - .build()) - .orElse(null); - } - - private FitbitActivityHeartRate getHeartRate(JsonNode activity) { - Optional mean = optInt(activity, "averageHeartRate"); - Optional> zones = optArray(activity, "heartRateZones"); - - if (mean.isEmpty() && zones.isEmpty()) { - return null; - } - - Builder heartRate = FitbitActivityHeartRate.newBuilder() - .setMean(mean.orElse(null)); - - zones.ifPresent(z -> { - for (JsonNode zone : z) { - Integer minValue = optInt(zone, "min").orElse(null); - Integer duration = optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null); - switch (optString(zone, "name").orElse("")) { - case "Out of Range": - heartRate.setMin(minValue); - heartRate.setDurationOutOfRange(duration); - break; - case "Fat Burn": - heartRate.setMinFatBurn(minValue); - heartRate.setDurationFatBurn(duration); - break; - case "Cardio": - heartRate.setMinCardio(minValue); - heartRate.setDurationCardio(duration); - break; - case "Peak": - heartRate.setMinPeak(minValue); - heartRate.setMax(optInt(zone, "max").orElse(null)); - heartRate.setDurationPeak(duration); - break; - default: - logger.warn("Cannot process unknown heart rate zone {}", zone.get("name").asText()); - break; - } - } - }); - - return heartRate.build(); - } - -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java deleted file mode 100644 index c48b4abe..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.java +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.confluent.connect.avro.AvroData; -import java.io.IOException; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import okhttp3.Headers; -import org.apache.avro.Schema.Field; -import org.apache.avro.generic.IndexedRecord; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.source.SourceRecord; -import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarbase.connect.rest.fitbit.user.User; -import org.radarbase.connect.rest.request.RestRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Abstract class to help convert Fitbit data to Avro Data. - */ -public abstract class FitbitAvroConverter implements PayloadToSourceRecordConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitAvroConverter.class); - private static final Map TIME_UNIT_MAP = new HashMap<>(); - - static { - TIME_UNIT_MAP.put("minute", TimeUnit.MINUTES); - TIME_UNIT_MAP.put("second", TimeUnit.SECONDS); - TIME_UNIT_MAP.put("hour", TimeUnit.HOURS); - TIME_UNIT_MAP.put("day", TimeUnit.DAYS); - TIME_UNIT_MAP.put("millisecond", TimeUnit.MILLISECONDS); - TIME_UNIT_MAP.put("nanosecond", TimeUnit.NANOSECONDS); - TIME_UNIT_MAP.put("microsecond", TimeUnit.MICROSECONDS); - } - - private final AvroData avroData; - - public FitbitAvroConverter(AvroData avroData) { - this.avroData = avroData; - } - - @Override - public Collection convert( - RestRequest restRequest, Headers headers, byte[] data) throws IOException { - if (data == null) { - throw new IOException("Failed to read body"); - } - JsonNode activities = JSON_READER.readTree(data); - - User user = ((FitbitRestRequest) restRequest).getUser(); - final SchemaAndValue key = user.getObservationKey(avroData); - double timeReceived = System.currentTimeMillis() / 1000d; - - return processRecords((FitbitRestRequest)restRequest, activities, timeReceived) - .filter(t -> validateRecord((FitbitRestRequest)restRequest, t)) - .map(t -> { - SchemaAndValue avro = avroData.toConnectData(t.value.getSchema(), t.value); - Map offset = Collections.singletonMap( - TIMESTAMP_OFFSET_KEY, t.sourceOffset.toEpochMilli()); - - return new SourceRecord(restRequest.getPartition(), offset, t.topic, - key.schema(), key.value(), avro.schema(), avro.value()); - }) - .collect(Collectors.toList()); - } - - private boolean validateRecord(FitbitRestRequest request, TopicData record) { - if (record == null) { - return false; - } - Instant endDate = request.getUser().getEndDate(); - if (endDate == null) { - return true; - } - Field timeField = record.value.getSchema().getField("time"); - if (timeField != null) { - long time = (long) (((Double)record.value.get(timeField.pos()) * 1000.0)); - return Instant.ofEpochMilli(time).isBefore(endDate); - } - return true; - } - - /** Process the JSON records generated by given request. */ - protected abstract Stream processRecords( - FitbitRestRequest request, - JsonNode root, - double timeReceived); - - /** Get Fitbit dataset interval used in some intraday API calls. */ - protected static int getRecordInterval(JsonNode root, int defaultValue) { - JsonNode type = root.get("datasetType"); - JsonNode interval = root.get("datasetInterval"); - if (type == null || interval == null) { - logger.warn("Failed to get data interval; using {} instead", defaultValue); - return defaultValue; - } - return (int)TIME_UNIT_MAP - .getOrDefault(type.asText(), TimeUnit.SECONDS) - .toSeconds(interval.asLong()); - } - - /** Converts an iterable (like a JsonNode containing an array) to a stream. */ - protected static Stream iterableToStream(Iterable iter) { - return StreamSupport.stream(iter.spliterator(), false); - } - - protected static Optional optLong(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.canConvertToLong() ? Optional.of(v.longValue()) : Optional.empty(); - } - - protected static Optional optDouble(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isNumber() ? Optional.of(v.doubleValue()) : Optional.empty(); - } - - protected static Optional optInt(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.canConvertToInt() ? Optional.of(v.intValue()) : Optional.empty(); - } - - protected static Optional optString(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isTextual() ? Optional.ofNullable(v.textValue()) : Optional.empty(); - } - - protected static Optional optBoolean(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isBoolean() ? Optional.of(v.booleanValue()) : Optional.empty(); - } - - protected static Optional optObject(JsonNode parent, String fieldName) { - JsonNode v = parent.get(fieldName); - return v != null && v.isObject() ? Optional.of((ObjectNode) v) : Optional.empty(); - } - - protected static Optional> optArray(JsonNode parent, String fieldName) { - JsonNode v = parent.get(fieldName); - return v != null && v.isArray() && v.size() != 0 ? - Optional.of(v) : Optional.empty(); - } - - /** Single value for a topic. */ - protected static class TopicData { - Instant sourceOffset; - final String topic; - final IndexedRecord value; - - public TopicData(Instant sourceOffset, String topic, IndexedRecord value) { - this.sourceOffset = sourceOffset; - this.topic = topic; - this.value = value; - } - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt new file mode 100644 index 00000000..75bab836 --- /dev/null +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt @@ -0,0 +1,97 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.connect.rest.fitbit.converter + +import io.confluent.connect.avro.AvroData +import okhttp3.Headers +import org.apache.kafka.connect.source.SourceRecord +import org.radarbase.connect.rest.RestSourceConnectorConfig +import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig +import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator +import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest +import org.radarbase.connect.rest.request.RestRequest +import org.radarbase.convert.fitbit.ConverterContext +import org.radarbase.convert.fitbit.FitbitDataConverter +import org.radarbase.convert.fitbit.TopicData +import org.slf4j.LoggerFactory +import java.io.IOException +import java.time.Instant +import java.util.* + +/** + * Abstract class to help convert Fitbit data to Avro Data. + */ +class FitbitAvroConverter( + private val avroData: AvroData, + private val dataConverterCreator: (FitbitRestSourceConnectorConfig) -> FitbitDataConverter +) : PayloadToSourceRecordConverter { + private lateinit var converter: FitbitDataConverter + override fun initialize(config: RestSourceConnectorConfig) { + converter = dataConverterCreator(config as FitbitRestSourceConnectorConfig) + } + @Throws(IOException::class) + override fun convert( + restRequest: RestRequest, headers: Headers, data: ByteArray? + ): Collection { + data ?: throw IOException("Failed to read body") + val activities = FitbitRequestGenerator.JSON_READER.readTree(data) + val user = (restRequest as FitbitRestRequest).user + val key = user.getObservationKey(avroData) + val timeReceived = System.currentTimeMillis() / 1000.0 + return converter.processRecords( + ConverterContext(user.userId, + restRequest.getRequest().url.toString(), + restRequest.dateRange), + activities, + timeReceived) + .mapNotNull { r -> r.fold( + { + it + }, + { + logger.error("Data conversion failed for {} of user {}", + restRequest.request.url, restRequest.user.userId, it) + null + }) + } + .filter { t -> validateRecord(restRequest, t) } + .map { t -> + val avro = avroData.toConnectData(t.value.schema, t.value) + val offset: Map = Collections.singletonMap( + PayloadToSourceRecordConverter.TIMESTAMP_OFFSET_KEY, + t.sourceOffset.toEpochMilli()) + SourceRecord(restRequest.getPartition(), offset, t.topic, + key.schema(), key.value(), avro.schema(), avro.value()) + } + .toList() + } + + private fun validateRecord(request: FitbitRestRequest, record: TopicData): Boolean { + val endDate = request.user.endDate ?: return true + val timeField = record.value.schema.getField("time") + return if (timeField != null) { + val time = (record.value[timeField.pos()] as Double * 1000.0).toLong() + Instant.ofEpochMilli(time) < endDate + } else true + } + + companion object { + private val logger = LoggerFactory.getLogger( + FitbitAvroConverter::class.java) + } +} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayCaloriesAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayCaloriesAvroConverter.java deleted file mode 100644 index 93fb8847..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayCaloriesAvroConverter.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradayCalories; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayCaloriesAvroConverter extends FitbitAvroConverter { - - private static final Logger logger = - LoggerFactory.getLogger(FitbitIntradayCaloriesAvroConverter.class); - - private String caloriesTopic; - - public FitbitIntradayCaloriesAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-calories-intraday"); - if (intraday == null) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 60); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().end(); - - return iterableToStream(dataset) - .map( - tryOrNull( - activity -> { - Instant time = - startDate.with(LocalTime.parse(activity.get("time").asText())).toInstant(); - - FitbitIntradayCalories calories = - new FitbitIntradayCalories( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asDouble(), - activity.get("level").asInt(), - activity.get("mets").asDouble()); - - return new TopicData(time, caloriesTopic, calories); - }, - (a, ex) -> - logger.warn( - "Failed to convert calories from request {} of user {}, {}", - request.getRequest().url(), - request.getUser(), - a, - ex))); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - caloriesTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayCaloriesTopic(); - logger.info("Using calories topic {}", caloriesTopic); - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java deleted file mode 100644 index 89bc4cf1..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayHeartRateAvroConverter.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradayHeartRate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayHeartRateAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger( - FitbitIntradayHeartRateAvroConverter.class); - private String heartRateTopic; - - public FitbitIntradayHeartRateAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - heartRateTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayHeartRateTopic(); - logger.info("Using heart rate topic {}", heartRateTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-heart-intraday"); - if (intraday == null || !intraday.isObject()) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null || !dataset.isArray()) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 1); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().start(); - - return iterableToStream(dataset) - .map(tryOrNull(activity -> { - Instant time = startDate.with(LocalTime.parse(activity.get("time").asText())) - .toInstant(); - - FitbitIntradayHeartRate heartRate = new FitbitIntradayHeartRate( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asInt()); - - return new TopicData(time, heartRateTopic, heartRate); - }, (a, ex) -> logger.warn( - "Failed to convert heart rate from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), a, ex))); - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java deleted file mode 100644 index 2381afa4..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitIntradayStepsAvroConverter.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradaySteps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayStepsAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger( - FitbitIntradayStepsAvroConverter.class); - - private String stepTopic; - - public FitbitIntradayStepsAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - stepTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayStepsTopic(); - logger.info("Using step topic {}", stepTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-steps-intraday"); - if (intraday == null) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 60); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().end(); - - return iterableToStream(dataset) - .map(tryOrNull(activity -> { - - Instant time = startDate - .with(LocalTime.parse(activity.get("time").asText())) - .toInstant(); - - FitbitIntradaySteps steps = new FitbitIntradaySteps( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asInt()); - - return new TopicData(time, stepTopic, steps); - }, (a, ex) -> logger.warn( - "Failed to convert steps from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), a, ex))); - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java deleted file mode 100644 index 976a348b..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitSleepAvroConverter.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute.DATE_TIME_FORMAT; -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.avro.generic.IndexedRecord; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitSleepClassic; -import org.radarcns.connector.fitbit.FitbitSleepClassicLevel; -import org.radarcns.connector.fitbit.FitbitSleepStage; -import org.radarcns.connector.fitbit.FitbitSleepStageLevel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitSleepAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitSleepAvroConverter.class); - - private static final Map CLASSIC_MAP = new HashMap<>(); - private static final Map STAGES_MAP = new HashMap<>(); - static { - CLASSIC_MAP.put("awake", FitbitSleepClassicLevel.AWAKE); - CLASSIC_MAP.put("asleep", FitbitSleepClassicLevel.ASLEEP); - CLASSIC_MAP.put("restless", FitbitSleepClassicLevel.RESTLESS); - - STAGES_MAP.put("wake", FitbitSleepStageLevel.AWAKE); - STAGES_MAP.put("rem", FitbitSleepStageLevel.REM); - STAGES_MAP.put("deep", FitbitSleepStageLevel.DEEP); - STAGES_MAP.put("light", FitbitSleepStageLevel.LIGHT); - } - - private String sleepStagesTopic; - private String sleepClassicTopic; - - public FitbitSleepAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - sleepStagesTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepStagesTopic(); - sleepClassicTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepClassicTopic(); - - logger.info("Using sleep topic {} and {}", sleepStagesTopic, sleepClassicTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode meta = root.get("meta"); - if (meta != null) { - JsonNode state = meta.get("state"); - if (state != null && meta.get("state").asText().equals("pending")) { - return Stream.empty(); - } - } - JsonNode sleepArray = root.get("sleep"); - if (sleepArray == null) { - return Stream.empty(); - } - - return iterableToStream(sleepArray) - .sorted(Comparator.comparing(s -> s.get("startTime").asText())) - .flatMap(tryOrNull(s -> { - Instant startTime = Instant.from(DATE_TIME_FORMAT.parse(s.get("startTime").asText())); - boolean isStages = s.get("type") == null || s.get("type").asText().equals("stages"); - - // use an intermediate offset for all records but the last. Since the query time - // depends only on the start time of a sleep stages group, this will reprocess the entire - // sleep stages group if something goes wrong while processing. - Instant intermediateOffset = startTime.minus(Duration.ofSeconds(1)); - - List allRecords = iterableToStream(s.get("levels").get("data")) - .map(d -> { - IndexedRecord sleep; - String topic; - - String dateTime = d.get("dateTime").asText(); - int duration = d.get("seconds").asInt(); - String level = d.get("level").asText(); - - if (isStages) { - sleep = new FitbitSleepStage( - dateTime, - timeReceived, - duration, - STAGES_MAP.getOrDefault(level, FitbitSleepStageLevel.UNKNOWN)); - topic = sleepStagesTopic; - } else { - sleep = new FitbitSleepClassic( - dateTime, - timeReceived, - duration, - CLASSIC_MAP.getOrDefault(level, FitbitSleepClassicLevel.UNKNOWN)); - topic = sleepClassicTopic; - } - - return new TopicData(intermediateOffset, topic, sleep); - }) - .collect(Collectors.toList()); - - if (allRecords.isEmpty()) { - return Stream.empty(); - } - - // The final group gets the actual offset, to ensure that the group does not get queried - // again. - allRecords.get(allRecords.size() - 1).sourceOffset = startTime; - - return allRecords.stream(); - }, (s, ex) -> logger.warn( - "Failed to convert sleep patterns from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), s, ex))); - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java deleted file mode 100644 index 01c77d8f..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitTimeZoneAvroConverter.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitTimeZone; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitTimeZoneAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitTimeZoneAvroConverter.class); - - private String timeZoneTopic; - - public FitbitTimeZoneAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - timeZoneTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitTimeZoneTopic(); - logger.info("Using timezone topic {}", timeZoneTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, - JsonNode root, - double timeReceived) { - JsonNode user = root.get("user"); - if (user == null) { - logger.warn("Failed to get timezone from {}, {}", request.getRequest().url(), root); - return Stream.empty(); - } - JsonNode offsetNode = user.get("offsetFromUTCMillis"); - Integer offset = offsetNode == null ? null : (int) (offsetNode.asLong() / 1000L); - - FitbitTimeZone timeZone = new FitbitTimeZone(timeReceived, offset); - - return Stream.of(new TopicData(request.getDateRange().start().toInstant(), - timeZoneTopic, timeZone)); - } -} diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java index 40635a5c..9638c402 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/request/FitbitRestRequest.java @@ -22,9 +22,9 @@ import okhttp3.OkHttpClient; import okhttp3.Request; import org.radarbase.connect.rest.fitbit.user.User; -import org.radarbase.connect.rest.fitbit.util.DateRange; import org.radarbase.connect.rest.request.RequestRoute; import org.radarbase.connect.rest.request.RestRequest; +import org.radarbase.convert.fitbit.DateRange; /** * REST request taking into account the user and offsets queried. The offsets are useful for diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitActivityLogRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitActivityLogRoute.java index 63a72c98..007a8e6a 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitActivityLogRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitActivityLogRoute.java @@ -17,32 +17,32 @@ package org.radarbase.connect.rest.fitbit.route; -import static java.time.ZoneOffset.UTC; -import static java.time.temporal.ChronoUnit.SECONDS; - import io.confluent.connect.avro.AvroData; -import java.time.Duration; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.stream.Stream; -import org.radarbase.connect.rest.fitbit.converter.FitbitActivityLogAvroConverter; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; -import org.radarbase.connect.rest.fitbit.util.DateRange; +import org.radarbase.convert.fitbit.DateRange; +import org.radarbase.convert.fitbit.FitbitActivityLogDataConverter; +import org.radarbase.convert.fitbit.FitbitDataConverter; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.SECONDS; public class FitbitActivityLogRoute extends FitbitPollingRoute { public static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE_TIME .withZone(UTC); private static final Duration ACTIVITY_LOG_POLL_INTERVAL = Duration.ofDays(1); - private final FitbitActivityLogAvroConverter converter; - public FitbitActivityLogRoute(FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "activity_log"); - converter = new FitbitActivityLogAvroConverter(avroData); + super(generator, userRepository, "activity_log", avroData); } @Override @@ -50,6 +50,11 @@ protected String getUrlFormat(String baseUrl) { return baseUrl + "/1/user/%s/activities/list.json?sort=asc&afterDate=%s&limit=20&offset=0"; } + @Override + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitActivityLogDataConverter(config.getActivityLogTopic()); + } + /** * Actually construct a request, based on the current offset * @param user Fitbit user @@ -69,9 +74,4 @@ protected Stream createRequests(User user) { protected Duration getPollIntervalPerUser() { return ACTIVITY_LOG_POLL_INTERVAL; } - - @Override - public FitbitActivityLogAvroConverter converter() { - return converter; - } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayCaloriesRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayCaloriesRoute.java index 13050c98..553211da 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayCaloriesRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayCaloriesRoute.java @@ -1,24 +1,27 @@ package org.radarbase.connect.rest.fitbit.route; -import static java.time.temporal.ChronoUnit.MINUTES; - import io.confluent.connect.avro.AvroData; -import java.util.stream.Stream; -import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; -import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayCaloriesAvroConverter; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.convert.fitbit.FitbitDataConverter; +import org.radarbase.convert.fitbit.FitbitIntradayCaloriesDataConverter; -public class FitbitIntradayCaloriesRoute extends FitbitPollingRoute { +import java.util.stream.Stream; - private final FitbitIntradayCaloriesAvroConverter caloriesAvroConverter; +import static java.time.temporal.ChronoUnit.MINUTES; +public class FitbitIntradayCaloriesRoute extends FitbitPollingRoute { public FitbitIntradayCaloriesRoute( FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "intraday_calories"); - caloriesAvroConverter = new FitbitIntradayCaloriesAvroConverter(avroData); + super(generator, userRepository, "intraday_calories", avroData); + } + + @Override + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitIntradayCaloriesDataConverter(config.getFitbitIntradayCaloriesTopic()); } @Override @@ -30,18 +33,13 @@ protected Stream createRequests(User user) { user, dateRange, user.getExternalUserId(), - DATE_FORMAT.format(dateRange.start()), - TIME_FORMAT.format(dateRange.start()), - TIME_FORMAT.format(dateRange.end()))); + DATE_FORMAT.format(dateRange.getStart()), + TIME_FORMAT.format(dateRange.getStart()), + TIME_FORMAT.format(dateRange.getEnd()))); } @Override protected String getUrlFormat(String baseUrl) { return baseUrl + "/1/user/%s/activities/calories/date/%s/1d/1min/time/%s/%s.json?timezone=UTC"; } - - @Override - public PayloadToSourceRecordConverter converter() { - return caloriesAvroConverter; - } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java index f7f06644..f2d50d83 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayHeartRateRoute.java @@ -17,24 +17,24 @@ package org.radarbase.connect.rest.fitbit.route; -import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; -import static java.time.temporal.ChronoUnit.SECONDS; - import io.confluent.connect.avro.AvroData; -import java.util.stream.Stream; -import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayHeartRateAvroConverter; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.convert.fitbit.FitbitDataConverter; +import org.radarbase.convert.fitbit.FitbitIntradayHeartRateDataConverter; -public class FitbitIntradayHeartRateRoute extends FitbitPollingRoute { - private final FitbitIntradayHeartRateAvroConverter converter; +import java.util.stream.Stream; + +import static java.time.format.DateTimeFormatter.ISO_LOCAL_TIME; +import static java.time.temporal.ChronoUnit.SECONDS; +public class FitbitIntradayHeartRateRoute extends FitbitPollingRoute { public FitbitIntradayHeartRateRoute(FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "heart_rate"); - this.converter = new FitbitIntradayHeartRateAvroConverter(avroData); + super(generator, userRepository, "heart_rate", avroData); } @Override @@ -42,16 +42,16 @@ protected String getUrlFormat(String baseUrl) { return baseUrl + "/1/user/%s/activities/heart/date/%s/1d/1sec/time/%s/%s.json?timezone=UTC"; } + @Override + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitIntradayHeartRateDataConverter(config.getFitbitIntradayHeartRateTopic()); + } + protected Stream createRequests(User user) { return startDateGenerator(getOffset(user).plus(ONE_SECOND).truncatedTo(SECONDS)) .map(dateRange -> newRequest(user, dateRange, - user.getExternalUserId(), DATE_FORMAT.format(dateRange.start()), - ISO_LOCAL_TIME.format(dateRange.start()), - ISO_LOCAL_TIME.format(dateRange.end().truncatedTo(SECONDS)))); - } - - @Override - public FitbitIntradayHeartRateAvroConverter converter() { - return converter; + user.getExternalUserId(), DATE_FORMAT.format(dateRange.getStart()), + ISO_LOCAL_TIME.format(dateRange.getStart()), + ISO_LOCAL_TIME.format(dateRange.getEnd().truncatedTo(SECONDS)))); } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java index 867e36fe..994a0f43 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitIntradayStepsRoute.java @@ -17,23 +17,23 @@ package org.radarbase.connect.rest.fitbit.route; -import static java.time.temporal.ChronoUnit.MINUTES; - import io.confluent.connect.avro.AvroData; -import java.util.stream.Stream; -import org.radarbase.connect.rest.fitbit.converter.FitbitIntradayStepsAvroConverter; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; +import org.radarbase.convert.fitbit.FitbitDataConverter; +import org.radarbase.convert.fitbit.FitbitIntradayStepsDataConverter; -public class FitbitIntradayStepsRoute extends FitbitPollingRoute { - private final FitbitIntradayStepsAvroConverter converter; +import java.util.stream.Stream; + +import static java.time.temporal.ChronoUnit.MINUTES; +public class FitbitIntradayStepsRoute extends FitbitPollingRoute { public FitbitIntradayStepsRoute(FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "intraday_steps"); - this.converter = new FitbitIntradayStepsAvroConverter(avroData); + super(generator, userRepository, "intraday_steps", avroData); } @Override @@ -41,15 +41,15 @@ protected String getUrlFormat(String baseUrl) { return baseUrl + "/1/user/%s/activities/steps/date/%s/1d/1min/time/%s/%s.json?timezone=UTC"; } + @Override + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitIntradayStepsDataConverter(config.getFitbitIntradayStepsTopic()); + } + protected Stream createRequests(User user) { return startDateGenerator(this.getOffset(user).plus(ONE_MINUTE).truncatedTo(MINUTES)) .map(dateRange -> newRequest(user, dateRange, - user.getExternalUserId(), DATE_FORMAT.format(dateRange.start()), - TIME_FORMAT.format(dateRange.start()), TIME_FORMAT.format(dateRange.end()))); - } - - @Override - public FitbitIntradayStepsAvroConverter converter() { - return converter; + user.getExternalUserId(), DATE_FORMAT.format(dateRange.getStart()), + TIME_FORMAT.format(dateRange.getStart()), TIME_FORMAT.format(dateRange.getEnd()))); } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java index 4a108491..89b4ffb5 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitPollingRoute.java @@ -43,19 +43,23 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import javax.ws.rs.NotAuthorizedException; + +import io.confluent.connect.avro.AvroData; import okhttp3.Request; import okhttp3.Response; import org.apache.kafka.connect.source.SourceRecord; import org.apache.kafka.connect.storage.OffsetStorageReader; import org.radarbase.connect.rest.RestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; +import org.radarbase.connect.rest.fitbit.converter.FitbitAvroConverter; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; -import org.radarbase.connect.rest.fitbit.util.DateRange; import org.radarbase.connect.rest.request.PollingRequestRoute; import org.radarbase.connect.rest.request.RestRequest; +import org.radarbase.convert.fitbit.DateRange; +import org.radarbase.convert.fitbit.FitbitDataConverter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,11 +119,13 @@ public abstract class FitbitPollingRoute implements PollingRequestRoute { private Duration pollIntervalPerUser; private final Set tooManyRequestsForUser; private Duration tooManyRequestsCooldown; + private FitbitAvroConverter converter; public FitbitPollingRoute( FitbitRequestGenerator generator, UserRepository userRepository, - String routeName) { + String routeName, + AvroData avroData) { this.generator = generator; this.userRepository = userRepository; this.offsets = new HashMap<>(); @@ -127,9 +133,12 @@ public FitbitPollingRoute( this.routeName = routeName; this.lastPoll = MIN_INSTANT; this.lastPollPerUser = new HashMap<>(); + this.converter = new FitbitAvroConverter(avroData, this::createConverter); this.tooManyRequestsForUser = ConcurrentHashMap.newKeySet(); } + protected abstract FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config); + @Override public void initialize(RestSourceConnectorConfig config) { FitbitRestSourceConnectorConfig fitbitConfig = (FitbitRestSourceConnectorConfig) config; @@ -138,7 +147,7 @@ public void initialize(RestSourceConnectorConfig config) { this.pollIntervalPerUser = fitbitConfig.getPollIntervalPerUser(); this.tooManyRequestsCooldown = fitbitConfig.getTooManyRequestsCooldownInterval() .minus(getPollIntervalPerUser()); - this.converter().initialize(fitbitConfig); + this.converter.initialize(fitbitConfig); } @Override @@ -153,7 +162,7 @@ public void requestSucceeded(RestRequest request, SourceRecord record) { public void requestEmpty(RestRequest request) { lastPollPerUser.put(((FitbitRestRequest) request).getUser().getId(), lastPoll); FitbitRestRequest fitbitRequest = (FitbitRestRequest) request; - Instant endOffset = fitbitRequest.getDateRange().end().toInstant(); + Instant endOffset = fitbitRequest.getDateRange().getEnd().toInstant(); if (DAYS.between(endOffset, lastPoll) >= HISTORICAL_TIME_DAYS) { String key = fitbitRequest.getUser().getVersionedId(); offsets.put(key, endOffset); @@ -362,4 +371,9 @@ Stream startDateGenerator(Instant startDate) { } } } + + @Override + public FitbitAvroConverter converter() { + return this.converter; + } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java index 0159cddc..479ec237 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitSleepRoute.java @@ -25,24 +25,24 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.stream.Stream; -import org.radarbase.connect.rest.fitbit.converter.FitbitSleepAvroConverter; + +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; -import org.radarbase.connect.rest.fitbit.util.DateRange; +import org.radarbase.convert.fitbit.DateRange; +import org.radarbase.convert.fitbit.FitbitDataConverter; +import org.radarbase.convert.fitbit.FitbitSleepDataConverter; public class FitbitSleepRoute extends FitbitPollingRoute { public static final DateTimeFormatter DATE_TIME_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE_TIME .withZone(UTC); private static final Duration SLEEP_POLL_INTERVAL = Duration.ofDays(1); - private final FitbitSleepAvroConverter converter; - public FitbitSleepRoute(FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "sleep"); - converter = new FitbitSleepAvroConverter(avroData); + super(generator, userRepository, "sleep", avroData); } @Override @@ -64,14 +64,14 @@ protected Stream createRequests(User user) { user.getExternalUserId(), DATE_TIME_FORMAT.format(startDate))); } - @Override - protected Duration getPollIntervalPerUser() { - return SLEEP_POLL_INTERVAL; + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitSleepDataConverter(config.getFitbitSleepStagesTopic(), + config.getFitbitSleepClassicTopic()); } @Override - public FitbitSleepAvroConverter converter() { - return converter; + protected Duration getPollIntervalPerUser() { + return SLEEP_POLL_INTERVAL; } } diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java index d16a4ef7..51f1c34e 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/route/FitbitTimeZoneRoute.java @@ -17,28 +17,28 @@ package org.radarbase.connect.rest.fitbit.route; -import static java.time.ZoneOffset.UTC; - import io.confluent.connect.avro.AvroData; -import java.time.Duration; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.fitbit.converter.FitbitTimeZoneAvroConverter; +import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator; import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; import org.radarbase.connect.rest.fitbit.user.User; import org.radarbase.connect.rest.fitbit.user.UserRepository; -import org.radarbase.connect.rest.fitbit.util.DateRange; +import org.radarbase.convert.fitbit.DateRange; +import org.radarbase.convert.fitbit.FitbitDataConverter; +import org.radarbase.convert.fitbit.FitbitTimeZoneDataConverter; + +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.stream.Stream; + +import static java.time.ZoneOffset.UTC; public class FitbitTimeZoneRoute extends FitbitPollingRoute { protected static final Duration TIME_ZONE_POLL_INTERVAL = Duration.ofHours(1); - private final FitbitTimeZoneAvroConverter converter; - public FitbitTimeZoneRoute(FitbitRequestGenerator generator, UserRepository userRepository, AvroData avroData) { - super(generator, userRepository, "timezone"); - this.converter = new FitbitTimeZoneAvroConverter(avroData); + super(generator, userRepository, "timezone", avroData); } @Override @@ -46,16 +46,16 @@ protected String getUrlFormat(String baseUrl) { return baseUrl + "/1/user/%s/profile.json"; } + @Override + protected FitbitDataConverter createConverter(FitbitRestSourceConnectorConfig config) { + return new FitbitTimeZoneDataConverter(config.getFitbitTimeZoneTopic()); + } + protected Stream createRequests(User user) { ZonedDateTime now = ZonedDateTime.now(UTC); return Stream.of(newRequest(user, new DateRange(now, now), user.getExternalUserId())); } - @Override - public FitbitTimeZoneAvroConverter converter() { - return converter; - } - @Override protected Duration getPollIntervalPerUser() { return TIME_ZONE_POLL_INTERVAL; diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java deleted file mode 100644 index d5f61565..00000000 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/util/DateRange.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.util; - -import java.time.ZonedDateTime; -import java.util.Objects; - -public class DateRange { - private final ZonedDateTime start; - private final ZonedDateTime end; - - public DateRange(ZonedDateTime start, ZonedDateTime end) { - this.start = start; - this.end = end; - } - - public ZonedDateTime start() { - return start; - } - - public ZonedDateTime end() { - return end; - } - - @Override - public String toString() { - return "DateRange{" - + "start=" + start - + ", end=" + end - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DateRange dateRange = (DateRange) o; - return Objects.equals(start, dateRange.start) && - Objects.equals(end, dateRange.end); - } - - @Override - public int hashCode() { - return Objects.hash(start, end); - } -} diff --git a/radar-fitbit-converter/build.gradle b/radar-fitbit-converter/build.gradle index 7d6ad79c..55f2fe3e 100644 --- a/radar-fitbit-converter/build.gradle +++ b/radar-fitbit-converter/build.gradle @@ -3,7 +3,10 @@ plugins { } dependencies { - implementation("org.slf4j:slf4j-api:1.7.32") + implementation('org.slf4j:slf4j-api:1.7.36') + api("org.jetbrains.kotlin:kotlin-stdlib-jdk8") + api group: 'org.radarbase', name: 'radar-schemas-commons', version: '0.7.3' + api("com.fasterxml.jackson.core:jackson-databind:$jacksonVersion") } tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt new file mode 100644 index 00000000..80fe2ae7 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.convert.fitbit + +data class ConverterContext( + val userId: String, + val url: String, + val dateRange: DateRange, +) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java deleted file mode 100644 index d5f61565..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.util; - -import java.time.ZonedDateTime; -import java.util.Objects; - -public class DateRange { - private final ZonedDateTime start; - private final ZonedDateTime end; - - public DateRange(ZonedDateTime start, ZonedDateTime end) { - this.start = start; - this.end = end; - } - - public ZonedDateTime start() { - return start; - } - - public ZonedDateTime end() { - return end; - } - - @Override - public String toString() { - return "DateRange{" - + "start=" + start - + ", end=" + end - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - DateRange dateRange = (DateRange) o; - return Objects.equals(start, dateRange.start) && - Objects.equals(end, dateRange.end); - } - - @Override - public int hashCode() { - return Objects.hash(start, end); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.kt new file mode 100644 index 00000000..e779fc69 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/DateRange.kt @@ -0,0 +1,25 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import java.time.ZonedDateTime +import java.util.* + +data class DateRange( + val start: ZonedDateTime, + val end: ZonedDateTime, +) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java deleted file mode 100644 index dd7256d1..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogAvroConverter.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.OffsetDateTime; -import java.util.Comparator; -import java.util.Optional; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitActivityHeartRate; -import org.radarcns.connector.fitbit.FitbitActivityHeartRate.Builder; -import org.radarcns.connector.fitbit.FitbitActivityLevels; -import org.radarcns.connector.fitbit.FitbitActivityLogRecord; -import org.radarcns.connector.fitbit.FitbitManualDataEntry; -import org.radarcns.connector.fitbit.FitbitSource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitActivityLogAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitActivityLogAvroConverter.class); - private static final float FOOD_CAL_TO_KJOULE_FACTOR = 4.1868f; - - private String activityLogTopic; - - public FitbitActivityLogAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - activityLogTopic = ((FitbitRestSourceConnectorConfig)config).getActivityLogTopic(); - - logger.info("Using activity log topic {}", activityLogTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - - JsonNode array = root.get("activities"); - if (array == null || !array.isArray()) { - return Stream.empty(); - } - - return iterableToStream(array) - .sorted(Comparator.comparing(s -> s.get("startTime").textValue())) - .map(tryOrNull(s -> { - OffsetDateTime startTime = OffsetDateTime.parse(s.get("startTime").textValue()); - FitbitActivityLogRecord record = getRecord(s, startTime); - return new TopicData(startTime.toInstant(), activityLogTopic, record); - }, (s, ex) -> logger.warn( - "Failed to convert sleep patterns from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), s, ex))); - } - - private FitbitActivityLogRecord getRecord(JsonNode s, OffsetDateTime startTime) { - return FitbitActivityLogRecord.newBuilder() - .setTime(startTime.toInstant().toEpochMilli() / 1000d) - .setTimeReceived(System.currentTimeMillis() / 1000d) - .setTimeLastModified(Instant.parse(s.get("lastModified").asText()).toEpochMilli() / 1000d) - .setId(optLong(s, "logId").orElseThrow( - () -> new IllegalArgumentException("Activity log ID not specified"))) - .setLogType(optString(s, "logType").orElse(null)) - .setType(optLong(s, "activityType").orElse(null)) - .setSpeed(optDouble(s, "speed").orElse(null)) - .setDistance(optDouble(s, "distance").map(Double::floatValue).orElse(null)) - .setSteps(optInt(s, "steps").orElse(null)) - .setEnergy(optInt(s, "calories").map(e -> e * FOOD_CAL_TO_KJOULE_FACTOR) - .orElse(null)) - .setDuration(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( - () -> new IllegalArgumentException("Activity log duration not specified"))) - .setDurationActive(optLong(s, "duration").map(d -> d / 1000f).orElseThrow( - () -> new IllegalArgumentException("Activity duration active not specified"))) - .setTimeZoneOffset(startTime.getOffset().getTotalSeconds()) - .setName(optString(s, "activityName").orElse(null)) - .setHeartRate(getHeartRate(s)) - .setManualDataEntry(getManualDataEntry(s)) - .setLevels(getActivityLevels(s)) - .setSource(getSource(s)) - .build(); - } - - private FitbitSource getSource(JsonNode s) { - return optObject(s, "source") - .flatMap(source -> optString(source, "id") - .map(id -> FitbitSource.newBuilder() - .setId(id) - .setName(optString(source, "name").orElse(null)) - .setType(optString(source, "type").orElse(null)) - .setUrl(optString(source, "url").orElse(null)) - .build())) - .orElse(null); - } - - private FitbitActivityLevels getActivityLevels(JsonNode s) { - return optArray(s, "activityLevels") - .map(levels -> { - FitbitActivityLevels.Builder activityLevels = FitbitActivityLevels.newBuilder(); - for (JsonNode level : levels) { - Integer duration = optInt(level, "minutes") - .map(t -> t * 60) - .orElse(null); - switch (optString(level, "name").orElse("")) { - case "sedentary": - activityLevels.setDurationSedentary(duration); - break; - case "lightly": - activityLevels.setDurationLightly(duration); - break; - case "fairly": - activityLevels.setDurationFairly(duration); - break; - case "very": - activityLevels.setDurationVery(duration); - } - } - - return activityLevels.build(); - }) - .orElse(null); - } - - private FitbitManualDataEntry getManualDataEntry(JsonNode s) { - return optObject(s, "manualValuesSpecified") - .map(manual -> FitbitManualDataEntry.newBuilder() - .setSteps(optBoolean(manual, "steps").orElse(null)) - .setDistance(optBoolean(manual, "distance").orElse(null)) - .setEnergy(optBoolean(manual, "calorie").orElse(null)) - .build()) - .orElse(null); - } - - private FitbitActivityHeartRate getHeartRate(JsonNode activity) { - Optional mean = optInt(activity, "averageHeartRate"); - Optional> zones = optArray(activity, "heartRateZones"); - - if (mean.isEmpty() && zones.isEmpty()) { - return null; - } - - Builder heartRate = FitbitActivityHeartRate.newBuilder() - .setMean(mean.orElse(null)); - - zones.ifPresent(z -> { - for (JsonNode zone : z) { - Integer minValue = optInt(zone, "min").orElse(null); - Integer duration = optInt(zone, "minutes") - .map(m -> m * 60) - .orElse(null); - switch (optString(zone, "name").orElse("")) { - case "Out of Range": - heartRate.setMin(minValue); - heartRate.setDurationOutOfRange(duration); - break; - case "Fat Burn": - heartRate.setMinFatBurn(minValue); - heartRate.setDurationFatBurn(duration); - break; - case "Cardio": - heartRate.setMinCardio(minValue); - heartRate.setDurationCardio(duration); - break; - case "Peak": - heartRate.setMinPeak(minValue); - heartRate.setMax(optInt(zone, "max").orElse(null)); - heartRate.setDurationPeak(duration); - break; - default: - logger.warn("Cannot process unknown heart rate zone {}", zone.get("name").asText()); - break; - } - } - }); - - return heartRate.build(); - } - -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt new file mode 100644 index 00000000..5df7262b --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt @@ -0,0 +1,145 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.* +import org.slf4j.LoggerFactory +import java.time.Instant +import java.time.OffsetDateTime +import java.time.ZoneOffset + +class FitbitActivityLogDataConverter( + private val activityLogTopic: String +) : FitbitDataConverter { + override fun processRecords( + context: ConverterContext, root: JsonNode, timeReceived: Double + ): Sequence> { + val array = root.optArray("activities") + ?: return emptySequence() + + return array.asSequence() + .sortedBy { it["startTime"].textValue() } + .mapCatching { s -> + val startTime = OffsetDateTime.parse(s["startTime"].textValue()) + val startInstant = startTime.toInstant() + TopicData( + sourceOffset = startInstant, + topic = activityLogTopic, + value = s.toActivityLogRecord(startInstant, startTime.offset), + ) + } + } + + private fun JsonNode.toActivityLogRecord( + startTime: Instant, + offset: ZoneOffset, + ): FitbitActivityLogRecord { + return FitbitActivityLogRecord.newBuilder().apply { + time = startTime.toEpochMilli() / 1000.0 + timeReceived = System.currentTimeMillis() / 1000.0 + timeLastModified = Instant.parse(get("lastModified").asText()).toEpochMilli() / 1000.0 + id = requireNotNull(optLong("logId")) { "Activity log ID not specified" } + logType = optString("logType") + type = optLong("activityType") + speed = optDouble("speed") + distance = optDouble("distance")?.toFloat() + steps = optInt("steps") + energy = optInt("calories")?.let { it * FOOD_CAL_TO_KJOULE_FACTOR } + duration = (requireNotNull(optLong("duration")) { "Activity log duration not specified" } + / 1000f) + durationActive = requireNotNull(optLong("durationActive")) { "Activity active log duration not specified" } / 1000f + timeZoneOffset = offset.totalSeconds + name = optString("activityName") + heartRate = toHeartRate() + manualDataEntry = optObject("manualValuesSpecified")?.toManualDataEntry() + levels = optArray("activityLevels")?.toActivityLevels() + source = optObject("source")?.toSource() + }.build() + } + + private fun JsonNode.toSource(): FitbitSource? = + optString("id")?.let { sourceId -> + FitbitSource.newBuilder().apply { + id = sourceId + name = optString("name") + type = optString("type") + url = optString("url") + }.build() + } + + private fun Iterable.toActivityLevels(): FitbitActivityLevels = + FitbitActivityLevels.newBuilder().apply { + forEach { level -> + val durationMinutes = level.optInt("minutes") ?: return@forEach + val duration = durationMinutes * 60 + when (level.optString("name")) { + "sedentary" -> durationSedentary = duration + "lightly" -> durationLightly = duration + "fairly" -> durationFairly = duration + "very" -> durationVery = duration + } + } + }.build() + + private fun JsonNode.toManualDataEntry(): FitbitManualDataEntry = + FitbitManualDataEntry.newBuilder().apply { + steps = optBoolean("steps") + distance = optBoolean("distance") + energy = optBoolean("calorie") + }.build() + + private fun JsonNode.toHeartRate(): FitbitActivityHeartRate? { + val averageHeartRate: Int? = optInt("averageHeartRate") + val zones = optArray("heartRateZones") + if (averageHeartRate == null && zones == null) { + return null + } + return FitbitActivityHeartRate.newBuilder().apply { + mean = averageHeartRate + zones?.forEach { zone -> + val minValue = zone.optInt("min") + val duration = zone.optInt("minutes")?.let { it * 60 } + when (val zoneText = zone.optString("name")) { + "Out of Range" -> { + min = minValue + durationOutOfRange = duration + } + "Fat Burn" -> { + minFatBurn = minValue + durationFatBurn = duration + } + "Cardio" -> { + minCardio = minValue + durationCardio = duration + } + "Peak" -> { + minPeak = minValue + max = zone.optInt("max") + durationPeak = duration + } + else -> logger.warn("Cannot process unknown heart rate zone {}", zoneText) + } + } + }.build() + } + + companion object { + private val logger = LoggerFactory.getLogger(FitbitActivityLogDataConverter::class.java) + private const val FOOD_CAL_TO_KJOULE_FACTOR = 4.1868f + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java deleted file mode 100644 index 7b2e248c..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitAvroConverter.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator.JSON_READER; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import io.confluent.connect.avro.AvroData; -import java.io.IOException; -import java.time.Instant; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; -import okhttp3.Headers; -import org.apache.avro.Schema.Field; -import org.apache.avro.generic.IndexedRecord; -import org.apache.kafka.connect.data.SchemaAndValue; -import org.apache.kafka.connect.source.SourceRecord; -import org.radarbase.connect.rest.converter.PayloadToSourceRecordConverter; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarbase.connect.rest.fitbit.user.User; -import org.radarbase.connect.rest.request.RestRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Abstract class to help convert Fitbit data to Avro Data. - */ -public abstract class FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitAvroConverter.class); - private static final Map TIME_UNIT_MAP = new HashMap<>(); - - static { - TIME_UNIT_MAP.put("minute", TimeUnit.MINUTES); - TIME_UNIT_MAP.put("second", TimeUnit.SECONDS); - TIME_UNIT_MAP.put("hour", TimeUnit.HOURS); - TIME_UNIT_MAP.put("day", TimeUnit.DAYS); - TIME_UNIT_MAP.put("millisecond", TimeUnit.MILLISECONDS); - TIME_UNIT_MAP.put("nanosecond", TimeUnit.NANOSECONDS); - TIME_UNIT_MAP.put("microsecond", TimeUnit.MICROSECONDS); - } - - private final AvroData avroData; - - public FitbitAvroConverter(AvroData avroData) { - this.avroData = avroData; - } - - public Collection convert( - DateRange userRange, Headers headers, byte[] data) throws IOException { - if (data == null) { - throw new IOException("Failed to read body"); - } - JsonNode activities = JSON_READER.readTree(data); - - User user = ((FitbitRestRequest) restRequest).getUser(); - final SchemaAndValue key = user.getObservationKey(avroData); - double timeReceived = System.currentTimeMillis() / 1000d; - - return processRecords((FitbitRestRequest)restRequest, activities, timeReceived) - .filter(t -> validateRecord((FitbitRestRequest)restRequest, t)) - .map(t -> { - SchemaAndValue avro = avroData.toConnectData(t.value.getSchema(), t.value); - Map offset = Collections.singletonMap( - TIMESTAMP_OFFSET_KEY, t.sourceOffset.toEpochMilli()); - - return new SourceRecord(restRequest.getPartition(), offset, t.topic, - key.schema(), key.value(), avro.schema(), avro.value()); - }) - .collect(Collectors.toList()); - } - - private boolean validateRecord(FitbitRestRequest request, TopicData record) { - if (record == null) { - return false; - } - Instant endDate = request.getUser().getEndDate(); - if (endDate == null) { - return true; - } - Field timeField = record.value.getSchema().getField("time"); - if (timeField != null) { - long time = (long) (((Double)record.value.get(timeField.pos()) * 1000.0)); - return Instant.ofEpochMilli(time).isBefore(endDate); - } - return true; - } - - /** Process the JSON records generated by given request. */ - protected abstract Stream processRecords( - FitbitRestRequest request, - JsonNode root, - double timeReceived); - - /** Get Fitbit dataset interval used in some intraday API calls. */ - protected static int getRecordInterval(JsonNode root, int defaultValue) { - JsonNode type = root.get("datasetType"); - JsonNode interval = root.get("datasetInterval"); - if (type == null || interval == null) { - logger.warn("Failed to get data interval; using {} instead", defaultValue); - return defaultValue; - } - return (int)TIME_UNIT_MAP - .getOrDefault(type.asText(), TimeUnit.SECONDS) - .toSeconds(interval.asLong()); - } - - /** Converts an iterable (like a JsonNode containing an array) to a stream. */ - protected static Stream iterableToStream(Iterable iter) { - return StreamSupport.stream(iter.spliterator(), false); - } - - protected static Optional optLong(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.canConvertToLong() ? Optional.of(v.longValue()) : Optional.empty(); - } - - protected static Optional optDouble(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isNumber() ? Optional.of(v.doubleValue()) : Optional.empty(); - } - - protected static Optional optInt(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.canConvertToInt() ? Optional.of(v.intValue()) : Optional.empty(); - } - - protected static Optional optString(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isTextual() ? Optional.ofNullable(v.textValue()) : Optional.empty(); - } - - protected static Optional optBoolean(JsonNode node, String fieldName) { - JsonNode v = node.get(fieldName); - return v != null && v.isBoolean() ? Optional.of(v.booleanValue()) : Optional.empty(); - } - - protected static Optional optObject(JsonNode parent, String fieldName) { - JsonNode v = parent.get(fieldName); - return v != null && v.isObject() ? Optional.of((ObjectNode) v) : Optional.empty(); - } - - protected static Optional> optArray(JsonNode parent, String fieldName) { - JsonNode v = parent.get(fieldName); - return v != null && v.isArray() && v.size() != 0 ? - Optional.of(v) : Optional.empty(); - } - - /** Single value for a topic. */ - protected static class TopicData { - Instant sourceOffset; - final String topic; - final IndexedRecord value; - - public TopicData(Instant sourceOffset, String topic, IndexedRecord value) { - this.sourceOffset = sourceOffset; - this.topic = topic; - this.value = value; - } - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt new file mode 100644 index 00000000..aea7e375 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode + +/** + * Abstract class to help convert Fitbit data to Avro Data. + */ +interface FitbitDataConverter { + /** Process the JSON records generated by given request. */ + fun processRecords( + context: ConverterContext, + root: JsonNode, + timeReceived: Double + ): Sequence> +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java deleted file mode 100644 index 93fb8847..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesAvroConverter.java +++ /dev/null @@ -1,79 +0,0 @@ -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradayCalories; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayCaloriesAvroConverter extends FitbitAvroConverter { - - private static final Logger logger = - LoggerFactory.getLogger(FitbitIntradayCaloriesAvroConverter.class); - - private String caloriesTopic; - - public FitbitIntradayCaloriesAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-calories-intraday"); - if (intraday == null) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 60); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().end(); - - return iterableToStream(dataset) - .map( - tryOrNull( - activity -> { - Instant time = - startDate.with(LocalTime.parse(activity.get("time").asText())).toInstant(); - - FitbitIntradayCalories calories = - new FitbitIntradayCalories( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asDouble(), - activity.get("level").asInt(), - activity.get("mets").asDouble()); - - return new TopicData(time, caloriesTopic, calories); - }, - (a, ex) -> - logger.warn( - "Failed to convert calories from request {} of user {}, {}", - request.getRequest().url(), - request.getUser(), - a, - ex))); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - caloriesTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayCaloriesTopic(); - logger.info("Using calories topic {}", caloriesTopic); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt new file mode 100644 index 00000000..0c9ac4cc --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt @@ -0,0 +1,41 @@ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.FitbitIntradayCalories +import java.time.LocalTime +import java.time.ZonedDateTime + +class FitbitIntradayCaloriesDataConverter( + private val caloriesTopic: String, +) : FitbitDataConverter { + + override fun processRecords( + context: ConverterContext, root: JsonNode, timeReceived: Double + ): Sequence> { + val intraday = root.optObject("activities-calories-intraday") + ?: return emptySequence() + val dataset = intraday.optArray("dataset") + ?: return emptySequence() + val interval: Int = intraday.getRecordInterval(60) + + // Used as the date to convert the local times in the dataset to absolute times. + val startDate: ZonedDateTime = context.dateRange.end + return dataset.asSequence() + .mapCatching { activity -> + val localTime = LocalTime.parse(activity.get("time").asText()) + val time = startDate.with(localTime).toInstant() + TopicData( + sourceOffset = time, + topic = caloriesTopic, + value = FitbitIntradayCalories( + time.toEpochMilli() / 1000.0, + timeReceived, + interval, + activity.get("value").asDouble(), + activity.get("level").asInt(), + activity.get("mets").asDouble(), + ) + ) + } + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java deleted file mode 100644 index 89bc4cf1..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateAvroConverter.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradayHeartRate; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayHeartRateAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger( - FitbitIntradayHeartRateAvroConverter.class); - private String heartRateTopic; - - public FitbitIntradayHeartRateAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - heartRateTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayHeartRateTopic(); - logger.info("Using heart rate topic {}", heartRateTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-heart-intraday"); - if (intraday == null || !intraday.isObject()) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null || !dataset.isArray()) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 1); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().start(); - - return iterableToStream(dataset) - .map(tryOrNull(activity -> { - Instant time = startDate.with(LocalTime.parse(activity.get("time").asText())) - .toInstant(); - - FitbitIntradayHeartRate heartRate = new FitbitIntradayHeartRate( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asInt()); - - return new TopicData(time, heartRateTopic, heartRate); - }, (a, ex) -> logger.warn( - "Failed to convert heart rate from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), a, ex))); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt new file mode 100644 index 00000000..27264904 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.FitbitIntradayHeartRate +import org.slf4j.LoggerFactory +import java.time.LocalTime +import java.time.ZonedDateTime + +class FitbitIntradayHeartRateDataConverter( + private val heartRateTopic: String +) : FitbitDataConverter { + override fun processRecords( + context: ConverterContext, root: JsonNode, timeReceived: Double + ): Sequence> { + val intraday = root.optObject("activities-heart-intraday") + ?: return emptySequence() + val dataset = intraday.optArray("dataset") + ?: return emptySequence() + val interval: Int = intraday.getRecordInterval(1) + + // Used as the date to convert the local times in the dataset to absolute times. + val startDate: ZonedDateTime = context.dateRange.end + return dataset.asSequence() + .mapCatching { activity -> + val localTime = LocalTime.parse(activity.get("time").asText()) + val time = startDate.with(localTime).toInstant() + TopicData( + sourceOffset = time, + topic = heartRateTopic, + value = FitbitIntradayHeartRate( + time.toEpochMilli() / 1000.0, + timeReceived, + interval, + activity.get("value").asInt(), + ) + ) + } + } + + companion object { + private val logger = LoggerFactory.getLogger( + FitbitIntradayHeartRateDataConverter::class.java) + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java deleted file mode 100644 index 2381afa4..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsAvroConverter.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Instant; -import java.time.LocalTime; -import java.time.ZonedDateTime; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitIntradaySteps; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitIntradayStepsAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger( - FitbitIntradayStepsAvroConverter.class); - - private String stepTopic; - - public FitbitIntradayStepsAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - stepTopic = ((FitbitRestSourceConnectorConfig) config).getFitbitIntradayStepsTopic(); - logger.info("Using step topic {}", stepTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode intraday = root.get("activities-steps-intraday"); - if (intraday == null) { - return Stream.empty(); - } - - JsonNode dataset = intraday.get("dataset"); - if (dataset == null) { - return Stream.empty(); - } - - int interval = getRecordInterval(intraday, 60); - - // Used as the date to convert the local times in the dataset to absolute times. - ZonedDateTime startDate = request.getDateRange().end(); - - return iterableToStream(dataset) - .map(tryOrNull(activity -> { - - Instant time = startDate - .with(LocalTime.parse(activity.get("time").asText())) - .toInstant(); - - FitbitIntradaySteps steps = new FitbitIntradaySteps( - time.toEpochMilli() / 1000d, - timeReceived, - interval, - activity.get("value").asInt()); - - return new TopicData(time, stepTopic, steps); - }, (a, ex) -> logger.warn( - "Failed to convert steps from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), a, ex))); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt new file mode 100644 index 00000000..eecaf661 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.FitbitIntradaySteps +import org.slf4j.LoggerFactory +import java.time.LocalTime +import java.time.ZonedDateTime + +class FitbitIntradayStepsDataConverter(private val stepTopic: String) : FitbitDataConverter { + override fun processRecords( + context: ConverterContext, root: JsonNode, timeReceived: Double + ): Sequence> { + val intraday = root.optObject("activities-steps-intraday") + ?: return emptySequence() + val dataset = intraday.optArray("dataset") + ?: return emptySequence() + val interval = intraday.getRecordInterval(60) + + // Used as the date to convert the local times in the dataset to absolute times. + val startDate: ZonedDateTime = context.dateRange.end + return dataset.asSequence() + .mapCatching { activity -> + val localTime = LocalTime.parse(activity.get("time").asText()) + val time = startDate.with(localTime).toInstant() + TopicData( + sourceOffset = time, + topic = stepTopic, + value = FitbitIntradaySteps( + time.toEpochMilli() / 1000.0, + timeReceived, + interval, + activity.get("value").asInt(), + ), + ) + } + } + + companion object { + private val logger = LoggerFactory.getLogger( + FitbitIntradayStepsDataConverter::class.java) + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java deleted file mode 100644 index 976a348b..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepAvroConverter.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import static org.radarbase.connect.rest.fitbit.route.FitbitSleepRoute.DATE_TIME_FORMAT; -import static org.radarbase.connect.rest.util.ThrowingFunction.tryOrNull; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.time.Duration; -import java.time.Instant; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import org.apache.avro.generic.IndexedRecord; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitSleepClassic; -import org.radarcns.connector.fitbit.FitbitSleepClassicLevel; -import org.radarcns.connector.fitbit.FitbitSleepStage; -import org.radarcns.connector.fitbit.FitbitSleepStageLevel; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitSleepAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitSleepAvroConverter.class); - - private static final Map CLASSIC_MAP = new HashMap<>(); - private static final Map STAGES_MAP = new HashMap<>(); - static { - CLASSIC_MAP.put("awake", FitbitSleepClassicLevel.AWAKE); - CLASSIC_MAP.put("asleep", FitbitSleepClassicLevel.ASLEEP); - CLASSIC_MAP.put("restless", FitbitSleepClassicLevel.RESTLESS); - - STAGES_MAP.put("wake", FitbitSleepStageLevel.AWAKE); - STAGES_MAP.put("rem", FitbitSleepStageLevel.REM); - STAGES_MAP.put("deep", FitbitSleepStageLevel.DEEP); - STAGES_MAP.put("light", FitbitSleepStageLevel.LIGHT); - } - - private String sleepStagesTopic; - private String sleepClassicTopic; - - public FitbitSleepAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - sleepStagesTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepStagesTopic(); - sleepClassicTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitSleepClassicTopic(); - - logger.info("Using sleep topic {} and {}", sleepStagesTopic, sleepClassicTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, JsonNode root, double timeReceived) { - JsonNode meta = root.get("meta"); - if (meta != null) { - JsonNode state = meta.get("state"); - if (state != null && meta.get("state").asText().equals("pending")) { - return Stream.empty(); - } - } - JsonNode sleepArray = root.get("sleep"); - if (sleepArray == null) { - return Stream.empty(); - } - - return iterableToStream(sleepArray) - .sorted(Comparator.comparing(s -> s.get("startTime").asText())) - .flatMap(tryOrNull(s -> { - Instant startTime = Instant.from(DATE_TIME_FORMAT.parse(s.get("startTime").asText())); - boolean isStages = s.get("type") == null || s.get("type").asText().equals("stages"); - - // use an intermediate offset for all records but the last. Since the query time - // depends only on the start time of a sleep stages group, this will reprocess the entire - // sleep stages group if something goes wrong while processing. - Instant intermediateOffset = startTime.minus(Duration.ofSeconds(1)); - - List allRecords = iterableToStream(s.get("levels").get("data")) - .map(d -> { - IndexedRecord sleep; - String topic; - - String dateTime = d.get("dateTime").asText(); - int duration = d.get("seconds").asInt(); - String level = d.get("level").asText(); - - if (isStages) { - sleep = new FitbitSleepStage( - dateTime, - timeReceived, - duration, - STAGES_MAP.getOrDefault(level, FitbitSleepStageLevel.UNKNOWN)); - topic = sleepStagesTopic; - } else { - sleep = new FitbitSleepClassic( - dateTime, - timeReceived, - duration, - CLASSIC_MAP.getOrDefault(level, FitbitSleepClassicLevel.UNKNOWN)); - topic = sleepClassicTopic; - } - - return new TopicData(intermediateOffset, topic, sleep); - }) - .collect(Collectors.toList()); - - if (allRecords.isEmpty()) { - return Stream.empty(); - } - - // The final group gets the actual offset, to ensure that the group does not get queried - // again. - allRecords.get(allRecords.size() - 1).sourceOffset = startTime; - - return allRecords.stream(); - }, (s, ex) -> logger.warn( - "Failed to convert sleep patterns from request {} of user {}, {}", - request.getRequest().url(), request.getUser(), s, ex))); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt new file mode 100644 index 00000000..49a62762 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt @@ -0,0 +1,120 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.FitbitSleepClassic +import org.radarcns.connector.fitbit.FitbitSleepClassicLevel +import org.radarcns.connector.fitbit.FitbitSleepStage +import org.radarcns.connector.fitbit.FitbitSleepStageLevel +import java.time.Duration +import java.time.Instant +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import kotlin.Result.Companion.failure +import kotlin.Result.Companion.success + +class FitbitSleepDataConverter( + private val sleepStagesTopic: String, + private val sleepClassicTopic: String, +) : FitbitDataConverter { + override fun processRecords( + context: ConverterContext, root: JsonNode, timeReceived: Double, + ): Sequence> { + val meta = root.optObject("meta") + if (meta != null && meta["state"]?.asText() == "pending") { + return emptySequence() + } + val sleepArray = root.optArray("sleep") + ?: return emptySequence() + return sleepArray.asSequence() + .sortedBy { s -> s.get("startTime").asText() } + .mapCatching { s -> + val startTime = Instant.from(DATE_TIME_FORMAT.parse(s.get("startTime").asText())) + val type = s.optString("type") + val isStages = type == null || type == "stages" + + // use an intermediate offset for all records but the last. Since the query time + // depends only on the start time of a sleep stages group, this will reprocess the entire + // sleep stages group if something goes wrong while processing. + val intermediateOffset = startTime.minus(Duration.ofSeconds(1)) + val allRecords: List = s.optObject("levels") + ?.optArray("data") + ?.map { d -> + val dateTime: String = d.get("dateTime").asText() + val duration: Int = d.get("seconds").asInt() + val level: String = d.get("level").asText() + if (isStages) { + TopicData( + sourceOffset = intermediateOffset, + topic = sleepStagesTopic, + value = FitbitSleepStage( + dateTime, + timeReceived, + duration, + level.toStagesLevel() + ), + ) + } else { + TopicData( + sourceOffset = intermediateOffset, + topic = sleepClassicTopic, + value = FitbitSleepClassic( + dateTime, + timeReceived, + duration, + level.toClassicLevel(), + ) + ) + } + } + ?: emptyList() + + // The final group gets the actual offset, to ensure that the group does not get queried + // again. + allRecords.lastOrNull()?.sourceOffset = startTime + allRecords + } + .flatMap { res -> + res.fold( + onSuccess = { data -> data.asSequence().map { success(it) } }, + onFailure = { sequenceOf(failure(it)) }, + ) + } + } + + companion object { + private val DATE_TIME_FORMAT = DateTimeFormatter.ISO_LOCAL_DATE_TIME + .withZone(ZoneOffset.UTC) + + + private fun String.toClassicLevel() = when (this) { + "awake" -> FitbitSleepClassicLevel.AWAKE + "asleep" -> FitbitSleepClassicLevel.ASLEEP + "restless" -> FitbitSleepClassicLevel.RESTLESS + else -> FitbitSleepClassicLevel.UNKNOWN + } + + private fun String.toStagesLevel() = when (this) { + "wake" -> FitbitSleepStageLevel.AWAKE + "rem" -> FitbitSleepStageLevel.REM + "deep" -> FitbitSleepStageLevel.DEEP + "light" -> FitbitSleepStageLevel.LIGHT + else -> FitbitSleepStageLevel.UNKNOWN + } + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java deleted file mode 100644 index 01c77d8f..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneAvroConverter.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.connect.rest.fitbit.converter; - -import com.fasterxml.jackson.databind.JsonNode; -import io.confluent.connect.avro.AvroData; -import java.util.stream.Stream; -import org.radarbase.connect.rest.RestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig; -import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest; -import org.radarcns.connector.fitbit.FitbitTimeZone; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class FitbitTimeZoneAvroConverter extends FitbitAvroConverter { - private static final Logger logger = LoggerFactory.getLogger(FitbitTimeZoneAvroConverter.class); - - private String timeZoneTopic; - - public FitbitTimeZoneAvroConverter(AvroData avroData) { - super(avroData); - } - - @Override - public void initialize(RestSourceConnectorConfig config) { - timeZoneTopic = ((FitbitRestSourceConnectorConfig)config).getFitbitTimeZoneTopic(); - logger.info("Using timezone topic {}", timeZoneTopic); - } - - @Override - protected Stream processRecords( - FitbitRestRequest request, - JsonNode root, - double timeReceived) { - JsonNode user = root.get("user"); - if (user == null) { - logger.warn("Failed to get timezone from {}, {}", request.getRequest().url(), root); - return Stream.empty(); - } - JsonNode offsetNode = user.get("offsetFromUTCMillis"); - Integer offset = offsetNode == null ? null : (int) (offsetNode.asLong() / 1000L); - - FitbitTimeZone timeZone = new FitbitTimeZone(timeReceived, offset); - - return Stream.of(new TopicData(request.getDateRange().start().toInstant(), - timeZoneTopic, timeZone)); - } -} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt new file mode 100644 index 00000000..70cff155 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt @@ -0,0 +1,51 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import org.radarcns.connector.fitbit.FitbitTimeZone +import org.slf4j.LoggerFactory +import kotlin.Result.Companion.failure +import kotlin.Result.Companion.success + +class FitbitTimeZoneDataConverter(private val timeZoneTopic: String) : FitbitDataConverter { + override fun processRecords( + context: ConverterContext, + root: JsonNode, + timeReceived: Double, + ): Sequence> { + val user = root.optObject("user") ?: run { + return sequenceOf( + failure(IllegalArgumentException("Failed to get timezone from $root")) + ) + } + val offset = user.optInt("offsetFromUTCMillis")?.let { it / 1000 } + return sequenceOf( + success( + TopicData( + sourceOffset = context.dateRange.start.toInstant(), + topic = timeZoneTopic, + value = FitbitTimeZone(timeReceived, offset), + ) + ) + ) + } + + companion object { + private val logger = LoggerFactory.getLogger(FitbitTimeZoneDataConverter::class.java) + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt new file mode 100644 index 00000000..4f7f5374 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.convert.fitbit + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.node.ObjectNode +import org.slf4j.LoggerFactory +import java.util.concurrent.TimeUnit + +private val logger = LoggerFactory.getLogger("org.radarbase.convert.fitbit.JsonNodeExtensionsKt") + +/** Get Fitbit dataset interval used in some intraday API calls. */ +fun JsonNode.getRecordInterval(defaultValue: Int): Int { + val type = this["datasetType"] + val interval = this["datasetInterval"] + if (type == null || interval == null) { + logger.warn("Failed to get data interval; using {} instead", defaultValue) + return defaultValue + } + return when (type.asText()) { + "minute" -> TimeUnit.MINUTES + "second" -> TimeUnit.SECONDS + "hour" -> TimeUnit.HOURS + "day" -> TimeUnit.DAYS + "millisecond" -> TimeUnit.MILLISECONDS + "nanosecond" -> TimeUnit.NANOSECONDS + "microsecond" -> TimeUnit.MICROSECONDS + else -> { + logger.warn("Failed to parse dataset interval type {} for {}; using {} seconds instead", type.asText(), interval.asLong(), defaultValue) + return defaultValue + } + }.toSeconds(interval.asLong()).toInt() +} + +fun JsonNode.optLong(fieldName: String): Long? = this[fieldName] + ?.takeIf { it.canConvertToLong() } + ?.longValue() + +fun JsonNode.optDouble(fieldName: String): Double? = this[fieldName] + ?.takeIf { it.isNumber } + ?.doubleValue() + +fun JsonNode.optInt(fieldName: String): Int? = this[fieldName] + ?.takeIf { it.canConvertToInt() } + ?.intValue() + +fun JsonNode.optString(fieldName: String?): String? = this[fieldName] + ?.takeIf { it.isTextual } + ?.textValue() + +fun JsonNode.optBoolean(fieldName: String?): Boolean? = this[fieldName] + ?.takeIf { it.isBoolean } + ?.booleanValue() + +fun JsonNode.optObject(fieldName: String?): ObjectNode? = this[fieldName] + ?.takeIf { it.isObject } as ObjectNode? + +fun JsonNode.optArray(fieldName: String?): Iterable? = this[fieldName] + ?.takeIf { it.isArray && it.size() > 0 } + + +fun Sequence.mapCatching(fn: (T) -> S): Sequence> = map { t -> + runCatching { + fn(t) + } +} diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/TopicData.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/TopicData.kt new file mode 100644 index 00000000..4fa24367 --- /dev/null +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/TopicData.kt @@ -0,0 +1,28 @@ +/* + * Copyright 2018 The Hyve + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.radarbase.convert.fitbit + +import org.apache.avro.generic.IndexedRecord +import java.time.Instant + +/** Single value for a topic. */ +data class TopicData( + var sourceOffset: Instant, + val topic: String, + val value: IndexedRecord, +) From 2b8126430745a138485e6c2f7115e61012dd8738 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 8 Mar 2022 14:47:19 +0100 Subject: [PATCH 4/5] Some simplifications --- .../fitbit/converter/FitbitAvroConverter.kt | 5 +--- .../convert/fitbit/ConverterContext.kt | 24 ------------------- .../fitbit/FitbitActivityLogDataConverter.kt | 2 +- .../convert/fitbit/FitbitDataConverter.kt | 2 +- .../FitbitIntradayCaloriesDataConverter.kt | 4 ++-- .../FitbitIntradayHeartRateDataConverter.kt | 4 ++-- .../FitbitIntradayStepsDataConverter.kt | 4 ++-- .../fitbit/FitbitSleepDataConverter.kt | 2 +- .../fitbit/FitbitTimeZoneDataConverter.kt | 4 ++-- 9 files changed, 12 insertions(+), 39 deletions(-) delete mode 100644 radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt diff --git a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt index 75bab836..a987ca1e 100644 --- a/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt +++ b/kafka-connect-fitbit-source/src/main/java/org/radarbase/connect/rest/fitbit/converter/FitbitAvroConverter.kt @@ -25,7 +25,6 @@ import org.radarbase.connect.rest.fitbit.FitbitRestSourceConnectorConfig import org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator import org.radarbase.connect.rest.fitbit.request.FitbitRestRequest import org.radarbase.connect.rest.request.RestRequest -import org.radarbase.convert.fitbit.ConverterContext import org.radarbase.convert.fitbit.FitbitDataConverter import org.radarbase.convert.fitbit.TopicData import org.slf4j.LoggerFactory @@ -54,9 +53,7 @@ class FitbitAvroConverter( val key = user.getObservationKey(avroData) val timeReceived = System.currentTimeMillis() / 1000.0 return converter.processRecords( - ConverterContext(user.userId, - restRequest.getRequest().url.toString(), - restRequest.dateRange), + restRequest.dateRange, activities, timeReceived) .mapNotNull { r -> r.fold( diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt deleted file mode 100644 index 80fe2ae7..00000000 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/ConverterContext.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright 2018 The Hyve - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.radarbase.convert.fitbit - -data class ConverterContext( - val userId: String, - val url: String, - val dateRange: DateRange, -) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt index 5df7262b..897ddad7 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitActivityLogDataConverter.kt @@ -27,7 +27,7 @@ class FitbitActivityLogDataConverter( private val activityLogTopic: String ) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, root: JsonNode, timeReceived: Double + dateRange: DateRange, root: JsonNode, timeReceived: Double ): Sequence> { val array = root.optArray("activities") ?: return emptySequence() diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt index aea7e375..62eec837 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitDataConverter.kt @@ -24,7 +24,7 @@ import com.fasterxml.jackson.databind.JsonNode interface FitbitDataConverter { /** Process the JSON records generated by given request. */ fun processRecords( - context: ConverterContext, + dateRange: DateRange, root: JsonNode, timeReceived: Double ): Sequence> diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt index 0c9ac4cc..a53d1b9e 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayCaloriesDataConverter.kt @@ -10,7 +10,7 @@ class FitbitIntradayCaloriesDataConverter( ) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, root: JsonNode, timeReceived: Double + dateRange: DateRange, root: JsonNode, timeReceived: Double ): Sequence> { val intraday = root.optObject("activities-calories-intraday") ?: return emptySequence() @@ -19,7 +19,7 @@ class FitbitIntradayCaloriesDataConverter( val interval: Int = intraday.getRecordInterval(60) // Used as the date to convert the local times in the dataset to absolute times. - val startDate: ZonedDateTime = context.dateRange.end + val startDate: ZonedDateTime = dateRange.end return dataset.asSequence() .mapCatching { activity -> val localTime = LocalTime.parse(activity.get("time").asText()) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt index 27264904..676368fa 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayHeartRateDataConverter.kt @@ -26,7 +26,7 @@ class FitbitIntradayHeartRateDataConverter( private val heartRateTopic: String ) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, root: JsonNode, timeReceived: Double + dateRange: DateRange, root: JsonNode, timeReceived: Double ): Sequence> { val intraday = root.optObject("activities-heart-intraday") ?: return emptySequence() @@ -35,7 +35,7 @@ class FitbitIntradayHeartRateDataConverter( val interval: Int = intraday.getRecordInterval(1) // Used as the date to convert the local times in the dataset to absolute times. - val startDate: ZonedDateTime = context.dateRange.end + val startDate: ZonedDateTime = dateRange.end return dataset.asSequence() .mapCatching { activity -> val localTime = LocalTime.parse(activity.get("time").asText()) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt index eecaf661..2fe2ca92 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitIntradayStepsDataConverter.kt @@ -24,7 +24,7 @@ import java.time.ZonedDateTime class FitbitIntradayStepsDataConverter(private val stepTopic: String) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, root: JsonNode, timeReceived: Double + dateRange: DateRange, root: JsonNode, timeReceived: Double ): Sequence> { val intraday = root.optObject("activities-steps-intraday") ?: return emptySequence() @@ -33,7 +33,7 @@ class FitbitIntradayStepsDataConverter(private val stepTopic: String) : FitbitDa val interval = intraday.getRecordInterval(60) // Used as the date to convert the local times in the dataset to absolute times. - val startDate: ZonedDateTime = context.dateRange.end + val startDate: ZonedDateTime = dateRange.end return dataset.asSequence() .mapCatching { activity -> val localTime = LocalTime.parse(activity.get("time").asText()) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt index 49a62762..f70bea58 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitSleepDataConverter.kt @@ -33,7 +33,7 @@ class FitbitSleepDataConverter( private val sleepClassicTopic: String, ) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, root: JsonNode, timeReceived: Double, + dateRange: DateRange, root: JsonNode, timeReceived: Double, ): Sequence> { val meta = root.optObject("meta") if (meta != null && meta["state"]?.asText() == "pending") { diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt index 70cff155..b06ad4f9 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/FitbitTimeZoneDataConverter.kt @@ -24,7 +24,7 @@ import kotlin.Result.Companion.success class FitbitTimeZoneDataConverter(private val timeZoneTopic: String) : FitbitDataConverter { override fun processRecords( - context: ConverterContext, + dateRange: DateRange, root: JsonNode, timeReceived: Double, ): Sequence> { @@ -37,7 +37,7 @@ class FitbitTimeZoneDataConverter(private val timeZoneTopic: String) : FitbitDat return sequenceOf( success( TopicData( - sourceOffset = context.dateRange.start.toInstant(), + sourceOffset = dateRange.start.toInstant(), topic = timeZoneTopic, value = FitbitTimeZone(timeReceived, offset), ) From 58763bb2747c73285b675547cd52aaf96efe6ae8 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 8 Mar 2022 14:50:06 +0100 Subject: [PATCH 5/5] Make helper functions internal --- .../convert/fitbit/JsonNodeExtensions.kt | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt index 4f7f5374..ea41adc7 100644 --- a/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt +++ b/radar-fitbit-converter/src/main/kotlin/org/radarbase/convert/fitbit/JsonNodeExtensions.kt @@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit private val logger = LoggerFactory.getLogger("org.radarbase.convert.fitbit.JsonNodeExtensionsKt") /** Get Fitbit dataset interval used in some intraday API calls. */ -fun JsonNode.getRecordInterval(defaultValue: Int): Int { +internal fun JsonNode.getRecordInterval(defaultValue: Int): Int { val type = this["datasetType"] val interval = this["datasetInterval"] if (type == null || interval == null) { @@ -47,34 +47,34 @@ fun JsonNode.getRecordInterval(defaultValue: Int): Int { }.toSeconds(interval.asLong()).toInt() } -fun JsonNode.optLong(fieldName: String): Long? = this[fieldName] +internal fun JsonNode.optLong(fieldName: String): Long? = this[fieldName] ?.takeIf { it.canConvertToLong() } ?.longValue() -fun JsonNode.optDouble(fieldName: String): Double? = this[fieldName] +internal fun JsonNode.optDouble(fieldName: String): Double? = this[fieldName] ?.takeIf { it.isNumber } ?.doubleValue() -fun JsonNode.optInt(fieldName: String): Int? = this[fieldName] +internal fun JsonNode.optInt(fieldName: String): Int? = this[fieldName] ?.takeIf { it.canConvertToInt() } ?.intValue() -fun JsonNode.optString(fieldName: String?): String? = this[fieldName] +internal fun JsonNode.optString(fieldName: String?): String? = this[fieldName] ?.takeIf { it.isTextual } ?.textValue() -fun JsonNode.optBoolean(fieldName: String?): Boolean? = this[fieldName] +internal fun JsonNode.optBoolean(fieldName: String?): Boolean? = this[fieldName] ?.takeIf { it.isBoolean } ?.booleanValue() -fun JsonNode.optObject(fieldName: String?): ObjectNode? = this[fieldName] +internal fun JsonNode.optObject(fieldName: String?): ObjectNode? = this[fieldName] ?.takeIf { it.isObject } as ObjectNode? -fun JsonNode.optArray(fieldName: String?): Iterable? = this[fieldName] +internal fun JsonNode.optArray(fieldName: String?): Iterable? = this[fieldName] ?.takeIf { it.isArray && it.size() > 0 } -fun Sequence.mapCatching(fn: (T) -> S): Sequence> = map { t -> +internal fun Sequence.mapCatching(fn: (T) -> S): Sequence> = map { t -> runCatching { fn(t) }