Xiaomi: Persist and overlay sleep stages

GBDaoGenerator/src/nodomain/freeyourgadget/gadgetbridge/daogen/GBDaoGenerator.java

@@ -45,7 +45,7 @@ public class GBDaoGenerator {
 
 
     public static void main(String[] args) throws Exception {
-        final Schema schema = new Schema(65, MAIN_PACKAGE + ".entities");
+        final Schema schema = new Schema(66, MAIN_PACKAGE + ".entities");
 
         Entity userAttributes = addUserAttributes(schema);
         Entity user = addUserInfo(schema, userAttributes);
@@ -72,6 +72,7 @@ public class GBDaoGenerator {
         addHuamiSleepRespiratoryRateSample(schema, user, device);
         addXiaomiActivitySample(schema, user, device);
         addXiaomiSleepTimeSamples(schema, user, device);
+        addXiaomiSleepStageSamples(schema, user, device);
         addXiaomiDailySummarySamples(schema, user, device);
         addPebbleHealthActivitySample(schema, user, device);
         addPebbleHealthActivityKindOverlay(schema, user, device);
@@ -345,6 +346,18 @@ public class GBDaoGenerator {
         addCommonTimeSampleProperties("AbstractTimeSample", sample, user, device);
         sample.addLongProperty("wakeupTime");
         sample.addBooleanProperty("isAwake");
+        sample.addIntProperty("totalDuration");
+        sample.addIntProperty("deepSleepDuration");
+        sample.addIntProperty("lightSleepDuration");
+        sample.addIntProperty("remSleepDuration");
+        sample.addIntProperty("awakeDuration");
+        return sample;
+    }
+
+    private static Entity addXiaomiSleepStageSamples(Schema schema, Entity user, Entity device) {
+        Entity sample = addEntity(schema, "XiaomiSleepStageSample");
+        addCommonTimeSampleProperties("AbstractTimeSample", sample, user, device);
+        sample.addIntProperty("stage");
         return sample;
     }
 

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/database/schema/GadgetbridgeUpdate_66.java

@@ -0,0 +1,56 @@
+/*  Copyright (C) 2023 José Rebelo
+
+    This file is part of Gadgetbridge.
+
+    Gadgetbridge is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Gadgetbridge is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+package nodomain.freeyourgadget.gadgetbridge.database.schema;
+
+import android.database.sqlite.SQLiteDatabase;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
+
+import nodomain.freeyourgadget.gadgetbridge.database.DBHelper;
+import nodomain.freeyourgadget.gadgetbridge.database.DBUpdateScript;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepTimeSampleDao;
+
+public class GadgetbridgeUpdate_66 implements DBUpdateScript {
+    @Override
+    public void upgradeSchema(final SQLiteDatabase db) {
+        final List<String> newColumns = Arrays.asList(
+                XiaomiSleepTimeSampleDao.Properties.TotalDuration.columnName,
+                XiaomiSleepTimeSampleDao.Properties.DeepSleepDuration.columnName,
+                XiaomiSleepTimeSampleDao.Properties.LightSleepDuration.columnName,
+                XiaomiSleepTimeSampleDao.Properties.RemSleepDuration.columnName,
+                XiaomiSleepTimeSampleDao.Properties.AwakeDuration.columnName
+        );
+
+        for (final String newColumn : newColumns) {
+            if (!DBHelper.existsColumn(XiaomiSleepTimeSampleDao.TABLENAME, newColumn, db)) {
+                final String SQL_ALTER_TABLE = String.format(
+                        Locale.ROOT,
+                        "ALTER TABLE %s ADD COLUMN %s INTEGER",
+                        XiaomiSleepTimeSampleDao.TABLENAME,
+                        newColumn
+                );
+                db.execSQL(SQL_ALTER_TABLE);
+            }
+        }
+    }
+
+    @Override
+    public void downgradeSchema(SQLiteDatabase db) {
+    }
+}

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/xiaomi/XiaomiSampleProvider.java

@@ -30,9 +30,11 @@ import nodomain.freeyourgadget.gadgetbridge.devices.AbstractSampleProvider;
 import nodomain.freeyourgadget.gadgetbridge.entities.DaoSession;
 import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiActivitySample;
 import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiActivitySampleDao;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepStageSample;
 import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepTimeSample;
 import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
 import nodomain.freeyourgadget.gadgetbridge.model.ActivityKind;
+import nodomain.freeyourgadget.gadgetbridge.util.RangeMap;
 
 public class XiaomiSampleProvider extends AbstractSampleProvider<XiaomiActivitySample> {
     private static final Logger LOG = LoggerFactory.getLogger(XiaomiSampleProvider.class);
@@ -90,18 +92,68 @@ public class XiaomiSampleProvider extends AbstractSampleProvider<XiaomiActivityS
     protected List<XiaomiActivitySample> getGBActivitySamples(final int timestamp_from, final int timestamp_to, final int activityType) {
         final List<XiaomiActivitySample> samples = super.getGBActivitySamples(timestamp_from, timestamp_to, activityType);
 
-        // Fetch bed and wakeup times and overlay them on the activity
+        final RangeMap<Long, Integer> stagesMap = new RangeMap<>();
-        final XiaomiSleepTimeSampleProvider sleepTimeSampleProvider = new XiaomiSleepTimeSampleProvider(getDevice(), getSession());
+
-        final List<XiaomiSleepTimeSample> sleepSamples = sleepTimeSampleProvider.getAllSamples(timestamp_from * 1000L, timestamp_to * 1000L);
+        final XiaomiSleepStageSampleProvider sleepStagesSampleProvider = new XiaomiSleepStageSampleProvider(getDevice(), getSession());
-        if (!sleepSamples.isEmpty()) {
+        final List<XiaomiSleepStageSample> stageSamples = sleepStagesSampleProvider.getAllSamples(timestamp_from * 1000L, timestamp_to * 1000L);
-            LOG.debug("Found {} sleep samples between {} and {}", sleepSamples.size(), timestamp_from, timestamp_to);
+        if (!stageSamples.isEmpty()) {
+            // We got actual sleep stages
+            LOG.debug("Found {} sleep stage samples between {} and {}", stageSamples.size(), timestamp_from, timestamp_to);
+
+            for (final XiaomiSleepStageSample stageSample : stageSamples) {
+                final int activityKind;
+
+                switch (stageSample.getStage()) {
+                    case 2: // deep
+                        activityKind = ActivityKind.TYPE_DEEP_SLEEP;
+                        break;
+                    case 3: // light
+                        activityKind = ActivityKind.TYPE_LIGHT_SLEEP;
+                        break;
+                    case 4: // rem
+                        activityKind = ActivityKind.TYPE_REM_SLEEP;
+                        break;
+                    case 0: // final awake
+                    case 1: // ?
+                    case 5: // awake during the night
+                    default:
+                        activityKind = ActivityKind.TYPE_UNKNOWN;
+                        break;
+                }
+                stagesMap.put(stageSample.getTimestamp(), activityKind);
+            }
+        } else {
+            // Fetch bed and wakeup times and overlay as light sleep on the activity
+            final XiaomiSleepTimeSampleProvider sleepTimeSampleProvider = new XiaomiSleepTimeSampleProvider(getDevice(), getSession());
+            final List<XiaomiSleepTimeSample> sleepSamples = sleepTimeSampleProvider.getAllSamples(timestamp_from * 1000L, timestamp_to * 1000L);
+            if (!sleepSamples.isEmpty()) {
+                LOG.debug("Found {} sleep samples between {} and {}", sleepSamples.size(), timestamp_from, timestamp_to);
+                for (final XiaomiSleepTimeSample stageSample : sleepSamples) {
+                    stagesMap.put(stageSample.getTimestamp(), ActivityKind.TYPE_LIGHT_SLEEP);
+                    stagesMap.put(stageSample.getWakeupTime(), ActivityKind.TYPE_UNKNOWN);
+                }
+            }
+        }
+
+        if (!stagesMap.isEmpty()) {
+            LOG.debug("Found {} sleep samples between {} and {}", stagesMap.size(), timestamp_from, timestamp_to);
 
             for (final XiaomiActivitySample sample : samples) {
                 final long ts = sample.getTimestamp() * 1000L;
-                for (final XiaomiSleepTimeSample sleepSample : sleepSamples) {
+                final Integer sleepType = stagesMap.get(ts);
-                    if (ts >= sleepSample.getTimestamp() && ts <= sleepSample.getWakeupTime()) {
+                if (sleepType != null) {
-                        sample.setRawKind(ActivityKind.TYPE_LIGHT_SLEEP);
+                    sample.setRawKind(sleepType);
-                        sample.setRawIntensity(30);
+
+                    switch (sleepType) {
+                        case ActivityKind.TYPE_DEEP_SLEEP:
+                            sample.setRawIntensity(10);
+                            break;
+                        case ActivityKind.TYPE_LIGHT_SLEEP:
+                            sample.setRawIntensity(30);
+                            break;
+                        case ActivityKind.TYPE_REM_SLEEP:
+                            sample.setRawIntensity(40);
+                            break;
                     }
                 }
             }

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/devices/xiaomi/XiaomiSleepStageSampleProvider.java

@@ -0,0 +1,56 @@
+/*  Copyright (C) 2023 José Rebelo
+
+    This file is part of Gadgetbridge.
+
+    Gadgetbridge is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Gadgetbridge is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+package nodomain.freeyourgadget.gadgetbridge.devices.xiaomi;
+
+import androidx.annotation.NonNull;
+
+import de.greenrobot.dao.AbstractDao;
+import de.greenrobot.dao.Property;
+import nodomain.freeyourgadget.gadgetbridge.devices.AbstractTimeSampleProvider;
+import nodomain.freeyourgadget.gadgetbridge.entities.DaoSession;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepStageSample;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepStageSampleDao;
+import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
+
+public class XiaomiSleepStageSampleProvider extends AbstractTimeSampleProvider<XiaomiSleepStageSample> {
+    public XiaomiSleepStageSampleProvider(final GBDevice device, final DaoSession session) {
+        super(device, session);
+    }
+
+    @NonNull
+    @Override
+    public AbstractDao<XiaomiSleepStageSample, ?> getSampleDao() {
+        return getSession().getXiaomiSleepStageSampleDao();
+    }
+
+    @NonNull
+    @Override
+    protected Property getTimestampSampleProperty() {
+        return XiaomiSleepStageSampleDao.Properties.Timestamp;
+    }
+
+    @NonNull
+    @Override
+    protected Property getDeviceIdentifierSampleProperty() {
+        return XiaomiSleepStageSampleDao.Properties.DeviceId;
+    }
+
+    @Override
+    public XiaomiSleepStageSample createSample() {
+        return new XiaomiSleepStageSample();
+    }
+}

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/service/devices/xiaomi/activity/impl/SleepStagesParser.java

@@ -16,15 +16,31 @@
     along with this program.  If not, see <http://www.gnu.org/licenses/>. */
 package nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.impl;
 
+import android.widget.Toast;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
+import java.util.ArrayList;
+import java.util.List;
 
+import nodomain.freeyourgadget.gadgetbridge.GBApplication;
+import nodomain.freeyourgadget.gadgetbridge.database.DBHandler;
+import nodomain.freeyourgadget.gadgetbridge.database.DBHelper;
+import nodomain.freeyourgadget.gadgetbridge.devices.xiaomi.XiaomiSleepStageSampleProvider;
+import nodomain.freeyourgadget.gadgetbridge.devices.xiaomi.XiaomiSleepTimeSampleProvider;
+import nodomain.freeyourgadget.gadgetbridge.entities.DaoSession;
+import nodomain.freeyourgadget.gadgetbridge.entities.Device;
+import nodomain.freeyourgadget.gadgetbridge.entities.User;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepStageSample;
+import nodomain.freeyourgadget.gadgetbridge.entities.XiaomiSleepTimeSample;
+import nodomain.freeyourgadget.gadgetbridge.impl.GBDevice;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.XiaomiSupport;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.XiaomiActivityFileId;
 import nodomain.freeyourgadget.gadgetbridge.service.devices.xiaomi.activity.XiaomiActivityParser;
+import nodomain.freeyourgadget.gadgetbridge.util.GB;
 
 public class SleepStagesParser extends XiaomiActivityParser {
     private static final Logger LOG = LoggerFactory.getLogger(SleepStagesParser.class);
@@ -51,7 +67,7 @@ public class SleepStagesParser extends XiaomiActivityParser {
         // timestamp when watch counts "real" sleep start, might be later than first phase change
         final int bedTime = buf.getInt();
         // timestamp when sleep ended (have not observed, but may also be earlier than last phase?)
-        final int wakeUpTime = buf.getInt();
+        final int wakeupTime = buf.getInt();
 
         // byte 8 medium
         // bytes 9,10 look like a short
@@ -67,14 +83,89 @@ public class SleepStagesParser extends XiaomiActivityParser {
         // sum of all "real" awake durations
         final short wakeDuration = buf.getShort();
 
+        LOG.debug("Sleep stages sample: bedTime: {}, wakeupTime: {}, sleepDuration: {}", bedTime, wakeupTime, sleepDuration);
+
+        if (bedTime == 0 || wakeupTime == 0 || sleepDuration == 0) {
+            LOG.warn("Ignoring sleep stages sample with no data");
+            return true;
+        }
+
+        final XiaomiSleepTimeSample sample = new XiaomiSleepTimeSample();
+        sample.setTimestamp(bedTime * 1000L);
+        sample.setWakeupTime(wakeupTime * 1000L);
+        sample.setIsAwake(false);
+        sample.setTotalDuration((int) sleepDuration);
+        sample.setDeepSleepDuration((int) deepSleepDuration);
+        sample.setLightSleepDuration((int) lightSleepDuration);
+        sample.setRemSleepDuration((int) REMDuration);
+        sample.setAwakeDuration((int) wakeDuration);
+
+        final List<XiaomiSleepStageSample> stages = new ArrayList<>();
+
         // byte 11 small-medium
         final byte unk3 = buf.get();
         while (buf.position() < buf.limit()) {
             // when the change to the phase occurs
             final int time = buf.getInt();
             // what phase state changed to
-            final byte sleepPhase = buf.get();
+            final int sleepPhase = buf.get() & 0xff;
+
+            final XiaomiSleepStageSample stageSample = new XiaomiSleepStageSample();
+            stageSample.setTimestamp(time * 1000L);
+            stageSample.setStage(sleepPhase);
+            stages.add(stageSample);
         }
+
+        // Save the sleep time sample
+        try (DBHandler handler = GBApplication.acquireDB()) {
+            final DaoSession session = handler.getDaoSession();
+            final GBDevice gbDevice = support.getDevice();
+
+            sample.setDevice(DBHelper.getDevice(gbDevice, session));
+            sample.setUser(DBHelper.getUser(session));
+
+            final XiaomiSleepTimeSampleProvider sampleProvider = new XiaomiSleepTimeSampleProvider(gbDevice, session);
+
+            // Check if there is already a later sleep sample - if so, ignore this one
+            // Samples for the same sleep will always have the same bedtime (timestamp), but we might get
+            // multiple bedtimes until the user wakes up
+            final List<XiaomiSleepTimeSample> existingSamples = sampleProvider.getAllSamples(sample.getTimestamp(), sample.getTimestamp());
+            if (!existingSamples.isEmpty()) {
+                final XiaomiSleepTimeSample existingSample = existingSamples.get(0);
+                if (existingSample.getWakeupTime() > sample.getWakeupTime()) {
+                    LOG.warn("Ignoring sleep sample - existing sample is more recent ({})", existingSample.getWakeupTime());
+                    return true;
+                }
+            }
+
+            sampleProvider.addSample(sample);
+        } catch (final Exception e) {
+            GB.toast(support.getContext(), "Error saving sleep sample", Toast.LENGTH_LONG, GB.ERROR);
+            LOG.error("Error saving sleep sample", e);
+            return false;
+        }
+
+        // Save the sleep stage samples
+        try (DBHandler handler = GBApplication.acquireDB()) {
+            final DaoSession session = handler.getDaoSession();
+            final GBDevice gbDevice = support.getDevice();
+            final Device device = DBHelper.getDevice(gbDevice, session);
+            final User user = DBHelper.getUser(session);
+
+            final XiaomiSleepStageSampleProvider sampleProvider = new XiaomiSleepStageSampleProvider(gbDevice, session);
+
+            for (final XiaomiSleepStageSample stageSample : stages) {
+                stageSample.setDevice(device);
+                stageSample.setUser(user);
+            }
+
+            sampleProvider.addSamples(stages);
+        } catch (final Exception e) {
+            GB.toast(support.getContext(), "Error saving sleep stage samples", Toast.LENGTH_LONG, GB.ERROR);
+            LOG.error("Error saving sleep stage samples", e);
+            return false;
+        }
+
         return true;
     }
 }

app/src/main/java/nodomain/freeyourgadget/gadgetbridge/util/RangeMap.java

@@ -0,0 +1,64 @@
+/*  Copyright (C) 2023 José Rebelo
+
+    This file is part of Gadgetbridge.
+
+    Gadgetbridge is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published
+    by the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    Gadgetbridge is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>. */
+package nodomain.freeyourgadget.gadgetbridge.util;
+
+import android.util.Pair;
+
+import androidx.annotation.Nullable;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/**
+ * A map of lower bounds for ranges.
+ */
+public class RangeMap<K extends Comparable<K>, V> {
+    private final List<Pair<K, V>> list = new ArrayList<>();
+    private boolean isSorted = false;
+
+    public void put(final K key, final V value) {
+        list.add(Pair.create(key, value));
+        isSorted = false;
+    }
+
+    @Nullable
+    public V get(final K key) {
+        if (!isSorted) {
+            Collections.sort(list, (a, b) -> {
+                return a.first.compareTo(b.first);
+            });
+            isSorted = true;
+        }
+
+        for (int i = list.size() - 1; i >= 0; i--) {
+            if (key.compareTo(list.get(i).first) > 0) {
+                return list.get(i).second;
+            }
+        }
+
+        return null;
+    }
+
+    public boolean isEmpty() {
+        return list.isEmpty();
+    }
+
+    public int size() {
+        return list.size();
+    }
+}