Simplify status interface

solar_relay/storage.py

@@ -13,11 +13,12 @@ from pathlib import Path
 import tempfile
 import threading
 import time
+from zoneinfo import ZoneInfo
 
 import pyarrow as pa
 import pyarrow.parquet as pq
 
-from .config import utc_text
+from .config import retention_cutoff, utc_text
 from .coverage import daily_coverage, milliseconds, sample_coverage
 
 NUMBERS = (
@@ -45,12 +46,13 @@ class StoreError(OSError):
 
 
 class Store:
-    def __init__(self, path: Path, retention_days=30):
+    def __init__(self, path: Path, retention_days=30, *, timezone_name="Europe/Zurich"):
         self.path = Path(path)
         if self.path.is_file():
             raise StoreError("collector.archive must be a directory")
         self.path.mkdir(parents=True, exist_ok=True, mode=0o700)
         self.retention_days = retention_days
+        self.zone = ZoneInfo(timezone_name)
         self._thread_lock = threading.RLock()
         descriptor = os.open(self.path / ".lock", os.O_CREAT | os.O_WRONLY, 0o600)
         os.close(descriptor)
@@ -190,16 +192,32 @@ class Store:
         return count
 
     def prune(self, *, now=None):
-        """Retain pending records; age acknowledged records from their receipt time."""
-        cutoff = utc_text((now or datetime.now(timezone.utc)) - timedelta(days=self.retention_days))
+        """Remove whole local observation days only after all their acknowledgements expire."""
+        boundary = retention_cutoff(now or datetime.now(timezone.utc), self.retention_days, self.zone.key)
+        cutoff, first_day = utc_text(boundary), boundary.astimezone(self.zone).date()
         deleted = 0
+        local_day = lambda row: datetime.fromtimestamp(row["observed_ms"] / 1000, self.zone).date()
         with self._locked(write=True):
+            partitions, expired, protected = [], set(), set()
+            # A local day can span two UTC partitions. One pending or recently
+            # acknowledged record protects the rest of its local day as well.
             for path in self._files():
-                index = self._read(path, ["uploaded_at"])
-                if not any(row["uploaded_at"] is not None and row["uploaded_at"] < cutoff for row in index):
+                index = self._read(path, ["observed_ms", "uploaded_at"])
+                days = set()
+                for row in index:
+                    day = local_day(row)
+                    days.add(day)
+                    if day < first_day and row["uploaded_at"] is not None and row["uploaded_at"] < cutoff:
+                        expired.add(day)
+                    else:
+                        protected.add(day)
+                partitions.append((path, days))
+            expired -= protected
+            for path, days in partitions:
+                if not days & expired:
                     continue
                 rows = self._read(path)
-                retained = [row for row in rows if row["uploaded_at"] is None or row["uploaded_at"] >= cutoff]
+                retained = [row for row in rows if local_day(row) not in expired]
                 self._write(path, retained)
                 deleted += len(rows) - len(retained)
         return deleted
@@ -229,7 +247,6 @@ class Store:
         }
 
     def coverage(self, source_id, timezone_name, days, now=None):
-        from zoneinfo import ZoneInfo
         now = now or datetime.now(timezone.utc)
         first = now.astimezone(ZoneInfo(timezone_name)).date() - timedelta(days=days)
         lower = round(datetime.combine(first, datetime.min.time(), ZoneInfo(timezone_name)).timestamp() * 1000)
@@ -247,8 +264,8 @@ class Store:
     def last_success(self, source_id):
         with self._locked():
             for path in reversed(self._files()):
-                rows = self._read(path, ["source_id", "observed_ms", "power_ok", "meter_ok", "boiler_ok"])
-                stamps = [row["observed_ms"] for row in rows if row["source_id"] == source_id and row["power_ok"] and row["meter_ok"] and row["boiler_ok"]]
+                rows = self._read(path, ["source_id", "observed_ms", "power_ok", "boiler_ok"])
+                stamps = [row["observed_ms"] for row in rows if row["source_id"] == source_id and row["power_ok"] and row["boiler_ok"]]
                 if stamps:
                     return utc_text(datetime.fromtimestamp(max(stamps) / 1000, timezone.utc))
         return None