Migrate records to solar web format

solar_relay/cli.py

@@ -40,6 +40,9 @@ def next_poll_at(now, interval):
 
 
 def _run(config, store):
+    normalized = store.normalize_history()
+    if normalized:
+        LOG.info("Normalized %d recovered records to common power readings", normalized)
     stop = threading.Event()
     previous = {}
     uploader = Uploader(config.upload, store) if config.upload.url else None
@@ -103,6 +106,7 @@ def _parser():
     commands.add_parser("upload", help="Upload the pending queue, stopping on the first failure")
     commands.add_parser("status", help="Show local archive/queue status without contacting the inverter")
     commands.add_parser("prune", help="Delete expired whole local days after acknowledgement retention")
+    commands.add_parser("normalize-history", help="Backfill common power readings in previously recovered records")
     for name, help_text in [
         ("export", "Export archived samples as JSON Lines, including already uploaded samples"),
         ("replay", "Requeue archived samples for upload, preserving their IDs"),
@@ -137,6 +141,8 @@ def main(argv=None):
             print(json.dumps(store.status(), indent=2))
         elif args.command == "prune":
             print(json.dumps({"deleted": store.prune()}))
+        elif args.command == "normalize-history":
+            print(json.dumps({"normalized": store.normalize_history()}))
         elif args.command == "recover":
             if not config.history.enabled:
                 raise ValueError("Enable history and configure Solar.web account credentials before recovery")

solar_relay/history.py

@@ -23,6 +23,40 @@ CHANNELS = {
 }
 
 
+def normalize_history_sample(sample):
+    """Expose recovered power in the same fields as live samples, including old archives."""
+    if "history_interval" not in sample:
+        return sample
+    interval = sample["history_interval"]
+    if interval["duration_seconds"] != 300:
+        raise ValueError("Historical record is not a five-minute interval")
+    energy = interval["energy_wh"]
+    if any(isinstance(energy[field], bool) or not isinstance(energy[field], (int, float))
+           or not math.isfinite(energy[field]) or not 0 <= energy[field] <= 1e18 for field in CHANNELS):
+        raise ValueError("Historical energy is missing or invalid")
+    power = {field: energy[field] * 12 for field in CHANNELS}
+    chart = sample.get("raw", {}).get("solarweb_history", {}).get("chart_power_w")
+    if chart is not None:
+        # Retain the chart's original watts instead of a W -> Wh -> W round trip.
+        generated, consumed = chart["production"], chart["consumption"]
+        if any(isinstance(value, bool) or not isinstance(value, (int, float))
+               or not math.isfinite(value) or value < 0 for value in (generated, consumed)):
+            raise ValueError("Historical chart power is missing or invalid")
+        if not all(math.isclose(value, power[field]) for field, value in
+                   (("production", generated), ("consumption", consumed))):
+            raise ValueError("Historical chart power disagrees with interval energy")
+        power = {"production": generated, "consumption": consumed,
+                 "bought": max(0, consumed - generated), "sold": max(0, generated - consumed)}
+    return {
+        **sample, "poll_interval_seconds": 300,
+        "readings": {**sample["readings"],
+                     "production_ac_w": power["production"], "consumption_w": power["consumption"],
+                     "grid_w": power["bought"] - power["sold"],
+                     "grid_import_w": power["bought"], "grid_export_w": power["sold"]},
+        "source_timestamps": {**sample.get("source_timestamps", {}), "powerflow": sample["observed_at"]},
+    }
+
+
 def history_sample(entry, source_id, system_id):
     stamp = milliseconds(entry["logDateTime"])
     if stamp % 300_000 or entry.get("logDuration") != 300:
@@ -39,7 +73,7 @@ def history_sample(entry, source_id, system_id):
         channels[name] = value
     energy = {field: channels[name] for field, name in CHANNELS.items()}
     when = utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc))
-    return {
+    return normalize_history_sample({
         "schema_version": 1,
         # Preserve IDs of intervals already recovered with the former Query API.
         "id": str(uuid5(NAMESPACE_URL, f"https://swqapi.solarweb.com/{system_id}/{source_id}/{when}")),
@@ -53,7 +87,7 @@ def history_sample(entry, source_id, system_id):
                    "boiler": "Historical boiler switch state is unavailable"},
         "raw": {"solarweb_history": entry},
         "history_interval": {"provider": "fronius-solarweb", "start": when, "duration_seconds": 300, "energy_wh": energy},
-    }
+    })
 
 
 def chart_entries(production, consumption):
@@ -106,12 +140,15 @@ class HistoryRecovery:
         missing = set(range(lower, upper, 300)) - present
         LOG.info("Checking %d historical energy intervals; %d are missing", (upper - lower) // 300, len(missing))
         # Chart dates use the installation's local timezone, including 23/25-hour DST days.
-        days = sorted({datetime.fromtimestamp(epoch, self.zone).date() for epoch in missing})
+        days = {}
+        for epoch in sorted(missing):
+            local = datetime.fromtimestamp(epoch, self.zone)
+            days.setdefault(local.date(), []).append(local.strftime("%H:%M:%S"))
         recovered = 0
-        for day in days:
+        for day in sorted(days):
             if stop and stop.is_set():
                 break
-            LOG.info("Recovering Solar.web energy history for %s", day)
+            LOG.info("Recovering Solar.web energy history for %s %s", day, ", ".join(days[day]))
             try:
                 production = self.client.get_chart(self.config.pv_system_id, "production", day)
                 if stop and stop.is_set():

solar_relay/storage.py

@@ -20,6 +20,7 @@ import pyarrow.parquet as pq
 
 from .config import retention_cutoff, utc_text
 from .coverage import daily_coverage, milliseconds, sample_coverage
+from .history import normalize_history_sample
 
 NUMBERS = (
     "production_w", "production_ac_w", "consumption_w", "grid_w",
@@ -136,11 +137,32 @@ class Store:
             rows = self._read(path)
             for existing in rows:
                 if existing["id"] == row["id"]:
-                    if existing["payload"] != row["payload"]:
+                    if normalize_history_sample(json.loads(existing["payload"])) != normalize_history_sample(sample):
                         raise StoreError("Sample ID already contains different data")
                     return
             self._write(path, [*rows, row])
 
+    def normalize_history(self):
+        """Backfill common power columns and queued payloads without changing IDs or acknowledgements."""
+        changed = 0
+        with self._locked(write=True):
+            for path in self._files():
+                rows = self._read(path)
+                dirty = False
+                for index, row in enumerate(rows):
+                    if row["record_kind"] != "history":
+                        continue
+                    sample = json.loads(row["payload"])
+                    normalized = normalize_history_sample(sample)
+                    if normalized == sample:
+                        continue
+                    rows[index] = {**self._row(normalized, row["uploaded_at"]), "recorded_ns": row["recorded_ns"]}
+                    changed += 1
+                    dirty = True
+                if dirty:
+                    self._write(path, rows)
+        return changed
+
     def pending(self, limit):
         with self._locked():
             rows = []

solar_relay/web/index.html

@@ -4,15 +4,15 @@
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <meta name="color-scheme" content="dark">
-  <title>Solar relay · Collection status</title>
-  <link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' rx='16' fill='%23171d20'/%3E%3Ccircle cx='32' cy='32' r='15' fill='%23e8ba70'/%3E%3C/svg%3E">
+  <title>Solar relay</title>
+  <link rel="icon" href="data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCA2NCA2NCI+CiAgPHRpdGxlPlNvbGFyIFJlbGF5PC90aXRsZT4KICA8cmVjdCB3aWR0aD0iNjQiIGhlaWdodD0iNjQiIHJ4PSIxNSIgZmlsbD0iIzBBMTMzMCIvPgogIDxjaXJjbGUgY3g9IjI4IiBjeT0iMjkiIHI9IjEzIiBmaWxsPSIjRkZDQTQ1Ii8+CiAgPHBhdGggZD0iTTExIDQzQzIwIDM0IDM4IDMwIDUzIDI5IiBmaWxsPSJub25lIiBzdHJva2U9IiNGRkY3RTgiIHN0cm9rZS13aWR0aD0iNSIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIi8+CiAgPHBhdGggZD0iTTQ2IDIzTDU0IDI5TDQ2IDM1IiBmaWxsPSJub25lIiBzdHJva2U9IiNGRkY3RTgiIHN0cm9rZS13aWR0aD0iNCIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIi8+Cjwvc3ZnPg==">
   <link rel="stylesheet" href="assets/status.css">
   <script src="assets/status.js" defer></script>
 </head>
 <body>
   <main>
     <header class="masthead">
-      <div class="brand"><span class="sun" aria-hidden="true">☀</span><span>SOLAR RELAY<small>RPI OFFICE</small></span></div>
+      <div class="brand"><span class="sun" aria-hidden="true">☀️</span><span>SOLAR RELAY<small>Raspberry Pi · Office</small></span></div>
       <div class="header-actions"><span id="overall" class="badge neutral" role="status">Connecting</span><button id="refresh" class="refresh">↻ <span>Refresh</span></button></div>
     </header>
     <section class="intro">