Migrate records to solar web format

solar_relay/history.py

@@ -23,6 +23,40 @@ CHANNELS = {
 }
 
 
+def normalize_history_sample(sample):
+    """Expose recovered power in the same fields as live samples, including old archives."""
+    if "history_interval" not in sample:
+        return sample
+    interval = sample["history_interval"]
+    if interval["duration_seconds"] != 300:
+        raise ValueError("Historical record is not a five-minute interval")
+    energy = interval["energy_wh"]
+    if any(isinstance(energy[field], bool) or not isinstance(energy[field], (int, float))
+           or not math.isfinite(energy[field]) or not 0 <= energy[field] <= 1e18 for field in CHANNELS):
+        raise ValueError("Historical energy is missing or invalid")
+    power = {field: energy[field] * 12 for field in CHANNELS}
+    chart = sample.get("raw", {}).get("solarweb_history", {}).get("chart_power_w")
+    if chart is not None:
+        # Retain the chart's original watts instead of a W -> Wh -> W round trip.
+        generated, consumed = chart["production"], chart["consumption"]
+        if any(isinstance(value, bool) or not isinstance(value, (int, float))
+               or not math.isfinite(value) or value < 0 for value in (generated, consumed)):
+            raise ValueError("Historical chart power is missing or invalid")
+        if not all(math.isclose(value, power[field]) for field, value in
+                   (("production", generated), ("consumption", consumed))):
+            raise ValueError("Historical chart power disagrees with interval energy")
+        power = {"production": generated, "consumption": consumed,
+                 "bought": max(0, consumed - generated), "sold": max(0, generated - consumed)}
+    return {
+        **sample, "poll_interval_seconds": 300,
+        "readings": {**sample["readings"],
+                     "production_ac_w": power["production"], "consumption_w": power["consumption"],
+                     "grid_w": power["bought"] - power["sold"],
+                     "grid_import_w": power["bought"], "grid_export_w": power["sold"]},
+        "source_timestamps": {**sample.get("source_timestamps", {}), "powerflow": sample["observed_at"]},
+    }
+
+
 def history_sample(entry, source_id, system_id):
     stamp = milliseconds(entry["logDateTime"])
     if stamp % 300_000 or entry.get("logDuration") != 300:
@@ -39,7 +73,7 @@ def history_sample(entry, source_id, system_id):
         channels[name] = value
     energy = {field: channels[name] for field, name in CHANNELS.items()}
     when = utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc))
-    return {
+    return normalize_history_sample({
         "schema_version": 1,
         # Preserve IDs of intervals already recovered with the former Query API.
         "id": str(uuid5(NAMESPACE_URL, f"https://swqapi.solarweb.com/{system_id}/{source_id}/{when}")),
@@ -53,7 +87,7 @@ def history_sample(entry, source_id, system_id):
                    "boiler": "Historical boiler switch state is unavailable"},
         "raw": {"solarweb_history": entry},
         "history_interval": {"provider": "fronius-solarweb", "start": when, "duration_seconds": 300, "energy_wh": energy},
-    }
+    })
 
 
 def chart_entries(production, consumption):
@@ -106,12 +140,15 @@ class HistoryRecovery:
         missing = set(range(lower, upper, 300)) - present
         LOG.info("Checking %d historical energy intervals; %d are missing", (upper - lower) // 300, len(missing))
         # Chart dates use the installation's local timezone, including 23/25-hour DST days.
-        days = sorted({datetime.fromtimestamp(epoch, self.zone).date() for epoch in missing})
+        days = {}
+        for epoch in sorted(missing):
+            local = datetime.fromtimestamp(epoch, self.zone)
+            days.setdefault(local.date(), []).append(local.strftime("%H:%M:%S"))
         recovered = 0
-        for day in days:
+        for day in sorted(days):
             if stop and stop.is_set():
                 break
-            LOG.info("Recovering Solar.web energy history for %s", day)
+            LOG.info("Recovering Solar.web energy history for %s %s", day, ", ".join(days[day]))
             try:
                 production = self.client.get_chart(self.config.pv_system_id, "production", day)
                 if stop and stop.is_set():