Migrate records to solar web format

solar_relay/storage.py

@@ -20,6 +20,7 @@ import pyarrow.parquet as pq
 
 from .config import retention_cutoff, utc_text
 from .coverage import daily_coverage, milliseconds, sample_coverage
+from .history import normalize_history_sample
 
 NUMBERS = (
     "production_w", "production_ac_w", "consumption_w", "grid_w",
@@ -136,11 +137,32 @@ class Store:
             rows = self._read(path)
             for existing in rows:
                 if existing["id"] == row["id"]:
-                    if existing["payload"] != row["payload"]:
+                    if normalize_history_sample(json.loads(existing["payload"])) != normalize_history_sample(sample):
                         raise StoreError("Sample ID already contains different data")
                     return
             self._write(path, [*rows, row])
 
+    def normalize_history(self):
+        """Backfill common power columns and queued payloads without changing IDs or acknowledgements."""
+        changed = 0
+        with self._locked(write=True):
+            for path in self._files():
+                rows = self._read(path)
+                dirty = False
+                for index, row in enumerate(rows):
+                    if row["record_kind"] != "history":
+                        continue
+                    sample = json.loads(row["payload"])
+                    normalized = normalize_history_sample(sample)
+                    if normalized == sample:
+                        continue
+                    rows[index] = {**self._row(normalized, row["uploaded_at"]), "recorded_ns": row["recorded_ns"]}
+                    changed += 1
+                    dirty = True
+                if dirty:
+                    self._write(path, rows)
+        return changed
+
     def pending(self, limit):
         with self._locked():
             rows = []