Reshape boiler data

solar_relay/storage.py

@@ -25,6 +25,7 @@ NUMBERS = (
     "production_w", "production_ac_w", "consumption_w", "grid_w",
     "grid_import_w", "grid_export_w", "production_ac_total_wh",
     "grid_import_total_wh", "grid_export_total_wh",
+    "boiler_threshold_w",
 )
 SCHEMA = pa.schema([
     ("id", pa.string()), ("source_id", pa.string()), ("observed_at", pa.string()),
@@ -32,12 +33,14 @@ SCHEMA = pa.schema([
     ("uploaded_at", pa.string()), ("payload", pa.string()),
     *((field, pa.float64()) for field in NUMBERS),
     ("boiler_enabled", pa.bool_()), ("boiler_reason", pa.string()),
+    ("boiler_threshold_source", pa.string()),
     ("record_kind", pa.string()),
     ("power_ms", pa.int64()), ("power_ok", pa.bool_()),
     ("meter_ms", pa.int64()), ("meter_ok", pa.bool_()),
     ("boiler_ms", pa.int64()), ("boiler_ok", pa.bool_()),
 ], metadata={b"solar_relay_schema": b"1"})
 COVERAGE_COLUMNS = ["observed_ms", "power_ms", "power_ok", "meter_ms", "meter_ok", "boiler_ms", "boiler_ok"]
+OPTIONAL_COLUMNS = {"boiler_threshold_w", "boiler_threshold_source"}  # Absent in archives written before threshold recording.
 
 
 class StoreError(OSError):
@@ -83,10 +86,16 @@ class Store:
         schema = parquet.schema_arrow
         # Validate the fields this version uses; read only those columns.
         if schema.metadata != SCHEMA.metadata or any(
-            schema.get_field_index(field.name) < 0 or schema.field(field.name) != field for field in SCHEMA
+            (schema.get_field_index(field.name) < 0 and field.name not in OPTIONAL_COLUMNS) or
+            (schema.get_field_index(field.name) >= 0 and schema.field(field.name) != field) for field in SCHEMA
         ):
             raise StoreError("Unsupported Parquet archive schema")
-        return parquet.read(columns=SCHEMA.names if columns is None else columns).to_pylist()
+        wanted = SCHEMA.names if columns is None else columns
+        missing = set(wanted) - set(schema.names)
+        rows = parquet.read(columns=[name for name in wanted if name not in missing]).to_pylist()
+        for row in rows:
+            row.update({name: None for name in missing})
+        return rows
 
     @staticmethod
     def _sync_directory(path):
@@ -127,6 +136,7 @@ class Store:
             "payload": json.dumps(sample, allow_nan=False, separators=(",", ":")),
             **{name: readings.get(name) for name in NUMBERS},
             "boiler_enabled": readings.get("boiler_enabled"), "boiler_reason": readings.get("boiler_reason"),
+            "boiler_threshold_source": readings.get("boiler_threshold_source"),
             "record_kind": "history" if "solarweb_history" in sample.get("raw", {}) else "live",
             **{name: bool(value) if name.endswith("_ok") else value
                for name, value in zip(COVERAGE_COLUMNS[1:], coverage[3:])},