Reshape boiler data

tests/test_parquet.py

@@ -27,12 +27,16 @@ class ParquetTests(unittest.TestCase):
     def test_real_parquet_has_typed_readings_and_original_payload(self):
         value = observed(self.now)
         value["readings"]["boiler_enabled"] = False
+        value["readings"]["boiler_threshold_w"] = 2600
+        value["readings"]["boiler_threshold_source"] = "config"
         self.store.add(value)
         path = next(self.path.glob("date=*/samples.parquet"))
         self.assertEqual(path.read_bytes()[:4], b"PAR1")
         row = pq.ParquetFile(path).read().to_pylist()[0]
         self.assertEqual(row["production_ac_w"], 2000)
         self.assertIs(row["boiler_enabled"], False)
+        self.assertEqual(row["boiler_threshold_w"], 2600)
+        self.assertEqual(row["boiler_threshold_source"], "config")
         self.assertIsNone(row["uploaded_at"])
         self.assertEqual(json.loads(row["payload"]), value)
         self.assertEqual(self.store.status()["format"], "parquet")
@@ -77,6 +81,18 @@ class ParquetTests(unittest.TestCase):
                 with self.assertRaises(StoreError):
                     self.store.pending(100)
 
+    def test_pre_threshold_archive_is_readable_and_upgraded_on_write(self):
+        value = observed(self.now)
+        self.store.add(value)
+        path = next(self.path.glob("date=*/samples.parquet"))
+        legacy = pq.ParquetFile(path).read().drop_columns(["boiler_threshold_w", "boiler_threshold_source"])
+        pq.write_table(legacy, path)
+        self.assertEqual(self.store.pending(100), [value])
+        self.assertEqual(list(self.store.records()), [value])
+        self.store.acknowledge([value["id"]], now=self.now)
+        self.assertTrue(pq.ParquetFile(path).schema_arrow.equals(SCHEMA, check_metadata=True))
+        self.assertIsNone(pq.ParquetFile(path).read().to_pylist()[0]["boiler_threshold_w"])
+
     def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
         zone = ZoneInfo("Europe/Zurich")
         for hour in (0, 12, 23):