Remove legacy code for format conversion

tests/test_parquet.py

@@ -7,11 +7,12 @@ import unittest
 from unittest.mock import patch
 from zoneinfo import ZoneInfo
 
+import pyarrow as pa
 import pyarrow.parquet as pq
 
 from solar_relay.cli import next_poll_at
 from solar_relay.config import retention_cutoff
-from solar_relay.storage import Store, StoreError
+from solar_relay.storage import SCHEMA, Store, StoreError
 from tests.test_status import observed
 
 
@@ -49,6 +50,33 @@ class ParquetTests(unittest.TestCase):
         self.assertEqual(self.store.prune(now=self.now + timedelta(days=31)), 2)
         self.assertEqual(list(self.store.records()), [items[3]])
 
+    def test_additional_archive_columns_do_not_change_records_or_acknowledgements(self):
+        value = observed(self.now)
+        self.store.add(value)
+        path = next(self.path.glob("date=*/samples.parquet"))
+        table = pq.ParquetFile(path).read()
+        table = table.append_column("unused_estimate", pa.array([100.0]))
+        pq.write_table(table, path)
+        self.assertEqual(self.store.pending(100), [value])
+        self.assertEqual(list(self.store.records()), [value])
+        self.store.acknowledge([value["id"]], now=self.now)
+        self.assertEqual(self.store.pending(100), [])
+        self.assertEqual(list(self.store.records()), [value])
+        self.assertTrue(pq.ParquetFile(path).schema_arrow.equals(SCHEMA, check_metadata=True))
+
+    def test_missing_or_mistyped_archive_columns_are_rejected(self):
+        value = observed(self.now)
+        self.store.add(value)
+        path = next(self.path.glob("date=*/samples.parquet"))
+        table = pq.ParquetFile(path).read()
+        index = table.schema.get_field_index("production_ac_w")
+        for invalid in (table.drop_columns(["production_ac_w"]),
+                        table.set_column(index, "production_ac_w", pa.array(["invalid"]))):
+            with self.subTest(schema=invalid.schema):
+                pq.write_table(invalid, path)
+                with self.assertRaises(StoreError):
+                    self.store.pending(100)
+
     def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
         zone = ZoneInfo("Europe/Zurich")
         for hour in (0, 12, 23):