franzz committed
1 parent
36180dffd7
commit
ec75ac15b0
30 files changed
+3042
-1
No files matched your search
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Exercise the workflow's real rsync script with a mocked service manager."""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import textwrap
|
||||
import unittest
|
||||
|
||||
|
||||
@unittest.skipUnless(shutil.which("rsync") and shutil.which("flock"), "deployment tools are not installed")
|
||||
class DeploymentTests(unittest.TestCase):
|
||||
def test_deploy_preserves_runtime_and_archive_and_restarts_service(self):
|
||||
workflow = (Path(__file__).resolve().parents[1] / ".gitea/workflows/deploy.yml").read_text()
|
||||
deploy_step = workflow.split("- name: Deploy and restart\n", 1)[1]
|
||||
script = textwrap.dedent(deploy_step.split("run: |\n", 1)[1])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
checkout, deployment, binaries = [root / name for name in ("checkout", "deployment", "bin")]
|
||||
for path in (checkout, deployment, binaries):
|
||||
path.mkdir()
|
||||
(checkout / "solar_relay").mkdir()
|
||||
(checkout / "solar_relay" / "new.py").write_text("# new release\n")
|
||||
(checkout / "requirements.txt").write_text("pyarrow==26.0.0\n")
|
||||
(deployment / "obsolete.py").write_text("# obsolete\n")
|
||||
protected = {
|
||||
"config.toml": "private configuration",
|
||||
".env": "private token",
|
||||
"data/archive/date=2026-11-01/samples.parquet": "durable parquet observations",
|
||||
"archive/date=2026-11-01/samples.parquet": "archived observations",
|
||||
".venv/keep.txt": "runtime venv",
|
||||
".git/keep.txt": "existing checkout metadata",
|
||||
}
|
||||
for relative, content in protected.items():
|
||||
target = deployment / relative
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(content)
|
||||
source = checkout / relative
|
||||
source.parent.mkdir(parents=True, exist_ok=True)
|
||||
source.write_text("must not replace local state")
|
||||
runtime = deployment / ".venv/bin/python"
|
||||
runtime.parent.mkdir(parents=True)
|
||||
runtime.write_text('#!/bin/sh\nif [ "$1" = "-m" ] && [ "$2" = pip ]; then printf "pip %s\\n" "$*" >> "$SOLAR_DEPLOY_TEST_LOG"; exit 0; fi\nexec ' + shlex.quote(sys.executable) + ' "$@"\n')
|
||||
runtime.chmod(0o755)
|
||||
mocks = {
|
||||
"sudo": '#!/bin/sh\n[ "$1" = "-n" ] || exit 99\nshift\nexec "$@"\n',
|
||||
"systemctl": '#!/bin/sh\nprintf "%s\\n" "$*" >> "$SOLAR_DEPLOY_TEST_LOG"\nif [ "$1" = show ]; then printf "0\\n"; fi\n',
|
||||
"sleep": "#!/bin/sh\nexit 0\n",
|
||||
"curl": '#!/bin/sh\nprintf "curl %s\\n" "$*" >> "$SOLAR_DEPLOY_TEST_LOG"\n',
|
||||
}
|
||||
for name, content in mocks.items():
|
||||
target = binaries / name
|
||||
target.write_text(content)
|
||||
target.chmod(0o755)
|
||||
log = root / "systemctl.log"
|
||||
env = {**os.environ, "DEPLOY_PATH": str(deployment), "SERVICE": "solar-relay.service",
|
||||
"PATH": str(binaries) + os.pathsep + os.environ["PATH"], "SOLAR_DEPLOY_TEST_LOG": str(log),
|
||||
"HEALTH_URL": "https://rpioffice.nest.lutran.ch/solar-relay/healthz"}
|
||||
result = subprocess.run(["bash", "-c", script], cwd=checkout, env=env, capture_output=True, text=True, timeout=15)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertTrue((deployment / "solar_relay/new.py").exists())
|
||||
self.assertFalse((deployment / "obsolete.py").exists())
|
||||
for relative, content in protected.items():
|
||||
self.assertEqual((deployment / relative).read_text(), content, relative)
|
||||
self.assertIn("restart solar-relay.service", log.read_text())
|
||||
self.assertIn("is-active --quiet solar-relay.service", log.read_text())
|
||||
self.assertIn("pip -m pip install --only-binary=pyarrow -r", log.read_text())
|
||||
self.assertLess(log.read_text().index("pip -m pip"), log.read_text().index("restart solar-relay.service"))
|
||||
self.assertIn(env["HEALTH_URL"], log.read_text())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,122 @@
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
import pyarrow.parquet as pq
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from solar_relay.config import HistoryConfig, utc_text
|
||||
from solar_relay.history import CHANNELS, HistoryRecovery, history_sample
|
||||
from solar_relay.http import TransportError
|
||||
from solar_relay.storage import Store
|
||||
from tests.test_status import observed
|
||||
|
||||
SYSTEM = "73733052-6f10-47a5-9746-7f7e76ebcb8c"
|
||||
|
||||
|
||||
def entry(when):
|
||||
values = {"production": 100, "consumption": 120, "self_consumption": 80, "bought": 40, "sold": 20}
|
||||
return {"logDateTime": utc_text(when), "logDuration": 300,
|
||||
"channels": [{"channelName": channel, "channelType": "Energy", "unit": "Wh", "value": values[field]}
|
||||
for field, channel in CHANNELS.items()]}
|
||||
|
||||
|
||||
class FakeHistoryHttp:
|
||||
def __init__(self, entries):
|
||||
self.entries, self.requests = entries, []
|
||||
|
||||
def request(self, url, **kwargs):
|
||||
self.requests.append((url, kwargs))
|
||||
return {"pvSystemId": SYSTEM, "data": deepcopy(self.entries)}
|
||||
|
||||
|
||||
class HistoryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.store = Store(Path(self.directory.name) / "archive")
|
||||
self.when = datetime(2026, 10, 30, 10, tzinfo=timezone.utc)
|
||||
self.source = "test-house"
|
||||
self.http = FakeHistoryHttp([entry(self.when + timedelta(minutes=offset)) for offset in [0, 5, 10]])
|
||||
self.config = HistoryConfig(enabled=True, pv_system_id=SYSTEM)
|
||||
self.recovery = HistoryRecovery(self.config, self.source, self.store, http=self.http, credentials=("id-test", "value-test"))
|
||||
|
||||
def recover(self):
|
||||
return self.recovery.recover(utc_text(self.when), utc_text(self.when + timedelta(minutes=15)), now=self.when + timedelta(hours=4))
|
||||
|
||||
def test_wifi_gap_recovers_only_missing_energy_without_inventing_boiler(self):
|
||||
for offset in [0, 10]:
|
||||
self.store.add(observed(self.when + timedelta(minutes=offset)))
|
||||
self.assertEqual(self.recover(), 1)
|
||||
records = list(self.store.records())
|
||||
recovered = [s for s in records if "history_interval" in s]
|
||||
self.assertEqual(len(recovered), 1)
|
||||
self.assertEqual(recovered[0]["history_interval"]["energy_wh"]["production"], 100)
|
||||
self.assertIsNone(recovered[0]["readings"]["boiler_enabled"])
|
||||
self.assertNotIn("boiler", recovered[0]["source_timestamps"])
|
||||
self.assertIn("boiler", recovered[0]["errors"])
|
||||
url, kwargs = self.http.requests[0]
|
||||
self.assertEqual(urlsplit(url).hostname, "swqapi.solarweb.com")
|
||||
self.assertEqual(kwargs["extra_headers"], {"AccessKeyId": "id-test", "AccessKeyValue": "value-test"})
|
||||
self.assertNotIn("value-test", url)
|
||||
self.assertEqual(self.recover(), 0)
|
||||
self.assertEqual(len(self.http.requests), 1)
|
||||
|
||||
def test_cloud_energy_does_not_mark_boiler_or_meter_history_complete(self):
|
||||
self.assertEqual(self.recover(), 3)
|
||||
rows = self.store.coverage(self.source, "UTC", 1, self.when + timedelta(days=1))
|
||||
day = rows[1]
|
||||
self.assertEqual(day["energy_complete"], 3)
|
||||
self.assertEqual(day["boiler_complete"], 0)
|
||||
self.assertEqual(day["meter_complete"], 0)
|
||||
self.assertEqual(day["complete"], 0)
|
||||
row = pq.ParquetFile(next(self.store.path.glob("date=*/samples.parquet"))).read().to_pylist()[0]
|
||||
self.assertEqual(row["record_kind"], "history")
|
||||
self.assertEqual(row["interval_energy_wh"]["production"], 100)
|
||||
|
||||
def test_partial_cloud_records_remain_missing_for_retry(self):
|
||||
self.http.entries[1]["channels"][0]["value"] = None
|
||||
self.assertEqual(self.recover(), 2)
|
||||
self.assertEqual(self.recover(), 0)
|
||||
self.http.entries[1] = entry(self.when + timedelta(minutes=5))
|
||||
self.assertEqual(self.recover(), 1)
|
||||
|
||||
def test_recovered_identifiers_are_stable_for_duplicate_delivery(self):
|
||||
first = history_sample(entry(self.when), self.source, SYSTEM)
|
||||
second = history_sample(entry(self.when), self.source, SYSTEM)
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_non_five_minute_and_bad_units_are_rejected(self):
|
||||
for change in [{"logDuration": 599}, {"logDateTime": utc_text(self.when + timedelta(seconds=1))}]:
|
||||
value = entry(self.when)
|
||||
value.update(change)
|
||||
with self.assertRaises(ValueError):
|
||||
history_sample(value, self.source, SYSTEM)
|
||||
value = entry(self.when)
|
||||
value["channels"][0]["unit"] = "W"
|
||||
with self.assertRaises(ValueError):
|
||||
history_sample(value, self.source, SYSTEM)
|
||||
|
||||
def test_missing_private_credentials_fail_before_requests(self):
|
||||
with self.assertRaisesRegex(ValueError, "SOLARWEB_ACCESS_KEY"):
|
||||
HistoryRecovery(self.config, self.source, self.store, credentials=("", ""))
|
||||
|
||||
def test_requests_are_split_at_utc_midnight_even_across_dst(self):
|
||||
self.http.entries = []
|
||||
self.recovery.recover("2026-10-24T22:00:00Z", "2026-10-26T00:00:00Z", now=datetime(2026, 10, 27, tzinfo=timezone.utc))
|
||||
self.assertEqual(len(self.http.requests), 2)
|
||||
for url, _ in self.http.requests:
|
||||
query = parse_qs(urlsplit(url).query)
|
||||
start, end = [datetime.fromisoformat(query[name][0]) for name in ("from", "to")]
|
||||
self.assertLessEqual((end - start).total_seconds(), 86400)
|
||||
|
||||
def test_recent_intervals_wait_for_the_hourly_cloud_upload(self):
|
||||
self.http.entries = []
|
||||
self.recovery.recover(utc_text(self.when), utc_text(self.when + timedelta(hours=4)), now=self.when + timedelta(hours=4))
|
||||
query = parse_qs(urlsplit(self.http.requests[0][0]).query)
|
||||
self.assertEqual(query["to"], [utc_text(self.when + timedelta(hours=2))])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,107 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from solar_relay.cli import next_poll_at
|
||||
from solar_relay.storage import Store, StoreError
|
||||
from tests.test_status import observed
|
||||
|
||||
|
||||
class ParquetTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.path = Path(self.directory.name) / "archive"
|
||||
self.store = Store(self.path)
|
||||
self.now = datetime(2026, 11, 1, 12, tzinfo=timezone.utc)
|
||||
|
||||
def test_real_parquet_has_typed_readings_and_original_payload(self):
|
||||
value = observed(self.now)
|
||||
value["readings"]["boiler_enabled"] = False
|
||||
self.store.add(value)
|
||||
path = next(self.path.glob("date=*/samples.parquet"))
|
||||
self.assertEqual(path.read_bytes()[:4], b"PAR1")
|
||||
row = pq.ParquetFile(path).read().to_pylist()[0]
|
||||
self.assertEqual(row["production_ac_w"], 2000)
|
||||
self.assertIs(row["boiler_enabled"], False)
|
||||
self.assertIsNone(row["uploaded_at"])
|
||||
self.assertEqual(json.loads(row["payload"]), value)
|
||||
self.assertEqual(self.store.status()["format"], "parquet")
|
||||
|
||||
def test_retention_uses_ack_time_and_never_deletes_pending(self):
|
||||
items = [observed(self.now - timedelta(days=90, minutes=index)) for index in range(4)]
|
||||
for item in items:
|
||||
self.store.add(item)
|
||||
self.store.acknowledge([items[0]["id"]], now=self.now - timedelta(days=31))
|
||||
self.store.acknowledge([items[1]["id"]], now=self.now - timedelta(days=30))
|
||||
self.store.acknowledge([items[2]["id"]], now=self.now)
|
||||
self.assertEqual(self.store.prune(now=self.now), 1)
|
||||
self.assertEqual({s["id"] for s in self.store.records()}, {s["id"] for s in items[1:]})
|
||||
self.assertEqual(self.store.pending(100), [items[3]])
|
||||
self.assertEqual(self.store.prune(now=self.now + timedelta(days=31)), 2)
|
||||
self.assertEqual(list(self.store.records()), [items[3]])
|
||||
|
||||
def test_replay_cancels_expiry_until_the_next_acknowledgement(self):
|
||||
value = observed(self.now)
|
||||
self.store.add(value)
|
||||
self.store.acknowledge([value["id"]], now=self.now - timedelta(days=40))
|
||||
self.store.replay()
|
||||
self.assertEqual(self.store.prune(now=self.now), 0)
|
||||
self.store.acknowledge([value["id"]], now=self.now)
|
||||
self.assertEqual(self.store.prune(now=self.now), 0)
|
||||
|
||||
def test_interrupted_atomic_write_preserves_the_previous_partition(self):
|
||||
first, second = observed(self.now), observed(self.now + timedelta(minutes=5))
|
||||
self.store.add(first)
|
||||
with patch("solar_relay.storage.os.replace", side_effect=OSError("simulated write failure")):
|
||||
with self.assertRaises(OSError):
|
||||
self.store.add(second)
|
||||
self.assertEqual(list(Store(self.path).records()), [first])
|
||||
self.assertEqual(list(self.path.glob("date=*/.samples-*.tmp")), [])
|
||||
|
||||
def test_two_store_instances_cannot_lose_concurrent_records(self):
|
||||
other = Store(self.path)
|
||||
values = [observed(self.now + timedelta(minutes=index * 5)) for index in range(12)]
|
||||
def add(pair):
|
||||
index, value = pair
|
||||
(other if index % 2 else self.store).add(value)
|
||||
with ThreadPoolExecutor(max_workers=2) as workers:
|
||||
list(workers.map(add, enumerate(values)))
|
||||
self.assertEqual({value["id"] for value in self.store.records()}, {value["id"] for value in values})
|
||||
|
||||
def test_pruning_an_entire_day_removes_its_parquet_file(self):
|
||||
value = observed(self.now - timedelta(days=60))
|
||||
self.store.add(value)
|
||||
self.store.acknowledge([value["id"]], now=self.now - timedelta(days=31))
|
||||
self.assertEqual(self.store.prune(now=self.now), 1)
|
||||
self.assertEqual(list(self.path.glob("date=*/samples.parquet")), [])
|
||||
self.assertEqual(self.store.status()["samples"], 0)
|
||||
|
||||
def test_corrupt_archive_fails_without_silently_dropping_records(self):
|
||||
value = observed(self.now)
|
||||
self.store.add(value)
|
||||
next(self.path.glob("date=*/samples.parquet")).write_bytes(b"broken")
|
||||
with self.assertRaises(StoreError):
|
||||
self.store.pending(100)
|
||||
|
||||
def test_archive_path_requires_a_directory(self):
|
||||
occupied = self.path.parent / "existing-file"
|
||||
occupied.write_bytes(b"existing data")
|
||||
with self.assertRaises(StoreError):
|
||||
Store(occupied)
|
||||
self.assertEqual(occupied.read_bytes(), b"existing data")
|
||||
|
||||
def test_next_poll_is_aligned_to_the_five_minute_clock(self):
|
||||
self.assertEqual(next_poll_at(301, 300), 600)
|
||||
self.assertEqual(next_poll_at(599.5, 300), 600)
|
||||
self.assertEqual(next_poll_at(600, 300), 900)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,386 @@
|
||||
from contextlib import contextmanager
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
import subprocess
|
||||
import sys
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from solar_relay.config import InverterConfig, UploadConfig, date_bound, load_config
|
||||
from solar_relay.fronius import FroniusClient, powerflow_readings
|
||||
from solar_relay.http import JsonHttp, TransportError
|
||||
from solar_relay.storage import Store
|
||||
from solar_relay.upload import Uploader, UploadError
|
||||
|
||||
|
||||
def api(data):
|
||||
return {
|
||||
"Body": {"Data": data},
|
||||
"Head": {"Status": {"Code": 0}, "Timestamp": "2026-11-01T12:00:00+00:00"},
|
||||
}
|
||||
|
||||
|
||||
POWER = api({
|
||||
"Site": {"P_PV": 2100, "P_Load": -2500, "P_Grid": 500, "E_Total": 100000},
|
||||
"Inverters": {"1": {"P": 2000, "E_Total": 100000}},
|
||||
})
|
||||
METER = api({"0": {
|
||||
"Enable": 1, "Meter_Location_Current": 0,
|
||||
"EnergyReal_WAC_Sum_Consumed": 50000, "EnergyReal_WAC_Sum_Produced": 80000,
|
||||
}})
|
||||
BOILER = {"pin1": {"state": True, "reason": "PSTR_EMRS_REASON_3"}}
|
||||
RESPONSES = {
|
||||
"/solar_api/v1/GetPowerFlowRealtimeData.fcgi": POWER,
|
||||
"/solar_api/v1/GetMeterRealtimeData.cgi": METER,
|
||||
"/api/status/emrs": BOILER,
|
||||
}
|
||||
|
||||
|
||||
class FakeHttp:
|
||||
def __init__(self, responses=None):
|
||||
self.responses = deepcopy(RESPONSES if responses is None else responses)
|
||||
|
||||
def request(self, url):
|
||||
result = self.responses[urlsplit(url).path]
|
||||
if isinstance(result, Exception):
|
||||
raise result
|
||||
return deepcopy(result)
|
||||
|
||||
|
||||
def sample():
|
||||
return FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp()).collect("test-house")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def server():
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def respond(self, status, data):
|
||||
body = json.dumps(data).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_GET(self):
|
||||
path = urlsplit(self.path).path
|
||||
self.server.get_counts[path] = self.server.get_counts.get(path, 0) + 1
|
||||
failures = self.server.get_failures.get(path, [])
|
||||
if failures:
|
||||
self.respond(failures.pop(0), {"error": "temporary"})
|
||||
return
|
||||
if self.path == "/redirect":
|
||||
self.send_response(302)
|
||||
self.send_header("Location", "/unexpected")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
elif self.path == "/unexpected":
|
||||
self.server.redirect_followed = True
|
||||
self.respond(200, {})
|
||||
elif self.path == "/nan":
|
||||
self.respond(200, {"reading": float("nan")})
|
||||
else:
|
||||
path = urlsplit(self.path).path
|
||||
self.respond(200, RESPONSES[path])
|
||||
|
||||
def do_POST(self):
|
||||
self.server.authorizations.append(self.headers.get("Authorization"))
|
||||
data = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
self.server.requests.append(data)
|
||||
ids = [s["id"] for s in data["samples"]]
|
||||
# Emulate an idempotent receiver that committed before its response failed.
|
||||
self.server.received_ids.update(ids)
|
||||
if self.server.fail_once:
|
||||
self.server.fail_once = False
|
||||
self.respond(503, {"error": "temporary"})
|
||||
elif self.server.ack_override is not None:
|
||||
self.respond(200, self.server.ack_override)
|
||||
else:
|
||||
self.respond(200, {"accepted_ids": ids})
|
||||
|
||||
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
httpd.fail_once = False
|
||||
httpd.ack_override = None
|
||||
httpd.requests = []
|
||||
httpd.get_counts = {}
|
||||
httpd.get_failures = {}
|
||||
httpd.authorizations = []
|
||||
httpd.received_ids = set()
|
||||
httpd.redirect_followed = False
|
||||
thread = threading.Thread(target=httpd.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
yield httpd, f"http://127.0.0.1:{httpd.server_port}"
|
||||
finally:
|
||||
httpd.shutdown()
|
||||
httpd.server_close()
|
||||
thread.join(timeout=2)
|
||||
|
||||
|
||||
class NormalizationTests(unittest.TestCase):
|
||||
def test_dc_ac_and_meter_directions_stay_distinct(self):
|
||||
result = sample()
|
||||
readings = result["readings"]
|
||||
self.assertEqual(readings["production_w"], 2100)
|
||||
self.assertEqual(readings["production_ac_w"], 2000)
|
||||
self.assertEqual(readings["consumption_w"], 2500)
|
||||
self.assertEqual(readings["grid_import_w"], 500)
|
||||
self.assertEqual(readings["grid_export_w"], 0)
|
||||
self.assertEqual(readings["grid_import_total_wh"], 50000)
|
||||
self.assertTrue(readings["boiler_enabled"])
|
||||
self.assertEqual(result["errors"], {})
|
||||
self.assertEqual(result["raw"]["powerflow"], POWER)
|
||||
self.assertEqual(result["source_timestamps"]["powerflow"], "2026-11-01T12:00:00.000Z")
|
||||
|
||||
def test_export_direction_and_nighttime_nulls(self):
|
||||
response = deepcopy(POWER)
|
||||
response["Body"]["Data"]["Site"].update(P_Grid=-1200, P_PV=None)
|
||||
response["Body"]["Data"]["Inverters"]["1"]["P"] = None
|
||||
readings, _ = powerflow_readings(response)
|
||||
self.assertEqual(readings["grid_import_w"], 0)
|
||||
self.assertEqual(readings["grid_export_w"], 1200)
|
||||
self.assertIsNone(readings["production_w"])
|
||||
self.assertIsNone(readings["production_ac_w"])
|
||||
|
||||
def test_boiler_failure_is_unknown_and_does_not_lose_energy(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/api/status/emrs"] = TransportError("HTTP 404")
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertIsNone(result["readings"]["boiler_enabled"])
|
||||
self.assertEqual(result["readings"]["production_w"], 2100)
|
||||
self.assertEqual(result["errors"], {"boiler": "HTTP 404"})
|
||||
|
||||
def test_false_is_a_valid_boiler_state(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/api/status/emrs"]["pin1"]["state"] = False
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertIs(result["readings"]["boiler_enabled"], False)
|
||||
self.assertNotIn("boiler", result["errors"])
|
||||
|
||||
def test_solar_api_application_error_does_not_accept_old_values(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/solar_api/v1/GetPowerFlowRealtimeData.fcgi"]["Head"]["Status"]["Code"] = 12
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertIsNone(result["readings"]["production_w"])
|
||||
self.assertIn("powerflow", result["errors"])
|
||||
self.assertTrue(result["readings"]["boiler_enabled"])
|
||||
|
||||
def test_load_branch_meter_cannot_be_mislabeled_as_grid_energy(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/solar_api/v1/GetMeterRealtimeData.cgi"]["Body"]["Data"]["0"]["Meter_Location_Current"] = 1
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertIsNone(result["readings"]["grid_import_total_wh"])
|
||||
self.assertIn("meter", result["errors"])
|
||||
|
||||
def test_meter_measurement_timestamp_is_preserved_instead_of_request_time(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/solar_api/v1/GetMeterRealtimeData.cgi"]["Body"]["Data"]["0"]["TimeStamp"] = 1793534100
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertEqual(result["source_timestamps"]["meter"], "2026-11-01T11:55:00.000Z")
|
||||
|
||||
def test_missing_power_fields_are_a_source_error(self):
|
||||
responses = deepcopy(RESPONSES)
|
||||
responses["/solar_api/v1/GetPowerFlowRealtimeData.fcgi"]["Body"]["Data"]["Site"] = {}
|
||||
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||
self.assertIn("powerflow", result["errors"])
|
||||
self.assertIsNone(result["readings"]["production_w"])
|
||||
|
||||
|
||||
class ArchiveTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.path = Path(self.directory.name) / "archive"
|
||||
self.store = Store(self.path)
|
||||
|
||||
def test_restart_and_acknowledgement_preserve_archive(self):
|
||||
first = sample()
|
||||
self.store.add(first)
|
||||
Store(self.path).acknowledge([first["id"]])
|
||||
reopened = Store(self.path)
|
||||
self.assertEqual(reopened.pending(100), [])
|
||||
self.assertEqual(list(reopened.records()), [first])
|
||||
self.assertEqual(reopened.status()["samples"], 1)
|
||||
if os.name == "posix":
|
||||
self.assertEqual(self.path.stat().st_mode & 0o777, 0o700)
|
||||
self.assertEqual(next(self.path.glob("date=*/samples.parquet")).stat().st_mode & 0o777, 0o600)
|
||||
|
||||
def test_local_date_export_and_replay_across_dst(self):
|
||||
start = date_bound("2026-10-25", "Europe/Zurich")
|
||||
until = date_bound("2026-10-26", "Europe/Zurich")
|
||||
self.assertEqual((datetime.fromisoformat(until) - datetime.fromisoformat(start)).total_seconds(), 25 * 3600)
|
||||
items = []
|
||||
for timestamp in ["2026-10-24T21:59:59.000Z", start, "2026-10-25T12:00:00.000Z", until]:
|
||||
item = sample()
|
||||
item["observed_at"] = timestamp
|
||||
items.append(item)
|
||||
self.store.add(item)
|
||||
self.store.acknowledge([item["id"] for item in items])
|
||||
self.assertEqual(list(self.store.records(start, until)), items[1:3])
|
||||
self.assertEqual(self.store.replay(start, until), 2)
|
||||
self.assertEqual(self.store.pending(100), items[1:3])
|
||||
self.assertEqual(self.store.status()["samples"], 4)
|
||||
|
||||
|
||||
class DeliveryTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.store = Store(Path(self.directory.name) / "archive")
|
||||
|
||||
def test_real_http_collection_delivery_and_lost_ack_retry(self):
|
||||
with server() as (httpd, base):
|
||||
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
|
||||
reading = client.collect("test")
|
||||
self.assertEqual(reading["errors"], {})
|
||||
self.store.add(reading)
|
||||
uploader = Uploader(UploadConfig(url=base + "/ingest"), self.store, token="test-token")
|
||||
httpd.fail_once = True
|
||||
with self.assertRaises(UploadError):
|
||||
uploader.upload_once()
|
||||
self.assertEqual(self.store.status()["pending"], 1)
|
||||
self.assertEqual(uploader.upload_once(), 1)
|
||||
self.assertEqual(self.store.status()["pending"], 0)
|
||||
self.assertEqual(self.store.status()["samples"], 1)
|
||||
self.assertEqual(httpd.requests[0], httpd.requests[1])
|
||||
self.assertEqual(httpd.received_ids, {reading["id"]})
|
||||
self.assertEqual(httpd.authorizations, ["Bearer test-token", "Bearer test-token"])
|
||||
|
||||
def test_plain_http_success_is_not_an_acknowledgement(self):
|
||||
with server() as (httpd, base):
|
||||
self.store.add(sample())
|
||||
uploader = Uploader(UploadConfig(url=base), self.store, token="test-token")
|
||||
httpd.ack_override = {"success": True}
|
||||
with self.assertRaises(UploadError):
|
||||
uploader.upload_once()
|
||||
self.assertEqual(self.store.status()["pending"], 1)
|
||||
|
||||
def test_partial_ack_only_marks_the_accepted_sample(self):
|
||||
with server() as (httpd, base):
|
||||
first, second = sample(), sample()
|
||||
self.store.add(first)
|
||||
self.store.add(second)
|
||||
httpd.ack_override = {"accepted_ids": [first["id"]]}
|
||||
with self.assertRaises(UploadError):
|
||||
Uploader(UploadConfig(url=base), self.store, token="test-token").upload_once()
|
||||
self.assertEqual(self.store.pending(100), [second])
|
||||
self.assertEqual(self.store.status()["samples"], 2)
|
||||
|
||||
def test_unknown_ack_cannot_clear_the_queue(self):
|
||||
with server() as (httpd, base):
|
||||
self.store.add(sample())
|
||||
httpd.ack_override = {"accepted_ids": ["unknown-id"]}
|
||||
with self.assertRaises(UploadError):
|
||||
Uploader(UploadConfig(url=base), self.store, token="test-token").upload_once()
|
||||
self.assertEqual(self.store.status()["pending"], 1)
|
||||
|
||||
def test_slow_upload_does_not_lock_recording(self):
|
||||
entered, release = threading.Event(), threading.Event()
|
||||
|
||||
class BlockingHttp:
|
||||
def request(self, url, *, payload, token):
|
||||
entered.set()
|
||||
release.wait(timeout=3)
|
||||
return {"accepted_ids": [s["id"] for s in payload["samples"]]}
|
||||
|
||||
self.store.add(sample())
|
||||
uploader = Uploader(UploadConfig(url="https://example.test/ingest"), self.store, token="test-token", http=BlockingHttp())
|
||||
result = []
|
||||
worker = threading.Thread(target=lambda: result.append(uploader.upload_once()))
|
||||
worker.start()
|
||||
try:
|
||||
self.assertTrue(entered.wait(timeout=2))
|
||||
second = sample()
|
||||
self.store.add(second)
|
||||
self.assertEqual(self.store.status()["samples"], 2)
|
||||
finally:
|
||||
release.set()
|
||||
worker.join(timeout=4)
|
||||
self.assertEqual(result, [1])
|
||||
self.assertEqual(self.store.pending(100), [second])
|
||||
|
||||
def test_redirects_and_invalid_json_numbers_are_rejected(self):
|
||||
with server() as (httpd, base):
|
||||
http = JsonHttp(timeout=2)
|
||||
with self.assertRaises(TransportError):
|
||||
http.request(base + "/redirect", token="test-token")
|
||||
self.assertFalse(httpd.redirect_followed)
|
||||
with self.assertRaises(TransportError):
|
||||
http.request(base + "/nan")
|
||||
|
||||
def test_missing_upload_token_fails_before_delivery(self):
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
with self.assertRaisesRegex(ValueError, "SOLAR_RELAY_TOKEN"):
|
||||
Uploader(UploadConfig(url="https://example.test/ingest"), self.store)
|
||||
|
||||
def test_cli_sample_and_export_use_the_persistent_archive(self):
|
||||
with server() as (_, base):
|
||||
config = Path(self.directory.name) / "config.toml"
|
||||
config.write_text(
|
||||
'[collector]\nsource_id="cli-test"\narchive="cli-archive"\n'
|
||||
f'\n[inverter]\nbase_url="{base}"\n'
|
||||
)
|
||||
command = [sys.executable, "-m", "solar_relay", "--config", str(config)]
|
||||
cwd = Path(__file__).resolve().parents[1]
|
||||
result = subprocess.run(command + ["sample"], cwd=cwd, capture_output=True, text=True, timeout=20)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
normalized = json.loads(result.stdout)
|
||||
result = subprocess.run(command + ["export"], cwd=cwd, capture_output=True, text=True, timeout=5)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
archived = json.loads(result.stdout)
|
||||
self.assertEqual(normalized["id"], archived["id"])
|
||||
self.assertEqual(archived["raw"]["powerflow"], POWER)
|
||||
|
||||
|
||||
class ConfigTests(unittest.TestCase):
|
||||
def load(self, extra=""):
|
||||
with tempfile.TemporaryDirectory() as name:
|
||||
path = Path(name) / "config.toml"
|
||||
path.write_text('[collector]\nsource_id="home"\narchive="state/archive"\n' + extra)
|
||||
return load_config(path), path.parent
|
||||
|
||||
def test_archive_is_relative_to_config(self):
|
||||
config, directory = self.load()
|
||||
self.assertEqual(config.collector.archive, directory / "state/archive")
|
||||
self.assertEqual(config.collector.poll_interval_seconds, 300)
|
||||
self.assertEqual(config.collector.retention_days, 30)
|
||||
self.assertEqual(config.upload.url, "")
|
||||
|
||||
def test_remote_upload_requires_https(self):
|
||||
with self.assertRaisesRegex(ValueError, "HTTPS"):
|
||||
self.load('\n[upload]\nurl="http://example.test/ingest"\n')
|
||||
config, _ = self.load('\n[upload]\nurl="http://127.0.0.1:8000/ingest"\n')
|
||||
self.assertTrue(config.upload.url)
|
||||
|
||||
def test_mistyped_configuration_is_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError, "Unknown inverter"):
|
||||
self.load('\n[inverter]\nboiler_pni="pin2"\n')
|
||||
with self.assertRaises(ValueError):
|
||||
self.load('\n[inverter]\nboiler_pin="pin9"\n')
|
||||
|
||||
def test_status_listener_defaults_and_loopback_binding(self):
|
||||
config, _ = self.load()
|
||||
self.assertEqual(config.http.port, 8765)
|
||||
self.assertEqual(config.http.bind, "127.0.0.1")
|
||||
with self.assertRaisesRegex(ValueError, "127.0.0.1"):
|
||||
self.load('\n[http]\nbind="0.0.0.0"\n')
|
||||
|
||||
def test_status_settings_can_disable_http_and_reject_invalid_paths(self):
|
||||
config, _ = self.load('\n[http]\nenabled=false\n')
|
||||
self.assertFalse(config.http.enabled)
|
||||
with self.assertRaises(ValueError):
|
||||
self.load('\n[http]\nbase_path="/../config"\n')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,114 @@
|
||||
from copy import deepcopy
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from solar_relay.cli import _run
|
||||
from solar_relay.config import CollectorConfig, Config, HttpConfig, InverterConfig, UploadConfig, utc_text
|
||||
from solar_relay.fronius import FroniusClient
|
||||
from solar_relay.http import TransportError
|
||||
from solar_relay.storage import Store
|
||||
from tests.test_relay import RESPONSES, server
|
||||
from tests.test_schedule import Clock
|
||||
|
||||
POWER_PATH = "/solar_api/v1/GetPowerFlowRealtimeData.fcgi"
|
||||
METER_PATH = "/solar_api/v1/GetMeterRealtimeData.cgi"
|
||||
BOILER_PATH = "/api/status/emrs"
|
||||
|
||||
|
||||
class ScriptedHttp:
|
||||
def __init__(self, clock):
|
||||
self.clock = clock
|
||||
self.requests = []
|
||||
self.failures = {POWER_PATH: [503, 503, 503]}
|
||||
|
||||
def request(self, url):
|
||||
path = urlsplit(url).path
|
||||
self.requests.append((self.clock.now, path))
|
||||
failures = self.failures.get(path, [])
|
||||
if failures:
|
||||
raise TransportError(f"HTTP {failures.pop(0)}")
|
||||
response = deepcopy(RESPONSES[path])
|
||||
if "Head" in response:
|
||||
response["Head"]["Timestamp"] = utc_text(datetime.fromtimestamp(self.clock.now, timezone.utc))
|
||||
return response
|
||||
|
||||
|
||||
class RetryTests(unittest.TestCase):
|
||||
def test_http_errors_retry_only_the_failed_endpoint_and_stop_on_success(self):
|
||||
for retries in (1, 3):
|
||||
with self.subTest(retries=retries), server() as (httpd, base):
|
||||
httpd.get_failures[POWER_PATH] = [503] * retries
|
||||
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
|
||||
with self.assertLogs("solar_relay.fronius", level="WARNING") as messages:
|
||||
sample = client.collect("test-house")
|
||||
self.assertEqual(sample["errors"], {})
|
||||
self.assertEqual(sample["readings"]["production_ac_w"], 2000)
|
||||
self.assertEqual(httpd.get_counts, {POWER_PATH: retries + 1, METER_PATH: 1, BOILER_PATH: 1})
|
||||
self.assertEqual(len(messages.output), retries)
|
||||
|
||||
def test_giving_up_after_three_retries_preserves_other_sources(self):
|
||||
for failures in ([500, 502, 503, 504, 503], [404] * 5):
|
||||
with self.subTest(statuses=failures), server() as (httpd, base):
|
||||
httpd.get_failures[BOILER_PATH] = list(failures)
|
||||
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
|
||||
with self.assertLogs("solar_relay.fronius", level="WARNING"):
|
||||
sample = client.collect("test-house")
|
||||
self.assertEqual(httpd.get_counts, {POWER_PATH: 1, METER_PATH: 1, BOILER_PATH: 4})
|
||||
self.assertEqual(len(httpd.get_failures[BOILER_PATH]), 1)
|
||||
self.assertEqual(sample["errors"], {"boiler": f"HTTP {failures[3]}"})
|
||||
self.assertIsNone(sample["readings"]["boiler_enabled"])
|
||||
self.assertEqual(sample["readings"]["production_ac_w"], 2000)
|
||||
self.assertEqual(sample["readings"]["grid_import_total_wh"], 50000)
|
||||
|
||||
def test_retry_attempts_keep_the_four_second_request_spacing(self):
|
||||
clock = Clock(0)
|
||||
http = ScriptedHttp(clock)
|
||||
client = FroniusClient(InverterConfig(), http=http)
|
||||
with patch("solar_relay.fronius.time.monotonic", side_effect=lambda: clock.now), \
|
||||
patch("solar_relay.fronius.time.sleep", side_effect=clock.wait), \
|
||||
self.assertLogs("solar_relay.fronius", level="WARNING"):
|
||||
result = client.collect("test-house")
|
||||
self.assertEqual(result["errors"], {})
|
||||
self.assertEqual([stamp for stamp, _ in http.requests], [0, 4, 8, 12, 16, 20])
|
||||
|
||||
def test_retries_preserve_clock_boundaries_and_one_saved_observation_per_poll(self):
|
||||
origin = datetime(2026, 11, 1, 12, tzinfo=timezone.utc).timestamp()
|
||||
clock = Clock(origin + 180) # Start at 12:03; first observation must be at 12:05.
|
||||
http = ScriptedHttp(clock)
|
||||
client = FroniusClient(InverterConfig(), http=http)
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
store = Store(Path(directory) / "archive")
|
||||
config = Config(CollectorConfig("test-house", store.path), InverterConfig(), UploadConfig(), HttpConfig(enabled=False))
|
||||
added = []
|
||||
original_add = store.add
|
||||
|
||||
def add(sample):
|
||||
original_add(sample)
|
||||
added.append(sample)
|
||||
if len(added) == 3:
|
||||
clock.set()
|
||||
|
||||
with patch("solar_relay.cli.threading.Event", return_value=clock), \
|
||||
patch("solar_relay.cli.time.time", side_effect=lambda: clock.now), \
|
||||
patch("solar_relay.cli.time.monotonic", side_effect=lambda: clock.now), \
|
||||
patch("solar_relay.fronius.time.sleep", side_effect=clock.wait), \
|
||||
patch("solar_relay.fronius.datetime", wraps=datetime) as dates, \
|
||||
patch("solar_relay.cli.signal.signal"), \
|
||||
patch("solar_relay.cli.FroniusClient", return_value=client), \
|
||||
patch.object(store, "add", side_effect=add), \
|
||||
self.assertLogs("solar_relay.fronius", level="WARNING"):
|
||||
dates.now.side_effect = lambda zone: datetime.fromtimestamp(clock.now, zone)
|
||||
self.assertEqual(_run(config, store), 0)
|
||||
records = list(store.records())
|
||||
self.assertEqual(len(records), 3)
|
||||
self.assertEqual([datetime.fromisoformat(s["observed_at"]).timestamp() for s in records],
|
||||
[origin + 300, origin + 600, origin + 900])
|
||||
self.assertTrue(all(not record["errors"] for record in records))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,91 @@
|
||||
"""Exercise the real recording loop with an advancing clock instead of real waits."""
|
||||
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from solar_relay.cli import _run
|
||||
from solar_relay.config import CollectorConfig, Config, HttpConfig, InverterConfig, UploadConfig, load_config
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self, now):
|
||||
self.now = now
|
||||
self.stopped = False
|
||||
self.waits = []
|
||||
|
||||
def is_set(self):
|
||||
return self.stopped
|
||||
|
||||
def set(self):
|
||||
self.stopped = True
|
||||
|
||||
def wait(self, delay):
|
||||
self.waits.append((self.now, delay))
|
||||
if not self.stopped:
|
||||
self.now += delay
|
||||
return self.stopped
|
||||
|
||||
|
||||
class ScheduleTests(unittest.TestCase):
|
||||
def run_collector(self, start, count=3, duration=8, cancel=False):
|
||||
clock = Clock(start)
|
||||
recorded = []
|
||||
config = Config(CollectorConfig("test-house", Path("unused")), InverterConfig(), UploadConfig(), HttpConfig(enabled=False))
|
||||
store = Mock()
|
||||
store.prune.return_value = 0
|
||||
|
||||
def record(*args):
|
||||
recorded.append(clock.now)
|
||||
clock.now += duration
|
||||
if len(recorded) == count:
|
||||
clock.set()
|
||||
|
||||
def cancel_wait(delay):
|
||||
clock.set()
|
||||
return True
|
||||
|
||||
if cancel:
|
||||
clock.wait = cancel_wait
|
||||
with patch("solar_relay.cli.threading.Event", return_value=clock), \
|
||||
patch("solar_relay.cli.time.time", side_effect=lambda: clock.now), \
|
||||
patch("solar_relay.cli.time.monotonic", side_effect=lambda: clock.now), \
|
||||
patch("solar_relay.cli.signal.signal"), \
|
||||
patch("solar_relay.cli.FroniusClient"), \
|
||||
patch("solar_relay.cli._record", side_effect=record):
|
||||
self.assertEqual(_run(config, store), 0)
|
||||
return recorded, clock
|
||||
|
||||
def test_startup_waits_then_records_once_on_each_boundary(self):
|
||||
recorded, clock = self.run_collector(301)
|
||||
self.assertEqual(recorded, [600, 900, 1200])
|
||||
self.assertEqual(clock.waits[0], (301, 299))
|
||||
self.assertTrue(all(stamp % 300 == 0 for stamp in recorded))
|
||||
|
||||
def test_restart_in_the_same_interval_adds_no_extra_observation(self):
|
||||
before, _ = self.run_collector(590, count=1)
|
||||
after, _ = self.run_collector(615, count=2)
|
||||
self.assertEqual(before + after, [600, 900, 1200])
|
||||
|
||||
def test_slow_collection_skips_missed_boundaries_without_catching_up(self):
|
||||
recorded, _ = self.run_collector(301, count=2, duration=370)
|
||||
self.assertEqual(recorded, [600, 1200])
|
||||
|
||||
def test_shutdown_while_waiting_does_not_take_a_final_observation(self):
|
||||
recorded, _ = self.run_collector(301, cancel=True)
|
||||
self.assertEqual(recorded, [])
|
||||
|
||||
def test_configuration_requires_exactly_five_minutes(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / "config.toml"
|
||||
for interval in (30, 60, 299, 301, 600):
|
||||
path.write_text(f"[collector]\npoll_interval_seconds = {interval}\n")
|
||||
with self.assertRaisesRegex(ValueError, "must be 300"):
|
||||
load_config(path)
|
||||
path.write_text("[collector]\npoll_interval_seconds = 300\n")
|
||||
self.assertEqual(load_config(path).collector.poll_interval_seconds, 300)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,196 @@
|
||||
from dataclasses import replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import unittest
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import ProxyHandler, Request, build_opener
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from solar_relay.config import Config, CollectorConfig, HttpConfig, InverterConfig, UploadConfig, utc_text
|
||||
from solar_relay.coverage import daily_coverage, sample_coverage
|
||||
from solar_relay.http import NoRedirects
|
||||
from solar_relay.status import StatusServer
|
||||
from solar_relay.storage import Store
|
||||
from tests.test_relay import sample, server
|
||||
|
||||
|
||||
def observed(when):
|
||||
value = sample()
|
||||
value["observed_at"] = utc_text(when)
|
||||
value["source_timestamps"] = {source: value["observed_at"] for source in ("powerflow", "meter", "boiler")}
|
||||
return value
|
||||
|
||||
|
||||
def rows_for_day(day):
|
||||
start = datetime.fromisoformat(day).replace(tzinfo=ZoneInfo("Europe/Zurich"))
|
||||
end = start + timedelta(days=1)
|
||||
first, last = round(start.timestamp() * 1000), round(end.timestamp() * 1000)
|
||||
return [(stamp, stamp, 1, stamp, 1, stamp, 1) for stamp in range(first, last + 1, 300_000)], end
|
||||
|
||||
|
||||
class CoverageTests(unittest.TestCase):
|
||||
def test_full_day_includes_all_288_intervals(self):
|
||||
rows, end = rows_for_day("2026-10-30")
|
||||
result = daily_coverage(rows, "Europe/Zurich", 1, end + timedelta(minutes=3))
|
||||
yesterday = result[1]
|
||||
self.assertEqual(yesterday["expected"], 288)
|
||||
self.assertEqual(yesterday["complete"], 288)
|
||||
self.assertEqual(yesterday["status"], "complete")
|
||||
self.assertEqual(result[0]["elapsed"], 0)
|
||||
self.assertTrue(all(mask == -1 for mask in result[0]["slots"]))
|
||||
|
||||
def test_dst_days_expect_276_and_300_intervals(self):
|
||||
for date, expected in [("2026-03-29", 276), ("2026-10-25", 300)]:
|
||||
with self.subTest(date=date):
|
||||
rows, end = rows_for_day(date)
|
||||
yesterday = daily_coverage(rows, "Europe/Zurich", 1, end + timedelta(minutes=3))[1]
|
||||
self.assertEqual(yesterday["expected"], expected)
|
||||
self.assertEqual(yesterday["complete"], expected)
|
||||
|
||||
def test_missing_five_minute_extractions_remain_missing(self):
|
||||
rows, end = rows_for_day("2026-10-30")
|
||||
del rows[10:13]
|
||||
yesterday = daily_coverage(rows, "Europe/Zurich", 1, end + timedelta(minutes=3))[1]
|
||||
self.assertEqual(yesterday["complete"], 285)
|
||||
self.assertEqual(yesterday["missing"], 3)
|
||||
self.assertEqual(yesterday["status"], "partial")
|
||||
|
||||
def test_energy_failure_does_not_erase_meter_or_boiler_coverage(self):
|
||||
rows, end = rows_for_day("2026-10-30")
|
||||
stamp = rows[1][0]
|
||||
rows[1] = (stamp, stamp, 0, stamp, 1, stamp, 1)
|
||||
yesterday = daily_coverage(rows, "Europe/Zurich", 1, end + timedelta(minutes=3))[1]
|
||||
self.assertEqual(yesterday["slots"][1], 6)
|
||||
self.assertEqual(yesterday["energy_complete"], 287)
|
||||
self.assertEqual(yesterday["meter_complete"], 288)
|
||||
self.assertEqual(yesterday["boiler_complete"], 288)
|
||||
self.assertEqual(yesterday["missing"], 1)
|
||||
|
||||
def test_source_errors_stale_readings_and_false_boiler_state(self):
|
||||
when = datetime(2026, 10, 30, tzinfo=timezone.utc)
|
||||
value = observed(when)
|
||||
value["readings"]["boiler_enabled"] = False
|
||||
self.assertEqual(sample_coverage(value)[-1], 1)
|
||||
value["errors"] = {"boiler": "unavailable"}
|
||||
self.assertEqual(sample_coverage(value)[-1], 0)
|
||||
value["source_timestamps"]["powerflow"] = utc_text(when - timedelta(seconds=120))
|
||||
self.assertEqual(sample_coverage(value)[4], 0)
|
||||
|
||||
def test_parquet_coverage_uses_the_selected_source_and_stored_extractions(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
archive = Store(Path(directory) / "archive")
|
||||
start = datetime(2026, 10, 30, tzinfo=ZoneInfo("Europe/Zurich"))
|
||||
for seconds in range(0, 301, 30):
|
||||
value = observed(start + timedelta(seconds=seconds))
|
||||
archive.add(value)
|
||||
result = archive.coverage(value["source_id"], "Europe/Zurich", 1, start + timedelta(days=1, minutes=3))
|
||||
self.assertEqual(result[1]["complete"], 2)
|
||||
other = archive.coverage("another-source", "Europe/Zurich", 1, start + timedelta(days=1, minutes=3))
|
||||
self.assertEqual(other[1]["complete"], 0)
|
||||
|
||||
|
||||
class StatusHttpTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.directory.cleanup)
|
||||
self.store = Store(Path(self.directory.name) / "archive")
|
||||
self.config = Config(CollectorConfig("test-house", self.store.path), InverterConfig(), UploadConfig(), HttpConfig(port=0))
|
||||
self.status = StatusServer(self.config, self.store)
|
||||
self.status.start()
|
||||
self.addCleanup(self.status.close)
|
||||
self.base = "http://127.0.0.1:" + str(self.status.address[1])
|
||||
self.http = build_opener(ProxyHandler({}), NoRedirects())
|
||||
|
||||
def get(self, path, *, method="GET"):
|
||||
try:
|
||||
result = self.http.open(Request(self.base + path, method=method), timeout=3)
|
||||
except HTTPError as error:
|
||||
result = error
|
||||
with result:
|
||||
return result.code, result.headers, result.read()
|
||||
|
||||
def test_page_assets_prefix_and_health_routes(self):
|
||||
for path in ["/", "/solar-relay/", "/assets/status.css", "/solar-relay/assets/status.js"]:
|
||||
status, _, body = self.get(path)
|
||||
self.assertEqual(status, 200, path)
|
||||
self.assertTrue(body)
|
||||
status, headers, body = self.get("/solar-relay")
|
||||
self.assertEqual(status, 308)
|
||||
self.assertEqual(headers["Location"], "/solar-relay/")
|
||||
self.assertEqual(json.loads(self.get("/solar-relay/healthz")[2])["status"], "ok")
|
||||
|
||||
def test_status_reports_extraction_and_backlog_without_raw_data_or_secrets(self):
|
||||
value = observed(datetime.now(timezone.utc))
|
||||
value["readings"]["boiler_enabled"] = False
|
||||
self.store.add(value)
|
||||
status, headers, body = self.get("/solar-relay/api/status?days=7")
|
||||
result = json.loads(body)
|
||||
self.assertEqual(status, 200)
|
||||
self.assertEqual(headers["Cache-Control"], "no-store")
|
||||
self.assertEqual(result["status"], "ok")
|
||||
self.assertEqual(result["collector"]["last_attempt"], value["observed_at"])
|
||||
self.assertEqual(result["collector"]["last_success"], value["observed_at"])
|
||||
self.assertEqual(result["archive"]["pending"], 1)
|
||||
self.assertEqual(len(result["days"]), 8)
|
||||
for forbidden in ["production_w", "raw", "database", self.directory.name]:
|
||||
self.assertNotIn(forbidden, body.decode())
|
||||
|
||||
def test_source_errors_do_not_expose_exception_messages(self):
|
||||
value = observed(datetime.now(timezone.utc))
|
||||
value["errors"] = {"boiler": "secret canary must not appear in HTTP"}
|
||||
self.store.add(value)
|
||||
result = json.loads(self.get("/api/status")[2])
|
||||
self.assertEqual(result["status"], "partial")
|
||||
self.assertEqual(result["collector"]["source_errors"], ["boiler"])
|
||||
self.assertNotIn("secret canary", json.dumps(result))
|
||||
|
||||
def test_only_explicit_read_only_routes_are_available(self):
|
||||
for path in ["/config.toml", "/.env", "/data/archive/date=2026-11-01/samples.parquet", "/../config.toml", "/solar-relay/.git/config"]:
|
||||
self.assertEqual(self.get(path)[0], 404, path)
|
||||
self.assertEqual(self.get("/api/status?days=91")[0], 400)
|
||||
self.assertEqual(self.get("/api/status?days=1&days=2")[0], 400)
|
||||
self.assertEqual(self.get("/api/status", method="POST")[0], 405)
|
||||
status, headers, body = self.get("/solar-relay/", method="HEAD")
|
||||
self.assertEqual(status, 200)
|
||||
self.assertGreater(int(headers["Content-Length"]), 0)
|
||||
self.assertEqual(body, b"")
|
||||
|
||||
def test_run_command_starts_and_stops_the_http_listener(self):
|
||||
with server() as (_, inverter):
|
||||
with socket.socket() as free:
|
||||
free.bind(("127.0.0.1", 0))
|
||||
port = free.getsockname()[1]
|
||||
path = Path(self.directory.name) / "config.toml"
|
||||
path.write_text(f'[collector]\nsource_id="run-test"\narchive="run-archive"\n[inverter]\nbase_url="{inverter}"\n[http]\nport={port}\n')
|
||||
# Keep the first scheduled poll in the future regardless of the real
|
||||
# clock. Scheduling itself is exercised with a simulated clock separately.
|
||||
entry = 'import sys; from solar_relay import cli; cli.next_poll_at = lambda now, interval: now + 300; sys.exit(cli.main())'
|
||||
process = subprocess.Popen([sys.executable, "-c", entry, "--config", str(path), "run"], cwd=Path(__file__).resolve().parents[1], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
try:
|
||||
for attempt in range(30):
|
||||
try:
|
||||
with self.http.open(f"http://127.0.0.1:{port}/healthz", timeout=1) as response:
|
||||
self.assertEqual(response.code, 200)
|
||||
break
|
||||
except URLError:
|
||||
time.sleep(0.1)
|
||||
else:
|
||||
self.fail("Status listener did not start")
|
||||
with self.http.open(f"http://127.0.0.1:{port}/api/status", timeout=1) as response:
|
||||
snapshot = json.load(response)
|
||||
self.assertEqual(snapshot["status"], "waiting")
|
||||
self.assertEqual(snapshot["archive"]["samples"], 0)
|
||||
process.terminate()
|
||||
_, stderr = process.communicate(timeout=5)
|
||||
self.assertEqual(process.returncode, 0, stderr)
|
||||
self.assertEqual(Store(path.parent / "run-archive").status()["samples"], 0)
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.communicate(timeout=5)
|
||||
Reference in new issue
Block a user