This commit is contained in:
1 parent
9e0230731b
commit
e7905c667d
10 files changed
+224
-27
No files matched your search
@@ -52,6 +52,7 @@ Uploads default to disabled until `[upload].url` is configured.
|
|||||||
| Solar API: PV, AC output, household load, grid power | `/solar_api/v1/GetPowerFlowRealtimeData.fcgi` |
|
| Solar API: PV, AC output, household load, grid power | `/solar_api/v1/GetPowerFlowRealtimeData.fcgi` |
|
||||||
| Solar API: grid meter counters | `/solar_api/v1/GetMeterRealtimeData.cgi?Scope=System` |
|
| Solar API: grid meter counters | `/solar_api/v1/GetMeterRealtimeData.cgi?Scope=System` |
|
||||||
| Internal GEN24 API: boiler switch | `/api/status/emrs` |
|
| Internal GEN24 API: boiler switch | `/api/status/emrs` |
|
||||||
|
| Internal GEN24 API: boiler switch-on threshold | `/api/config/emrs` |
|
||||||
|
|
||||||
Solar API must be enabled in the inverter's Communication settings. The collector
|
Solar API must be enabled in the inverter's Communication settings. The collector
|
||||||
only reads these endpoints. The boiler endpoint is an internal firmware interface;
|
only reads these endpoints. The boiler endpoint is an internal firmware interface;
|
||||||
@@ -67,6 +68,7 @@ if it stops working, boiler state is recorded as unknown rather than off.
|
|||||||
| `production_ac_total_wh` | Lifetime AC production counter |
|
| `production_ac_total_wh` | Lifetime AC production counter |
|
||||||
| `grid_import_total_wh`, `grid_export_total_wh` | Grid meter lifetime import/export counters |
|
| `grid_import_total_wh`, `grid_export_total_wh` | Grid meter lifetime import/export counters |
|
||||||
| `boiler_enabled`, `boiler_reason` | Output state (`true`, `false`, or `null`) and reason code |
|
| `boiler_enabled`, `boiler_reason` | Output state (`true`, `false`, or `null`) and reason code |
|
||||||
|
| `boiler_threshold_w`, `boiler_threshold_source` | Switch-on threshold in watts and `api` or `config` provenance |
|
||||||
|
|
||||||
The meter must be installed at the grid feed-in point. Missing/stale readings and
|
The meter must be installed at the grid feed-in point. Missing/stale readings and
|
||||||
source errors never become zero energy or an off boiler. DC panel power differs
|
source errors never become zero energy or an off boiler. DC panel power differs
|
||||||
@@ -75,7 +77,19 @@ from AC output by conversion losses. The website uses AC output for its energy b
|
|||||||
The boiler switch enables heating but does not measure electricity use: the thermostat
|
The boiler switch enables heating but does not measure electricity use: the thermostat
|
||||||
can stop heating while the output remains enabled. Five-minute polling can miss short
|
can stop heating while the output remains enabled. Five-minute polling can miss short
|
||||||
transitions; enabled durations are estimates. A branch meter is needed for boiler kWh.
|
transitions; enabled durations are estimates. A branch meter is needed for boiler kWh.
|
||||||
The 2000 W threshold is power (2 kW), not energy (kWh).
|
The threshold is power (W), not energy (Wh).
|
||||||
|
|
||||||
|
The collector tries `/api/config/emrs` once per observation and reads
|
||||||
|
`rules.<boiler_pin>.thresholdOn`, matching the GEN24's own web interface. Negative
|
||||||
|
`pgrid` thresholds denote exported surplus; their absolute magnitude is used.
|
||||||
|
This endpoint can require authentication even while the switch state is public.
|
||||||
|
If it is protected, unavailable or has no valid threshold, the collector uses
|
||||||
|
`[inverter].boiler_threshold_w` (default `2000`) from its private configuration.
|
||||||
|
A settings failure does not invalidate a successfully read switch state. The
|
||||||
|
threshold and its source are saved with each new observation and sent to Solar.
|
||||||
|
Requests still respect the minimum four-second spacing. Existing archives remain
|
||||||
|
readable; legacy switch-only samples receive the configured fallback when uploaded,
|
||||||
|
without rewriting their original payload or replacing a threshold already recorded.
|
||||||
|
|
||||||
## Parquet archive, delivery and retention
|
## Parquet archive, delivery and retention
|
||||||
|
|
||||||
@@ -107,8 +121,9 @@ totals are calculated by the receiving application.
|
|||||||
These common power readings match the sibling `solar` application's existing
|
These common power readings match the sibling `solar` application's existing
|
||||||
model: its `stats` table stores timestamped watts for Production and Consumption,
|
model: its `stats` table stores timestamped watts for Production and Consumption,
|
||||||
then applies `5/60` only when calculating Wh totals. `production_ac_w` maps to
|
then applies `5/60` only when calculating Wh totals. `production_ac_w` maps to
|
||||||
Production and `consumption_w` maps to Consumption. The solar-side relay importer
|
Production and `consumption_w` maps to Consumption. Solar currently imports only
|
||||||
is separate work; this relay change prepares both record sources for ingestion.
|
the boiler switch from these uploads; it continues to obtain production and
|
||||||
|
consumption directly from Solar.web.
|
||||||
|
|
||||||
The upload request is a JSON object with `schema_version: 1` and a `samples` array,
|
The upload request is a JSON object with `schema_version: 1` and a `samples` array,
|
||||||
authenticated with `upload.token` from private `config.toml`, or the environment
|
authenticated with `upload.token` from private `config.toml`, or the environment
|
||||||
@@ -116,7 +131,7 @@ variable named by `upload.token_env` when that setting is empty. The website mus
|
|||||||
commit its data before responding:
|
commit its data before responding:
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{"accepted_ids": ["1599c148-612c-46ab-aa9a-cf9a13271a72"]}
|
{"result":"success","data":{"accepted_ids":["1599c148-612c-46ab-aa9a-cf9a13271a72"]},"desc_lang_id":"","desc_lang_params":[]}
|
||||||
```
|
```
|
||||||
|
|
||||||
Only explicitly acknowledged IDs receive an `uploaded_at` timestamp. A plain HTTP
|
Only explicitly acknowledged IDs receive an `uploaded_at` timestamp. A plain HTTP
|
||||||
@@ -124,8 +139,21 @@ Only explicitly acknowledged IDs receive an `uploaded_at` timestamp. A plain HTT
|
|||||||
acknowledgements mark only the accepted records; remaining records retry with
|
acknowledgements mark only the accepted records; remaining records retry with
|
||||||
exponential backoff. Delivery is **at least once**: a receiver must deduplicate stable
|
exponential backoff. Delivery is **at least once**: a receiver must deduplicate stable
|
||||||
UUIDs when a response is lost, so retries and replays do not double-count energy.
|
UUIDs when a response is lost, so retries and replays do not double-count energy.
|
||||||
The sibling solar app does not yet provide an HTTP receiver; leave uploads
|
Solar uses its standard JSON result envelope: `result` must be `success`, and
|
||||||
disabled until a compatible receiver is available.
|
acknowledgements are read from `data.accepted_ids`. Plain `accepted_ids` responses
|
||||||
|
from other compatible receivers are also supported.
|
||||||
|
The sibling Solar app provides `index.php?a=relay` as this receiver. Set `[upload].url`
|
||||||
|
to its HTTPS URL and `[upload].token` to the same private value as Solar's
|
||||||
|
`Settings::SOLAR_RELAY_TOKEN`. The collector's `source_id` must match Solar's
|
||||||
|
`Settings::SOLAR_RELAY_SOURCE_ID` (default `home-gen24`). Restart the relay after
|
||||||
|
changing its private configuration. Pending samples then upload automatically;
|
||||||
|
use `replay` to resend retained samples already acknowledged elsewhere.
|
||||||
|
|
||||||
|
Solar stores five-minute `boiler` stats in watts: the recorded threshold when enabled,
|
||||||
|
zero when off. It calculates daily Wh by summing watts × 5/60 and uses those totals
|
||||||
|
for monthly and yearly charts. It acknowledges other readings without importing them,
|
||||||
|
including historical recovery records whose boiler state is unknown. Solar's
|
||||||
|
existing Solar.web import continues to supply production and consumption.
|
||||||
|
|
||||||
`collector.retention_days` defaults to 30. All retention cutoffs are midnight in
|
`collector.retention_days` defaults to 30. All retention cutoffs are midnight in
|
||||||
`collector.timezone`. For example, on November 2 with 30 days retained, the cutoff
|
`collector.timezone`. For example, on November 2 with 30 days retained, the cutoff
|
||||||
|
|||||||
+8
-8
@@ -7,14 +7,15 @@ retention_days = 30 # Full local days after acknowledgement; pen
|
|||||||
|
|
||||||
[inverter]
|
[inverter]
|
||||||
base_url = "http://fronius.local"
|
base_url = "http://fronius.local"
|
||||||
meter_id = "0" # Primary meter, as reported by Solar API.
|
meter_id = "0" # Primary meter, as reported by Solar API.
|
||||||
boiler_pin = "pin1"
|
boiler_pin = "pin1" # Check fronius emrs status API: /api/status/emrs
|
||||||
|
boiler_threshold_w = 2000 # Fallback load-management on threshold, in W, when API settings cannot be read.
|
||||||
request_timeout_seconds = 5
|
request_timeout_seconds = 5
|
||||||
min_request_interval_seconds = 4
|
min_request_interval_seconds = 4
|
||||||
|
|
||||||
[upload]
|
[upload]
|
||||||
url = "" # Set a compatible HTTPS receiver URL to enable uploads.
|
url = "" # Solar website's HTTPS index.php?a=relay URL; empty disables uploads.
|
||||||
token = "" # Shared bearer token; enter the real value only in private config.toml.
|
token = "" # Match Solar's Settings::SOLAR_RELAY_TOKEN; real values belong only in private configuration.
|
||||||
batch_size = 100
|
batch_size = 100
|
||||||
request_timeout_seconds = 10
|
request_timeout_seconds = 10
|
||||||
retry_initial_seconds = 5
|
retry_initial_seconds = 5
|
||||||
@@ -29,10 +30,9 @@ history_days = 14
|
|||||||
|
|
||||||
[history]
|
[history]
|
||||||
enabled = false
|
enabled = false
|
||||||
pv_system_id = "" # Your Solar.web PV system UUID; required when enabled.
|
pv_system_id = "" # Your Solar.web PV system UUID; required when enabled.
|
||||||
username = "" # Account email; enter real credentials only in private config.toml.
|
username = "" # Account email; enter real credentials only in private config.toml.
|
||||||
password = ""
|
password = ""
|
||||||
# Recover from local midnight collector.retention_days ago, including empty days.
|
lag_seconds = 7200 # How long (in seconds) before trying to pick up historical data from solarweb
|
||||||
lag_seconds = 7200 # Allow the inverter to upload its hourly historical data.
|
|
||||||
check_interval_seconds = 3600
|
check_interval_seconds = 3600
|
||||||
request_timeout_seconds = 45
|
request_timeout_seconds = 45
|
||||||
+2
-2
@@ -42,7 +42,7 @@ def next_poll_at(now, interval):
|
|||||||
def _run(config, store):
|
def _run(config, store):
|
||||||
stop = threading.Event()
|
stop = threading.Event()
|
||||||
previous = {}
|
previous = {}
|
||||||
uploader = Uploader(config.upload, store) if config.upload.url else None
|
uploader = Uploader(config.upload, store, boiler_threshold_w=config.inverter.boiler_threshold_w) if config.upload.url else None
|
||||||
recovery = HistoryRecovery(config.history, config.collector.source_id, store,
|
recovery = HistoryRecovery(config.history, config.collector.source_id, store,
|
||||||
timezone_name=config.collector.timezone) if config.history.enabled else None
|
timezone_name=config.collector.timezone) if config.history.enabled else None
|
||||||
worker = None
|
worker = None
|
||||||
@@ -150,7 +150,7 @@ def main(argv=None):
|
|||||||
print(json.dumps({"requeued": store.replay(start, until)}))
|
print(json.dumps({"requeued": store.replay(start, until)}))
|
||||||
elif args.command == "upload":
|
elif args.command == "upload":
|
||||||
store.prune()
|
store.prune()
|
||||||
uploader = Uploader(config.upload, store)
|
uploader = Uploader(config.upload, store, boiler_threshold_w=config.inverter.boiler_threshold_w)
|
||||||
count = 0
|
count = 0
|
||||||
while uploaded := uploader.upload_once():
|
while uploaded := uploader.upload_once():
|
||||||
count += uploaded
|
count += uploaded
|
||||||
|
|||||||
@@ -51,6 +51,7 @@ class InverterConfig:
|
|||||||
boiler_pin: str = "pin1"
|
boiler_pin: str = "pin1"
|
||||||
request_timeout_seconds: float = 5
|
request_timeout_seconds: float = 5
|
||||||
min_request_interval_seconds: float = 4
|
min_request_interval_seconds: float = 4
|
||||||
|
boiler_threshold_w: float = 2000
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -163,6 +164,9 @@ def load_config(path: Path) -> Config:
|
|||||||
raise ValueError("inverter.boiler_pin must be pin1, pin2, pin3 or pin4")
|
raise ValueError("inverter.boiler_pin must be pin1, pin2, pin3 or pin4")
|
||||||
_number(inverter.request_timeout_seconds, "inverter.request_timeout_seconds", 0.1)
|
_number(inverter.request_timeout_seconds, "inverter.request_timeout_seconds", 0.1)
|
||||||
_number(inverter.min_request_interval_seconds, "inverter.min_request_interval_seconds", 4)
|
_number(inverter.min_request_interval_seconds, "inverter.min_request_interval_seconds", 4)
|
||||||
|
_number(inverter.boiler_threshold_w, "inverter.boiler_threshold_w", 0.01)
|
||||||
|
if inverter.boiler_threshold_w > 99_999_999.99:
|
||||||
|
raise ValueError("inverter.boiler_threshold_w exceeds the stored watts range")
|
||||||
upload = UploadConfig(**data.get("upload", {}))
|
upload = UploadConfig(**data.get("upload", {}))
|
||||||
if not isinstance(upload.url, str):
|
if not isinstance(upload.url, str):
|
||||||
raise ValueError("upload.url must be a string")
|
raise ValueError("upload.url must be a string")
|
||||||
|
|||||||
+34
-3
@@ -94,14 +94,31 @@ def boiler_readings(response, pin):
|
|||||||
return {"boiler_enabled": state["state"], "boiler_reason": reason}
|
return {"boiler_enabled": state["state"], "boiler_reason": reason}
|
||||||
|
|
||||||
|
|
||||||
|
def boiler_threshold(response, pin):
|
||||||
|
"""GEN24's UI reads rules.<pin>.thresholdOn from /api/config/emrs, in watts.
|
||||||
|
|
||||||
|
In pgrid mode, negative thresholds denote surplus exported to the grid.
|
||||||
|
The UI displays their absolute magnitude as the switch-on threshold.
|
||||||
|
"""
|
||||||
|
rule = response.get("rules", {}).get(pin)
|
||||||
|
if not isinstance(rule, dict):
|
||||||
|
raise ValueError("Boiler threshold rule is missing")
|
||||||
|
value = _number(rule.get("thresholdOn"))
|
||||||
|
if value is not None and rule.get("mode") == "pgrid":
|
||||||
|
value = abs(value)
|
||||||
|
if value is None or not 0 < value <= 99_999_999.99:
|
||||||
|
raise ValueError("Boiler switch-on threshold is missing or invalid")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
class FroniusClient:
|
class FroniusClient:
|
||||||
def __init__(self, config: InverterConfig, *, http=None):
|
def __init__(self, config: InverterConfig, *, http=None):
|
||||||
self.config = config
|
self.config = config
|
||||||
self.http = http or JsonHttp(config.request_timeout_seconds, local=True)
|
self.http = http or JsonHttp(config.request_timeout_seconds, local=True)
|
||||||
self.last_request_started = None
|
self.last_request_started = None
|
||||||
|
|
||||||
def _read(self, path):
|
def _read(self, path, retries=REQUEST_RETRIES):
|
||||||
for attempt in range(REQUEST_RETRIES + 1):
|
for attempt in range(retries + 1):
|
||||||
if self.last_request_started is not None:
|
if self.last_request_started is not None:
|
||||||
wait = self.config.min_request_interval_seconds - (time.monotonic() - self.last_request_started)
|
wait = self.config.min_request_interval_seconds - (time.monotonic() - self.last_request_started)
|
||||||
if wait > 0:
|
if wait > 0:
|
||||||
@@ -110,7 +127,7 @@ class FroniusClient:
|
|||||||
try:
|
try:
|
||||||
return self.http.request(self.config.base_url.rstrip("/") + path)
|
return self.http.request(self.config.base_url.rstrip("/") + path)
|
||||||
except TransportError as error:
|
except TransportError as error:
|
||||||
if attempt == REQUEST_RETRIES:
|
if attempt == retries:
|
||||||
raise
|
raise
|
||||||
LOG.warning("Fronius GET %s failed (%s); retry %d/%d", path, error, attempt + 1, REQUEST_RETRIES)
|
LOG.warning("Fronius GET %s failed (%s); retry %d/%d", path, error, attempt + 1, REQUEST_RETRIES)
|
||||||
|
|
||||||
@@ -151,4 +168,18 @@ class FroniusClient:
|
|||||||
sample["source_timestamps"][name] = timestamp
|
sample["source_timestamps"][name] = timestamp
|
||||||
except (TransportError, ValueError, KeyError, TypeError, AttributeError) as error:
|
except (TransportError, ValueError, KeyError, TypeError, AttributeError) as error:
|
||||||
sample["errors"][name] = str(error)
|
sample["errors"][name] = str(error)
|
||||||
|
# These settings may require inverter authentication (HTTP 401). A failed
|
||||||
|
# settings read must not turn a valid boiler state into a missing sample.
|
||||||
|
sample["readings"]["boiler_threshold_w"] = self.config.boiler_threshold_w
|
||||||
|
sample["readings"]["boiler_threshold_source"] = "config"
|
||||||
|
try:
|
||||||
|
response = self._read("/api/config/emrs", retries=0)
|
||||||
|
threshold = boiler_threshold(response, self.config.boiler_pin)
|
||||||
|
sample["readings"]["boiler_threshold_w"] = threshold
|
||||||
|
sample["readings"]["boiler_threshold_source"] = "api"
|
||||||
|
# Keep only the relevant load-management settings, never an authentication response.
|
||||||
|
rule = response["rules"][self.config.boiler_pin]
|
||||||
|
sample["raw"]["boiler_config"] = {"thresholdOn": rule["thresholdOn"], "mode": rule.get("mode")}
|
||||||
|
except (TransportError, ValueError, KeyError, TypeError, AttributeError):
|
||||||
|
pass
|
||||||
return sample
|
return sample
|
||||||
+12
-2
@@ -25,6 +25,7 @@ NUMBERS = (
|
|||||||
"production_w", "production_ac_w", "consumption_w", "grid_w",
|
"production_w", "production_ac_w", "consumption_w", "grid_w",
|
||||||
"grid_import_w", "grid_export_w", "production_ac_total_wh",
|
"grid_import_w", "grid_export_w", "production_ac_total_wh",
|
||||||
"grid_import_total_wh", "grid_export_total_wh",
|
"grid_import_total_wh", "grid_export_total_wh",
|
||||||
|
"boiler_threshold_w",
|
||||||
)
|
)
|
||||||
SCHEMA = pa.schema([
|
SCHEMA = pa.schema([
|
||||||
("id", pa.string()), ("source_id", pa.string()), ("observed_at", pa.string()),
|
("id", pa.string()), ("source_id", pa.string()), ("observed_at", pa.string()),
|
||||||
@@ -32,12 +33,14 @@ SCHEMA = pa.schema([
|
|||||||
("uploaded_at", pa.string()), ("payload", pa.string()),
|
("uploaded_at", pa.string()), ("payload", pa.string()),
|
||||||
*((field, pa.float64()) for field in NUMBERS),
|
*((field, pa.float64()) for field in NUMBERS),
|
||||||
("boiler_enabled", pa.bool_()), ("boiler_reason", pa.string()),
|
("boiler_enabled", pa.bool_()), ("boiler_reason", pa.string()),
|
||||||
|
("boiler_threshold_source", pa.string()),
|
||||||
("record_kind", pa.string()),
|
("record_kind", pa.string()),
|
||||||
("power_ms", pa.int64()), ("power_ok", pa.bool_()),
|
("power_ms", pa.int64()), ("power_ok", pa.bool_()),
|
||||||
("meter_ms", pa.int64()), ("meter_ok", pa.bool_()),
|
("meter_ms", pa.int64()), ("meter_ok", pa.bool_()),
|
||||||
("boiler_ms", pa.int64()), ("boiler_ok", pa.bool_()),
|
("boiler_ms", pa.int64()), ("boiler_ok", pa.bool_()),
|
||||||
], metadata={b"solar_relay_schema": b"1"})
|
], metadata={b"solar_relay_schema": b"1"})
|
||||||
COVERAGE_COLUMNS = ["observed_ms", "power_ms", "power_ok", "meter_ms", "meter_ok", "boiler_ms", "boiler_ok"]
|
COVERAGE_COLUMNS = ["observed_ms", "power_ms", "power_ok", "meter_ms", "meter_ok", "boiler_ms", "boiler_ok"]
|
||||||
|
OPTIONAL_COLUMNS = {"boiler_threshold_w", "boiler_threshold_source"} # Absent in archives written before threshold recording.
|
||||||
|
|
||||||
|
|
||||||
class StoreError(OSError):
|
class StoreError(OSError):
|
||||||
@@ -83,10 +86,16 @@ class Store:
|
|||||||
schema = parquet.schema_arrow
|
schema = parquet.schema_arrow
|
||||||
# Validate the fields this version uses; read only those columns.
|
# Validate the fields this version uses; read only those columns.
|
||||||
if schema.metadata != SCHEMA.metadata or any(
|
if schema.metadata != SCHEMA.metadata or any(
|
||||||
schema.get_field_index(field.name) < 0 or schema.field(field.name) != field for field in SCHEMA
|
(schema.get_field_index(field.name) < 0 and field.name not in OPTIONAL_COLUMNS) or
|
||||||
|
(schema.get_field_index(field.name) >= 0 and schema.field(field.name) != field) for field in SCHEMA
|
||||||
):
|
):
|
||||||
raise StoreError("Unsupported Parquet archive schema")
|
raise StoreError("Unsupported Parquet archive schema")
|
||||||
return parquet.read(columns=SCHEMA.names if columns is None else columns).to_pylist()
|
wanted = SCHEMA.names if columns is None else columns
|
||||||
|
missing = set(wanted) - set(schema.names)
|
||||||
|
rows = parquet.read(columns=[name for name in wanted if name not in missing]).to_pylist()
|
||||||
|
for row in rows:
|
||||||
|
row.update({name: None for name in missing})
|
||||||
|
return rows
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _sync_directory(path):
|
def _sync_directory(path):
|
||||||
@@ -127,6 +136,7 @@ class Store:
|
|||||||
"payload": json.dumps(sample, allow_nan=False, separators=(",", ":")),
|
"payload": json.dumps(sample, allow_nan=False, separators=(",", ":")),
|
||||||
**{name: readings.get(name) for name in NUMBERS},
|
**{name: readings.get(name) for name in NUMBERS},
|
||||||
"boiler_enabled": readings.get("boiler_enabled"), "boiler_reason": readings.get("boiler_reason"),
|
"boiler_enabled": readings.get("boiler_enabled"), "boiler_reason": readings.get("boiler_reason"),
|
||||||
|
"boiler_threshold_source": readings.get("boiler_threshold_source"),
|
||||||
"record_kind": "history" if "solarweb_history" in sample.get("raw", {}) else "live",
|
"record_kind": "history" if "solarweb_history" in sample.get("raw", {}) else "live",
|
||||||
**{name: bool(value) if name.endswith("_ok") else value
|
**{name: bool(value) if name.endswith("_ok") else value
|
||||||
for name, value in zip(COVERAGE_COLUMNS[1:], coverage[3:])},
|
for name, value in zip(COVERAGE_COLUMNS[1:], coverage[3:])},
|
||||||
|
|||||||
+18
-2
@@ -1,9 +1,10 @@
|
|||||||
"""Retryable delivery with explicit per-sample acknowledgement."""
|
"""Retryable delivery with explicit per-sample acknowledgement."""
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
import math
|
||||||
import os
|
import os
|
||||||
|
|
||||||
from .config import UploadConfig
|
from .config import InverterConfig, UploadConfig
|
||||||
from .http import JsonHttp, TransportError
|
from .http import JsonHttp, TransportError
|
||||||
from .storage import Store
|
from .storage import Store
|
||||||
|
|
||||||
@@ -15,7 +16,7 @@ class UploadError(Exception):
|
|||||||
|
|
||||||
|
|
||||||
class Uploader:
|
class Uploader:
|
||||||
def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None):
|
def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None, boiler_threshold_w=None):
|
||||||
if not config.url:
|
if not config.url:
|
||||||
raise ValueError("Set upload.url to the website's ingestion endpoint first")
|
raise ValueError("Set upload.url to the website's ingestion endpoint first")
|
||||||
self.config = config
|
self.config = config
|
||||||
@@ -24,17 +25,32 @@ class Uploader:
|
|||||||
if not isinstance(self.token, str) or not self.token or any(ord(c) < 33 or ord(c) > 126 for c in self.token):
|
if not isinstance(self.token, str) or not self.token or any(ord(c) < 33 or ord(c) > 126 for c in self.token):
|
||||||
raise ValueError(f"Set upload.token in config.toml or {config.token_env} to a nonempty bearer token without whitespace")
|
raise ValueError(f"Set upload.token in config.toml or {config.token_env} to a nonempty bearer token without whitespace")
|
||||||
self.http = http or JsonHttp(config.request_timeout_seconds)
|
self.http = http or JsonHttp(config.request_timeout_seconds)
|
||||||
|
self.boiler_threshold_w = InverterConfig().boiler_threshold_w if boiler_threshold_w is None else boiler_threshold_w
|
||||||
|
if (isinstance(self.boiler_threshold_w, bool) or not isinstance(self.boiler_threshold_w, (int, float))
|
||||||
|
or not math.isfinite(self.boiler_threshold_w) or not 0 < self.boiler_threshold_w <= 99_999_999.99):
|
||||||
|
raise ValueError("Invalid fallback boiler threshold")
|
||||||
|
|
||||||
def upload_once(self) -> int:
|
def upload_once(self) -> int:
|
||||||
batch = self.store.pending(self.config.batch_size)
|
batch = self.store.pending(self.config.batch_size)
|
||||||
if not batch:
|
if not batch:
|
||||||
return 0
|
return 0
|
||||||
|
# Older retained samples contain the switch state only. Supply the configured
|
||||||
|
# fallback on delivery; never replace thresholds already recorded with a sample.
|
||||||
|
for sample in batch:
|
||||||
|
readings = sample.get("readings", {})
|
||||||
|
if isinstance(readings.get("boiler_enabled"), bool) and "boiler_threshold_w" not in readings:
|
||||||
|
readings["boiler_threshold_w"] = self.boiler_threshold_w
|
||||||
|
readings["boiler_threshold_source"] = "config"
|
||||||
try:
|
try:
|
||||||
response = self.http.request(
|
response = self.http.request(
|
||||||
self.config.url, payload={"schema_version": 1, "samples": batch}, token=self.token,
|
self.config.url, payload={"schema_version": 1, "samples": batch}, token=self.token,
|
||||||
)
|
)
|
||||||
except TransportError as error:
|
except TransportError as error:
|
||||||
raise UploadError(str(error)) from None
|
raise UploadError(str(error)) from None
|
||||||
|
if "result" in response:
|
||||||
|
if response["result"] != "success" or not isinstance(response.get("data"), dict):
|
||||||
|
raise UploadError("Receiver returned an unsuccessful or malformed result")
|
||||||
|
response = response["data"]
|
||||||
accepted = response.get("accepted_ids")
|
accepted = response.get("accepted_ids")
|
||||||
if not isinstance(accepted, list) or not all(isinstance(value, str) for value in accepted):
|
if not isinstance(accepted, list) or not all(isinstance(value, str) for value in accepted):
|
||||||
raise UploadError("Receiver must return an accepted_ids array")
|
raise UploadError("Receiver must return an accepted_ids array")
|
||||||
|
|||||||
@@ -27,12 +27,16 @@ class ParquetTests(unittest.TestCase):
|
|||||||
def test_real_parquet_has_typed_readings_and_original_payload(self):
|
def test_real_parquet_has_typed_readings_and_original_payload(self):
|
||||||
value = observed(self.now)
|
value = observed(self.now)
|
||||||
value["readings"]["boiler_enabled"] = False
|
value["readings"]["boiler_enabled"] = False
|
||||||
|
value["readings"]["boiler_threshold_w"] = 2600
|
||||||
|
value["readings"]["boiler_threshold_source"] = "config"
|
||||||
self.store.add(value)
|
self.store.add(value)
|
||||||
path = next(self.path.glob("date=*/samples.parquet"))
|
path = next(self.path.glob("date=*/samples.parquet"))
|
||||||
self.assertEqual(path.read_bytes()[:4], b"PAR1")
|
self.assertEqual(path.read_bytes()[:4], b"PAR1")
|
||||||
row = pq.ParquetFile(path).read().to_pylist()[0]
|
row = pq.ParquetFile(path).read().to_pylist()[0]
|
||||||
self.assertEqual(row["production_ac_w"], 2000)
|
self.assertEqual(row["production_ac_w"], 2000)
|
||||||
self.assertIs(row["boiler_enabled"], False)
|
self.assertIs(row["boiler_enabled"], False)
|
||||||
|
self.assertEqual(row["boiler_threshold_w"], 2600)
|
||||||
|
self.assertEqual(row["boiler_threshold_source"], "config")
|
||||||
self.assertIsNone(row["uploaded_at"])
|
self.assertIsNone(row["uploaded_at"])
|
||||||
self.assertEqual(json.loads(row["payload"]), value)
|
self.assertEqual(json.loads(row["payload"]), value)
|
||||||
self.assertEqual(self.store.status()["format"], "parquet")
|
self.assertEqual(self.store.status()["format"], "parquet")
|
||||||
@@ -77,6 +81,18 @@ class ParquetTests(unittest.TestCase):
|
|||||||
with self.assertRaises(StoreError):
|
with self.assertRaises(StoreError):
|
||||||
self.store.pending(100)
|
self.store.pending(100)
|
||||||
|
|
||||||
|
def test_pre_threshold_archive_is_readable_and_upgraded_on_write(self):
|
||||||
|
value = observed(self.now)
|
||||||
|
self.store.add(value)
|
||||||
|
path = next(self.path.glob("date=*/samples.parquet"))
|
||||||
|
legacy = pq.ParquetFile(path).read().drop_columns(["boiler_threshold_w", "boiler_threshold_source"])
|
||||||
|
pq.write_table(legacy, path)
|
||||||
|
self.assertEqual(self.store.pending(100), [value])
|
||||||
|
self.assertEqual(list(self.store.records()), [value])
|
||||||
|
self.store.acknowledge([value["id"]], now=self.now)
|
||||||
|
self.assertTrue(pq.ParquetFile(path).schema_arrow.equals(SCHEMA, check_metadata=True))
|
||||||
|
self.assertIsNone(pq.ParquetFile(path).read().to_pylist()[0]["boiler_threshold_w"])
|
||||||
|
|
||||||
def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
|
def test_retention_boundary_is_local_midnight_regardless_of_execution_time(self):
|
||||||
zone = ZoneInfo("Europe/Zurich")
|
zone = ZoneInfo("Europe/Zurich")
|
||||||
for hour in (0, 12, 23):
|
for hour in (0, 12, 23):
|
||||||
|
|||||||
+81
-1
@@ -14,7 +14,7 @@ from unittest.mock import Mock, patch
|
|||||||
from urllib.parse import urlsplit
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from solar_relay.config import InverterConfig, UploadConfig, date_bound, load_config
|
from solar_relay.config import InverterConfig, UploadConfig, date_bound, load_config
|
||||||
from solar_relay.fronius import FroniusClient, powerflow_readings
|
from solar_relay.fronius import FroniusClient, boiler_threshold, powerflow_readings
|
||||||
from solar_relay.http import JsonHttp, TransportError
|
from solar_relay.http import JsonHttp, TransportError
|
||||||
from solar_relay.storage import Store
|
from solar_relay.storage import Store
|
||||||
from solar_relay.upload import Uploader, UploadError
|
from solar_relay.upload import Uploader, UploadError
|
||||||
@@ -36,10 +36,12 @@ METER = api({"0": {
|
|||||||
"EnergyReal_WAC_Sum_Consumed": 50000, "EnergyReal_WAC_Sum_Produced": 80000,
|
"EnergyReal_WAC_Sum_Consumed": 50000, "EnergyReal_WAC_Sum_Produced": 80000,
|
||||||
}})
|
}})
|
||||||
BOILER = {"pin1": {"state": True, "reason": "PSTR_EMRS_REASON_3"}}
|
BOILER = {"pin1": {"state": True, "reason": "PSTR_EMRS_REASON_3"}}
|
||||||
|
BOILER_CONFIG = {"rules": {"pin1": {"mode": "pgrid", "thresholdOn": -2000, "thresholdOff": -500}}}
|
||||||
RESPONSES = {
|
RESPONSES = {
|
||||||
"/solar_api/v1/GetPowerFlowRealtimeData.fcgi": POWER,
|
"/solar_api/v1/GetPowerFlowRealtimeData.fcgi": POWER,
|
||||||
"/solar_api/v1/GetMeterRealtimeData.cgi": METER,
|
"/solar_api/v1/GetMeterRealtimeData.cgi": METER,
|
||||||
"/api/status/emrs": BOILER,
|
"/api/status/emrs": BOILER,
|
||||||
|
"/api/config/emrs": BOILER_CONFIG,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -138,6 +140,8 @@ class NormalizationTests(unittest.TestCase):
|
|||||||
self.assertEqual(readings["grid_export_w"], 0)
|
self.assertEqual(readings["grid_export_w"], 0)
|
||||||
self.assertEqual(readings["grid_import_total_wh"], 50000)
|
self.assertEqual(readings["grid_import_total_wh"], 50000)
|
||||||
self.assertTrue(readings["boiler_enabled"])
|
self.assertTrue(readings["boiler_enabled"])
|
||||||
|
self.assertEqual(readings["boiler_threshold_w"], 2000)
|
||||||
|
self.assertEqual(readings["boiler_threshold_source"], "api")
|
||||||
self.assertEqual(result["errors"], {})
|
self.assertEqual(result["errors"], {})
|
||||||
self.assertEqual(result["raw"]["powerflow"], POWER)
|
self.assertEqual(result["raw"]["powerflow"], POWER)
|
||||||
self.assertEqual(result["source_timestamps"]["powerflow"], "2026-11-01T12:00:00.000Z")
|
self.assertEqual(result["source_timestamps"]["powerflow"], "2026-11-01T12:00:00.000Z")
|
||||||
@@ -167,6 +171,27 @@ class NormalizationTests(unittest.TestCase):
|
|||||||
self.assertIs(result["readings"]["boiler_enabled"], False)
|
self.assertIs(result["readings"]["boiler_enabled"], False)
|
||||||
self.assertNotIn("boiler", result["errors"])
|
self.assertNotIn("boiler", result["errors"])
|
||||||
|
|
||||||
|
def test_configured_threshold_is_used_when_settings_require_authentication(self):
|
||||||
|
responses = deepcopy(RESPONSES)
|
||||||
|
responses["/api/config/emrs"] = TransportError("HTTP 401")
|
||||||
|
result = FroniusClient(InverterConfig(min_request_interval_seconds=0, boiler_threshold_w=2500), http=FakeHttp(responses)).collect("test")
|
||||||
|
self.assertTrue(result["readings"]["boiler_enabled"])
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_w"], 2500)
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
|
||||||
|
self.assertEqual(result["errors"], {})
|
||||||
|
|
||||||
|
def test_settings_are_matched_to_the_configured_pin_and_bad_values_use_fallback(self):
|
||||||
|
for value in (None, 0, True, "2000", float("nan"), 100_000_000):
|
||||||
|
with self.subTest(value=value):
|
||||||
|
responses = deepcopy(RESPONSES)
|
||||||
|
responses["/api/config/emrs"]["rules"]["pin1"]["thresholdOn"] = value
|
||||||
|
result = FroniusClient(InverterConfig(min_request_interval_seconds=0), http=FakeHttp(responses)).collect("test")
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_w"], 2000)
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
|
||||||
|
settings = {"rules": {"pin1": {"mode": "pgrid", "thresholdOn": -1500},
|
||||||
|
"pin2": {"mode": "ppv", "thresholdOn": 3200}}}
|
||||||
|
self.assertEqual(boiler_threshold(settings, "pin2"), 3200)
|
||||||
|
|
||||||
def test_solar_api_application_error_does_not_accept_old_values(self):
|
def test_solar_api_application_error_does_not_accept_old_values(self):
|
||||||
responses = deepcopy(RESPONSES)
|
responses = deepcopy(RESPONSES)
|
||||||
responses["/solar_api/v1/GetPowerFlowRealtimeData.fcgi"]["Head"]["Status"]["Code"] = 12
|
responses["/solar_api/v1/GetPowerFlowRealtimeData.fcgi"]["Head"]["Status"]["Code"] = 12
|
||||||
@@ -265,6 +290,53 @@ class DeliveryTests(unittest.TestCase):
|
|||||||
uploader.upload_once()
|
uploader.upload_once()
|
||||||
self.assertEqual(self.store.status()["pending"], 1)
|
self.assertEqual(self.store.status()["pending"], 1)
|
||||||
|
|
||||||
|
def test_solar_json_result_envelope_acknowledges_the_samples(self):
|
||||||
|
with server() as (httpd, base):
|
||||||
|
reading = sample()
|
||||||
|
self.store.add(reading)
|
||||||
|
httpd.ack_override = {"result": "success", "data": {"accepted_ids": [reading["id"]]},
|
||||||
|
"desc_lang_id": "", "desc_lang_params": []}
|
||||||
|
uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store)
|
||||||
|
self.assertEqual(uploader.upload_once(), 1)
|
||||||
|
self.assertEqual(self.store.status()["pending"], 0)
|
||||||
|
|
||||||
|
def test_failed_or_malformed_solar_result_never_clears_the_queue(self):
|
||||||
|
with server() as (httpd, base):
|
||||||
|
reading = sample()
|
||||||
|
self.store.add(reading)
|
||||||
|
uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store)
|
||||||
|
accepted = {"accepted_ids": [reading["id"]]}
|
||||||
|
for response in [
|
||||||
|
{"result": "error", "data": accepted},
|
||||||
|
{"result": "error", **accepted},
|
||||||
|
{"result": True, "data": accepted},
|
||||||
|
{"result": "success", "data": None},
|
||||||
|
{"result": "success", "data": []},
|
||||||
|
{"result": "success", "data": {}, **accepted},
|
||||||
|
]:
|
||||||
|
with self.subTest(response=response):
|
||||||
|
httpd.ack_override = response
|
||||||
|
with self.assertRaises(UploadError):
|
||||||
|
uploader.upload_once()
|
||||||
|
self.assertEqual(self.store.status()["pending"], 1)
|
||||||
|
|
||||||
|
def test_legacy_samples_receive_config_fallback_without_replacing_recorded_thresholds(self):
|
||||||
|
legacy = sample()
|
||||||
|
del legacy["readings"]["boiler_threshold_w"]
|
||||||
|
del legacy["readings"]["boiler_threshold_source"]
|
||||||
|
current = sample()
|
||||||
|
current["readings"]["boiler_threshold_w"] = 3100
|
||||||
|
self.store.add(legacy)
|
||||||
|
self.store.add(current)
|
||||||
|
with server() as (httpd, base):
|
||||||
|
uploader = Uploader(UploadConfig(url=base, token="test-token"), self.store, boiler_threshold_w=2500)
|
||||||
|
self.assertEqual(uploader.upload_once(), 2)
|
||||||
|
delivered = {s["id"]: s for s in httpd.requests[0]["samples"]}
|
||||||
|
self.assertEqual(delivered[legacy["id"]]["readings"]["boiler_threshold_w"], 2500)
|
||||||
|
self.assertEqual(delivered[legacy["id"]]["readings"]["boiler_threshold_source"], "config")
|
||||||
|
self.assertEqual(delivered[current["id"]]["readings"]["boiler_threshold_w"], 3100)
|
||||||
|
self.assertEqual(list(self.store.records()), [legacy, current])
|
||||||
|
|
||||||
def test_partial_ack_only_marks_the_accepted_sample(self):
|
def test_partial_ack_only_marks_the_accepted_sample(self):
|
||||||
with server() as (httpd, base):
|
with server() as (httpd, base):
|
||||||
first, second = sample(), sample()
|
first, second = sample(), sample()
|
||||||
@@ -376,6 +448,14 @@ class ConfigTests(unittest.TestCase):
|
|||||||
self.assertEqual(config.collector.poll_interval_seconds, 300)
|
self.assertEqual(config.collector.poll_interval_seconds, 300)
|
||||||
self.assertEqual(config.collector.retention_days, 30)
|
self.assertEqual(config.collector.retention_days, 30)
|
||||||
self.assertEqual(config.upload.url, "")
|
self.assertEqual(config.upload.url, "")
|
||||||
|
self.assertEqual(config.inverter.boiler_threshold_w, 2000)
|
||||||
|
|
||||||
|
def test_boiler_threshold_fallback_is_configurable_and_must_be_positive_watts(self):
|
||||||
|
config, _ = self.load('\n[inverter]\nboiler_threshold_w=2750\n')
|
||||||
|
self.assertEqual(config.inverter.boiler_threshold_w, 2750)
|
||||||
|
for value in ('0', '-100', 'true', '"2000"', 'nan', 'inf', '100000000'):
|
||||||
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||||
|
self.load(f'\n[inverter]\nboiler_threshold_w={value}\n')
|
||||||
|
|
||||||
def test_remote_upload_requires_https(self):
|
def test_remote_upload_requires_https(self):
|
||||||
with self.assertRaisesRegex(ValueError, "HTTPS"):
|
with self.assertRaisesRegex(ValueError, "HTTPS"):
|
||||||
|
|||||||
+15
-3
@@ -17,6 +17,7 @@ from tests.test_schedule import Clock
|
|||||||
POWER_PATH = "/solar_api/v1/GetPowerFlowRealtimeData.fcgi"
|
POWER_PATH = "/solar_api/v1/GetPowerFlowRealtimeData.fcgi"
|
||||||
METER_PATH = "/solar_api/v1/GetMeterRealtimeData.cgi"
|
METER_PATH = "/solar_api/v1/GetMeterRealtimeData.cgi"
|
||||||
BOILER_PATH = "/api/status/emrs"
|
BOILER_PATH = "/api/status/emrs"
|
||||||
|
BOILER_CONFIG_PATH = "/api/config/emrs"
|
||||||
|
|
||||||
|
|
||||||
class ScriptedHttp:
|
class ScriptedHttp:
|
||||||
@@ -47,7 +48,7 @@ class RetryTests(unittest.TestCase):
|
|||||||
sample = client.collect("test-house")
|
sample = client.collect("test-house")
|
||||||
self.assertEqual(sample["errors"], {})
|
self.assertEqual(sample["errors"], {})
|
||||||
self.assertEqual(sample["readings"]["production_ac_w"], 2000)
|
self.assertEqual(sample["readings"]["production_ac_w"], 2000)
|
||||||
self.assertEqual(httpd.get_counts, {POWER_PATH: retries + 1, METER_PATH: 1, BOILER_PATH: 1})
|
self.assertEqual(httpd.get_counts, {POWER_PATH: retries + 1, METER_PATH: 1, BOILER_PATH: 1, BOILER_CONFIG_PATH: 1})
|
||||||
self.assertEqual(len(messages.output), retries)
|
self.assertEqual(len(messages.output), retries)
|
||||||
|
|
||||||
def test_giving_up_after_three_retries_preserves_other_sources(self):
|
def test_giving_up_after_three_retries_preserves_other_sources(self):
|
||||||
@@ -57,7 +58,7 @@ class RetryTests(unittest.TestCase):
|
|||||||
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
|
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0))
|
||||||
with self.assertLogs("solar_relay.fronius", level="WARNING"):
|
with self.assertLogs("solar_relay.fronius", level="WARNING"):
|
||||||
sample = client.collect("test-house")
|
sample = client.collect("test-house")
|
||||||
self.assertEqual(httpd.get_counts, {POWER_PATH: 1, METER_PATH: 1, BOILER_PATH: 4})
|
self.assertEqual(httpd.get_counts, {POWER_PATH: 1, METER_PATH: 1, BOILER_PATH: 4, BOILER_CONFIG_PATH: 1})
|
||||||
self.assertEqual(len(httpd.get_failures[BOILER_PATH]), 1)
|
self.assertEqual(len(httpd.get_failures[BOILER_PATH]), 1)
|
||||||
self.assertEqual(sample["errors"], {"boiler": f"HTTP {failures[3]}"})
|
self.assertEqual(sample["errors"], {"boiler": f"HTTP {failures[3]}"})
|
||||||
self.assertIsNone(sample["readings"]["boiler_enabled"])
|
self.assertIsNone(sample["readings"]["boiler_enabled"])
|
||||||
@@ -73,7 +74,18 @@ class RetryTests(unittest.TestCase):
|
|||||||
self.assertLogs("solar_relay.fronius", level="WARNING"):
|
self.assertLogs("solar_relay.fronius", level="WARNING"):
|
||||||
result = client.collect("test-house")
|
result = client.collect("test-house")
|
||||||
self.assertEqual(result["errors"], {})
|
self.assertEqual(result["errors"], {})
|
||||||
self.assertEqual([stamp for stamp, _ in http.requests], [0, 4, 8, 12, 16, 20])
|
self.assertEqual([stamp for stamp, _ in http.requests], [0, 4, 8, 12, 16, 20, 24])
|
||||||
|
|
||||||
|
def test_protected_threshold_settings_use_fallback_without_retries_or_losing_state(self):
|
||||||
|
with server() as (httpd, base):
|
||||||
|
httpd.get_failures[BOILER_CONFIG_PATH] = [401, 401]
|
||||||
|
client = FroniusClient(InverterConfig(base_url=base, min_request_interval_seconds=0, boiler_threshold_w=2400))
|
||||||
|
result = client.collect("test")
|
||||||
|
self.assertEqual(httpd.get_counts[BOILER_CONFIG_PATH], 1)
|
||||||
|
self.assertEqual(result["errors"], {})
|
||||||
|
self.assertTrue(result["readings"]["boiler_enabled"])
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_w"], 2400)
|
||||||
|
self.assertEqual(result["readings"]["boiler_threshold_source"], "config")
|
||||||
|
|
||||||
def test_retries_preserve_clock_boundaries_and_one_saved_observation_per_poll(self):
|
def test_retries_preserve_clock_boundaries_and_one_saved_observation_per_poll(self):
|
||||||
origin = datetime(2026, 11, 1, 12, tzinfo=timezone.utc).timestamp()
|
origin = datetime(2026, 11, 1, 12, tzinfo=timezone.utc).timestamp()
|
||||||
|
|||||||
Reference in new issue
Block a user