franzz committed
1 parent
36180dffd7
commit
ec75ac15b0
30 files changed
+3042
-1
No files matched your search
@@ -0,0 +1,4 @@
|
||||
"""Local Fronius recorder and durable HTTP relay (AGPL-3.0-or-later)."""
|
||||
|
||||
__version__ = "0.2.0"
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
raise SystemExit(main())
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Commands for recording, uploading, inspecting and replaying the local archive."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from zoneinfo import ZoneInfoNotFoundError
|
||||
|
||||
from . import __version__
|
||||
from .config import date_bound, load_config
|
||||
from .fronius import FroniusClient
|
||||
from .history import HistoryRecovery
|
||||
from .http import TransportError
|
||||
from .storage import Store
|
||||
from .status import StatusServer
|
||||
from .upload import Uploader, UploadError
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _record(client, store, source_id, interval=300):
|
||||
sample = client.collect(source_id, interval)
|
||||
# Commit before attempting any upload. Even a completely failed poll is a gap marker.
|
||||
store.add(sample)
|
||||
if sample["errors"]:
|
||||
LOG.warning("Recorded observation %s with errors: %s", sample["id"], json.dumps(sample["errors"]))
|
||||
else:
|
||||
LOG.info("Recorded observation %s", sample["id"])
|
||||
return sample
|
||||
|
||||
|
||||
def next_poll_at(now, interval):
|
||||
"""UTC wall-clock boundaries: xx:00, xx:05, ... for the default interval."""
|
||||
return (math.floor(now / interval) + 1) * interval
|
||||
|
||||
|
||||
def _run(config, store):
|
||||
stop = threading.Event()
|
||||
previous = {}
|
||||
uploader = Uploader(config.upload, store) if config.upload.url else None
|
||||
recovery = HistoryRecovery(config.history, config.collector.source_id, store) if config.history.enabled else None
|
||||
worker = None
|
||||
history_worker = None
|
||||
status_server = None
|
||||
try:
|
||||
for name in (signal.SIGINT, signal.SIGTERM):
|
||||
previous[name] = signal.signal(name, lambda *_: stop.set())
|
||||
if config.http.enabled:
|
||||
status_server = StatusServer(config, store)
|
||||
status_server.start()
|
||||
LOG.info("Status page listening on http://%s:%s", *status_server.address)
|
||||
if uploader:
|
||||
# Remote timeouts and retry backoff must not interrupt local recording.
|
||||
worker = threading.Thread(target=uploader.run, args=(stop,), name="solar-upload", daemon=True)
|
||||
worker.start()
|
||||
else:
|
||||
LOG.info("Upload URL is empty; recording locally only")
|
||||
if recovery:
|
||||
history_worker = threading.Thread(target=recovery.run, args=(stop,), name="solar-history", daemon=True)
|
||||
history_worker.start()
|
||||
client = FroniusClient(config.inverter)
|
||||
next_prune = 0
|
||||
while not stop.is_set():
|
||||
if time.monotonic() >= next_prune:
|
||||
deleted = store.prune()
|
||||
if deleted:
|
||||
LOG.info("Deleted %d acknowledged observations older than the retention period", deleted)
|
||||
next_prune = time.monotonic() + 3600
|
||||
interval = config.collector.poll_interval_seconds
|
||||
now = time.time()
|
||||
# Wait before every observation, including the first after a restart.
|
||||
# Recompute after each poll so a slow poll never triggers a catch-up burst.
|
||||
if stop.wait(min(interval, max(0, next_poll_at(now, interval) - now))):
|
||||
break
|
||||
_record(client, store, config.collector.source_id, interval)
|
||||
finally:
|
||||
stop.set()
|
||||
if status_server:
|
||||
status_server.close()
|
||||
if worker:
|
||||
worker.join(timeout=config.upload.request_timeout_seconds + 1)
|
||||
if history_worker:
|
||||
history_worker.join(timeout=config.history.request_timeout_seconds + 1)
|
||||
for name, handler in previous.items():
|
||||
signal.signal(name, handler)
|
||||
return 0
|
||||
|
||||
|
||||
def _parser():
|
||||
parser = argparse.ArgumentParser(description="Record local Fronius readings and relay them to a website")
|
||||
parser.add_argument("--version", action="version", version=__version__)
|
||||
parser.add_argument("--config", type=Path, default=Path("config.toml"))
|
||||
parser.add_argument("--log-level", choices=("DEBUG", "INFO", "WARNING", "ERROR"), default="INFO")
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("run", help="Record continuously, uploading in a separate worker when configured")
|
||||
commands.add_parser("sample", help="Record one observation and print its normalized readings")
|
||||
commands.add_parser("upload", help="Upload the pending queue, stopping on the first failure")
|
||||
commands.add_parser("status", help="Show local archive/queue status without contacting the inverter")
|
||||
commands.add_parser("prune", help="Delete records acknowledged more than retention_days ago")
|
||||
for name, help_text in [
|
||||
("export", "Export archived samples as JSON Lines, including already uploaded samples"),
|
||||
("replay", "Requeue archived samples for upload, preserving their IDs"),
|
||||
("recover", "Retrieve missing energy intervals from the authenticated Solar.web API"),
|
||||
]:
|
||||
command = commands.add_parser(name, help=help_text)
|
||||
command.add_argument("--from", dest="from_date", help="Inclusive local date or ISO timestamp")
|
||||
command.add_argument("--until", help="Exclusive local date or ISO timestamp")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
args = _parser().parse_args(argv)
|
||||
logging.basicConfig(level=args.log_level, format="%(asctime)s %(levelname)s %(message)s")
|
||||
try:
|
||||
config = load_config(args.config)
|
||||
start = until = None
|
||||
if args.command in ("export", "replay", "recover"):
|
||||
start = date_bound(args.from_date, config.collector.timezone)
|
||||
until = date_bound(args.until, config.collector.timezone)
|
||||
if start is not None and until is not None and start >= until:
|
||||
raise ValueError("--from must be earlier than --until")
|
||||
store = Store(config.collector.archive, config.collector.retention_days)
|
||||
if args.command == "run":
|
||||
return _run(config, store)
|
||||
if args.command == "sample":
|
||||
sample = _record(FroniusClient(config.inverter), store, config.collector.source_id, config.collector.poll_interval_seconds)
|
||||
print(json.dumps({key: value for key, value in sample.items() if key != "raw"}, indent=2))
|
||||
return 1 if sample["errors"] else 0
|
||||
if args.command == "status":
|
||||
print(json.dumps(store.status(), indent=2))
|
||||
elif args.command == "prune":
|
||||
print(json.dumps({"deleted": store.prune()}))
|
||||
elif args.command == "recover":
|
||||
if not config.history.enabled:
|
||||
raise ValueError("Enable history and configure Solar.web API credentials before recovery")
|
||||
recovery = HistoryRecovery(config.history, config.collector.source_id, store)
|
||||
print(json.dumps({"recovered": recovery.recover(start, until)}))
|
||||
elif args.command == "export":
|
||||
for sample in store.records(start, until):
|
||||
print(json.dumps(sample, allow_nan=False, separators=(",", ":")))
|
||||
elif args.command == "replay":
|
||||
print(json.dumps({"requeued": store.replay(start, until)}))
|
||||
elif args.command == "upload":
|
||||
store.prune()
|
||||
uploader = Uploader(config.upload, store)
|
||||
count = 0
|
||||
while uploaded := uploader.upload_once():
|
||||
count += uploaded
|
||||
print(json.dumps({"uploaded": count, "pending": store.status()["pending"]}))
|
||||
return 0
|
||||
except BrokenPipeError:
|
||||
return 0
|
||||
except (ValueError, TypeError, OSError, UploadError, TransportError, ZoneInfoNotFoundError) as error:
|
||||
LOG.error("%s", error)
|
||||
return 1
|
||||
except KeyboardInterrupt:
|
||||
return 130
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Configuration contains environment variable names, never upload credentials."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tomllib
|
||||
from urllib.parse import urlsplit
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
|
||||
def utc_text(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("A timestamp must include its timezone")
|
||||
return value.astimezone(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def date_bound(value: str | None, timezone_name: str) -> str | None:
|
||||
"""Date bounds are local midnights; timestamp bounds may specify an offset."""
|
||||
if value is None:
|
||||
return None
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=ZoneInfo(timezone_name))
|
||||
return utc_text(parsed)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CollectorConfig:
|
||||
source_id: str
|
||||
archive: Path
|
||||
timezone: str = "Europe/Zurich"
|
||||
poll_interval_seconds: float = 300
|
||||
retention_days: int = 30
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InverterConfig:
|
||||
base_url: str = "http://fronius.local"
|
||||
meter_id: str = "0"
|
||||
boiler_pin: str = "pin1"
|
||||
request_timeout_seconds: float = 5
|
||||
min_request_interval_seconds: float = 4
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UploadConfig:
|
||||
url: str = ""
|
||||
token_env: str = "SOLAR_RELAY_TOKEN"
|
||||
batch_size: int = 100
|
||||
request_timeout_seconds: float = 10
|
||||
retry_initial_seconds: float = 5
|
||||
retry_max_seconds: float = 300
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HttpConfig:
|
||||
enabled: bool = True
|
||||
bind: str = "127.0.0.1"
|
||||
port: int = 8765
|
||||
base_path: str = "/solar-relay"
|
||||
history_days: int = 14
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HistoryConfig:
|
||||
enabled: bool = False
|
||||
pv_system_id: str = ""
|
||||
access_key_id_env: str = "SOLARWEB_ACCESS_KEY_ID"
|
||||
access_key_value_env: str = "SOLARWEB_ACCESS_KEY_VALUE"
|
||||
lookback_days: int = 7
|
||||
lag_seconds: int = 7200
|
||||
check_interval_seconds: int = 3600
|
||||
request_timeout_seconds: float = 10
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Config:
|
||||
collector: CollectorConfig
|
||||
inverter: InverterConfig
|
||||
upload: UploadConfig
|
||||
http: HttpConfig = HttpConfig()
|
||||
history: HistoryConfig = HistoryConfig()
|
||||
|
||||
|
||||
def _number(value: object, name: str, minimum: float) -> float:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
||||
raise ValueError(f"{name} must be a number")
|
||||
if not math.isfinite(value) or value < minimum:
|
||||
raise ValueError(f"{name} must be at least {minimum}")
|
||||
return float(value)
|
||||
|
||||
|
||||
def _url(value: str, name: str, *, upload: bool = False) -> None:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError(f"{name} must be a URL string")
|
||||
parsed = urlsplit(value)
|
||||
if parsed.scheme not in ("http", "https") or not parsed.hostname:
|
||||
raise ValueError(f"{name} must be an HTTP(S) URL")
|
||||
if parsed.username or parsed.password or parsed.fragment:
|
||||
raise ValueError(f"{name} cannot contain credentials or a fragment")
|
||||
if upload and parsed.scheme != "https" and parsed.hostname not in ("localhost", "127.0.0.1", "::1"):
|
||||
raise ValueError("The upload URL must use HTTPS (HTTP is allowed only on loopback for testing)")
|
||||
# Validate the port now, rather than failing inside the recorder loop.
|
||||
parsed.port
|
||||
|
||||
|
||||
def load_config(path: Path) -> Config:
|
||||
path = path.resolve()
|
||||
with path.open("rb") as handle:
|
||||
data = tomllib.load(handle)
|
||||
unknown = set(data) - {"collector", "inverter", "upload", "http", "history"}
|
||||
if unknown:
|
||||
raise ValueError(f"Unknown configuration sections: {', '.join(sorted(unknown))}")
|
||||
for section, keys in {
|
||||
"collector": set(CollectorConfig.__dataclass_fields__),
|
||||
"inverter": set(InverterConfig.__dataclass_fields__),
|
||||
"upload": set(UploadConfig.__dataclass_fields__),
|
||||
"http": set(HttpConfig.__dataclass_fields__),
|
||||
"history": set(HistoryConfig.__dataclass_fields__),
|
||||
}.items():
|
||||
if not isinstance(data.get(section, {}), dict):
|
||||
raise ValueError(f"{section} must be a table")
|
||||
extra = set(data.get(section, {})) - keys
|
||||
if extra:
|
||||
raise ValueError(f"Unknown {section} settings: {', '.join(sorted(extra))}")
|
||||
values = data.get("collector", {})
|
||||
source_id = values.get("source_id", "home-gen24")
|
||||
if not isinstance(source_id, str) or not re.fullmatch(r"[A-Za-z0-9_.-]{1,80}", source_id):
|
||||
raise ValueError("collector.source_id must contain 1–80 letters, digits, dots, underscores or hyphens")
|
||||
archive = Path(values.get("archive", "data/archive")).expanduser()
|
||||
if not archive.is_absolute():
|
||||
archive = path.parent / archive
|
||||
timezone_name = values.get("timezone", "Europe/Zurich")
|
||||
ZoneInfo(timezone_name)
|
||||
interval = _number(values.get("poll_interval_seconds", 300), "collector.poll_interval_seconds", 10)
|
||||
if interval != 300:
|
||||
raise ValueError("collector.poll_interval_seconds must be 300 (one observation every five minutes)")
|
||||
retention = values.get("retention_days", 30)
|
||||
if isinstance(retention, bool) or not isinstance(retention, int) or retention < 1:
|
||||
raise ValueError("collector.retention_days must be a positive integer")
|
||||
collector = CollectorConfig(source_id, archive.resolve(), timezone_name, interval, retention)
|
||||
inverter = InverterConfig(**data.get("inverter", {}))
|
||||
_url(inverter.base_url, "inverter.base_url")
|
||||
parsed = urlsplit(inverter.base_url)
|
||||
if parsed.path not in ("", "/") or parsed.query:
|
||||
raise ValueError("inverter.base_url must be the inverter origin, without a path or query")
|
||||
if not isinstance(inverter.meter_id, str) or not inverter.meter_id.isdecimal():
|
||||
raise ValueError('inverter.meter_id must be a string such as "0"')
|
||||
if not isinstance(inverter.boiler_pin, str) or not re.fullmatch(r"pin[1-4]", inverter.boiler_pin):
|
||||
raise ValueError("inverter.boiler_pin must be pin1, pin2, pin3 or pin4")
|
||||
_number(inverter.request_timeout_seconds, "inverter.request_timeout_seconds", 0.1)
|
||||
_number(inverter.min_request_interval_seconds, "inverter.min_request_interval_seconds", 4)
|
||||
upload = UploadConfig(**data.get("upload", {}))
|
||||
if not isinstance(upload.url, str):
|
||||
raise ValueError("upload.url must be a string")
|
||||
if upload.url:
|
||||
_url(upload.url, "upload.url", upload=True)
|
||||
if not isinstance(upload.token_env, str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", upload.token_env):
|
||||
raise ValueError("upload.token_env must be an environment variable name")
|
||||
if isinstance(upload.batch_size, bool) or not isinstance(upload.batch_size, int) or not 1 <= upload.batch_size <= 500:
|
||||
raise ValueError("upload.batch_size must be an integer between 1 and 500")
|
||||
_number(upload.request_timeout_seconds, "upload.request_timeout_seconds", 0.1)
|
||||
_number(upload.retry_initial_seconds, "upload.retry_initial_seconds", 1)
|
||||
_number(upload.retry_max_seconds, "upload.retry_max_seconds", upload.retry_initial_seconds)
|
||||
http = HttpConfig(**data.get("http", {}))
|
||||
if not isinstance(http.enabled, bool):
|
||||
raise ValueError("http.enabled must be true or false")
|
||||
if http.bind != "127.0.0.1":
|
||||
raise ValueError("http.bind must be 127.0.0.1; expose the page through a reverse proxy")
|
||||
if isinstance(http.port, bool) or not isinstance(http.port, int) or not 1024 <= http.port <= 65535:
|
||||
raise ValueError("http.port must be an integer between 1024 and 65535")
|
||||
if not isinstance(http.base_path, str) or (http.base_path and not re.fullmatch(r"/[A-Za-z0-9_-]+(?:/[A-Za-z0-9_-]+)*", http.base_path)):
|
||||
raise ValueError("http.base_path must be empty or a URL path without a trailing slash")
|
||||
if isinstance(http.history_days, bool) or not isinstance(http.history_days, int) or not 1 <= http.history_days <= 90:
|
||||
raise ValueError("http.history_days must be an integer between 1 and 90")
|
||||
history = HistoryConfig(**data.get("history", {}))
|
||||
if not isinstance(history.enabled, bool):
|
||||
raise ValueError("history.enabled must be true or false")
|
||||
if not isinstance(history.pv_system_id, str) or (history.pv_system_id and not re.fullmatch(r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", history.pv_system_id)):
|
||||
raise ValueError("history.pv_system_id must be a UUID")
|
||||
if history.enabled and not history.pv_system_id:
|
||||
raise ValueError("Set history.pv_system_id before enabling Solar.web recovery")
|
||||
for name in ("access_key_id_env", "access_key_value_env"):
|
||||
if not isinstance(getattr(history, name), str) or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", getattr(history, name)):
|
||||
raise ValueError(f"history.{name} must name an environment variable")
|
||||
for name, minimum in [("lookback_days", 1), ("lag_seconds", 3600), ("check_interval_seconds", 3600)]:
|
||||
value = getattr(history, name)
|
||||
if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
|
||||
raise ValueError(f"history.{name} must be an integer of at least {minimum}")
|
||||
if history.lookback_days > 30 or (history.enabled and history.lookback_days > retention):
|
||||
raise ValueError("history.lookback_days must be at most 30 and no longer than retention_days when enabled")
|
||||
_number(history.request_timeout_seconds, "history.request_timeout_seconds", 0.1)
|
||||
return Config(collector, inverter, upload, http, history)
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Five-minute extraction completeness, distinct from continuous observed coverage."""
|
||||
|
||||
from collections import Counter
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import math
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from .config import utc_text
|
||||
|
||||
BUCKET_MS = 300_000
|
||||
MAX_SOURCE_AGE_MS = 90_000
|
||||
|
||||
|
||||
def milliseconds(value):
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
if parsed.tzinfo is None:
|
||||
raise ValueError("Observation timestamp has no timezone")
|
||||
return round(parsed.timestamp() * 1000)
|
||||
|
||||
|
||||
def _finite(value):
|
||||
return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value)
|
||||
|
||||
|
||||
def sample_coverage(sample):
|
||||
observed = milliseconds(sample["observed_at"])
|
||||
if "history_interval" in sample:
|
||||
# Cloud history has measured interval energy, but no cumulative meter or relay history.
|
||||
return (sample["id"], sample["source_id"], observed, observed, 1, observed, 0, observed, 0)
|
||||
readings = sample.get("readings", {})
|
||||
result = [sample["id"], sample["source_id"], observed]
|
||||
for source, fields in [
|
||||
("powerflow", ["production_ac_w", "consumption_w", "grid_import_w", "grid_export_w"]),
|
||||
("meter", ["grid_import_total_wh", "grid_export_total_wh"]),
|
||||
("boiler", []),
|
||||
]:
|
||||
try:
|
||||
stamp = milliseconds(sample["source_timestamps"][source])
|
||||
valid = source not in sample.get("errors", {}) and abs(stamp - observed) <= MAX_SOURCE_AGE_MS
|
||||
except (KeyError, ValueError, TypeError, AttributeError):
|
||||
stamp, valid = observed, False
|
||||
valid = valid and all(_finite(readings.get(field)) for field in fields)
|
||||
if source == "boiler":
|
||||
valid = valid and isinstance(readings.get("boiler_enabled"), bool)
|
||||
result.extend([stamp if valid else observed, int(valid)])
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def daily_coverage(rows, timezone_name, days, now=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
zone = ZoneInfo(timezone_name)
|
||||
today = now.astimezone(zone).date()
|
||||
first = today - timedelta(days=days)
|
||||
upper = round(now.timestamp() * 1000)
|
||||
slots_by_time = {}
|
||||
observations = Counter()
|
||||
for row in rows:
|
||||
observed = row[0]
|
||||
if observed > upper + 1000:
|
||||
continue
|
||||
date = datetime.fromtimestamp(observed / 1000, zone).date().isoformat()
|
||||
observations[date] += 1
|
||||
bucket = observed // BUCKET_MS * BUCKET_MS
|
||||
mask = sum(1 << index for index in range(3) if row[2 + index * 2])
|
||||
slots_by_time[bucket] = slots_by_time.get(bucket, 0) | mask
|
||||
result = []
|
||||
for offset in range(days + 1):
|
||||
date = today - timedelta(days=offset)
|
||||
start = datetime.combine(date, datetime.min.time(), zone)
|
||||
end = datetime.combine(date + timedelta(days=1), datetime.min.time(), zone)
|
||||
start_ms = round(start.timestamp() * 1000)
|
||||
expected = round((end.timestamp() - start.timestamp()) * 1000) // BUCKET_MS
|
||||
elapsed = expected if date < today else max(0, min(expected, (upper - start_ms) // BUCKET_MS))
|
||||
slots = []
|
||||
for index in range(expected):
|
||||
slots.append(slots_by_time.get(start_ms + index * BUCKET_MS, 0) if index < elapsed else -1)
|
||||
complete = sum(mask == 7 for mask in slots)
|
||||
count = observations[date.isoformat()]
|
||||
status = "in_progress" if date == today else "complete" if complete == expected else "partial" if count else "missing"
|
||||
result.append({
|
||||
"date": date.isoformat(), "day_start": utc_text(start), "expected": expected, "elapsed": elapsed,
|
||||
"complete": complete, "missing": elapsed - complete, "samples": count, "status": status,
|
||||
"energy_complete": sum(mask >= 0 and bool(mask & 1) for mask in slots),
|
||||
"meter_complete": sum(mask >= 0 and bool(mask & 2) for mask in slots),
|
||||
"boiler_complete": sum(mask >= 0 and bool(mask & 4) for mask in slots),
|
||||
"slots": slots,
|
||||
})
|
||||
return result
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Read Solar API and GEN24 load-management state without changing the inverter."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from uuid import uuid4
|
||||
|
||||
from .config import InverterConfig, utc_text
|
||||
from .http import JsonHttp, TransportError
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
REQUEST_RETRIES = 3
|
||||
|
||||
|
||||
def _number(value):
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise ValueError("Invalid numeric reading")
|
||||
return value
|
||||
|
||||
|
||||
def _sum_known(values):
|
||||
values = list(values)
|
||||
return sum(values) if values and all(v is not None for v in values) else None
|
||||
|
||||
|
||||
def _data(response):
|
||||
head = response.get("Head", {})
|
||||
if head.get("Status", {}).get("Code") != 0:
|
||||
raise ValueError("Solar API reported an unsuccessful status")
|
||||
data = response.get("Body", {}).get("Data")
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError("Solar API data is missing")
|
||||
timestamp = datetime.fromisoformat(head["Timestamp"].replace("Z", "+00:00"))
|
||||
return data, utc_text(timestamp)
|
||||
|
||||
|
||||
def powerflow_readings(response):
|
||||
data, timestamp = _data(response)
|
||||
site = data.get("Site")
|
||||
inverters = data.get("Inverters", {})
|
||||
if not isinstance(site, dict) or not isinstance(inverters, dict):
|
||||
raise ValueError("Solar API site/inverter data is missing")
|
||||
if not {"P_PV", "P_Load", "P_Grid"}.issubset(site):
|
||||
raise ValueError("Required Solar API power fields are missing")
|
||||
load = _number(site.get("P_Load"))
|
||||
grid = _number(site.get("P_Grid"))
|
||||
total = _number(site.get("E_Total"))
|
||||
if total is None:
|
||||
total = _sum_known(_number(device.get("E_Total")) for device in inverters.values())
|
||||
return {
|
||||
# GEN24 P_PV is DC production; keep AC output separately instead of mixing them.
|
||||
"production_w": _number(site.get("P_PV")),
|
||||
"production_ac_w": _sum_known(_number(device.get("P")) for device in inverters.values()),
|
||||
"consumption_w": -load if load is not None else None,
|
||||
"grid_w": grid, # Positive: import. Negative: export (feed-in point meter).
|
||||
"grid_import_w": max(0, grid) if grid is not None else None,
|
||||
"grid_export_w": max(0, -grid) if grid is not None else None,
|
||||
"production_ac_total_wh": total,
|
||||
}, timestamp
|
||||
|
||||
|
||||
def meter_readings(response, meter_id):
|
||||
data, timestamp = _data(response)
|
||||
meter = data.get(meter_id)
|
||||
if not isinstance(meter, dict):
|
||||
raise ValueError(f"Configured meter {meter_id} is missing")
|
||||
if meter.get("Enable") not in (1, 1.0):
|
||||
raise ValueError("Configured meter is disabled")
|
||||
if meter.get("Meter_Location_Current") != 0:
|
||||
raise ValueError("Configured meter must be at the grid feed-in point")
|
||||
if not {"EnergyReal_WAC_Sum_Consumed", "EnergyReal_WAC_Sum_Produced"}.issubset(meter):
|
||||
raise ValueError("Required meter energy counters are missing")
|
||||
if meter.get("TimeStamp") is not None:
|
||||
try:
|
||||
timestamp = utc_text(datetime.fromtimestamp(_number(meter["TimeStamp"]), timezone.utc))
|
||||
except (OverflowError, OSError):
|
||||
raise ValueError("Invalid meter measurement timestamp") from None
|
||||
return {
|
||||
"grid_import_total_wh": _number(meter.get("EnergyReal_WAC_Sum_Consumed")),
|
||||
"grid_export_total_wh": _number(meter.get("EnergyReal_WAC_Sum_Produced")),
|
||||
}, timestamp
|
||||
|
||||
|
||||
def boiler_readings(response, pin):
|
||||
state = response.get(pin)
|
||||
if not isinstance(state, dict) or not isinstance(state.get("state"), bool):
|
||||
raise ValueError(f"Boiler output {pin} has no boolean state")
|
||||
reason = state.get("reason")
|
||||
if reason is not None and not isinstance(reason, str):
|
||||
raise ValueError("Boiler reason is not a string")
|
||||
return {"boiler_enabled": state["state"], "boiler_reason": reason}
|
||||
|
||||
|
||||
class FroniusClient:
|
||||
def __init__(self, config: InverterConfig, *, http=None):
|
||||
self.config = config
|
||||
self.http = http or JsonHttp(config.request_timeout_seconds, local=True)
|
||||
self.last_request_started = None
|
||||
|
||||
def _read(self, path):
|
||||
for attempt in range(REQUEST_RETRIES + 1):
|
||||
if self.last_request_started is not None:
|
||||
wait = self.config.min_request_interval_seconds - (time.monotonic() - self.last_request_started)
|
||||
if wait > 0:
|
||||
time.sleep(wait)
|
||||
self.last_request_started = time.monotonic()
|
||||
try:
|
||||
return self.http.request(self.config.base_url.rstrip("/") + path)
|
||||
except TransportError as error:
|
||||
if attempt == REQUEST_RETRIES:
|
||||
raise
|
||||
LOG.warning("Fronius GET %s failed (%s); retry %d/%d", path, error, attempt + 1, REQUEST_RETRIES)
|
||||
|
||||
def collect(self, source_id, poll_interval_seconds=300):
|
||||
sample = {
|
||||
"schema_version": 1,
|
||||
"id": str(uuid4()),
|
||||
"source_id": source_id,
|
||||
"poll_interval_seconds": poll_interval_seconds,
|
||||
"observed_at": utc_text(datetime.now(timezone.utc)),
|
||||
"inverter": {"meter_id": self.config.meter_id, "boiler_pin": self.config.boiler_pin},
|
||||
"readings": dict.fromkeys([
|
||||
"production_w", "production_ac_w", "consumption_w", "grid_w",
|
||||
"grid_import_w", "grid_export_w", "production_ac_total_wh",
|
||||
"grid_import_total_wh", "grid_export_total_wh",
|
||||
"boiler_enabled", "boiler_reason",
|
||||
]),
|
||||
"source_timestamps": {},
|
||||
"errors": {},
|
||||
"raw": {},
|
||||
}
|
||||
for name, path in [
|
||||
("powerflow", "/solar_api/v1/GetPowerFlowRealtimeData.fcgi"),
|
||||
("meter", "/solar_api/v1/GetMeterRealtimeData.cgi?Scope=System"),
|
||||
("boiler", "/api/status/emrs"),
|
||||
]:
|
||||
try:
|
||||
response = self._read(path)
|
||||
sample["raw"][name] = response
|
||||
if name == "powerflow":
|
||||
readings, timestamp = powerflow_readings(response)
|
||||
elif name == "meter":
|
||||
readings, timestamp = meter_readings(response, self.config.meter_id)
|
||||
else:
|
||||
readings = boiler_readings(response, self.config.boiler_pin)
|
||||
timestamp = utc_text(datetime.now(timezone.utc))
|
||||
sample["readings"].update(readings)
|
||||
sample["source_timestamps"][name] = timestamp
|
||||
except (TransportError, ValueError, KeyError, TypeError, AttributeError) as error:
|
||||
sample["errors"][name] = str(error)
|
||||
return sample
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Optional private Solar.web Query API recovery of missing five-minute energy.
|
||||
|
||||
GEN24's local Solar API exposes no archive. This reads the independent cloud API
|
||||
after the inverter has uploaded its internal backlog; it never requests a guest page.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
from urllib.parse import urlencode
|
||||
from uuid import NAMESPACE_URL, uuid5
|
||||
|
||||
from .config import utc_text
|
||||
from .coverage import milliseconds
|
||||
from .http import JsonHttp, TransportError
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CHANNELS = {
|
||||
"production": "EnergyProductionTotal", "consumption": "EnergyConsumptionTotal",
|
||||
"self_consumption": "EnergySelfConsumptionTotal", "bought": "EnergyPurchased", "sold": "EnergyFeedIn",
|
||||
}
|
||||
|
||||
|
||||
def history_sample(entry, source_id, system_id):
|
||||
stamp = milliseconds(entry["logDateTime"])
|
||||
if stamp % 300_000 or entry.get("logDuration") != 300:
|
||||
raise ValueError("Historical record is not an aligned five-minute interval")
|
||||
channels = {}
|
||||
for channel in entry["channels"]:
|
||||
name, value = channel.get("channelName"), channel.get("value")
|
||||
if name not in CHANNELS.values():
|
||||
continue
|
||||
if name in channels or channel.get("unit") != "Wh" or channel.get("channelType") != "Energy":
|
||||
raise ValueError("Unexpected historical channel metadata")
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not 0 <= value <= 1e18:
|
||||
raise ValueError("Historical energy is missing or invalid")
|
||||
channels[name] = value
|
||||
energy = {field: channels[name] for field, name in CHANNELS.items()}
|
||||
when = utc_text(datetime.fromtimestamp(stamp / 1000, timezone.utc))
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"id": str(uuid5(NAMESPACE_URL, f"https://swqapi.solarweb.com/{system_id}/{source_id}/{when}")),
|
||||
"source_id": source_id, "observed_at": when,
|
||||
"readings": dict.fromkeys([
|
||||
"production_w", "production_ac_w", "consumption_w", "grid_w", "grid_import_w", "grid_export_w",
|
||||
"production_ac_total_wh", "grid_import_total_wh", "grid_export_total_wh", "boiler_enabled", "boiler_reason",
|
||||
]),
|
||||
"source_timestamps": {},
|
||||
"errors": {"meter": "Cumulative meter readings are unavailable in recovered energy intervals",
|
||||
"boiler": "Historical boiler switch state is unavailable"},
|
||||
"raw": {"solarweb_history": entry},
|
||||
"history_interval": {"provider": "fronius-solarweb", "start": when, "duration_seconds": 300, "energy_wh": energy},
|
||||
}
|
||||
|
||||
|
||||
class HistoryRecovery:
|
||||
def __init__(self, config, source_id, store, *, http=None, credentials=None):
|
||||
self.config, self.source_id, self.store = config, source_id, store
|
||||
names = (config.access_key_id_env, config.access_key_value_env)
|
||||
values = credentials if credentials is not None else [os.environ.get(name, "") for name in names]
|
||||
if len(values) != 2 or any(not isinstance(value, str) or not value or any(ord(c) < 33 or ord(c) > 126 for c in value) for value in values):
|
||||
raise ValueError(f"Set {names[0]} and {names[1]} before enabling Solar.web recovery")
|
||||
self.headers = dict(zip(("AccessKeyId", "AccessKeyValue"), values))
|
||||
self.http = http or JsonHttp(config.request_timeout_seconds)
|
||||
|
||||
def recover(self, start=None, until=None, *, now=None, stop=None):
|
||||
now = now or datetime.now(timezone.utc)
|
||||
latest = int((now - timedelta(seconds=self.config.lag_seconds)).timestamp()) // 300 * 300
|
||||
first = self.store.status()["first_observation"]
|
||||
earliest = max(int((now - timedelta(days=self.config.lookback_days)).timestamp()) // 300 * 300,
|
||||
milliseconds(first) // 300_000 * 300 if first else latest)
|
||||
lower = milliseconds(start) // 300_000 * 300 if start else earliest
|
||||
upper = min(milliseconds(until) // 300_000 * 300, latest) if until else latest
|
||||
if lower >= upper:
|
||||
return 0
|
||||
if upper - lower > 30 * 86400:
|
||||
raise ValueError("Recover at most 30 days at once")
|
||||
stamp = lambda epoch: utc_text(datetime.fromtimestamp(epoch, timezone.utc))
|
||||
present = self.store.energy_slots(self.source_id, stamp(lower), stamp(upper))
|
||||
missing = set(range(lower, upper, 300)) - present
|
||||
# Query only UTC days containing gaps. No request spans the API's 24-hour limit.
|
||||
days = sorted({epoch // 86400 * 86400 for epoch in missing})
|
||||
recovered = 0
|
||||
for day in days:
|
||||
if stop and stop.is_set():
|
||||
break
|
||||
begin, end = max(day, lower), min(day + 86400, upper)
|
||||
page = 0
|
||||
while True:
|
||||
if stop and stop.is_set():
|
||||
break
|
||||
query = urlencode({"from": stamp(begin), "to": stamp(end), "timezone": "zulu",
|
||||
"channel": ",".join(CHANNELS.values()), "offset": page * 300, "limit": 300})
|
||||
url = f"https://swqapi.solarweb.com/pvsystems/{self.config.pv_system_id}/histdata?{query}"
|
||||
response = self.http.request(url, extra_headers=self.headers)
|
||||
system_id = response.get("pvSystemId")
|
||||
if not isinstance(system_id, str) or system_id.lower() != self.config.pv_system_id.lower() or not isinstance(response.get("data"), list):
|
||||
raise ValueError("Unexpected Solar.web history response")
|
||||
entries = response["data"]
|
||||
if len(entries) > 300:
|
||||
raise ValueError("Solar.web history response exceeds the requested page size")
|
||||
for entry in entries:
|
||||
try:
|
||||
sample = history_sample(entry, self.source_id, self.config.pv_system_id)
|
||||
epoch = milliseconds(sample["observed_at"]) // 1000
|
||||
except (KeyError, TypeError, ValueError, AttributeError):
|
||||
# Partial/unavailable cloud records remain missing and are tried again later.
|
||||
continue
|
||||
if begin <= epoch < end and epoch in missing:
|
||||
self.store.add(sample)
|
||||
missing.remove(epoch)
|
||||
recovered += 1
|
||||
if len(entries) < 300:
|
||||
break
|
||||
page += 1
|
||||
if page > 2:
|
||||
raise ValueError("Solar.web history pagination did not finish")
|
||||
return recovered
|
||||
|
||||
def run(self, stop):
|
||||
while not stop.is_set():
|
||||
try:
|
||||
recovered = self.recover(stop=stop)
|
||||
if recovered:
|
||||
LOG.info("Recovered %d historical energy intervals from Solar.web", recovered)
|
||||
except (TransportError, OSError, ValueError) as error:
|
||||
LOG.warning("Historical recovery failed (%s); retrying on the next scheduled check", type(error).__name__)
|
||||
stop.wait(self.config.check_interval_seconds)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Bounded JSON requests; redirects cannot forward upload credentials."""
|
||||
|
||||
import json
|
||||
from http.client import HTTPException
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import HTTPRedirectHandler, ProxyHandler, Request, build_opener
|
||||
|
||||
from . import __version__
|
||||
|
||||
|
||||
class TransportError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class NoRedirects(HTTPRedirectHandler):
|
||||
def redirect_request(self, request, response, code, message, headers, new_url):
|
||||
return None
|
||||
|
||||
|
||||
def _invalid_constant(value):
|
||||
raise ValueError("Non-finite JSON number")
|
||||
|
||||
|
||||
class JsonHttp:
|
||||
MAX_RESPONSE_BYTES = 2 * 1024 * 1024
|
||||
|
||||
def __init__(self, timeout: float, *, local: bool = False):
|
||||
handlers = [NoRedirects()]
|
||||
if local:
|
||||
# LAN traffic must not go through a workstation's HTTP(S) proxy.
|
||||
handlers.append(ProxyHandler({}))
|
||||
self.opener = build_opener(*handlers)
|
||||
self.timeout = timeout
|
||||
|
||||
def request(self, url: str, *, payload: dict | None = None, token: str | None = None, extra_headers=None) -> dict:
|
||||
headers = {"Accept": "application/json", "User-Agent": f"solar-relay/{__version__}"}
|
||||
headers.update(extra_headers or {})
|
||||
body = None
|
||||
if payload is not None:
|
||||
body = json.dumps(payload, allow_nan=False, separators=(",", ":")).encode("utf-8")
|
||||
headers["Content-Type"] = "application/json"
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = Request(url, data=body, headers=headers, method="POST" if body is not None else "GET")
|
||||
try:
|
||||
with self.opener.open(request, timeout=self.timeout) as response:
|
||||
raw = response.read(self.MAX_RESPONSE_BYTES + 1)
|
||||
except HTTPError as error:
|
||||
# Do not log server bodies: they can echo request headers or credentials.
|
||||
error.close()
|
||||
raise TransportError(f"HTTP {error.code}") from None
|
||||
except (URLError, OSError, TimeoutError, HTTPException) as error:
|
||||
raise TransportError(f"Network request failed ({type(error).__name__})") from None
|
||||
if len(raw) > self.MAX_RESPONSE_BYTES:
|
||||
raise TransportError("JSON response exceeds 2 MiB")
|
||||
try:
|
||||
value = json.loads(raw, parse_constant=_invalid_constant)
|
||||
except (ValueError, UnicodeError, RecursionError):
|
||||
raise TransportError("Response is not valid JSON") from None
|
||||
if not isinstance(value, dict):
|
||||
raise TransportError("Expected a JSON object")
|
||||
return value
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Loopback-only status page; Nginx provides HTTPS and the public path."""
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
from pathlib import Path
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from . import __version__
|
||||
from .config import utc_text
|
||||
from .coverage import milliseconds
|
||||
|
||||
|
||||
class StatusServer:
|
||||
def __init__(self, config, store):
|
||||
self.config, self.store = config, store
|
||||
self.started = time.monotonic()
|
||||
self.cache = {}
|
||||
self.cache_lock = threading.Lock()
|
||||
folder = Path(__file__).parent / "web"
|
||||
self.assets = {
|
||||
"/": ("text/html; charset=utf-8", (folder / "index.html").read_bytes()),
|
||||
"/assets/status.css": ("text/css; charset=utf-8", (folder / "status.css").read_bytes()),
|
||||
"/assets/status.js": ("text/javascript; charset=utf-8", (folder / "status.js").read_bytes()),
|
||||
}
|
||||
owner = self
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
server_version = "solar-relay"
|
||||
sys_version = ""
|
||||
|
||||
def log_message(self, *_):
|
||||
pass
|
||||
|
||||
def send(self, status, body, kind="application/json; charset=utf-8", *, head=False, headers=None):
|
||||
if not isinstance(body, bytes):
|
||||
body = json.dumps(body, allow_nan=False, separators=(",", ":")).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", kind)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.send_header("X-Content-Type-Options", "nosniff")
|
||||
self.send_header("Content-Security-Policy", "default-src 'self'; img-src 'self' data:; script-src 'self'; style-src 'self' 'unsafe-inline'; object-src 'none'; frame-ancestors 'none'; base-uri 'self'")
|
||||
for key, value in (headers or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
if not head:
|
||||
self.wfile.write(body)
|
||||
|
||||
def page(self, head=False):
|
||||
url = urlsplit(self.path)
|
||||
path = url.path
|
||||
prefix = owner.config.http.base_path
|
||||
if prefix and path == prefix:
|
||||
self.send(308, {}, head=head, headers={"Location": prefix + "/"})
|
||||
return
|
||||
if prefix and path.startswith(prefix + "/"):
|
||||
path = path[len(prefix):]
|
||||
if path in owner.assets:
|
||||
kind, body = owner.assets[path]
|
||||
self.send(200, body, kind, head=head)
|
||||
elif path == "/healthz":
|
||||
self.send(200, {"status": "ok", "service": "solar-relay", "version": __version__}, head=head)
|
||||
elif path == "/api/status":
|
||||
try:
|
||||
values = parse_qs(url.query).get("days", [str(owner.config.http.history_days)])
|
||||
if len(values) != 1 or not values[0].isdecimal() or not 1 <= int(values[0]) <= 90:
|
||||
self.send(400, {"error": "days must be between 1 and 90"}, head=head)
|
||||
return
|
||||
self.send(200, owner.snapshot(int(values[0])), head=head)
|
||||
except (OSError, ValueError):
|
||||
self.send(503, {"error": "Archive status is temporarily unavailable"}, head=head)
|
||||
else:
|
||||
self.send(404, {"error": "Not found"}, head=head)
|
||||
|
||||
def do_GET(self):
|
||||
self.page()
|
||||
|
||||
def do_HEAD(self):
|
||||
self.page(head=True)
|
||||
|
||||
def do_POST(self):
|
||||
self.send(405, {"error": "Read-only status endpoint"}, headers={"Allow": "GET, HEAD"})
|
||||
|
||||
self.server = ThreadingHTTPServer((config.http.bind, config.http.port), Handler)
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, kwargs={"poll_interval": 0.2}, name="solar-status", daemon=True)
|
||||
|
||||
@property
|
||||
def address(self):
|
||||
return self.server.server_address
|
||||
|
||||
def start(self):
|
||||
self.thread.start()
|
||||
|
||||
def close(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=2)
|
||||
|
||||
def snapshot(self, days):
|
||||
# Repeated page refreshes don't repeatedly scan the historical coverage index.
|
||||
with self.cache_lock:
|
||||
cached = self.cache.get(days)
|
||||
if cached and time.monotonic() - cached[0] < 10:
|
||||
return cached[1]
|
||||
archive = self.store.status()
|
||||
now = datetime.now(timezone.utc)
|
||||
attempt = archive["last_observation"]
|
||||
age = (now.timestamp() * 1000 - milliseconds(attempt)) / 1000 if attempt else None
|
||||
errors = list(archive["last_errors"])
|
||||
success = self.store.last_success(self.config.collector.source_id)
|
||||
status = "waiting" if age is None else "stale" if age > max(90, self.config.collector.poll_interval_seconds * 3) or age < -90 else "partial" if errors or success != attempt else "ok"
|
||||
body = {
|
||||
"service": "solar-relay", "version": __version__, "generated_at": utc_text(now),
|
||||
"timezone": self.config.collector.timezone, "source_id": self.config.collector.source_id,
|
||||
"status": status, "uptime_seconds": int(time.monotonic() - self.started),
|
||||
"collector": {"last_attempt": attempt, "last_success": success,
|
||||
"age_seconds": round(max(0, age), 1) if age is not None else None, "source_errors": errors,
|
||||
"poll_interval_seconds": self.config.collector.poll_interval_seconds},
|
||||
"archive": {"samples": archive["samples"], "pending": archive["pending"], "last_upload": archive["last_upload"],
|
||||
"upload_enabled": bool(self.config.upload.url), "format": "parquet", "retention_days": self.config.collector.retention_days},
|
||||
"history": {"enabled": self.config.history.enabled, "provider": "solarweb_query_api",
|
||||
"boiler_history_available": False},
|
||||
"days": self.store.coverage(self.config.collector.source_id, self.config.collector.timezone, days, now),
|
||||
}
|
||||
self.cache[days] = time.monotonic(), body
|
||||
return body
|
||||
@@ -0,0 +1,263 @@
|
||||
"""Durable, date-partitioned Parquet archive and acknowledged upload queue.
|
||||
|
||||
Each mutation replaces a daily partition atomically. A process/thread lock serializes
|
||||
readers and writers; it is never held during an HTTP request.
|
||||
"""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import fcntl
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from .config import utc_text
|
||||
from .coverage import daily_coverage, milliseconds, sample_coverage
|
||||
|
||||
NUMBERS = (
|
||||
"production_w", "production_ac_w", "consumption_w", "grid_w",
|
||||
"grid_import_w", "grid_export_w", "production_ac_total_wh",
|
||||
"grid_import_total_wh", "grid_export_total_wh",
|
||||
)
|
||||
SCHEMA = pa.schema([
|
||||
("id", pa.string()), ("source_id", pa.string()), ("observed_at", pa.string()),
|
||||
("observed_ms", pa.int64()), ("recorded_ns", pa.int64()),
|
||||
("uploaded_at", pa.string()), ("payload", pa.string()),
|
||||
*((field, pa.float64()) for field in NUMBERS),
|
||||
("boiler_enabled", pa.bool_()), ("boiler_reason", pa.string()),
|
||||
("record_kind", pa.string()),
|
||||
("interval_energy_wh", pa.struct([(field, pa.float64()) for field in ("production", "consumption", "self_consumption", "bought", "sold")])),
|
||||
("power_ms", pa.int64()), ("power_ok", pa.bool_()),
|
||||
("meter_ms", pa.int64()), ("meter_ok", pa.bool_()),
|
||||
("boiler_ms", pa.int64()), ("boiler_ok", pa.bool_()),
|
||||
], metadata={b"solar_relay_schema": b"1"})
|
||||
COVERAGE_COLUMNS = ["observed_ms", "power_ms", "power_ok", "meter_ms", "meter_ok", "boiler_ms", "boiler_ok"]
|
||||
|
||||
|
||||
class StoreError(OSError):
|
||||
"""An archive could not be read or updated safely."""
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, path: Path, retention_days=30):
|
||||
self.path = Path(path)
|
||||
if self.path.is_file():
|
||||
raise StoreError("collector.archive must be a directory")
|
||||
self.path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.retention_days = retention_days
|
||||
self._thread_lock = threading.RLock()
|
||||
descriptor = os.open(self.path / ".lock", os.O_CREAT | os.O_WRONLY, 0o600)
|
||||
os.close(descriptor)
|
||||
|
||||
@contextmanager
|
||||
def _locked(self, *, write=False):
|
||||
with self._thread_lock, (self.path / ".lock").open("rb") as handle:
|
||||
fcntl.flock(handle, fcntl.LOCK_EX if write else fcntl.LOCK_SH)
|
||||
try:
|
||||
yield
|
||||
except pa.ArrowException as error:
|
||||
raise StoreError(f"Parquet archive error ({type(error).__name__})") from error
|
||||
finally:
|
||||
fcntl.flock(handle, fcntl.LOCK_UN)
|
||||
|
||||
def _files(self):
|
||||
return sorted(self.path.glob("date=????-??-??/samples.parquet"))
|
||||
|
||||
def _partition(self, stamp):
|
||||
date = datetime.fromtimestamp(milliseconds(stamp) / 1000, timezone.utc).date()
|
||||
return self.path / f"date={date.isoformat()}" / "samples.parquet"
|
||||
|
||||
@staticmethod
|
||||
def _read(path, columns=None):
|
||||
# Avoid implicit Hive partition/schema inference from the directory name.
|
||||
if not path.exists():
|
||||
return []
|
||||
parquet = pq.ParquetFile(path, page_checksum_verification=True)
|
||||
if not parquet.schema_arrow.equals(SCHEMA, check_metadata=True):
|
||||
raise StoreError("Unsupported Parquet archive schema")
|
||||
return parquet.read(columns=columns).to_pylist()
|
||||
|
||||
@staticmethod
|
||||
def _sync_directory(path):
|
||||
descriptor = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
|
||||
try:
|
||||
os.fsync(descriptor)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
|
||||
def _write(self, path, rows):
|
||||
new_directory = not path.parent.exists()
|
||||
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
if new_directory:
|
||||
self._sync_directory(self.path)
|
||||
if not rows:
|
||||
path.unlink(missing_ok=True)
|
||||
self._sync_directory(path.parent)
|
||||
return
|
||||
descriptor, temporary = tempfile.mkstemp(prefix=".samples-", suffix=".tmp", dir=path.parent)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as handle:
|
||||
table = pa.Table.from_pylist(sorted(rows, key=lambda row: (row["observed_ms"], row["recorded_ns"], row["id"])), schema=SCHEMA)
|
||||
pq.write_table(table, handle, compression="zstd", write_page_checksum=True)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(temporary, path)
|
||||
self._sync_directory(path.parent)
|
||||
finally:
|
||||
Path(temporary).unlink(missing_ok=True)
|
||||
|
||||
@staticmethod
|
||||
def _row(sample, uploaded_at=None):
|
||||
coverage = sample_coverage(sample)
|
||||
readings = sample.get("readings", {})
|
||||
return {
|
||||
"id": sample["id"], "source_id": sample["source_id"], "observed_at": sample["observed_at"],
|
||||
"observed_ms": coverage[2], "recorded_ns": time.time_ns(), "uploaded_at": uploaded_at,
|
||||
"payload": json.dumps(sample, allow_nan=False, separators=(",", ":")),
|
||||
**{name: readings.get(name) for name in NUMBERS},
|
||||
"boiler_enabled": readings.get("boiler_enabled"), "boiler_reason": readings.get("boiler_reason"),
|
||||
"record_kind": "history" if "history_interval" in sample else "live",
|
||||
"interval_energy_wh": sample.get("history_interval", {}).get("energy_wh"),
|
||||
**{name: bool(value) if name.endswith("_ok") else value
|
||||
for name, value in zip(COVERAGE_COLUMNS[1:], coverage[3:])},
|
||||
}
|
||||
|
||||
def add(self, sample):
|
||||
path, row = self._partition(sample["observed_at"]), self._row(sample)
|
||||
with self._locked(write=True):
|
||||
rows = self._read(path)
|
||||
for existing in rows:
|
||||
if existing["id"] == row["id"]:
|
||||
if existing["payload"] != row["payload"]:
|
||||
raise StoreError("Sample ID already contains different data")
|
||||
return
|
||||
self._write(path, [*rows, row])
|
||||
|
||||
def pending(self, limit):
|
||||
with self._locked():
|
||||
rows = []
|
||||
for path in self._files():
|
||||
index = self._read(path, ["uploaded_at"])
|
||||
if any(row["uploaded_at"] is None for row in index):
|
||||
rows.extend(row for row in self._read(path, ["recorded_ns", "id", "payload", "uploaded_at"]) if row["uploaded_at"] is None)
|
||||
rows.sort(key=lambda row: (row["recorded_ns"], row["id"]))
|
||||
return [json.loads(row["payload"]) for row in rows[:limit]]
|
||||
|
||||
def acknowledge(self, ids, *, now=None):
|
||||
wanted, stamp = set(ids), utc_text(now or datetime.now(timezone.utc))
|
||||
if not wanted:
|
||||
return
|
||||
with self._locked(write=True):
|
||||
for path in self._files():
|
||||
index = self._read(path, ["id", "uploaded_at"])
|
||||
if not any(row["id"] in wanted and row["uploaded_at"] is None for row in index):
|
||||
continue
|
||||
rows = self._read(path)
|
||||
for row in rows:
|
||||
if row["id"] in wanted and row["uploaded_at"] is None:
|
||||
row["uploaded_at"] = stamp
|
||||
self._write(path, rows)
|
||||
|
||||
def records(self, start=None, until=None):
|
||||
lower, upper = milliseconds(start) if start else None, milliseconds(until) if until else None
|
||||
with self._locked():
|
||||
rows = []
|
||||
for path in self._files():
|
||||
rows.extend(row for row in self._read(path, ["observed_ms", "recorded_ns", "id", "payload"])
|
||||
if (lower is None or row["observed_ms"] >= lower) and (upper is None or row["observed_ms"] < upper))
|
||||
rows.sort(key=lambda row: (row["observed_ms"], row["recorded_ns"], row["id"]))
|
||||
return (json.loads(row["payload"]) for row in rows)
|
||||
|
||||
def replay(self, start=None, until=None):
|
||||
lower, upper = milliseconds(start) if start else None, milliseconds(until) if until else None
|
||||
count = 0
|
||||
with self._locked(write=True):
|
||||
for path in self._files():
|
||||
rows, changed = self._read(path), False
|
||||
for row in rows:
|
||||
if (lower is None or row["observed_ms"] >= lower) and (upper is None or row["observed_ms"] < upper):
|
||||
row["uploaded_at"] = None
|
||||
count += 1
|
||||
changed = True
|
||||
if changed:
|
||||
self._write(path, rows)
|
||||
return count
|
||||
|
||||
def prune(self, *, now=None):
|
||||
"""Retain pending records; age acknowledged records from their receipt time."""
|
||||
cutoff = utc_text((now or datetime.now(timezone.utc)) - timedelta(days=self.retention_days))
|
||||
deleted = 0
|
||||
with self._locked(write=True):
|
||||
for path in self._files():
|
||||
index = self._read(path, ["uploaded_at"])
|
||||
if not any(row["uploaded_at"] is not None and row["uploaded_at"] < cutoff for row in index):
|
||||
continue
|
||||
rows = self._read(path)
|
||||
retained = [row for row in rows if row["uploaded_at"] is None or row["uploaded_at"] >= cutoff]
|
||||
self._write(path, retained)
|
||||
deleted += len(rows) - len(retained)
|
||||
return deleted
|
||||
|
||||
def status(self):
|
||||
with self._locked():
|
||||
rows = []
|
||||
latest = None
|
||||
for path in self._files():
|
||||
index = self._read(path, ["observed_at", "observed_ms", "recorded_ns", "uploaded_at"])
|
||||
rows.extend(index)
|
||||
if index:
|
||||
candidate = max(index, key=lambda row: (row["observed_ms"], row["recorded_ns"]))
|
||||
if latest is None or (candidate["observed_ms"], candidate["recorded_ns"]) > latest[0]:
|
||||
latest = ((candidate["observed_ms"], candidate["recorded_ns"]), path)
|
||||
errors = {}
|
||||
if latest:
|
||||
newest = max(self._read(latest[1], ["observed_ms", "recorded_ns", "payload"]), key=lambda row: (row["observed_ms"], row["recorded_ns"]))
|
||||
errors = json.loads(newest["payload"]).get("errors", {})
|
||||
return {
|
||||
"archive": str(self.path), "format": "parquet", "retention_days": self.retention_days,
|
||||
"samples": len(rows), "pending": sum(row["uploaded_at"] is None for row in rows),
|
||||
"first_observation": min((row["observed_at"] for row in rows), default=None),
|
||||
"last_observation": max((row["observed_at"] for row in rows), default=None),
|
||||
"last_upload": max((row["uploaded_at"] for row in rows if row["uploaded_at"]), default=None),
|
||||
"last_errors": errors,
|
||||
}
|
||||
|
||||
def coverage(self, source_id, timezone_name, days, now=None):
|
||||
from zoneinfo import ZoneInfo
|
||||
now = now or datetime.now(timezone.utc)
|
||||
first = now.astimezone(ZoneInfo(timezone_name)).date() - timedelta(days=days)
|
||||
lower = round(datetime.combine(first, datetime.min.time(), ZoneInfo(timezone_name)).timestamp() * 1000)
|
||||
upper = round(now.timestamp() * 1000)
|
||||
with self._locked():
|
||||
rows = []
|
||||
for path in self._files():
|
||||
if path.parent.name[5:] < (now - timedelta(days=days + 2)).date().isoformat():
|
||||
continue
|
||||
rows.extend(tuple(row[name] for name in COVERAGE_COLUMNS)
|
||||
for row in self._read(path, ["source_id", *COVERAGE_COLUMNS])
|
||||
if row["source_id"] == source_id and lower <= row["observed_ms"] <= upper)
|
||||
return daily_coverage(sorted(rows), timezone_name, days, now)
|
||||
|
||||
def last_success(self, source_id):
|
||||
with self._locked():
|
||||
for path in reversed(self._files()):
|
||||
rows = self._read(path, ["source_id", "observed_ms", "power_ok", "meter_ok", "boiler_ok"])
|
||||
stamps = [row["observed_ms"] for row in rows if row["source_id"] == source_id and row["power_ok"] and row["meter_ok"] and row["boiler_ok"]]
|
||||
if stamps:
|
||||
return utc_text(datetime.fromtimestamp(max(stamps) / 1000, timezone.utc))
|
||||
return None
|
||||
|
||||
def energy_slots(self, source_id, start, until):
|
||||
"""Return UTC bucket starts already having valid local or recovered energy."""
|
||||
lower, upper = milliseconds(start), milliseconds(until)
|
||||
with self._locked():
|
||||
return {row["observed_ms"] // 300_000 * 300
|
||||
for path in self._files()
|
||||
for row in self._read(path, ["source_id", "observed_ms", "power_ok"])
|
||||
if row["source_id"] == source_id and row["power_ok"] and lower <= row["observed_ms"] < upper}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Retryable delivery with explicit per-sample acknowledgement."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
from .config import UploadConfig
|
||||
from .http import JsonHttp, TransportError
|
||||
from .storage import Store
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class UploadError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Uploader:
|
||||
def __init__(self, config: UploadConfig, store: Store, *, token=None, http=None):
|
||||
if not config.url:
|
||||
raise ValueError("Set upload.url to the website's ingestion endpoint first")
|
||||
self.config = config
|
||||
self.store = store
|
||||
self.token = token if token is not None else os.environ.get(config.token_env, "")
|
||||
if not self.token or any(ord(c) < 33 or ord(c) > 126 for c in self.token):
|
||||
raise ValueError(f"Set {config.token_env} to a nonempty bearer token without whitespace")
|
||||
self.http = http or JsonHttp(config.request_timeout_seconds)
|
||||
|
||||
def upload_once(self) -> int:
|
||||
batch = self.store.pending(self.config.batch_size)
|
||||
if not batch:
|
||||
return 0
|
||||
try:
|
||||
response = self.http.request(
|
||||
self.config.url, payload={"schema_version": 1, "samples": batch}, token=self.token,
|
||||
)
|
||||
except TransportError as error:
|
||||
raise UploadError(str(error)) from None
|
||||
accepted = response.get("accepted_ids")
|
||||
if not isinstance(accepted, list) or not all(isinstance(value, str) for value in accepted):
|
||||
raise UploadError("Receiver must return an accepted_ids array")
|
||||
requested = {sample["id"] for sample in batch}
|
||||
if len(set(accepted)) != len(accepted) or not set(accepted).issubset(requested):
|
||||
raise UploadError("Receiver acknowledged duplicate or unknown sample IDs")
|
||||
self.store.acknowledge(accepted)
|
||||
if set(accepted) != requested:
|
||||
raise UploadError(f"Receiver accepted {len(accepted)}/{len(batch)} samples; others remain queued")
|
||||
return len(accepted)
|
||||
|
||||
def run(self, stop):
|
||||
delay = self.config.retry_initial_seconds
|
||||
while not stop.is_set():
|
||||
try:
|
||||
count = self.upload_once()
|
||||
except (UploadError, OSError) as error:
|
||||
LOG.warning("Upload failed: %s; retrying in %.0fs", error, delay)
|
||||
stop.wait(delay)
|
||||
delay = min(delay * 2, self.config.retry_max_seconds)
|
||||
else:
|
||||
delay = self.config.retry_initial_seconds
|
||||
if count:
|
||||
LOG.info("Uploaded and acknowledged %d samples", count)
|
||||
else:
|
||||
stop.wait(5)
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<meta name="color-scheme" content="dark">
|
||||
<title>Solar relay · Collection status</title>
|
||||
<link rel="icon" href="data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 64 64'%3E%3Crect width='64' height='64' rx='16' fill='%23171d20'/%3E%3Ccircle cx='32' cy='32' r='15' fill='%23e8ba70'/%3E%3C/svg%3E">
|
||||
<link rel="stylesheet" href="assets/status.css">
|
||||
<script src="assets/status.js" defer></script>
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<header class="masthead">
|
||||
<div class="brand"><span class="sun" aria-hidden="true">☀</span><span>SOLAR RELAY<small>RPI OFFICE</small></span></div>
|
||||
<div class="header-actions"><span id="overall" class="badge neutral" role="status">Connecting</span><button id="refresh" class="refresh">↻ <span>Refresh</span></button></div>
|
||||
</header>
|
||||
<section class="intro">
|
||||
<p class="eyebrow">COLLECTION MONITOR</p>
|
||||
<h1>Solar relay status</h1>
|
||||
<p class="description">Extraction, delivery and five-minute coverage of your local Fronius data.</p>
|
||||
<div class="context"><span id="source">—</span><span class="separator">/</span><span id="timezone">Europe/Zurich</span><span class="separator">/</span><span id="updated">Waiting for the relay</span></div>
|
||||
</section>
|
||||
<p id="notice" class="notice" role="alert" hidden></p>
|
||||
<section class="metrics" aria-label="Current collection status">
|
||||
<article class="metric"><p class="metric-label">Last extraction</p><strong id="last-extraction">—</strong><p id="last-extraction-date">No observation received yet</p><small id="last-success">Last successful extraction: —</small></article>
|
||||
<article class="metric"><p class="metric-label">Yesterday’s coverage</p><strong id="yesterday">— <span>/ —</span></strong><p id="yesterday-detail">Five-minute intervals</p><small>Energy, meter and boiler status</small></article>
|
||||
<article class="metric"><p class="metric-label">Waiting to upload</p><strong id="pending">—</strong><p id="upload-detail">Checking delivery</p><small id="last-upload">Last acknowledgement: —</small></article>
|
||||
<article class="metric"><p class="metric-label">Local archive</p><strong id="samples">—</strong><p>Observations retained in Parquet</p><small id="retention">Acknowledged readings kept for 30 days</small></article>
|
||||
</section>
|
||||
<section class="panel">
|
||||
<div class="panel-heading"><div><p class="eyebrow">DAILY COVERAGE</p><h2>Five-minute completeness</h2></div><label class="range">History <select id="history"><option value="7">7 days</option><option value="14" selected>14 days</option><option value="30">30 days</option><option value="90">90 days</option></select></label></div>
|
||||
<p class="panel-note">Complete means a valid energy, meter and boiler extraction in that five-minute interval. It does not mean continuous measurement. Recovered cloud history supplies energy only.</p>
|
||||
<div class="table-scroll"><table><thead><tr><th>Day</th><th>Complete intervals</th><th class="source-column">Energy</th><th class="source-column">Meter</th><th class="source-column">Boiler</th><th>Status</th></tr></thead><tbody id="days"><tr><td colspan="6" class="empty">Waiting for the local archive…</td></tr></tbody></table></div>
|
||||
</section>
|
||||
<section class="panel detail" aria-label="Selected day interval details">
|
||||
<div class="panel-heading"><div><p class="eyebrow">FIVE-MINUTE INTERVALS</p><h2 id="selected-date">Select a day</h2></div><div class="legend"><span><i class="complete"></i>Complete</span><span><i class="partial"></i>Partial</span><span><i class="missing"></i>Missing</span><span><i class="future"></i>Upcoming</span></div></div>
|
||||
<p id="selected-summary" class="panel-note">Each column is one hour. Each cell is a five-minute interval.</p>
|
||||
<div id="slots" class="slots" aria-label="Five-minute coverage"></div>
|
||||
<p id="missing-ranges" class="gap-summary"></p>
|
||||
</section>
|
||||
<footer><span>Read-only status · refreshes every 15 seconds</span><span id="version">solar-relay</span></footer>
|
||||
<noscript><p class="notice">Enable JavaScript to load extraction and coverage status.</p></noscript>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,4 @@
|
||||
:root{color-scheme:dark;--bg:#101518;--panel:#171e22;--line:#2b353a;--ink:#eef1ed;--muted:#9ba9ac;--amber:#e8ba70;--green:#7ad6ac;--red:#e68d86;--radius:12px;font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif;font-synthesis:none}
|
||||
*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink)}button,select{font:inherit}button{cursor:pointer}button:focus-visible,select:focus-visible{outline:2px solid var(--amber);outline-offset:4px}main{max-width:1216px;margin:auto;padding:0 32px}.masthead{height:100px;display:flex;align-items:center;justify-content:space-between;border-bottom:1px solid var(--line)}.brand{display:flex;align-items:center;gap:12px;font-size:13px;letter-spacing:.14em;font-weight:750}.brand small{display:block;color:var(--muted);font-size:9px;letter-spacing:.22em;margin-top:5px}.sun{font-size:36px;color:var(--amber);line-height:1}.header-actions{display:flex;align-items:center;gap:16px}.badge{display:inline-flex;align-items:center;gap:7px;white-space:nowrap;border-radius:20px;padding:6px 11px;font-size:11px;font-weight:650;line-height:1.3;background:#252e32;color:var(--muted)}.badge::before{content:"";width:5px;height:5px;border-radius:50%;background:currentColor}.badge.ok,.badge.complete{color:var(--green);background:#193a2e}.badge.partial,.badge.in_progress{color:var(--amber);background:#3a3121}.badge.stale{color:var(--red);background:#3d2829}.badge.missing,.badge.waiting,.badge.neutral{color:var(--muted);background:#252e32}.refresh{border:1px solid var(--line);background:transparent;color:var(--ink);border-radius:7px;padding:8px 12px;font-size:13px}.refresh:hover{border-color:var(--muted)}.refresh:disabled{opacity:.45;cursor:wait}.intro{padding:42px 0 28px}.eyebrow{color:var(--amber);font-size:10px;font-weight:700;letter-spacing:.17em;margin:0 0 13px}h1{font-size:clamp(27px,3.4vw,42px);letter-spacing:-.045em;font-weight:550;line-height:1.14;margin:0 0 13px}.description{color:var(--muted);font-size:14px;line-height:1.6;margin:0}.context{display:flex;align-items:center;gap:10px;font-size:11px;color:var(--muted);margin-top:20px;flex-wrap:wrap}.separator{color:#48565c}.metrics{display:grid;grid-template-columns:repeat(4,minmax(0,1fr));gap:14px;margin:0 0 28px}.metric{border:1px solid var(--line);border-radius:var(--radius);padding:23px 20px;background:var(--panel)}.metric-label{font-size:11px;font-weight:550;color:var(--muted);margin:0 0 18px}.metric strong{display:block;font-size:30px;font-weight:550;font-variant-numeric:tabular-nums;letter-spacing:-.04em;line-height:1.1;min-height:34px}.metric strong span{color:var(--muted);font-size:17px;font-weight:400;letter-spacing:normal}.metric p:not(.metric-label){font-size:11px;margin:12px 0 7px;color:#d1d9d7}.metric small{font-size:10px;color:var(--muted);line-height:1.6;display:block}.panel{background:var(--panel);border:1px solid var(--line);border-radius:var(--radius);margin-bottom:22px;overflow:hidden}.panel-heading{display:flex;align-items:center;justify-content:space-between;gap:16px;padding:24px 24px 0}.panel-heading .eyebrow{margin-bottom:7px}h2{font-size:19px;font-weight:550;letter-spacing:-.025em;margin:0}.panel-note{font-size:11px;line-height:1.7;color:var(--muted);padding:0 24px;margin:12px 0 22px}.range{display:flex;align-items:center;gap:9px;color:var(--muted);font-size:11px;white-space:nowrap}select{background:#20292e;border:1px solid #354149;border-radius:6px;color:var(--ink);padding:7px 27px 7px 9px;font-size:11px}.table-scroll{overflow:auto}table{width:100%;border-collapse:collapse;text-align:left;font-size:12px;font-variant-numeric:tabular-nums}th{background:#1c252a;border-top:1px solid var(--line);border-bottom:1px solid var(--line);padding:11px 18px;font-size:9px;font-weight:650;letter-spacing:.07em;text-transform:uppercase;color:var(--muted)}th:first-child,td:first-child{padding-left:24px}td{padding:13px 18px;border-bottom:1px solid #283137;white-space:nowrap}tbody tr:last-child td{border-bottom:0}tr.selected{background:#222d32}tr:hover{background:#202a2f}.day-select{display:block;text-align:left;color:var(--ink);background:transparent;border:0;padding:0;line-height:1.55;font-size:12px}.day-select small{display:block;color:var(--muted);font-size:10px}.coverage-number{display:flex;align-items:baseline;gap:5px;font-size:12px}.coverage-number small{font-size:10px;color:var(--muted)}.progress{height:3px;width:126px;border-radius:2px;background:#303c42;margin-top:7px;overflow:hidden}.progress span{display:block;height:100%;background:var(--green)}.source-count{color:#c1ceca;font-size:11px}.source-count.incomplete{color:var(--amber)}.empty{padding:40px!important;text-align:center;color:var(--muted)}.legend{display:flex;gap:14px;flex-wrap:wrap;font-size:10px;color:var(--muted)}.legend span{display:flex;align-items:center;gap:6px}.legend i{display:block;width:7px;height:7px;border-radius:2px}.slots{display:flex;gap:5px;padding:0 24px;overflow-x:auto}.hour{display:grid;grid-template-rows:16px repeat(12,5px);gap:3px;flex:1;min-width:14px}.hour-label{font-size:9px;color:var(--muted);font-variant-numeric:tabular-nums;text-align:center}.slot{display:block;border-radius:1px;min-height:5px}.complete{background:#76c9a4}.partial{background:#bd9558}.missing{bacLine truncated
|
||||
@media(max-width:900px){.metrics{grid-template-columns:repeat(2,minmax(0,1fr))}.metric{padding:20px}.source-column{display:none}.legend{max-width:215px;gap:10px}.slots{gap:4px}.progress{width:115px}}
|
||||
@media(max-width:560px){main{padding:0 16px}.masthead{height:82px}.brand{font-size:11px;gap:8px}.sun{font-size:30px}.header-actions{gap:8px}.refresh span{display:none}.refresh{font-size:17px;padding:4px 9px}.badge{font-size:10px;padding:6px 8px}.intro{padding-top:30px}.metrics{gap:10px;margin-bottom:20px}.metric{padding:17px 14px}.metric strong{font-size:26px}.metric strong span{font-size:14px}.metric-label{margin-bottom:14px}.metric small{font-size:9px}.panel-heading{padding:20px 16px 0;align-items:flex-start}h2{font-size:16px}.panel-note{padding:0 16px;font-size:10px}th,td{padding-left:11px;padding-right:11px}th:first-child,td:first-child{padding-left:16px}.progress{width:88px}.range{flex-direction:column;align-items:flex-end;gap:5px}.legend{font-size:9px;gap:7px;max-width:150px}.slots{padding:0 16px;gap:3px}.hour{min-width:10px;grid-template-rows:16px repeat(12,5px);gap:3px}.hour-label{font-size:8px}.gap-summary{padding:16px;font-size:10px}.day-select{font-size:11px}footer{font-size:9px;flex-wrap:wrap}.detail .panel-heading{gap:8px}}
|
||||
@@ -0,0 +1,148 @@
|
||||
/* Read-only local archive dashboard. All URLs remain relative to the proxy prefix. */
|
||||
(() => {
|
||||
'use strict';
|
||||
const $ = id => document.getElementById(id);
|
||||
const labels = {ok:'Collecting',waiting:'Waiting for data',partial:'Partial extraction',stale:'Extraction overdue',complete:'Complete',in_progress:'In progress',missing:'No data'};
|
||||
let current = null;
|
||||
let selected = null;
|
||||
let busy = false;
|
||||
let zone = 'Europe/Zurich';
|
||||
const number = value => new Intl.NumberFormat('en-GB').format(value);
|
||||
const format = (value, options) => value ? new Intl.DateTimeFormat('en-GB', {timeZone:zone,...options}).format(new Date(value)) : '—';
|
||||
const time = value => format(value,{hour:'2-digit',minute:'2-digit',second:'2-digit',hour12:false});
|
||||
const date = value => format(value,{day:'2-digit',month:'short',year:'numeric'});
|
||||
function badge(text, status) {
|
||||
const element = document.createElement('span');
|
||||
element.className = 'badge ' + status;
|
||||
element.textContent = text;
|
||||
return element;
|
||||
}
|
||||
function td(className) {
|
||||
const element = document.createElement('td');
|
||||
if(className) element.className = className;
|
||||
return element;
|
||||
}
|
||||
function addText(parent, tag, text, className) {
|
||||
const element = document.createElement(tag);
|
||||
if(className) element.className = className;
|
||||
element.textContent = text;
|
||||
parent.append(element);
|
||||
return element;
|
||||
}
|
||||
function renderDays() {
|
||||
$('days').replaceChildren();
|
||||
for(const day of current.days) {
|
||||
const row = document.createElement('tr');
|
||||
row.classList.toggle('selected',day.date===selected);
|
||||
const dayCell = td();
|
||||
const button = addText(dayCell,'button',format(day.date+'T12:00:00Z',{day:'2-digit',month:'short'}),'day-select');
|
||||
button.type = 'button';
|
||||
button.setAttribute('aria-expanded',String(day.date===selected));
|
||||
addText(button,'small',day.status==='in_progress'?'Today':format(day.date+'T12:00:00Z',{weekday:'long'}));
|
||||
button.addEventListener('click',() => {selected=day.date;renderDays();renderSlots();});
|
||||
row.append(dayCell);
|
||||
const amount = td();
|
||||
const count = addText(amount,'div',number(day.complete),'coverage-number');
|
||||
addText(count,'small','/ '+number(day.elapsed)+(day.status==='in_progress'?' so far':''));
|
||||
const progress = addText(amount,'div','','progress');
|
||||
const fill = addText(progress,'span','');
|
||||
fill.style.width = (day.elapsed ? day.complete/day.elapsed*100:0)+'%';
|
||||
row.append(amount);
|
||||
for(const field of ['energy_complete','meter_complete','boiler_complete']) {
|
||||
const cell = td('source-column');
|
||||
addText(cell,'span',number(day[field])+'/'+number(day.elapsed),'source-count'+(day[field]<day.elapsed?' incomplete':''));
|
||||
row.append(cell);
|
||||
}
|
||||
const state = td();
|
||||
state.append(badge(labels[day.status]||day.status,day.status));
|
||||
row.append(state);
|
||||
$('days').append(row);
|
||||
}
|
||||
}
|
||||
function renderSlots() {
|
||||
const day = current.days.find(item => item.date===selected);
|
||||
if(!day) return;
|
||||
$('selected-date').textContent = format(day.date+'T12:00:00Z',{weekday:'long',day:'numeric',month:'long',year:'numeric'});
|
||||
$('selected-summary').textContent = `${number(day.complete)} complete of ${number(day.elapsed)} elapsed intervals · ${number(day.expected)} intervals in this calendar day. Each cell is five minutes.`;
|
||||
const start = Date.parse(day.day_start);
|
||||
const stamp = index => format(new Date(start+index*300000),{hour:'2-digit',minute:'2-digit',timeZoneName:'shortOffset',hour12:false});
|
||||
$('slots').replaceChildren();
|
||||
let hour;
|
||||
for(let index=0;index<day.slots.length;index++) {
|
||||
if(index%12===0) {
|
||||
hour = addText($('slots'),'div','','hour');
|
||||
addText(hour,'span',format(new Date(start+index*300000),{hour:'2-digit',hour12:false}),'hour-label');
|
||||
}
|
||||
const mask = day.slots[index];
|
||||
const state = mask===7?'complete':mask===-1?'future':mask>0?'partial':'missing';
|
||||
const missing = ['Energy','Meter','Boiler'].filter((_,bit) => !(mask & (1<<bit)));
|
||||
const text = `${stamp(index)}–${stamp(index+1)} · ${mask===7?'Complete':mask===-1?'Upcoming':'Incomplete: '+missing.join(', ')}`;
|
||||
const cell = addText(hour,'span','','slot '+state);
|
||||
cell.title = text;
|
||||
cell.setAttribute('role','img');
|
||||
cell.setAttribute('aria-label',text);
|
||||
}
|
||||
const ranges = [];
|
||||
for(let index=0;index<day.elapsed;) {
|
||||
if(day.slots[index]===7) {index++;continue;}
|
||||
const from = index;
|
||||
while(index<day.elapsed && day.slots[index]!==7) index++;
|
||||
ranges.push(`${stamp(from)}–${stamp(index)}`);
|
||||
}
|
||||
$('missing-ranges').textContent = ranges.length ? `${number(day.missing)} incomplete intervals. Gaps: ${ranges.slice(0,6).join('; ')}${ranges.length>6?`; plus ${number(ranges.length-6)} more ranges.`:'.'}` : day.status==='in_progress'?'All elapsed intervals are complete. The current interval is still being recorded.':'All five-minute intervals have complete data.';
|
||||
}
|
||||
function render(data) {
|
||||
current = data;
|
||||
zone = data.timezone;
|
||||
$('source').textContent = data.source_id;
|
||||
$('timezone').textContent = zone;
|
||||
$('updated').textContent = 'Updated '+time(data.generated_at);
|
||||
$('overall').textContent = labels[data.status]||data.status;
|
||||
$('overall').className = 'badge '+data.status;
|
||||
$('last-extraction').textContent = data.collector.last_attempt?time(data.collector.last_attempt):'Waiting';
|
||||
$('last-extraction-date').textContent = data.collector.last_attempt?date(data.collector.last_attempt):'No observation received yet';
|
||||
$('last-success').textContent = 'Last successful: '+(data.collector.last_success?date(data.collector.last_success)+' '+time(data.collector.last_success):'not yet recorded');
|
||||
const yesterday = data.days[1];
|
||||
if(yesterday) {
|
||||
$('yesterday').replaceChildren(document.createTextNode(number(yesterday.complete)+' '));
|
||||
addText($('yesterday'),'span','/ '+number(yesterday.expected));
|
||||
$('yesterday-detail').textContent = labels[yesterday.status]+' · '+date(yesterday.date+'T12:00:00Z');
|
||||
}
|
||||
$('pending').textContent = number(data.archive.pending);
|
||||
$('upload-detail').textContent = !data.archive.upload_enabled?'Uploads not configured':data.archive.pending?'Saved locally, awaiting delivery':'All observations acknowledged';
|
||||
$('last-upload').textContent = 'Last acknowledgement: '+(data.archive.last_upload?date(data.archive.last_upload)+' '+time(data.archive.last_upload):'none yet');
|
||||
$('samples').textContent = number(data.archive.samples);
|
||||
$('retention').textContent = `Acknowledged readings kept for ${number(data.archive.retention_days)} days · pending readings retained`;
|
||||
$('version').textContent = 'solar-relay '+data.version;
|
||||
if(!selected || !data.days.some(day=>day.date===selected)) selected=(data.days.find(day=>day.samples>0)||yesterday||data.days[0]).date;
|
||||
renderDays();
|
||||
renderSlots();
|
||||
}
|
||||
async function refresh() {
|
||||
if(busy) return;
|
||||
busy=true;
|
||||
$('refresh').disabled=true;
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(()=>controller.abort(),12000);
|
||||
try {
|
||||
const response = await fetch('api/status?days='+encodeURIComponent($('history').value),{cache:'no-store',signal:controller.signal});
|
||||
if(!response.ok) throw new Error('Status request failed');
|
||||
render(await response.json());
|
||||
$('notice').hidden=true;
|
||||
} catch {
|
||||
$('notice').textContent = current?'The relay cannot be reached. Showing the last received status; automatic refresh will retry.':'Waiting for the relay status service. Check that solar-relay.service is running and the reverse proxy forwards this path to port 8765.';
|
||||
$('notice').hidden=false;
|
||||
$('overall').textContent='Connection lost';
|
||||
$('overall').className='badge stale';
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
busy=false;
|
||||
$('refresh').disabled=false;
|
||||
}
|
||||
}
|
||||
$('refresh').addEventListener('click',refresh);
|
||||
$('history').addEventListener('change',refresh);
|
||||
document.addEventListener('visibilitychange',()=>{if(!document.hidden) refresh();});
|
||||
setInterval(()=>{if(!document.hidden) refresh();},15000);
|
||||
refresh();
|
||||
})();
|
||||
Reference in new issue
Block a user