89 lines
3.9 KiB
Python
89 lines
3.9 KiB
Python
"""Five-minute extraction completeness, distinct from continuous observed coverage."""
|
|
|
|
from collections import Counter
|
|
from datetime import datetime, timedelta, timezone
|
|
import math
|
|
from zoneinfo import ZoneInfo
|
|
|
|
from .config import utc_text
|
|
|
|
BUCKET_MS = 300_000
|
|
MAX_SOURCE_AGE_MS = 90_000
|
|
|
|
|
|
def milliseconds(value):
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
raise ValueError("Observation timestamp has no timezone")
|
|
return round(parsed.timestamp() * 1000)
|
|
|
|
|
|
def _finite(value):
|
|
return not isinstance(value, bool) and isinstance(value, (int, float)) and math.isfinite(value)
|
|
|
|
|
|
def sample_coverage(sample):
|
|
observed = milliseconds(sample["observed_at"])
|
|
if "history_interval" in sample:
|
|
# Cloud history has measured interval energy, but no cumulative meter or relay history.
|
|
return (sample["id"], sample["source_id"], observed, observed, 1, observed, 0, observed, 0)
|
|
readings = sample.get("readings", {})
|
|
result = [sample["id"], sample["source_id"], observed]
|
|
for source, fields in [
|
|
("powerflow", ["production_ac_w", "consumption_w", "grid_import_w", "grid_export_w"]),
|
|
("meter", ["grid_import_total_wh", "grid_export_total_wh"]),
|
|
("boiler", []),
|
|
]:
|
|
try:
|
|
stamp = milliseconds(sample["source_timestamps"][source])
|
|
valid = source not in sample.get("errors", {}) and abs(stamp - observed) <= MAX_SOURCE_AGE_MS
|
|
except (KeyError, ValueError, TypeError, AttributeError):
|
|
stamp, valid = observed, False
|
|
valid = valid and all(_finite(readings.get(field)) for field in fields)
|
|
if source == "boiler":
|
|
valid = valid and isinstance(readings.get("boiler_enabled"), bool)
|
|
result.extend([stamp if valid else observed, int(valid)])
|
|
return tuple(result)
|
|
|
|
|
|
def daily_coverage(rows, timezone_name, days, now=None):
|
|
now = now or datetime.now(timezone.utc)
|
|
zone = ZoneInfo(timezone_name)
|
|
today = now.astimezone(zone).date()
|
|
first = today - timedelta(days=days)
|
|
upper = round(now.timestamp() * 1000)
|
|
slots_by_time = {}
|
|
observations = Counter()
|
|
for row in rows:
|
|
observed = row[0]
|
|
if observed > upper + 1000:
|
|
continue
|
|
date = datetime.fromtimestamp(observed / 1000, zone).date().isoformat()
|
|
observations[date] += 1
|
|
bucket = observed // BUCKET_MS * BUCKET_MS
|
|
mask = sum(1 << index for index in range(3) if row[2 + index * 2])
|
|
slots_by_time[bucket] = slots_by_time.get(bucket, 0) | mask
|
|
result = []
|
|
for offset in range(days + 1):
|
|
date = today - timedelta(days=offset)
|
|
start = datetime.combine(date, datetime.min.time(), zone)
|
|
end = datetime.combine(date + timedelta(days=1), datetime.min.time(), zone)
|
|
start_ms = round(start.timestamp() * 1000)
|
|
expected = round((end.timestamp() - start.timestamp()) * 1000) // BUCKET_MS
|
|
elapsed = expected if date < today else max(0, min(expected, (upper - start_ms) // BUCKET_MS))
|
|
slots = []
|
|
for index in range(expected):
|
|
slots.append(slots_by_time.get(start_ms + index * BUCKET_MS, 0) if index < elapsed else -1)
|
|
complete = sum(mask == 7 for mask in slots)
|
|
count = observations[date.isoformat()]
|
|
status = "in_progress" if date == today else "complete" if complete == expected else "partial" if count else "missing"
|
|
result.append({
|
|
"date": date.isoformat(), "day_start": utc_text(start), "expected": expected, "elapsed": elapsed,
|
|
"complete": complete, "missing": elapsed - complete, "samples": count, "status": status,
|
|
"energy_complete": sum(mask >= 0 and bool(mask & 1) for mask in slots),
|
|
"meter_complete": sum(mask >= 0 and bool(mask & 2) for mask in slots),
|
|
"boiler_complete": sum(mask >= 0 and bool(mask & 4) for mask in slots),
|
|
"slots": slots,
|
|
})
|
|
return result
|