Implement dashboard parsing and delivery sensors

Reads delivery day, next arrival date, both upcoming carts, box contents,
add-ons and order totals from a single /p/dashboard/details fetch. Adds five
sensors and parser tests over a synthetic fixture mirroring the live markup.

Records two portal quirks: every /p/ path returns 200 so signed-in state is
detected by a Sign Out control, and cart-contents-skipped marks the locked
cart rather than a skipped order.
This commit is contained in:
flan
2026-08-03 17:23:35 +00:00
parent eb50a9ecc1
commit e1d8ba720c
10 changed files with 588 additions and 123 deletions
+253 -54
View File
@@ -1,7 +1,12 @@
"""HTTP client for the freshharvest.com customer portal.
"""HTTP client and HTML parser for the freshharvest.com customer portal.
The site is server-rendered (no JSON API), so this client drives the same form
flow a browser does and parses HTML out the other side.
The site is server-rendered with no JSON API, so this client drives the same
form flow a browser does and parses the dashboard markup.
Everything the integration needs lives on a single page,
``/p/dashboard/details``: the account's delivery day and next arrival date, the
upcoming carts with their contents, and the order totals. Keeping this to one
request per refresh is deliberate — the portal is a small business's website.
"""
from __future__ import annotations
@@ -12,6 +17,7 @@ from dataclasses import dataclass, field
from datetime import date
import aiohttp
from bs4 import BeautifulSoup
from yarl import URL
_LOGGER = logging.getLogger(__name__)
@@ -19,18 +25,26 @@ _LOGGER = logging.getLogger(__name__)
BASE = URL("https://freshharvest.com")
LOGIN_FORM = "/s/popup/login"
LOGIN_SUBMIT = "/s/submit/login"
DASHBOARD = "/p/dashboard/details"
# Browser UA: the portal serves a reduced/blocked page to obvious scripts.
# The portal serves a reduced page to obviously-scripted clients.
USER_AGENT = (
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/126.0 Safari/537.36"
)
# Hidden anti-replay fields, minted fresh on every GET of the login form and
# only valid for the session cookie they were issued against.
# Hidden anti-replay fields, minted per session on each GET of the login form.
_HIDDEN_RE = re.compile(
r"name='(?P<name>LoginSecurity|SubmitToken)'[^>]*value='(?P<value>[^']*)'"
)
_NEXT_ARRIVING_RE = re.compile(
r"Your deliveries are\s*(?P<day>[A-Za-z]+)\.\s*"
r"Next Arriving:\s*(?P<date>[A-Za-z]+\s+\d+(?:st|nd|rd|th)?,\s*\d{4})",
re.IGNORECASE,
)
_TAB_DATE_RE = re.compile(r"(?P<month>\d{1,2})/(?P<day>\d{1,2})")
_ORDINAL_RE = re.compile(r"(\d+)(?:st|nd|rd|th)", re.IGNORECASE)
_MONEY_RE = re.compile(r"-?\$\s*([\d,]+\.\d{2})")
class FreshHarvestError(Exception):
@@ -41,16 +55,210 @@ class FreshHarvestAuthError(FreshHarvestError):
"""Credentials rejected, or the session expired and could not be renewed."""
@dataclass
class Delivery:
"""A scheduled delivery."""
def _money(text: str | None) -> float | None:
"""Pull a dollar amount out of a label. Non-amounts (e.g. 'Add Tip') -> None."""
if not text:
return None
match = _MONEY_RE.search(text)
if not match:
return None
return float(match.group(1).replace(",", ""))
def _text(node, selector: str) -> str | None:
found = node.select_one(selector)
if found is None:
return None
value = found.get_text(" ", strip=True)
return value or None
@dataclass
class OrderItem:
"""One line in a cart: a produce-box component or a paid add-on."""
name: str
quantity: int | None = None
unit: str | None = None
price: float | None = None
@dataclass
class DeliveryOrder:
"""A single upcoming delivery."""
delivery_id: str
delivery_date: date | None = None
window: str | None = None
status: str | None = None
box_name: str | None = None
box_price: float | None = None
items: list[OrderItem] = field(default_factory=list)
addons: list[OrderItem] = field(default_factory=list)
subtotal: float | None = None
tax: float | None = None
delivery_fee: float | None = None
total: float | None = None
items: list[str] = field(default_factory=list)
cutoff: str | None = None
# Non-empty only while the order can still be changed, e.g. "Shop tomorrow"
# or "Shop thru Sunday 8/9". Empty once the order is locked for packing.
shop_window: str | None = None
@property
def is_open(self) -> bool:
"""Whether the order can still be customized."""
return bool(self.shop_window)
@property
def all_items(self) -> list[OrderItem]:
return [*self.items, *self.addons]
@dataclass
class AccountSnapshot:
"""Everything read from one dashboard fetch."""
next_delivery: date | None = None
delivery_day: str | None = None
orders: list[DeliveryOrder] = field(default_factory=list)
@property
def next_order(self) -> DeliveryOrder | None:
"""The cart for the next arriving delivery.
Matched against the account's stated next-arrival date. Note that this
order is usually already locked: the *open* cart is the one after it.
"""
if self.next_delivery is not None:
for order in self.orders:
if order.delivery_date == self.next_delivery:
return order
dated = [o for o in self.orders if o.delivery_date]
return min(dated, key=lambda o: o.delivery_date) if dated else None
@property
def open_order(self) -> DeliveryOrder | None:
"""The earliest cart that can still be customized."""
candidates = [o for o in self.orders if o.is_open and o.delivery_date]
return min(candidates, key=lambda o: o.delivery_date) if candidates else None
def _parse_full_date(raw: str) -> date | None:
"""Parse 'August 4th, 2026'."""
from datetime import datetime
cleaned = _ORDINAL_RE.sub(r"\1", raw).replace(",", " ")
cleaned = re.sub(r"\s+", " ", cleaned).strip()
try:
return datetime.strptime(cleaned, "%B %d %Y").date()
except ValueError:
_LOGGER.debug("could not parse date %r", raw)
return None
def _parse_tab_date(raw: str, anchor: date | None) -> date | None:
"""Parse a cart tab label like 'Tue 8/11', which carries no year.
The year is taken from ``anchor`` (the account's next-arrival date), rolling
forward when the tab falls far enough behind it to be a December/January
boundary rather than a genuinely earlier delivery.
"""
match = _TAB_DATE_RE.search(raw or "")
if not match:
return None
month, day = int(match["month"]), int(match["day"])
year = anchor.year if anchor else date.today().year
try:
candidate = date(year, month, day)
except ValueError:
return None
if anchor and (anchor - candidate).days > 180:
try:
candidate = date(year + 1, month, day)
except ValueError:
return None
return candidate
def parse_dashboard(html: str) -> AccountSnapshot:
"""Parse ``/p/dashboard/details`` into a snapshot."""
soup = BeautifulSoup(html, "html.parser")
snapshot = AccountSnapshot()
account = soup.select_one(".account")
if account is not None:
match = _NEXT_ARRIVING_RE.search(account.get_text(" ", strip=True))
if match:
snapshot.delivery_day = match["day"]
snapshot.next_delivery = _parse_full_date(match["date"])
# Tab labels carry the delivery dates; the cart bodies carry everything else.
tab_dates = {
node.get("data-cart-select"): node.get_text(strip=True)
for node in soup.select(".cart-selector-options-wrapper > div[data-cart-select]")
}
for cart in soup.select("div.cart-contents[data-cart-select]"):
delivery_id = cart.get("data-cart-select")
order = DeliveryOrder(
delivery_id=delivery_id,
delivery_date=_parse_tab_date(
tab_dates.get(delivery_id, ""), snapshot.next_delivery
),
box_name=_text(cart, ".cart-basket-columns h6"),
box_price=_money(_text(cart, ".cart-basket-columns .item-total")),
shop_window=_text(cart, ".cart-customize-wrapper"),
)
for row in cart.select(".basket-item"):
name = _text(row, ".basket-item-name")
if not name:
continue
quantity = _text(row, ".basket-item-quantity")
order.items.append(
OrderItem(
name=name,
quantity=int(quantity) if (quantity or "").isdigit() else None,
unit=_text(row, ".basket-item-uom"),
)
)
for row in cart.select(".cart-item"):
classes = row.get("class") or []
if "basket-item" in classes:
continue
name = _text(row, ".item-name")
if not name:
continue
quantity = _text(row, ".item-order-quantity")
order.addons.append(
OrderItem(
name=name,
# .item-total is the extended price: 4 muffins -> $17.96.
price=_money(_text(row, ".item-total")),
quantity=int(quantity) if (quantity or "").isdigit() else None,
unit=_text(row, ".item-uom"),
)
)
totals = soup.select_one(f"#OrderTotals-{delivery_id}")
if totals is not None:
labels = totals.select(".summary-item.label")
values = totals.select(".summary-item.value")
for label, value in zip(labels, values):
key = label.get_text(" ", strip=True).lower()
amount = _money(value.get_text(" ", strip=True))
if key.startswith("order total"):
order.total = amount
elif key.startswith("subtotal"):
order.subtotal = amount
elif key.startswith("tax"):
order.tax = amount
elif key.startswith("delivery"):
order.delivery_fee = amount
snapshot.orders.append(order)
if not snapshot.orders and snapshot.next_delivery is None:
raise FreshHarvestError("dashboard markup not recognised")
return snapshot
class FreshHarvestClient:
@@ -75,7 +283,7 @@ class FreshHarvestClient:
"""Run the two-step handshake: fetch tokens, then post credentials.
The tokens are bound to the session cookie issued by the same GET, so
the fetch and the post cannot be split across sessions or cached.
the fetch and the post must share a cookie jar.
"""
form = await self._get(LOGIN_FORM)
hidden = {m["name"]: m["value"] for m in _HIDDEN_RE.finditer(form)}
@@ -92,53 +300,44 @@ class FreshHarvestClient:
"SubmitToken": hidden["SubmitToken"],
"Redirect": "",
}
async with self._session.post(
BASE.join(URL(LOGIN_SUBMIT)),
data=payload,
headers={"User-Agent": USER_AGENT},
) as resp:
resp.raise_for_status()
body = await resp.text()
try:
async with self._session.post(
BASE.join(URL(LOGIN_SUBMIT)),
data=payload,
headers={"User-Agent": USER_AGENT},
) as resp:
resp.raise_for_status()
body = await resp.text()
except aiohttp.ClientError as err:
raise FreshHarvestError(f"login request failed: {err}") from err
if not self._looks_authenticated(body):
if not self._signed_in(body):
raise FreshHarvestAuthError("login rejected")
self._authenticated = True
@staticmethod
def _looks_authenticated(body: str) -> bool:
"""Distinguish a good login from a rejected one.
def _signed_in(body: str) -> bool:
"""A signed-in page carries a Sign Out control; a signed-out one does not.
NOTE: provisional. Every /p/* path returns 200 even for nonsense URLs,
so an HTTP status is not a signal here. Needs confirming against a real
authenticated response before this integration can be trusted.
HTTP status is useless here — the portal answers 200 for every /p/ path,
including invented ones.
"""
lowered = body.lower()
if "invalid" in lowered or "incorrect" in lowered:
return False
return "sign out" in lowered or "log out" in lowered
return "sign out" in body.lower()
async def async_get_next_delivery(self) -> Delivery:
"""Return the account's next scheduled delivery.
UNIMPLEMENTED. The portal serves a catch-all 200 for every /p/* path
when signed out, so the real account pages could not be located or
parsed without an authenticated session. Fill this in against a live
session rather than guessing at selectors.
"""
raise FreshHarvestError(
"delivery parsing is not implemented yet: the portal HTML has not "
"been mapped against a signed-in session"
)
async def async_fetch(self, path: str) -> str:
"""Fetch a portal page, re-authenticating once if the session lapsed."""
async def async_get_snapshot(self) -> AccountSnapshot:
"""Fetch and parse the dashboard, re-authenticating once if needed."""
if not self._authenticated:
await self.async_login()
body = await self._get(path)
if not self._looks_authenticated(body):
self._authenticated = False
await self.async_login()
body = await self._get(path)
if not self._looks_authenticated(body):
raise FreshHarvestAuthError(f"could not hold a session for {path}")
return body
try:
body = await self._get(DASHBOARD)
if not self._signed_in(body):
self._authenticated = False
await self.async_login()
body = await self._get(DASHBOARD)
if not self._signed_in(body):
raise FreshHarvestAuthError("could not hold a signed-in session")
except aiohttp.ClientError as err:
raise FreshHarvestError(f"dashboard request failed: {err}") from err
return parse_dashboard(body)
+10 -5
View File
@@ -9,14 +9,19 @@ from homeassistant.core import HomeAssistant
from homeassistant.exceptions import ConfigEntryAuthFailed
from homeassistant.helpers.update_coordinator import DataUpdateCoordinator, UpdateFailed
from .api import Delivery, FreshHarvestAuthError, FreshHarvestClient, FreshHarvestError
from .api import (
AccountSnapshot,
FreshHarvestAuthError,
FreshHarvestClient,
FreshHarvestError,
)
from .const import DOMAIN, UPDATE_INTERVAL
_LOGGER = logging.getLogger(__name__)
class FreshHarvestCoordinator(DataUpdateCoordinator[Delivery]):
"""Fetch the account's next delivery."""
class FreshHarvestCoordinator(DataUpdateCoordinator[AccountSnapshot]):
"""Fetch the account dashboard on a slow interval."""
def __init__(
self, hass: HomeAssistant, entry: ConfigEntry, client: FreshHarvestClient
@@ -30,9 +35,9 @@ class FreshHarvestCoordinator(DataUpdateCoordinator[Delivery]):
)
self.client = client
async def _async_update_data(self) -> Delivery:
async def _async_update_data(self) -> AccountSnapshot:
try:
return await self.client.async_get_next_delivery()
return await self.client.async_get_snapshot()
except FreshHarvestAuthError as err:
raise ConfigEntryAuthFailed(str(err)) from err
except FreshHarvestError as err:
+1 -1
View File
@@ -7,6 +7,6 @@
"integration_type": "service",
"iot_class": "cloud_polling",
"issue_tracker": "https://git.onetick.ninja/flan/ha-freshharvest/issues",
"requirements": [],
"requirements": ["beautifulsoup4>=4.12"],
"version": "0.1.0"
}
+63 -19
View File
@@ -4,11 +4,13 @@ from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
from homeassistant.components.sensor import (
SensorDeviceClass,
SensorEntity,
SensorEntityDescription,
SensorStateClass,
)
from homeassistant.core import HomeAssistant
from homeassistant.helpers.device_registry import DeviceEntryType, DeviceInfo
@@ -16,16 +18,30 @@ from homeassistant.helpers.entity_platform import AddEntitiesCallback
from homeassistant.helpers.update_coordinator import CoordinatorEntity
from . import FreshHarvestConfigEntry
from .api import Delivery
from .api import AccountSnapshot, DeliveryOrder
from .const import DOMAIN
from .coordinator import FreshHarvestCoordinator
def _items_attrs(order: DeliveryOrder | None) -> dict[str, Any] | None:
if order is None:
return None
return {
"box": order.box_name,
"produce": [
" ".join(part for part in (str(i.quantity or ""), i.name, i.unit) if part)
for i in order.items
],
"add_ons": [a.name for a in order.addons],
}
@dataclass(frozen=True, kw_only=True)
class FreshHarvestSensorDescription(SensorEntityDescription):
"""Describes a Fresh Harvest sensor."""
value_fn: Callable[[Delivery], object]
value_fn: Callable[[AccountSnapshot], Any]
attrs_fn: Callable[[AccountSnapshot], dict[str, Any] | None] | None = None
SENSORS: tuple[FreshHarvestSensorDescription, ...] = (
@@ -33,24 +49,52 @@ SENSORS: tuple[FreshHarvestSensorDescription, ...] = (
key="next_delivery",
translation_key="next_delivery",
device_class=SensorDeviceClass.DATE,
value_fn=lambda d: d.delivery_date,
value_fn=lambda s: s.next_delivery,
attrs_fn=lambda s: {
"delivery_day": s.delivery_day,
"box": s.next_order.box_name if s.next_order else None,
},
),
FreshHarvestSensorDescription(
key="order_total",
translation_key="order_total",
key="next_delivery_total",
translation_key="next_delivery_total",
device_class=SensorDeviceClass.MONETARY,
state_class=SensorStateClass.TOTAL,
native_unit_of_measurement="USD",
value_fn=lambda d: d.total,
value_fn=lambda s: s.next_order.total if s.next_order else None,
attrs_fn=lambda s: None
if s.next_order is None
else {
"subtotal": s.next_order.subtotal,
"tax": s.next_order.tax,
"delivery_fee": s.next_order.delivery_fee,
},
),
FreshHarvestSensorDescription(
key="status",
translation_key="status",
value_fn=lambda d: d.status,
key="next_delivery_items",
translation_key="next_delivery_items",
state_class=SensorStateClass.MEASUREMENT,
native_unit_of_measurement="items",
value_fn=lambda s: len(s.next_order.all_items) if s.next_order else None,
attrs_fn=lambda s: _items_attrs(s.next_order),
),
FreshHarvestSensorDescription(
key="item_count",
translation_key="item_count",
value_fn=lambda d: len(d.items) if d.items else None,
key="open_order_delivery",
translation_key="open_order_delivery",
device_class=SensorDeviceClass.DATE,
value_fn=lambda s: s.open_order.delivery_date if s.open_order else None,
attrs_fn=lambda s: None
if s.open_order is None
else {
"total": s.open_order.total,
"box": s.open_order.box_name,
},
),
FreshHarvestSensorDescription(
key="shop_window",
translation_key="shop_window",
value_fn=lambda s: (s.open_order.shop_window if s.open_order else None)
or "closed",
),
)
@@ -68,7 +112,7 @@ async def async_setup_entry(
class FreshHarvestSensor(CoordinatorEntity[FreshHarvestCoordinator], SensorEntity):
"""A value read off the Fresh Harvest portal."""
"""A value read off the Fresh Harvest dashboard."""
_attr_has_entity_name = True
entity_description: FreshHarvestSensorDescription
@@ -87,17 +131,17 @@ class FreshHarvestSensor(CoordinatorEntity[FreshHarvestCoordinator], SensorEntit
name="Fresh Harvest",
manufacturer="Fresh Harvest",
entry_type=DeviceEntryType.SERVICE,
configuration_url="https://freshharvest.com/",
configuration_url="https://freshharvest.com/p/dashboard/details",
)
@property
def native_value(self):
def native_value(self) -> Any:
"""Return the sensor value."""
return self.entity_description.value_fn(self.coordinator.data)
@property
def extra_state_attributes(self) -> dict[str, object] | None:
"""Expose box contents on the item-count sensor."""
if self.entity_description.key != "item_count":
def extra_state_attributes(self) -> dict[str, Any] | None:
"""Return the sensor's extra attributes."""
if self.entity_description.attrs_fn is None:
return None
return {"items": self.coordinator.data.items}
return self.entity_description.attrs_fn(self.coordinator.data)
+4 -3
View File
@@ -20,9 +20,10 @@
"entity": {
"sensor": {
"next_delivery": { "name": "Next delivery" },
"order_total": { "name": "Order total" },
"status": { "name": "Order status" },
"item_count": { "name": "Items in box" }
"next_delivery_total": { "name": "Next delivery total" },
"next_delivery_items": { "name": "Next delivery items" },
"open_order_delivery": { "name": "Open order delivery" },
"shop_window": { "name": "Shopping window" }
}
}
}