feat(config)!: drop legacy layouts and support marshmallow 3 (#132)

Dropping the legacy microgrid config layouts (breaking) and fixing
config loading under marshmallow 3.

## Breaking change

The top-level microgrid-ID and `meta`-nested layouts are no longer
migrated on load. A file with no `assets` table now loads as empty and
contributes nothing to a merge, instead of having its top-level keys
read as microgrid IDs. This lets a mixed list of files (some carrying an
`[app]` table for their own config, some an `assets` table) be passed to
`AssetsConfig.load_from_files` / `load_configs`, which reads only the
`assets`-bearing ones. Rebuild any legacy-layout files with
`generate-config`. `_STEPS` is now empty; `assets.version` validation
and stamping remain.

 ## Bug fix

`tomllib` parses TOML datetimes to native `datetime` objects, which
marshmallow 3's `DateTime` field rejected (marshmallow 4 accepts them).
A `TomlAwareDateTime` field passes an already-parsed `datetime` through
and otherwise defers to the parent for ISO strings, applied to every
bare datetime config field sourced from TOML.

  ## Build

Now declares `marshmallow` as a direct dependency. It was already used
transitively via `marshmallow-dataclass` but never declared, so the
minimum-version job left it free to resolve to marshmallow 4. Declaring
it pins the lower bound so the marshmallow-3 path is tested.

Author: cwasicki

RELEASE_NOTES.md

@@ -8,6 +8,14 @@ validate, patch, and query merged config files.
 
 ## Upgrading
 
+- The legacy config layouts are no longer migrated on load: top-level microgrid
+  IDs and `meta`-nested microgrid fields are gone. A file with no `assets` table
+  now loads as empty and contributes nothing to a merge, rather than having its
+  top-level keys read as microgrid IDs. This lets a mixed list of files (some
+  carrying an `[app]` table for their own config, some an `assets` table) be
+  passed to `AssetsConfig.load_from_files` / `load_configs`, which reads only the
+  `assets`-bearing ones. Rebuild any legacy-layout files with `generate-config`.
+
 - `generate-config` now writes the `assets.microgrids` layout and stamps
   `assets.version`, instead of the legacy layout. Regenerate configs to get the
   current format; generated files then load back without a migration.
@@ -40,3 +48,10 @@ validate, patch, and query merged config files.
 - `generate-config --inplace` refreshes managed values while preserving
   formatting; `--fill-missing` only adds absent values. Generated patches are
   validated before writing.
+
+## Bug Fixes
+
+- Config files with validity periods now load under marshmallow 3, not only
+  marshmallow 4. `tomllib` yields native `datetime` objects, which marshmallow
+  3's `DateTime` field rejected. `marshmallow` is now a direct dependency so the
+  minimum-version test exercises this path.

pyproject.toml

@@ -26,6 +26,10 @@ classifiers = [
 ]
 requires-python = ">= 3.11, < 4"
 dependencies = [
+  # Used transitively via marshmallow-dataclass (every config schema is a
+  # `marshmallow.Schema`); declared directly so `pytest_min` pins its lower
+  # bound and the minimum-version path is actually tested.
+  "marshmallow >= 3.18.0, < 5",
   "marshmallow-dataclass >= 8.7.1, < 9",
   "typing-extensions >= 4.14.1, < 5",
   "frequenz-microgrid-component-graph >= 0.5.0, < 0.6",

src/frequenz/gridpool/cli/_dump_config.py

@@ -17,6 +17,7 @@
 fields are done here, since TOML has no null and `tomlkit` rejects it.
 """
 
+from dataclasses import asdict
 from typing import Any
 
 import tomlkit
@@ -86,13 +87,10 @@ def dump_map(configs: dict[int, MicrogridConfig]) -> str:
         The TOML representation as a string, with one blank line between
         microgrids and entries sorted by numeric microgrid ID.
     """
-    schema = MicrogridConfig.Schema()
     doc = tomlkit.document()
     doc.append(tomlkit.key(["assets", "version"]), _CURRENT_VERSION)
     for mid in sorted(configs):
-        dumped = schema.dump(configs[mid])
-        assert isinstance(dumped, dict)
-        leaves = _iter_leaves([*_MICROGRID_PREFIX, str(mid)], dumped)
+        leaves = _iter_leaves([*_MICROGRID_PREFIX, str(mid)], asdict(configs[mid]))
         if not leaves:
             continue
         doc.add(tomlkit.nl())

src/frequenz/gridpool/cli/_patch_config.py

@@ -10,6 +10,7 @@
 """
 
 from collections.abc import Mapping
+from dataclasses import asdict
 from pathlib import Path
 from typing import Any
 
@@ -54,7 +55,6 @@ def patch_text(
     """
     doc = tomlkit.parse(original)
     _reject_legacy_layout(doc)
-    schema = MicrogridConfig.Schema()
 
     # Leaves needing a whole new sub-table can't be inserted via item assignment
     # without tomlkit falling back to bracket-header syntax, so they are spliced
@@ -63,9 +63,7 @@ def patch_text(
 
     for microgrid_id, cfg in configs.items():
         mid = str(microgrid_id)
-        dumped = schema.dump(cfg)
-        assert isinstance(dumped, dict)
-        leaves = _iter_leaves([], dumped)
+        leaves = _iter_leaves([], asdict(cfg))
         if not leaves:
             continue
 

src/frequenz/gridpool/config/_assets.py

@@ -443,23 +443,28 @@ def _read_assets_table(cls, config_path: Path) -> dict[str, Any]:
         """Read the raw `assets` table from a TOML file.
 
         The document is migrated to the current format before its `assets`
-        table is returned for merging.
+        table is returned for merging. A file with no `assets` table contributes
+        nothing to the merge, so consumers can pass a mixed list of files and
+        gridpool reads only the `assets`-bearing ones.
 
         Args:
             config_path: The path to the TOML configuration file.
 
         Returns:
-            The raw `assets` table, unvalidated, for merging before it is loaded.
+            The raw `assets` table, unvalidated, for merging before it is loaded,
+            or an empty table if the file has none.
 
         Raises:
-            TypeError: If `assets` is not a table.
+            TypeError: If `assets` is present but not a table.
         """
         with config_path.open("rb") as f:
             data: dict[str, Any] = tomllib.load(f)
 
         data = migrate(data, config_path)
 
-        assets = data["assets"]
+        assets = data.get("assets")
+        if assets is None:
+            return {}
         if not isinstance(assets, dict):
             raise TypeError(
                 f"{config_path}: `assets` must be a table, got {type(assets)}"

src/frequenz/gridpool/config/_fields.py

@@ -0,0 +1,29 @@
+# License: MIT
+# Copyright © 2026 Frequenz Energy-as-a-Service GmbH
+
+"""Reusable marshmallow fields for config data models."""
+
+from datetime import datetime
+from typing import Any
+
+import marshmallow
+
+
+class TomlAwareDateTime(marshmallow.fields.DateTime):
+    """A `DateTime` that also accepts an already-parsed `datetime`.
+
+    marshmallow 3's `DateTime` deserialises only ISO strings, but `tomllib`
+    yields native `datetime` objects before the value reaches marshmallow, so
+    those inputs must pass through unchanged. marshmallow 4 accepts them itself;
+    this keeps both behaving the same.
+    """
+
+    def _deserialize(self, value: Any, *args: Any, **kwargs: Any) -> datetime:
+        if isinstance(value, datetime):
+            return value
+        return super()._deserialize(value, *args, **kwargs)
+
+
+def toml_datetime_metadata() -> dict[str, Any]:
+    """Field metadata mapping a TOML-sourced `datetime` to `TomlAwareDateTime`."""
+    return {"marshmallow_field": TomlAwareDateTime(allow_none=True)}

src/frequenz/gridpool/config/_microgrid.py

@@ -11,6 +11,8 @@
 from marshmallow import Schema
 from marshmallow_dataclass import dataclass
 
+from ._fields import toml_datetime_metadata
+
 ComponentType = Literal["grid", "pv", "battery", "consumption", "chp", "ev"]
 """Valid component types."""
 
@@ -98,10 +100,10 @@ def is_valid_type(cls, ctype: str) -> bool:
 class PVConfig:
     """Configuration of a PV system in a microgrid."""
 
-    start_time: datetime | None = None
+    start_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Start time of the PV system installation."""
 
-    end_time: datetime | None = None
+    end_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """End time of the PV system installation."""
 
     peak_power: float | None = None
@@ -119,10 +121,10 @@ class WindConfig:
     # pylint: disable=too-many-instance-attributes
     """Configuration of a wind turbine in a microgrid."""
 
-    start_time: datetime | None = None
+    start_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Start time of the wind turbine installation."""
 
-    end_time: datetime | None = None
+    end_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """End time of the wind turbine installation."""
 
     turbine_model: str | None = None
@@ -151,10 +153,10 @@ class WindConfig:
 class BatteryConfig:
     """Configuration of a battery in a microgrid."""
 
-    start_time: datetime | None = None
+    start_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Start time of the battery installation."""
 
-    end_time: datetime | None = None
+    end_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """End time of the battery installation."""
 
     capacity: float | None = None
@@ -187,10 +189,10 @@ class MicrogridConfig:
     altitude: float | None = None
     """Geographic altitude of the microgrid."""
 
-    start_time: datetime | None = None
+    start_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Start time of the microgrid operation."""
 
-    end_time: datetime | None = None
+    end_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """End time of the microgrid operation."""
 
     pv: dict[str, PVConfig] | None = None

src/frequenz/gridpool/config/_migrations.py

@@ -3,18 +3,16 @@
 
 """Format migrations for raw config documents.
 
-Each step rewrites the raw `dict` a file parses to, before it is loaded into the
-typed model. Steps are self-detecting and idempotent, so the `version` field
-does not gate whether a step runs; it only records how old a file may be, which
-bounds when a step may be dropped.
+Loading validates `assets.version` and stamps the current version, so a file
+written for a newer reader is rejected rather than silently misread. Each
+migration step rewrites the raw `dict` before it is loaded into the typed
+model; steps are self-detecting and idempotent, so the `version` field does not
+gate whether a step runs. No legacy layouts remain, so `_STEPS` is empty.
 """
 
-import logging
 from pathlib import Path
 from typing import Any, Callable
 
-_logger = logging.getLogger(__name__)
-
 _CURRENT_VERSION = 1
 
 
@@ -27,7 +25,7 @@ def migrate(raw: dict[str, Any], source: Path) -> dict[str, Any]:
 
     Args:
         raw: The raw document as parsed from a file.
-        source: The file it was parsed from, for warnings and errors.
+        source: The file it was parsed from, named in error messages.
 
     Returns:
         The document in the current format.
@@ -56,84 +54,5 @@ def migrate(raw: dict[str, Any], source: Path) -> dict[str, Any]:
     return raw
 
 
-def _nest_top_level_microgrids(raw: dict[str, Any], source: Path) -> dict[str, Any]:
-    """Wrap deprecated top-level microgrid entries under `assets.microgrids`.
-
-    Args:
-        raw: The raw document.
-        source: The file it was parsed from.
-
-    Returns:
-        The document with its entries under `assets`.
-
-    Raises:
-        ValueError: If both layouts are present, a half-migrated file rather
-            than a merge.
-    """
-    if "assets" in raw:
-        stray = sorted(k for k in raw if k != "assets")
-        if stray:
-            raise ValueError(
-                f"{source}: keys {stray} sit outside `assets` while the file "
-                "already has an `assets` table; move them under `assets.microgrids`."
-            )
-        return raw
-
-    _logger.warning(
-        "%s: top-level microgrid IDs are deprecated, "
-        "nest the entries under `assets.microgrids` instead.",
-        source,
-    )
-    return {"assets": {"microgrids": dict(raw)}}
-
-
-def _lift_microgrid_meta(raw: dict[str, Any], source: Path) -> dict[str, Any]:
-    """Merge a deprecated nested `meta` table onto its microgrid entry.
-
-    Earlier files nested a microgrid's fields under `meta`; they now sit
-    directly on the entry.
-
-    Args:
-        raw: The raw document.
-        source: The file it was parsed from.
-
-    Returns:
-        The document with microgrid fields directly on each entry.
-
-    Raises:
-        TypeError: If a `meta` value is not a table.
-        ValueError: If a field is set both under `meta` and on the entry.
-    """
-    assets = raw.get("assets")
-    if not isinstance(assets, dict):
-        return raw
-    microgrids = assets.get("microgrids")
-    if not isinstance(microgrids, dict):
-        return raw
-    for mid, entry in microgrids.items():
-        if not isinstance(entry, dict) or "meta" not in entry:
-            continue
-        meta = entry.pop("meta")
-        if not isinstance(meta, dict):
-            raise TypeError(f"{source}: microgrid {mid} `meta` must be a table")
-        _logger.warning(
-            "%s: microgrid %s nests fields under `meta`; that is deprecated, "
-            "set them directly on the entry instead.",
-            source,
-            mid,
-        )
-        for key, value in meta.items():
-            if key in entry:
-                raise ValueError(
-                    f"{source}: microgrid {mid} sets `{key}` both under `meta` "
-                    "and directly; keep only the direct one"
-                )
-            entry[key] = value
-    return raw
-
-
-_STEPS: list[tuple[int, Callable[[dict[str, Any], Path], dict[str, Any]]]] = [
-    (0, _nest_top_level_microgrids),
-    (0, _lift_microgrid_meta),
-]
+_STEPS: list[tuple[int, Callable[[dict[str, Any], Path], dict[str, Any]]]] = []
 """Migration steps as `(from_version, step)`, oldest first."""

src/frequenz/gridpool/config/_topology.py

@@ -26,6 +26,8 @@
 from marshmallow import Schema
 from marshmallow_dataclass import dataclass
 
+from ._fields import toml_datetime_metadata
+
 _EIC_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ-"
 _MARKET_AREA_EU_DE = 101
 
@@ -165,10 +167,10 @@ class ValidityConfig:
     participation: MarketParticipationType | None = None
     """The market use case, unset for a plain validity window."""
 
-    start: datetime | None = None
+    start: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Inclusive start of the period."""
 
-    end: datetime | None = None
+    end: datetime | None = field(default=None, metadata=toml_datetime_metadata())
     """Exclusive end of the period."""
 
     Schema: ClassVar[Type[Schema]] = Schema