From 22e2c686df5e5eeacc054ee28878be40d842d697 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 17 Sep 2026 20:37:53 +0200 Subject: [PATCH 1/3] Fill the repeated DST hour from the next forecast value The fall-back day has 100 ISPs, but a forecast producer that writes a fixed 96-slot day leaves the four ISPs of the repeated hour without a predicted datapoint of their own. Those positions were submitted as 0.0, which posts a real zero trading position for an hour that genuinely exists. They now reuse the next forecast value of the day instead. The scope is deliberately narrow: an ISP qualifies only when its market local time occurs more than once in the day's ISP grid, which is true of the repeated hour and nothing else. Every other gap still goes out as 0.0, since it means the producer skipped intervals it covered or its horizon ended, and both are already logged. A repeated-hour ISP with nothing left in the day to borrow from also falls back to 0.0, because the API requires a volume on every entry. The interior/trailing gap counters move into the fill pass. The old formula derived interior gaps by subtracting the trailing ones from the total, which would now count every position filled from a neighbour as a gap that went out as 0.0, over-reporting by exactly that many. A filled position is always interior, since filling requires a real value later in the day, so the subtraction could not go negative. It would just be wrong. The collapse test can no longer enumerate every position holding the seeded value: the positions between the two passes of the repeated hour now borrow that same value, so a read and a fill are indistinguishable by value alone. It asserts the two passes directly instead. --- .../distroenergy/DistroEnergyHandler.java | 89 +++++++++++++++---- .../DistroEnergyHandlerTest.groovy | 3 +- 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java index 5f641cf..cc82f81 100644 --- a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java +++ b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java @@ -319,8 +319,14 @@ protected boolean submitDayAheadForecast(LocalDate marketDate) { *

An empty result means "nothing to submit" and cannot be confused with a real day, which is * never shorter than 92 entries. A day beyond the external forecast producer's horizon * legitimately has no data, and submitting it would post a zero net power trading position for a - * day we know nothing about. Within a day that does have a forecast every gap is filled with 0.0, - * interior and trailing alike, because the API requires the complete day. + * day we know nothing about. Within a day that does have a forecast every gap is filled, interior + * and trailing alike, because the API requires the complete day and a volume on every entry. + * + *

A gap goes out as 0.0, which is a real trading position, except in the repeated hour of the + * fall-back day. Those four ISPs exist only because of the transition, and a producer writing a + * fixed 96-slot day leaves them without a row of their own, so they reuse the next forecast value + * of the day rather than trade the hour away. With nothing left in the day to reuse they fall + * back to 0.0 like any other gap. * *

The decision is taken from the ISP grid rather than from whatever the query returned, so a * value belonging to a neighbouring day can never make this day look covered. @@ -340,7 +346,7 @@ static List buildSubmissionData( Map valuesByStorageKey = new HashMap<>(); for (ValueDatapoint datapoint : datapoints) { - // Gap-filled buckets carry a null value; skip them so they fall through to the default below. + // Gap-filled buckets carry a null value; skip them so they fall through to the fill below. if (datapoint.getValue() instanceof Number value) { valuesByStorageKey.put( Instant.ofEpochMilli(datapoint.getTimestamp()).atZone(storageZone).toLocalDateTime(), @@ -351,27 +357,37 @@ static List buildSubmissionData( ZonedDateTime dayStart = marketDate.atStartOfDay(marketZone); ZonedDateTime dayEnd = dayStart.plusDays(1); - List submissionData = new ArrayList<>(); + List isps = new ArrayList<>(); + for (ZonedDateTime isp = dayStart; isp.isBefore(dayEnd); isp = isp.plus(ISP_DURATION)) { + isps.add(isp); + } + + // The repeated hour of the fall-back day is the only stretch whose market local time is not + // unique within the day, so on every other day this stays empty and nothing borrows a value. + Set seenLocalTimes = new HashSet<>(); + Set repeatedLocalTimes = new HashSet<>(); + for (ZonedDateTime isp : isps) { + if (!seenLocalTimes.add(isp.toLocalDateTime())) { + repeatedLocalTimes.add(isp.toLocalDateTime()); + } + } + + Double[] values = new Double[isps.size()]; Set keysRead = new HashSet<>(); - int missing = 0; int collapsed = 0; int lastRealPosition = 0; - int position = 1; - for (ZonedDateTime isp = dayStart; isp.isBefore(dayEnd); isp = isp.plus(ISP_DURATION)) { - LocalDateTime storageKey = isp.withZoneSameInstant(storageZone).toLocalDateTime(); + for (int i = 0; i < isps.size(); i++) { + LocalDateTime storageKey = isps.get(i).withZoneSameInstant(storageZone).toLocalDateTime(); Double value = valuesByStorageKey.get(storageKey); + values[i] = value; - if (value == null) { - missing++; - } else { - lastRealPosition = position; + if (value != null) { + lastRealPosition = i + 1; if (!keysRead.add(storageKey)) { collapsed++; } } - - submissionData.add(new SubmissionData(position++, null, null, value != null ? value : 0.0)); } // Nothing at all was forecast for this day. Hand the caller the empty sentinel and stay silent @@ -380,6 +396,36 @@ static List buildSubmissionData( return List.of(); } + // Walk backwards so every gap has the next forecast value of the day to hand. Only the repeated + // DST hour takes it. A filled position never becomes a source, so what is borrowed is always a + // real forecast value rather than another gap's 0.0. + int dstFilled = 0; + int interiorMissing = 0; + int trailingMissing = 0; + Double nextRealValue = null; + + for (int i = values.length - 1; i >= 0; i--) { + if (values[i] != null) { + nextRealValue = values[i]; + } else if (nextRealValue != null + && repeatedLocalTimes.contains(isps.get(i).toLocalDateTime())) { + values[i] = nextRealValue; + dstFilled++; + } else { + values[i] = 0.0; + if (i + 1 > lastRealPosition) { + trailingMissing++; + } else { + interiorMissing++; + } + } + } + + List submissionData = new ArrayList<>(values.length); + for (int i = 0; i < values.length; i++) { + submissionData.add(new SubmissionData(i + 1, null, null, values[i])); + } + if (collapsed > 0) { // One line per day rather than per position: on the fall-back day every position in the // repeated hour reads the same stored row. @@ -394,8 +440,19 @@ static List buildSubmissionData( + " predicted datapoint row (openremote/openremote#3292)"); } - int trailingMissing = submissionData.size() - lastRealPosition; - int interiorMissing = missing - trailingMissing; + if (dstFilled > 0) { + // Expected against a producer that writes a fixed 96-slot day: the four extra ISPs of the + // fall-back hour have no row of their own to read. + LOG.warning( + "Day-ahead submission for " + + marketDate + + " has " + + dstFilled + + " of " + + submissionData.size() + + " positions in the repeated DST hour without a predicted datapoint; each reuses the" + + " next forecast value of the day"); + } if (interiorMissing > 0) { // A hole before the end of the forecast means the producer skipped intervals it did cover, diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy index 441e7e5..9e9a4f1 100644 --- a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy @@ -279,6 +279,7 @@ class DistroEnergyHandlerTest extends Specification { data.size() == 100 and: "both passes read the surviving row, so the collapse only ever adds a read" - data.findAll { it.volume == 3.25d }*.position == [9, 13] + data[8].volume == 3.25d // 02:00 CEST, the pass that wrote the row + data[12].volume == 3.25d // 02:00 CET, the same row read a second time } } From 8fe9b38b4dcaa5dd9eefe2b171adc667fd69e933 Mon Sep 17 00:00:00 2001 From: Michael Date: Thu, 17 Sep 2026 20:39:36 +0200 Subject: [PATCH 2/3] Cover the repeated DST hour fill Three cases, all under a UTC storage frame so the two passes of the fall-back hour keep separate rows and a dropped ISP is a genuinely empty position rather than a collapsed one. Dropping either pass of the repeated hour leaves 96 of 100 ISPs covered, the shape a producer writing a fixed 96-slot day produces, and the emptied positions take the next real value. A forecast that ends before the repeated hour has nothing to borrow, so those positions stay 0.0. A gap in the afternoon of the same day stays 0.0 too, which pins the scope to the repeated hour rather than to the DST day as a whole. --- .../DistroEnergyHandlerTest.groovy | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy index 9e9a4f1..70458bf 100644 --- a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy @@ -82,6 +82,19 @@ class DistroEnergyHandlerTest extends Specification { new ArrayList<>(byStorageKey.values()) } + /** + * The same datapoints with the given ISP indices of the day removed, the way a producer that + * writes a fixed 96-slot day leaves the extra ISPs of the fall-back hour unwritten. + * + * Only meaningful under a storage zone that keeps every ISP on its own row, so the index dropped + * is the position emptied. + */ + static List> withoutIsps(List> datapoints, LocalDate marketDate, + List indices) { + def dropped = indices.collect { ispInstants(marketDate)[it].toInstant().toEpochMilli() } as Set + datapoints.findAll { !dropped.contains(it.timestamp) } + } + static long expectedIsps(LocalDate marketDate) { DateTimeCalculation.numberOfIspsOnDay(marketDate, ISP, MARKET.id) } @@ -140,6 +153,64 @@ class DistroEnergyHandlerTest extends Specification { data*.volume.every { it != 0.0d } } + @Unroll + def "a repeated DST hour with no datapoints of its own reuses the next forecast value: #label"() { + given: "a UTC storage frame, where the two passes of the fall-back hour keep separate rows" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, dropped) + + expect: "the producer covered 96 of the 100 ISPs, as a fixed 96-slot day would" + datapoints.size() == 96 + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "the submission is still the full 100 entries the API requires" + data.size() == 100 + + and: "the emptied positions took the next real value rather than trading the hour away" + data[dropped]*.volume.every { it == expectedVolume } + + and: "nothing outside the repeated hour moved" + data[0..7]*.volume == (1..8).collect { it * 1.0d } + data[16..99]*.volume == (17..100).collect { it * 1.0d } + + where: + label | dropped || expectedVolume + "first pass, 02:00-02:45 CEST" | [8, 9, 10, 11] || 13.0d + "second pass, 02:00-02:45 CET" | [12, 13, 14, 15] || 17.0d + } + + def "a repeated DST hour with nothing left to reuse falls back to 0.0"() { + given: "the forecast horizon ends before the repeated hour begins" + def datapoints = datapointsFor(FALL_BACK, UTC_STORAGE).take(8) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "the day is still submitted in full" + data.size() == 100 + + and: "the API requires a volume, so the repeated hour is 0.0 like the rest of the tail" + data[8..15]*.volume.every { it == 0.0d } + data[16..99]*.volume.every { it == 0.0d } + } + + def "a gap outside the repeated hour is still 0.0 on the fall-back day"() { + given: "one afternoon ISP is missing on a day that does carry a DST transition" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, [60]) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "only the repeated hour borrows a neighbour; an ordinary gap is a real trading position" + data.size() == 100 + data[60].volume == 0.0d + + and: "its neighbours are untouched" + data[59].volume == 60.0d + data[61].volume == 62.0d + } + def "spring-forward day never looks up the non-existent local hour"() { given: def datapoints = datapointsFor(SPRING_FORWARD, AMSTERDAM_STORAGE) From 607c60016969386d95f54c130554a491e2cc2ce6 Mon Sep 17 00:00:00 2001 From: Michael Date: Fri, 18 Sep 2026 12:25:40 +0200 Subject: [PATCH 3/3] Tighten the repeated-hour tests and correct the DST javadoc The javadoc claimed the method "becomes exact without changing" once predicted datapoints move to UTC. That is not true of the repeated hour. While the storage frame collapses the hour, both passes read the one surviving row and carry its distinct values. Under UTC each instant gets its own row, a producer that writes the hour once leaves the other pass empty, and that pass is filled flat on the next forecast value. The submitted day changes on the fall-back date when openremote/openremote#3292 lands, so the javadoc now says so and says the flat hour is deliberate. Also drops the claim that the rule covers "those four ISPs": detection is generic over any repeated market local time, which is two ISPs against a half-hour transition such as Australia/Lord_Howe. Test gaps: - Nothing pinned that a filled 0.0 is never itself a fill source. A refactor updating the carried value on every position, not just real ones, passed the whole suite and flattened the repeated hour to 0.0. The new case leaves the hour and the two ISPs after it unwritten, so the scan has to cross two filled zeros to reach 03:30. - The parameterised case asserted the dropped positions and everything outside the hour, never the pass that kept its rows, so a fill overwriting real values inside the hour went unnoticed. - The Amsterdam collapse guard only asserted "not 0.0", which holds for almost any wrong value. It now pins what each pass reads. - Comparing against a repeated list rather than calling every() also pins the element count. Hoists a duplicated toLocalDateTime() call in the detection loop. --- .../distroenergy/DistroEnergyHandler.java | 27 ++++++++++----- .../DistroEnergyHandlerTest.groovy | 33 ++++++++++++++++--- 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java index cc82f81..2849b1e 100644 --- a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java +++ b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java @@ -323,20 +323,30 @@ protected boolean submitDayAheadForecast(LocalDate marketDate) { * and trailing alike, because the API requires the complete day and a volume on every entry. * *

A gap goes out as 0.0, which is a real trading position, except in the repeated hour of the - * fall-back day. Those four ISPs exist only because of the transition, and a producer writing a + * fall-back day. An ISP qualifies when its market local time occurs more than once in the day's + * grid: four ISPs against a whole-hour transition, two against a half-hour one such as + * Australia/Lord_Howe. Those ISPs exist only because of the transition, and a producer writing a * fixed 96-slot day leaves them without a row of their own, so they reuse the next forecast value * of the day rather than trade the hour away. With nothing left in the day to reuse they fall * back to 0.0 like any other gap. * + *

They all reuse that same next value, so the hour goes out flat. That is deliberate. + * Borrowing from the ISP that shares the market local time would keep the hour's shape, at the + * price of a second fill rule, and one quarter-hour of the repeated hour is no better a guess for + * the others than the value that follows them. + * *

The decision is taken from the ISP grid rather than from whatever the query returned, so a * value belonging to a neighbouring day can never make this day look covered. * *

Under a JVM zone that observes DST the storage frame is not monotonic, so on the fall-back - * day the two instants of the repeated hour collapse onto a single stored row and both read the - * same value. That is a consequence of the naive primary key upstream - * (openremote/openremote#3292); once predicted datapoints are stored in UTC every instant maps to - * a distinct row and this method becomes exact without changing. The collapse can only duplicate - * a read, never erase one, so it cannot turn a day with a forecast into a skip. + * day the two instants of the repeated hour collapse onto a single stored row and both read it. + * That is a consequence of the naive primary key upstream (openremote/openremote#3292). The + * collapse can only duplicate a read, never erase one, so it cannot turn a day with a forecast + * into a skip, and while it lasts the repeated hour is never a gap at all: both passes read the + * one surviving row, so the hour carries as many distinct values as it has ISPs. Once predicted + * datapoints are stored in UTC every instant maps to a row of its own, a producer that writes the + * hour once leaves the other pass empty, and that pass is filled flat as above. The day submitted + * on the fall-back date therefore changes when #3292 lands; no other day is affected. */ static List buildSubmissionData( LocalDate marketDate, @@ -367,8 +377,9 @@ static List buildSubmissionData( Set seenLocalTimes = new HashSet<>(); Set repeatedLocalTimes = new HashSet<>(); for (ZonedDateTime isp : isps) { - if (!seenLocalTimes.add(isp.toLocalDateTime())) { - repeatedLocalTimes.add(isp.toLocalDateTime()); + LocalDateTime localTime = isp.toLocalDateTime(); + if (!seenLocalTimes.add(localTime)) { + repeatedLocalTimes.add(localTime); } } diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy index 70458bf..26e029e 100644 --- a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy @@ -151,6 +151,10 @@ class DistroEnergyHandlerTest extends Specification { and: "and none of them defaulted to 0.0, they reuse the surviving twin" data*.volume.every { it != 0.0d } + + and: "each pass carries the four distinct values of the row that survived the collapse" + data[8..11]*.volume == [13.0d, 14.0d, 15.0d, 16.0d] + data[12..15]*.volume == [13.0d, 14.0d, 15.0d, 16.0d] } @Unroll @@ -167,17 +171,36 @@ class DistroEnergyHandlerTest extends Specification { then: "the submission is still the full 100 entries the API requires" data.size() == 100 - and: "the emptied positions took the next real value rather than trading the hour away" - data[dropped]*.volume.every { it == expectedVolume } + and: "the emptied positions all took the next real value rather than trading the hour away" + data[dropped]*.volume == [expectedVolume] * dropped.size() + + and: "the pass that kept its rows is untouched, so the fill never overwrites a real value" + data[kept]*.volume == kept.collect { (it + 1) * 1.0d } and: "nothing outside the repeated hour moved" data[0..7]*.volume == (1..8).collect { it * 1.0d } data[16..99]*.volume == (17..100).collect { it * 1.0d } where: - label | dropped || expectedVolume - "first pass, 02:00-02:45 CEST" | [8, 9, 10, 11] || 13.0d - "second pass, 02:00-02:45 CET" | [12, 13, 14, 15] || 17.0d + label | dropped | kept || expectedVolume + "first pass, 02:00-02:45 CEST" | [8, 9, 10, 11] | [12, 13, 14, 15] || 13.0d + "second pass, 02:00-02:45 CET" | [12, 13, 14, 15] | [8, 9, 10, 11] || 17.0d + } + + def "the repeated hour looks past filled 0.0 positions for a real value"() { + given: "the repeated hour and the two ISPs after it are both unwritten" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, + [12, 13, 14, 15, 16, 17]) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "03:00 and 03:15 are ordinary gaps, so they are 0.0 and not a source to borrow from" + data[16..17]*.volume == [0.0d, 0.0d] + + and: "the repeated hour reaches past them to 03:30, the next value that is real" + data[12..15]*.volume == [19.0d, 19.0d, 19.0d, 19.0d] + data[18].volume == 19.0d } def "a repeated DST hour with nothing left to reuse falls back to 0.0"() {