diff --git a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java index 5f641cf..2849b1e 100644 --- a/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java +++ b/ems/src/main/java/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandler.java @@ -319,18 +319,34 @@ protected boolean submitDayAheadForecast(LocalDate marketDate) { *

An empty result means "nothing to submit" and cannot be confused with a real day, which is * never shorter than 92 entries. A day beyond the external forecast producer's horizon * legitimately has no data, and submitting it would post a zero net power trading position for a - * day we know nothing about. Within a day that does have a forecast every gap is filled with 0.0, - * interior and trailing alike, because the API requires the complete day. + * day we know nothing about. Within a day that does have a forecast every gap is filled, interior + * and trailing alike, because the API requires the complete day and a volume on every entry. + * + *

A gap goes out as 0.0, which is a real trading position, except in the repeated hour of the + * fall-back day. An ISP qualifies when its market local time occurs more than once in the day's + * grid: four ISPs against a whole-hour transition, two against a half-hour one such as + * Australia/Lord_Howe. Those ISPs exist only because of the transition, and a producer writing a + * fixed 96-slot day leaves them without a row of their own, so they reuse the next forecast value + * of the day rather than trade the hour away. With nothing left in the day to reuse they fall + * back to 0.0 like any other gap. + * + *

They all reuse that same next value, so the hour goes out flat. That is deliberate. + * Borrowing from the ISP that shares the market local time would keep the hour's shape, at the + * price of a second fill rule, and one quarter-hour of the repeated hour is no better a guess for + * the others than the value that follows them. * *

The decision is taken from the ISP grid rather than from whatever the query returned, so a * value belonging to a neighbouring day can never make this day look covered. * *

Under a JVM zone that observes DST the storage frame is not monotonic, so on the fall-back - * day the two instants of the repeated hour collapse onto a single stored row and both read the - * same value. That is a consequence of the naive primary key upstream - * (openremote/openremote#3292); once predicted datapoints are stored in UTC every instant maps to - * a distinct row and this method becomes exact without changing. The collapse can only duplicate - * a read, never erase one, so it cannot turn a day with a forecast into a skip. + * day the two instants of the repeated hour collapse onto a single stored row and both read it. + * That is a consequence of the naive primary key upstream (openremote/openremote#3292). The + * collapse can only duplicate a read, never erase one, so it cannot turn a day with a forecast + * into a skip, and while it lasts the repeated hour is never a gap at all: both passes read the + * one surviving row, so the hour carries as many distinct values as it has ISPs. Once predicted + * datapoints are stored in UTC every instant maps to a row of its own, a producer that writes the + * hour once leaves the other pass empty, and that pass is filled flat as above. The day submitted + * on the fall-back date therefore changes when #3292 lands; no other day is affected. */ static List buildSubmissionData( LocalDate marketDate, @@ -340,7 +356,7 @@ static List buildSubmissionData( Map valuesByStorageKey = new HashMap<>(); for (ValueDatapoint datapoint : datapoints) { - // Gap-filled buckets carry a null value; skip them so they fall through to the default below. + // Gap-filled buckets carry a null value; skip them so they fall through to the fill below. if (datapoint.getValue() instanceof Number value) { valuesByStorageKey.put( Instant.ofEpochMilli(datapoint.getTimestamp()).atZone(storageZone).toLocalDateTime(), @@ -351,27 +367,38 @@ static List buildSubmissionData( ZonedDateTime dayStart = marketDate.atStartOfDay(marketZone); ZonedDateTime dayEnd = dayStart.plusDays(1); - List submissionData = new ArrayList<>(); + List isps = new ArrayList<>(); + for (ZonedDateTime isp = dayStart; isp.isBefore(dayEnd); isp = isp.plus(ISP_DURATION)) { + isps.add(isp); + } + + // The repeated hour of the fall-back day is the only stretch whose market local time is not + // unique within the day, so on every other day this stays empty and nothing borrows a value. + Set seenLocalTimes = new HashSet<>(); + Set repeatedLocalTimes = new HashSet<>(); + for (ZonedDateTime isp : isps) { + LocalDateTime localTime = isp.toLocalDateTime(); + if (!seenLocalTimes.add(localTime)) { + repeatedLocalTimes.add(localTime); + } + } + + Double[] values = new Double[isps.size()]; Set keysRead = new HashSet<>(); - int missing = 0; int collapsed = 0; int lastRealPosition = 0; - int position = 1; - for (ZonedDateTime isp = dayStart; isp.isBefore(dayEnd); isp = isp.plus(ISP_DURATION)) { - LocalDateTime storageKey = isp.withZoneSameInstant(storageZone).toLocalDateTime(); + for (int i = 0; i < isps.size(); i++) { + LocalDateTime storageKey = isps.get(i).withZoneSameInstant(storageZone).toLocalDateTime(); Double value = valuesByStorageKey.get(storageKey); + values[i] = value; - if (value == null) { - missing++; - } else { - lastRealPosition = position; + if (value != null) { + lastRealPosition = i + 1; if (!keysRead.add(storageKey)) { collapsed++; } } - - submissionData.add(new SubmissionData(position++, null, null, value != null ? value : 0.0)); } // Nothing at all was forecast for this day. Hand the caller the empty sentinel and stay silent @@ -380,6 +407,36 @@ static List buildSubmissionData( return List.of(); } + // Walk backwards so every gap has the next forecast value of the day to hand. Only the repeated + // DST hour takes it. A filled position never becomes a source, so what is borrowed is always a + // real forecast value rather than another gap's 0.0. + int dstFilled = 0; + int interiorMissing = 0; + int trailingMissing = 0; + Double nextRealValue = null; + + for (int i = values.length - 1; i >= 0; i--) { + if (values[i] != null) { + nextRealValue = values[i]; + } else if (nextRealValue != null + && repeatedLocalTimes.contains(isps.get(i).toLocalDateTime())) { + values[i] = nextRealValue; + dstFilled++; + } else { + values[i] = 0.0; + if (i + 1 > lastRealPosition) { + trailingMissing++; + } else { + interiorMissing++; + } + } + } + + List submissionData = new ArrayList<>(values.length); + for (int i = 0; i < values.length; i++) { + submissionData.add(new SubmissionData(i + 1, null, null, values[i])); + } + if (collapsed > 0) { // One line per day rather than per position: on the fall-back day every position in the // repeated hour reads the same stored row. @@ -394,8 +451,19 @@ static List buildSubmissionData( + " predicted datapoint row (openremote/openremote#3292)"); } - int trailingMissing = submissionData.size() - lastRealPosition; - int interiorMissing = missing - trailingMissing; + if (dstFilled > 0) { + // Expected against a producer that writes a fixed 96-slot day: the four extra ISPs of the + // fall-back hour have no row of their own to read. + LOG.warning( + "Day-ahead submission for " + + marketDate + + " has " + + dstFilled + + " of " + + submissionData.size() + + " positions in the repeated DST hour without a predicted datapoint; each reuses the" + + " next forecast value of the day"); + } if (interiorMissing > 0) { // A hole before the end of the forecast means the producer skipped intervals it did cover, diff --git a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy index 441e7e5..26e029e 100644 --- a/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy +++ b/ems/src/test/groovy/org/openremote/extension/ems/manager/distroenergy/DistroEnergyHandlerTest.groovy @@ -82,6 +82,19 @@ class DistroEnergyHandlerTest extends Specification { new ArrayList<>(byStorageKey.values()) } + /** + * The same datapoints with the given ISP indices of the day removed, the way a producer that + * writes a fixed 96-slot day leaves the extra ISPs of the fall-back hour unwritten. + * + * Only meaningful under a storage zone that keeps every ISP on its own row, so the index dropped + * is the position emptied. + */ + static List> withoutIsps(List> datapoints, LocalDate marketDate, + List indices) { + def dropped = indices.collect { ispInstants(marketDate)[it].toInstant().toEpochMilli() } as Set + datapoints.findAll { !dropped.contains(it.timestamp) } + } + static long expectedIsps(LocalDate marketDate) { DateTimeCalculation.numberOfIspsOnDay(marketDate, ISP, MARKET.id) } @@ -138,6 +151,87 @@ class DistroEnergyHandlerTest extends Specification { and: "and none of them defaulted to 0.0, they reuse the surviving twin" data*.volume.every { it != 0.0d } + + and: "each pass carries the four distinct values of the row that survived the collapse" + data[8..11]*.volume == [13.0d, 14.0d, 15.0d, 16.0d] + data[12..15]*.volume == [13.0d, 14.0d, 15.0d, 16.0d] + } + + @Unroll + def "a repeated DST hour with no datapoints of its own reuses the next forecast value: #label"() { + given: "a UTC storage frame, where the two passes of the fall-back hour keep separate rows" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, dropped) + + expect: "the producer covered 96 of the 100 ISPs, as a fixed 96-slot day would" + datapoints.size() == 96 + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "the submission is still the full 100 entries the API requires" + data.size() == 100 + + and: "the emptied positions all took the next real value rather than trading the hour away" + data[dropped]*.volume == [expectedVolume] * dropped.size() + + and: "the pass that kept its rows is untouched, so the fill never overwrites a real value" + data[kept]*.volume == kept.collect { (it + 1) * 1.0d } + + and: "nothing outside the repeated hour moved" + data[0..7]*.volume == (1..8).collect { it * 1.0d } + data[16..99]*.volume == (17..100).collect { it * 1.0d } + + where: + label | dropped | kept || expectedVolume + "first pass, 02:00-02:45 CEST" | [8, 9, 10, 11] | [12, 13, 14, 15] || 13.0d + "second pass, 02:00-02:45 CET" | [12, 13, 14, 15] | [8, 9, 10, 11] || 17.0d + } + + def "the repeated hour looks past filled 0.0 positions for a real value"() { + given: "the repeated hour and the two ISPs after it are both unwritten" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, + [12, 13, 14, 15, 16, 17]) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "03:00 and 03:15 are ordinary gaps, so they are 0.0 and not a source to borrow from" + data[16..17]*.volume == [0.0d, 0.0d] + + and: "the repeated hour reaches past them to 03:30, the next value that is real" + data[12..15]*.volume == [19.0d, 19.0d, 19.0d, 19.0d] + data[18].volume == 19.0d + } + + def "a repeated DST hour with nothing left to reuse falls back to 0.0"() { + given: "the forecast horizon ends before the repeated hour begins" + def datapoints = datapointsFor(FALL_BACK, UTC_STORAGE).take(8) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "the day is still submitted in full" + data.size() == 100 + + and: "the API requires a volume, so the repeated hour is 0.0 like the rest of the tail" + data[8..15]*.volume.every { it == 0.0d } + data[16..99]*.volume.every { it == 0.0d } + } + + def "a gap outside the repeated hour is still 0.0 on the fall-back day"() { + given: "one afternoon ISP is missing on a day that does carry a DST transition" + def datapoints = withoutIsps(datapointsFor(FALL_BACK, UTC_STORAGE), FALL_BACK, [60]) + + when: + def data = DistroEnergyHandler.buildSubmissionData(FALL_BACK, MARKET, UTC_STORAGE, datapoints) + + then: "only the repeated hour borrows a neighbour; an ordinary gap is a real trading position" + data.size() == 100 + data[60].volume == 0.0d + + and: "its neighbours are untouched" + data[59].volume == 60.0d + data[61].volume == 62.0d } def "spring-forward day never looks up the non-existent local hour"() { @@ -279,6 +373,7 @@ class DistroEnergyHandlerTest extends Specification { data.size() == 100 and: "both passes read the surviving row, so the collapse only ever adds a read" - data.findAll { it.volume == 3.25d }*.position == [9, 13] + data[8].volume == 3.25d // 02:00 CEST, the pass that wrote the row + data[12].volume == 3.25d // 02:00 CET, the same row read a second time } }