From c2cc3d0ba1014d0cd2b42b6b9ad5b904b2268a88 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:13 +0200 Subject: [PATCH 01/87] feat(cmdb-import): read the archive sheet and let module and usage be archived Adds the OpenSpec change cmdb-import-archive-reconciliation (REQ-CMDB-015..017, REQ-CMDB-012 and REQ-CMDB-014 modified). The register fragment opts module (0.3.9) and usage (1.5.7) into OpenRegister's archive state, the import profile names the sheet "Gearchiveerde Applicaties" and the accepted missingRecords modes, and the workbook reader returns the APPIDs of that sheet, read under the same bounds as the CMDB sheets for its key column only. The archived-applications fixture covers it. First of three slices of #1281; nothing reads the archive sheet yet. Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbImportProfile.php | 55 ++++- lib/Service/Cmdb/CmdbWorkbookReader.php | 151 ++++++++++--- lib/Settings/cmdb-import/topdesk-profile.json | 7 +- .../register.d/topdesk-cmdb-import.json | 8 +- .../.openspec.yaml | 2 + .../design.md | 207 ++++++++++++++++++ .../proposal.md | 95 ++++++++ .../specs/cmdb-export-import/spec.md | 191 ++++++++++++++++ .../tasks.md | 89 ++++++++ openspec/specs/cmdb-export-import/spec.md | 5 +- .../Unit/Fixtures/CmdbFixtureHygieneTest.php | 2 +- .../Service/Cmdb/CmdbImportProfileTest.php | 24 ++ .../Service/Cmdb/CmdbWorkbookReaderTest.php | 104 +++++++++ .../Settings/PublicationFieldRulesTest.php | 2 +- .../Unit/Settings/TopdeskCmdbFragmentTest.php | 30 ++- tests/fixtures/cmdb/README.md | 4 +- tests/fixtures/cmdb/build-fixtures.py | 46 ++++ .../cmdb/topdesk-archived-applications.xlsx | Bin 0 -> 151786 bytes 18 files changed, 970 insertions(+), 52 deletions(-) create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/design.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/proposal.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md create mode 100644 openspec/changes/cmdb-import-archive-reconciliation/tasks.md create mode 100644 tests/fixtures/cmdb/topdesk-archived-applications.xlsx diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index 1e8805c90..183c02fe6 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -332,6 +332,42 @@ public function sheetNames(): array { return array_column($this->sheets(), 'name'); }//end sheetNames() + /** + * The name of the archive sheet, or null when the profile names none. + * + * The archive sheet lists the applications the municipality archived; the + * import reads only its key column (`archiveSheet.name` in the profile). + * + * @return string|null + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function archiveSheetName(): ?string { + $name = $this->profile()['archiveSheet']['name'] ?? null; + if (is_string($name) === false || $name === '' || in_array($name, $this->sheetNames(), true) === true) { + return null; + } + + return $name; + }//end archiveSheetName() + + /** + * Every sheet the reader may parse: the source sheets, then the archive sheet. + * + * @return array + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function readSheetNames(): array { + $names = $this->sheetNames(); + $archive = $this->archiveSheetName(); + if ($archive !== null) { + $names[] = $archive; + } + + return $names; + }//end readSheetNames() + /** * The rank of a sheet when an APPID is on more than one: lower wins. * @@ -574,19 +610,28 @@ public function neverWrittenOnUpdate(string $target): array { }//end neverWrittenOnUpdate() /** - * The accepted values of the missingRecords option. + * The accepted values of the missingRecords option, readable without validating the packs. + * + * The controller checks the option before the import runs, so, like + * maxFileBytes(), this must not depend on OpenRegister being available. * * @return array * - * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ public function missingRecordsModes(): array { - $modes = $this->stringList(key: 'missingRecords'); - if ($modes === []) { + try { + $profile = $this->profile ?? $this->decodeFile(fileName: $this->profileFile); + } catch (CmdbImportException $e) { + return ['keep']; + } + + $modes = $profile['missingRecords'] ?? []; + if (is_array($modes) === false || $modes === []) { return ['keep']; } - return $modes; + return array_values(array_map('strval', $modes)); }//end missingRecordsModes() /** diff --git a/lib/Service/Cmdb/CmdbWorkbookReader.php b/lib/Service/Cmdb/CmdbWorkbookReader.php index 3ff3e9184..9bdf9b9a6 100644 --- a/lib/Service/Cmdb/CmdbWorkbookReader.php +++ b/lib/Service/Cmdb/CmdbWorkbookReader.php @@ -42,6 +42,9 @@ * beyond twice the row limit is `TOO_MANY_ROWS`. A read filter then materialises only the * header row and the resolved columns of the rows up to that bound * (CmdbReadFilter), which bounds the cell objects, not the parse. + * 7. The archive sheet ("Gearchiveerde Applicaties"), when the workbook has + * it, counts as a read sheet for every bound above, and only its key + * column is resolved and read (openspec/changes/cmdb-import-archive-reconciliation). * * @category Service * @package OCA\Stackiq\Service\Cmdb @@ -151,17 +154,21 @@ public function isAvailable(): bool { * The sheets are then read twice through a read filter: once for the * header row, once for the resolved columns of the data rows, so no other * cell is ever materialised. + * The archive sheet, when the workbook has it, is read under the same + * bounds as a source sheet, for its key column only. * * @param string $path The xlsx file, already checked by assertXlsx(). * @param CmdbImportProfile $profile The import profile. * * @return array `rows` (list of {sheet, row, cells, uncached}), `importWarnings` - * (list of {sheet, message}) and `date1904` (bool). + * (list of {sheet, message}), `date1904` (bool) and `archive` + * ({sheet, present, keyColumnMissing, appIds}). * * @throws CmdbImportException WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN or TOO_MANY_ROWS. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function read(string $path, CmdbImportProfile $profile): array { $oversized = $this->assertUncompressedSize(path: $path, limit: $profile->maxUncompressedBytes(), partLimit: $profile->maxPartBytes()); @@ -177,7 +184,7 @@ public function read(string $path, CmdbImportProfile $profile): array { $blank = (new CmdbPartReferences())->assertPartSizes( path: $path, oversized: $oversized, - sourceSheets: $profile->sheetNames(), + sourceSheets: $profile->readSheetNames(), limit: $profile->maxPartBytes(), scanner: $scanner ); @@ -303,10 +310,81 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s $bounds->assertReferencedStringBytes( path: $path, limit: $profile->maxReferencedStringBytes(), - sheetCount: count($profile->sheetNames()), + sheetCount: count($profile->readSheetNames()), scanner: $scanner ); + ['present' => $present, 'available' => $available] = $this->sourceSheets(path: $path, profile: $profile); + + $archiveSheet = $profile->archiveSheetName(); + $loaded = $present; + if ($archiveSheet !== null && in_array($archiveSheet, $available, true) === true) { + $loaded[] = $archiveSheet; + } + + $limit = $profile->maxRowsPerSheet(); + $lastRow = self::lastReadableRow(limit: $limit); + $this->assertRowSpan(path: $path, sheetNames: $loaded, lastRow: $lastRow, limit: $limit); + + $headers = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: 1)); + try { + $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); + $archiveColumns = []; + if ($loaded !== $present) { + // Only the key column of the archive sheet is read; the allowlist does not grow. + $archiveColumns = $this->resolveColumns(worksheet: $headers->getSheetByName($archiveSheet), referenced: [$profile->keyColumn()]); + } + } finally { + $headers->disconnectWorksheets(); + } + + // A sheet without the key column is not read; the import reports it as missing its key column. + $archive = [ + 'sheet' => (string)$archiveSheet, + 'present' => false, + 'keyColumnMissing' => ($loaded !== $present && $archiveColumns === []), + 'appIds' => [], + ]; + $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); + $loaded = $present; + if ($archiveColumns !== []) { + $letters[$archiveSheet] = array_keys($archiveColumns); + $loaded[] = $archiveSheet; + } + + $spreadsheet = $this->load(path: $path, sheetNames: $loaded, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); + try { + $rows = $this->readSourceRows(spreadsheet: $spreadsheet, sheetNames: $present, columnsPerSheet: $resolved['columns'], limit: $limit); + if ($archiveColumns !== []) { + $archiveRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($archiveSheet), + columns: $archiveColumns, + sheetName: (string)$archiveSheet, + limit: $limit + ); + $archive['present'] = true; + $archive['appIds'] = array_column(array_column($archiveRows, 'cells'), $profile->keyColumn()); + } + + $date1904 = self::usesDate1904(spreadsheet: $spreadsheet); + } finally { + $spreadsheet->disconnectWorksheets(); + } + + return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904, 'archive' => $archive]; + }//end readBounded() + + /** + * The profile's source sheets that the workbook has, and every sheet it has. + * + * @param string $path The xlsx file. + * @param CmdbImportProfile $profile The import profile. + * + * @return array{present: array, available: array} + * + * @throws CmdbImportException NOT_XLSX when the sheet list cannot be read, NO_SOURCE_SHEET when no source sheet is there. + */ + private function sourceSheets(string $path, CmdbImportProfile $profile): array { $reader = $this->newReader(sheetNames: null); try { $available = $reader->listWorksheetNames($path); @@ -328,41 +406,50 @@ private function readBounded(string $path, CmdbImportProfile $profile, object $s ); } - $limit = $profile->maxRowsPerSheet(); - $lastRow = self::lastReadableRow(limit: $limit); - $this->assertRowSpan(path: $path, sheetNames: $present, lastRow: $lastRow, limit: $limit); + return ['present' => $present, 'available' => $available]; + }//end sourceSheets() - $headers = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: 1)); - try { - $resolved = $this->resolveSheets(spreadsheet: $headers, sheetNames: $present, profile: $profile); - } finally { - $headers->disconnectWorksheets(); + /** + * The data rows of the source sheets, in sheet order. + * + * @param object $spreadsheet The spreadsheet, loaded through the data-pass read filter. + * @param array $sheetNames The source sheets the workbook has. + * @param array> $columnsPerSheet Sheet name => column letter => column name. + * @param int $limit The row limit per sheet. + * + * @return array> See read(). + * + * @throws CmdbImportException TOO_MANY_ROWS. + */ + private function readSourceRows(object $spreadsheet, array $sheetNames, array $columnsPerSheet, int $limit): array { + $rows = []; + foreach ($sheetNames as $sheetName) { + $sheetRows = $this->readRows( + worksheet: $spreadsheet->getSheetByName($sheetName), + columns: $columnsPerSheet[$sheetName], + sheetName: $sheetName, + limit: $limit + ); + array_push($rows, ...$sheetRows); } - $letters = array_map(static fn (array $columns): array => array_keys($columns), $resolved['columns']); - $spreadsheet = $this->load(path: $path, sheetNames: $present, filter: new CmdbReadFilter(lastRow: $lastRow, columns: $letters)); - try { - $rows = []; - foreach ($present as $sheetName) { - $sheetRows = $this->readRows( - worksheet: $spreadsheet->getSheetByName($sheetName), - columns: $resolved['columns'][$sheetName], - sheetName: $sheetName, - limit: $limit - ); - array_push($rows, ...$sheetRows); - } + return $rows; + }//end readSourceRows() - $date1904 = false; - if (method_exists($spreadsheet, 'getExcelCalendar') === true) { - $date1904 = ((int)$spreadsheet->getExcelCalendar() === 1904); - } - } finally { - $spreadsheet->disconnectWorksheets(); + /** + * Whether the workbook uses the 1904 date system. + * + * @param object $spreadsheet The spreadsheet. + * + * @return bool + */ + private static function usesDate1904(object $spreadsheet): bool { + if (method_exists($spreadsheet, 'getExcelCalendar') === false) { + return false; } - return ['rows' => $rows, 'importWarnings' => $resolved['warnings'], 'date1904' => $date1904]; - }//end readBounded() + return ((int)$spreadsheet->getExcelCalendar() === 1904); + }//end usesDate1904() /** * The last row number the data pass reads. diff --git a/lib/Settings/cmdb-import/topdesk-profile.json b/lib/Settings/cmdb-import/topdesk-profile.json index 79dc43253..a6b3df2d3 100644 --- a/lib/Settings/cmdb-import/topdesk-profile.json +++ b/lib/Settings/cmdb-import/topdesk-profile.json @@ -1,8 +1,8 @@ { "id": "topdesk-cmdb", "name": "TOPdesk CMDB export", - "version": "2.0.0", - "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the key and required columns, date and id columns, values that mean empty, the pack per target and the limits. Columns that neither this profile nor a pack names are never read.", + "version": "2.1.0", + "description": "How stackiq reads a TOPdesk CMDB export (xlsx): the two CMDB sheets, the archive sheet (APPID column only), the key and required columns, date and id columns, values that mean empty, the pack per target, the accepted missingRecords modes and the limits. Columns that neither this profile nor a pack names are never read.", "maxFileBytes": 10485760, "maxRowsPerSheet": 10000, "maxUncompressedBytes": 52428800, @@ -21,6 +21,7 @@ } ], "sheetPrecedence": ["Beheerde Applicaties CMDB", "Onbeh Applicaties CMDB"], + "archiveSheet": { "name": "Gearchiveerde Applicaties" }, "keyColumn": "APPID", "nameColumn": "Applicatie Naam", "requiredColumns": ["APPID", "Applicatie Naam"], @@ -44,5 +45,5 @@ "neverWritten": { "module": ["publicationDate", "depublicationDate"] }, - "missingRecords": ["keep"] + "missingRecords": ["keep", "archive"] } diff --git a/lib/Settings/register.d/topdesk-cmdb-import.json b/lib/Settings/register.d/topdesk-cmdb-import.json index 5fb300099..977c5db24 100644 --- a/lib/Settings/register.d/topdesk-cmdb-import.json +++ b/lib/Settings/register.d/topdesk-cmdb-import.json @@ -2,7 +2,10 @@ "components": { "schemas": { "module": { - "version": "0.3.8", + "version": "0.3.9", + "configuration": { + "x-openregister-archive": { "enabled": true } + }, "properties": { "externalId": { "type": "string", @@ -72,8 +75,9 @@ } }, "usage": { - "version": "1.5.6", + "version": "1.5.7", "configuration": { + "x-openregister-archive": { "enabled": true }, "x-openregister-lifecycle": { "transitions": { "importAcquisition": { diff --git a/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml b/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml new file mode 100644 index 000000000..5c0f6674b --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: conduction +created: 2026-10-09 diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md new file mode 100644 index 000000000..f71cfcce6 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -0,0 +1,207 @@ +# Design: cmdb-import-archive-reconciliation + +## Context + +The CMDB import (archived change `2026-10-05-cmdb-export-import`) matches rows on `module.externalKey` = `topdesk::` and leaves records that are missing from a newer export untouched (REQ-CMDB-012). The municipality's export carries, next to the two CMDB sheets, a sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging), derived from the raw "Invoer gearchiveerde appl" sheet. The municipality wants an application that moved there to be archived, and one that is on no sheet any more to be soft-deleted (2026-10-01). + +What OpenRegister offers (origin/beta `ff4dad5c`, identical to development for `lib/`): + +- **Soft delete.** `ObjectServiceInterface::deleteObject(uuid, register, schema, ...)` writes `@self.deleted` (`deletedBy`, `deletedAt`, `purgeDate` 30 days on). Lists exclude deleted objects unless `_includeDeleted=true`, but the list search cannot return one even then (D9). `MagicMapper::findDeletedAcrossAllMagicTables()` lists the trash; `MagicMapper::restoreObject(uuid)` clears the marker. `DeletedController` uses both. +- **Archive.** `ObjectEntity::archive()` / `unarchive()` write and clear `@self.archived` (`by`, `at`, `reason`). `Service\Object\ArchiveHandler::archive()` / `unarchive()` is the service path behind `POST` / `DELETE /api/objects/{register}/{schema}/{id}/archive`: it resolves the object, refuses a schema without `x-openregister-archive: {enabled: true}` (`ArchiveNotOfferedException`), checks `update` on the object for the session user, persists the marker and writes an audit entry. Lists exclude archived objects unless `_archived=true` (only the archive) or `_archived=any`. A read by identifier is unaffected. `SaveObject` refuses a data write to an archived object (`ObjectStateWriteException`). +- **Search lenses** combine in SQL: `_ids`, `_includeDeleted` and `_archived` apply together in `MagicSearchHandler`. Its row conversion, however, drops a soft-deleted row and hydrates no archive marker (D9). + +OpenCatalogi (beta `df25ff75`) passes list parameters through to OpenRegister and forces `_includeDeleted=false`; it has no archive filter of its own, so `_archived=true` reaches OpenRegister unchanged. Portaliq (beta `8287e390`) lists usages with neither lens. Both hide archived and deleted records as they are (decided 2026-10-09). + +## Goals / Non-Goals + +**Goals** + +- A re-import archives what moved to the archive sheet and soft-deletes what left the export entirely, for the imported municipality only. +- A returning application is revived, never duplicated. +- The admin sees and chooses the behaviour, and the report accounts for every archived, unarchived, deleted and restored application. +- Archived applications and usages can be listed in stackiq on request and stay out of OpenCatalogi and Portaliq without a change there. + +**Non-Goals** + +- Hard deletion, the trash's retention, contact persons, suppliers, modules another organisation imported (a module carrying this municipality's import key is this municipality's record, also when another organisation uses it; D5), changes to OpenCatalogi or Portaliq, an archive widget on the detail pages. + +## Architecture Overview + +``` +CmdbImportController::import() missingRecords: keep | archive (default archive); remove → 422 + ▼ +CmdbExportImportService::import() + ├─ resolve ArchiveHandler + MagicMapper (archive mode only; missing → 503 ARCHIVE_UNAVAILABLE) + ├─ lock the register + ├─ CmdbWorkbookReader::read() CMDB sheets as before + the archive sheet's APPID column + ├─ rows, each in its own boundary match module and usage: working, then _archived=true, then the trash (D9) + │ archived match → unarchive, deleted match → restore (and unarchive), then the usual update + └─ reconcile() (archive mode, not cancelled) phase `reconciling`, same lock + page usages (consumer = municipality): working, then _archived=true; deleted ones from the trash + load their modules in batches (_ids: working, then _archived=true; the rest from the trash) + per application: key on CMDB sheets → nothing + key on archive sheet → archive module + usage (restore first when deleted) + key nowhere → soft-delete module + usage, or archive when the sheet is absent + each in its own boundary → report rows archived | deleted | failed + ▼ +OpenRegister: module, usage (archived / deleted markers); contactPerson, organization untouched + ├─▶ OpenCatalogi: archived modules only with _archived=true + └─▶ Portaliq: archived and deleted usages disappear from "Software we use" +``` + +## Decisions + +### D1. "Archived" is OpenRegister's archive state on the module and on the usage + +Decided by the product owner on 2026-10-09. No new field, no new `usage.status` value, no `depublicationDate`. + +- **Alternative: `usage.status` `Archived`.** Rejected: OpenCatalogi drops `status === 'archived'` only on `/api/search`, so `/api/{catalogSlug}` and the federation route would need an OpenCatalogi change and a new beta. +- **Alternative: `module.depublicationDate`.** Rejected: the module read rule does not look at it, and the import never writes it (D6 of the first change). + +Both schemas opt in through the register fragment: `module` 0.3.8 → 0.3.9 and `usage` 1.5.6 → 1.5.7 get `configuration.x-openregister-archive: {"enabled": true}`. Fragments are merged by key union and the loader keeps the highest `version` per schema, so the bump deploys with the existing register import. The `SCHEMA_OUTDATED` check is not widened: an installation whose schemas predate the annotation gets `ArchiveNotOfferedException` per object, which the reconciliation reports as a failed row (D7), and the docs point to Force Update. + +### D2. The option: `keep | archive`, default `archive` + +`CmdbImportProfile::missingRecordsModes()` reads the accepted values from the profile (`["keep", "archive"]`) without validating the packs, like `maxFileBytes()`, because the controller checks the option before the import runs. The default lives in `CmdbExportImportService::DEFAULT_MISSING_RECORDS`. `remove` and `mark` stay reserved: 422 `MISSING_RECORDS_UNSUPPORTED` with `details.accepted` naming both accepted values. The section sends the admin's choice as `missingRecords` and defaults to `archive`. + +With `updateExisting=false`, a matched row is skipped as `exists` before anything is revived, as before: the admin asked to leave existing records alone. + +### D3. Reading the archive sheet + +The profile names the sheet (`archiveSheet.name`: "Gearchiveerde Applicaties"). The reader lists the workbook's sheets, and when the archive sheet is present it is loaded with the CMDB sheets under the same bounds: it counts as a read sheet for `CmdbPartReferences` (an oversized archive sheet is refused, not blanked) and for the referenced-string budget, its row span is bounded like a source sheet, and `TOO_MANY_ROWS` names it. Only its APPID column is resolved and read; the allowlist does not grow. The reader returns `archive: {present, keyColumnMissing, appIds}` next to `rows`. + +- Sheet absent, or present without an APPID column: `present` false (with `keyColumnMissing`), and the service adds a report warning. The reconciliation then archives every missing application and soft-deletes nothing: a workbook saved without the sheet must not empty the trash-bound set by accident. +- The archive sheet's other columns are not read; `Applicatie Status` there is always `Verwijderd` in the municipality's export and adds nothing. + +### D4. Matching widens to archived and deleted objects, and revives what it matches + +`matchModule()` looks the module up by `externalKey` in every state, as D9 reads it; `moduleBelongsTo()` looks up this municipality's usage the same way (a module whose usage is archived still belongs to the municipality), and keeps the working lens for "does any other organisation use it". `upsertUsage()` finds the usage in every state. Whatever is matched is revived before it is written: a soft-deleted object is restored (`MagicMapper::restoreObject()`), and unarchived as well when the restored object says it was archived; an archived one is unarchived (`ArchiveHandler::unarchive()`), because `SaveObject` refuses a write to an archived object. The update starts from the object `restoreObject()` answers, not from the trash listing's copy. The row outcome is `restored` when anything was restored, else `unarchived` when anything was unarchived, else the usual `created | updated | unchanged`. When more than one object matches, `findInAnyState()` takes a working one before an archived one, and an archived one before one in the trash: before this change a module deleted by hand was invisible to the match, so the next import created a second one with the same `externalKey`, and reviving the deleted one would make two live modules. Organisations and contact persons keep the working lens: a deleted supplier is never matched. + +### D5. The reconciliation step + +Runs inside `runImport()` after the row loop, only when the loop finished (not cancelled) and `missingRecords` is `archive`, under the register lock the import already holds. It checks for a cancel once before it loads the scope (a cancel that came in during the last row) and again before each application; a cancel stops it there, keeps what was already archived or deleted, and marks the report cancelled. An exception outside an object's boundary fails the run like any other (`IMPORT_FAILED`, operation `failed`), so a half-done reconciliation is visible. + +Scope: the usages whose `consumer` is the municipality, in every state as D9 reads it: paged with `PAGE_SIZE` without a lens and with `_archived=true`, plus the deleted ones from the trash. Their modules are loaded in chunks of `PAGE_SIZE` with `_ids`, without a lens and with `_archived=true`, and the rest from the trash. Every usage and module keeps the state D9 gave it; the table below decides on that state. Only a module whose `externalKey` starts with `topdesk::` is an imported application; the key after the last `:` is the APPID's match key. A usage without such a module (a hand-made usage, or a module another organisation imported) is left alone. + +Per application, with `processed` = every APPID match key on the CMDB sheets (whatever the row's outcome: a row that failed or was skipped is still in the export) and `archived` = the match keys on the archive sheet: + +| key is | module/usage state | action | outcome | +|---|---|---|---| +| on a CMDB sheet | any | nothing (the row handled it) | — | +| on the archive sheet | working | archive usage, then module | `archived` | +| on the archive sheet | archived | nothing | — | +| on the archive sheet | deleted | restore, then archive | `archived` | +| nowhere, sheet present | working or archived | soft-delete usage, then module | `deleted` | +| nowhere, sheet present | deleted | nothing | — | +| nowhere, sheet absent | working | archive usage, then module | `archived` | +| nowhere, sheet absent | archived or deleted | nothing | — | + +The usage is written before the module, so a failure between the two leaves the municipality's usage in the new state and the module visible, never the reverse. A module another municipality still uses is archived or deleted all the same when this municipality's usage says so: the module carries this municipality's import key, so it is this municipality's record (REQ-CMDB-006 allows a shared module only when the other organisation's usage was added by hand). The archive reason is `cmdb-import: on sheet "Gearchiveerde Applicaties"` or `cmdb-import: not in the export`, plus the operation id, so the audit trail says why. + +Each application runs in its own boundary: a failure is a `failed` row naming the step (`archive`, `restore`, `delete`) and the exception class, as row failures do, and the other applications continue. Contact persons are never touched. + +Progress: the operation gets the phase `reconciling` (added to `ProgressTracker::PHASES` with weight 0, so the ArchiMate import's weighted percentage is unchanged), `total_items` grows by the number of usages in scope, and `processed_items` advances per application; the section's own percentage comes from those counts. + +### D6. The report + +`CmdbImportReport` gains the outcomes `archived`, `unarchived`, `deleted` and `restored`; `summary()` counts them like the others. Reconciliation rows carry the archive sheet's name (for `archived` from the sheet) or an empty sheet, row 0, the APPID (`externalNumber`), the module name and both uuids: no person data. `toStoredArray()` ranks them with the rest. The section shows the four counts as summary tiles when they are non-zero, and the outcome filter offers them. + +### D7. Failure modes + +- `ArchiveHandler` or `MagicMapper` missing, or without a method the import calls (`archive`, `unarchive`, `restoreObject`, `findDeletedAcrossAllMagicTables`): 503 `ARCHIVE_UNAVAILABLE` before the lock is taken, only in archive mode. `keep` needs neither: without them the trash cannot be read, so a soft-deleted module or usage is not matched (the log says so) and a row whose match must be unarchived fails at step `module` or `usage`. +- A schema without `x-openregister-archive`: `ArchiveNotOfferedException` from OpenRegister per object → a failed row with the exception class; the docs say to run Force Update. +- A failure while reviving a matched row: the row fails at step `module` or `usage` like any other row failure. + +### D8. Stackiq UI + +The Applications (`Modules`, a `FacetedCatalogIndexView` that forwards `quickFilters` to `CnIndexPage`) and Applications in use (`Gebruik`) pages get the quick filter `{"label": "Archived", "filter": {"_archived": "true"}}`. `useSelfFetchList` spreads the active tab's filter into the fetch parameters as they are, so `_archived=true` reaches OpenRegister and the list shows the archive alone; the facet narrowing's `_ids` composes with it (AND). The icon `ArchiveOutline` is registered in `src/icons.js`. `@conduction/nextcloud-vue` 2.65 renders no `@self.archived` on a detail page or in the sidebar metadata, so the detail pages show no archive state yet; that is a library follow-up, not a stackiq widget. + +### D9. How the import reads the state of a stored object + +Found on a Nextcloud 35 rig with OpenRegister 2.1.37-beta (origin/beta `ff4dad5c`): the second import of the original export left APPID 1234 archived (`unchanged`) and created a second module for APPID 2. `ObjectService::searchObjects()` (via `QueryHandler::searchObjects()` → `MagicMapper::searchObjects()` → `MagicSearchHandler::searchObjects()`) applies the lenses in SQL (`buildFilteredQuery()` → `applyBasicFilters()`, `MagicSearchHandler.php` 2135–2153), but converts each row with `MagicSearchHandler::convertRowToObjectEntity()` (3780), which: + +- reads no `_archived` column, so every result keeps `ObjectEntity::$archived = []` (`ObjectEntity.php` 334) and looks working; +- turns the `_deleted` marker into `new DateTime(...)` (3896–3898), but `DeleteObject` writes it as a JSON object (`DeleteObject.php` 361), so the conversion throws, the row becomes null (4035) and `executeSearchQuery()` skips it (3714): `_includeDeleted=true` returns no soft-deleted object. + +So the import never reads the state from a search result: + +- **Working**: found by a search without a lens. **Archived**: found by a search with `_archived=true` (the archive alone, not deleted). An object is looked up working first, then archived. +- **Deleted**: from `MagicMapper::findDeletedAcrossAllMagicTables(_rbac: false, _multitenancy: false)`, the listing behind the Deleted page, which reads the rows itself (`MagicMapper.php` 7001). It scans every magic table of the instance and pages only after merging them, so the import reads it once per run, lazily, and keeps this register's modules and usages. Its objects carry data but no markers (`MagicMapper::rowToObjectEntity()`, 9131), and that data has every string that parses as JSON decoded ("2" becomes 2), so it is never written back. +- **Deleted and archived**: the trash does not say. `MagicMapper::restoreObject()` clears `_deleted` (10078) and answers the object as the mapper reads it by uuid, through `MagicStatisticsHandler::convertRowToObjectEntity()`, which does decode `_archived` (`MagicStatisticsHandler.php` 631–652). A returning row unarchives it when that object is archived, and updates from it. The reconciliation does not need to know: an object in the trash is either left there or restored and archived, and `ArchiveHandler::archive()` leaves an archived object as it is (`ArchiveHandler.php` 111). + +`ArchiveHandler::unarchive()` clears the marker to SQL NULL, so the object is back in the working lists, and does nothing for an object that is not archived (161). A lookup costs one search for a working match and two for an archived one or none, plus the one trash read per run. When OpenRegister hydrates both markers on search results, this can go back to one search with both lenses; the lens-based reading stays correct either way. + +## API Design + +`POST /api/cmdb-import`: `missingRecords` accepts `keep` and `archive` (default `archive`). The report's `summary` gains `archived`, `unarchived`, `deleted` and `restored`; `rows[].outcome` may be any of the four; reconciliation rows have `row: 0`. New error code 503 `ARCHIVE_UNAVAILABLE`. `MISSING_RECORDS_UNSUPPORTED` keeps its status and gains both accepted values in `details.accepted`. `openapi.json` is updated accordingly. + +## Database Changes + +None. The archive and delete markers are OpenRegister's metadata columns. + +## Mixed-spec rationale (ADR-032) + +The change is `kind: code`. Its only schema delta is the annotation `x-openregister-archive: {enabled: true}` on `module` and `usage` in the existing fragment `topdesk-cmdb-import.json`, with a version bump of both. The annotation is glue for this code: OpenRegister refuses to archive an object of a schema without it, and nothing else in stackiq reads it. It follows the fragment convention (ADR-037) and deploys with the existing register import. + +## Declarative-vs-imperative decision (ADR-031) + +- **Declarative:** which schemas may be archived (`x-openregister-archive` on `module` and `usage`), which sheet is the archive sheet and which option values are accepted (`topdesk-profile.json`), and which list shows the archive (the manifest quick filter `_archived=true`). +- **Imperative, because it is the external-integration exception the first change already claimed:** comparing an uploaded workbook with the register and deciding per application between nothing, archive, unarchive, soft-delete and restore. No `x-openregister-*` block can express "archive when the source lists it as archived and delete when the source no longer lists it": that rule depends on a file the register never sees. The state transitions themselves are OpenRegister's (`ArchiveHandler`, `deleteObject()`, `restoreObject()`); stackiq only decides when to call them. +- **Rule stated once, enforced in code:** the table in D5. + +## Nextcloud Integration + +- Controller: `CmdbImportController::import()` (unchanged auth: Nextcloud admins only, CSRF). +- Services: `CmdbExportImportService` (lenses, revival, reconciliation), `Cmdb\CmdbWorkbookReader` and `Cmdb\CmdbImportProfile` (archive sheet), `Cmdb\CmdbImportReport` (outcomes), `ProgressTracker` (phase). +- OpenRegister: `ObjectServiceInterface::searchObjects()` with `_archived` and `_ids`, `::deleteObject()` (contract); `Service\Object\ArchiveHandler::archive()` / `unarchive()` and `Db\MagicMapper::restoreObject()` / `findDeletedAcrossAllMagicTables()` (container, guarded, like the mapping engine). The handler checks `update` for the session user with RBAC on; the import is admin-only, so the check passes, and the audit entry names the admin. + +## Security Considerations + +- Still admin-only, CSRF-protected, with the file bounds of the first change; the archive sheet is read under the same bounds, APPID column only. +- Scope is the imported municipality's usages; a supplier's or another municipality's module is never archived or deleted through this municipality's import. +- Nothing is hard-deleted; every transition is an OpenRegister audit entry with a reason. +- The report and the logs carry APPIDs, module names and uuids, never owner data; contact persons are untouched. + +## NL Design System + +The section adds one radio group with a help text, built from `NcCheckboxRadioSwitch` like the existing switches, labelled and translated. The quick filter is the library's own chip. + +## File Structure + +``` +lib/Service/CmdbExportImportService.php lenses, revival, reconcile(), state helpers +lib/Service/Cmdb/CmdbWorkbookReader.php the archive sheet +lib/Service/Cmdb/CmdbImportProfile.php archiveSheetName(), readSheetNames(), missingRecordsModes() +lib/Service/Cmdb/CmdbImportReport.php archived | unarchived | deleted | restored +lib/Controller/CmdbImportController.php default archive, accepted values, ARCHIVE_UNAVAILABLE +lib/Exception/CmdbImportException.php ARCHIVE_UNAVAILABLE +lib/Service/ProgressTracker.php phase reconciling +lib/Settings/cmdb-import/topdesk-profile.json archiveSheet, missingRecords +lib/Settings/register.d/topdesk-cmdb-import.json x-openregister-archive, module 0.3.9, usage 1.5.7 +src/utils/cmdbImport.js, src/views/settings/sections/CmdbImport.vue +src/manifest.json, src/manifest.d/usages.json, src/icons.js +tests/Unit/..., tests/fixtures/cmdb/topdesk-archived-applications.xlsx, tests/e2e/spec-coverage/cmdb-import.spec.ts +docs/features/cmdb-import.md, l10n/en.json, l10n/nl.json +``` + +## Seed Data + +No seed objects change. The fragment's three seed modules stay as they are; the `module` and `usage` schemas gain only the archive annotation. + +## Risks / Trade-offs + +- [The default archives on the first re-import after the upgrade] → The section shows the choice with help text; the report lists every archived and deleted application; nothing is hard-deleted. +- [A workbook saved without the archive sheet] → archive only, never delete, plus a warning (D3). +- [A failure between the usage and the module] → the usage is written first, so the municipality's view is right and the module is at worst still visible (D5). +- [Reviving before writing adds calls per returning row] → only for rows whose match is archived or deleted; a row whose module is working costs the same searches as before, a new row one more search (the archive lens). +- [The trash listing scans every magic table of the instance] → read once per import, not per row (D9). +- [OpenRegister internals change shape] → guarded resolution, 503 before the file is read (D7). + +## Migration Plan + +No data migration. The fragment deploys with the existing register import (Force Update on an installation that already has the register). Rollback is a revert of the PR; archived or deleted objects keep their state and can be unarchived or restored by hand. + +## Open Questions + +- Should the library render `@self.archived` on detail pages, so an admin sees why an application is not in the list? (library follow-up) +- Should a module that no usage points at any more be archived as well? Out of scope here. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md new file mode 100644 index 000000000..7c27fdd85 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md @@ -0,0 +1,95 @@ +--- +kind: code +depends_on: [] +--- + +# Proposal: cmdb-import-archive-reconciliation + +## Summary + +A repeated CMDB import now reconciles the applications a municipality uses with the newer export. An application whose APPID is no longer on the two CMDB sheets but is listed on the export's sheet "Gearchiveerde Applicaties" is archived: its `module` and the municipality's `usage` get OpenRegister's archive state, so they drop out of every list and search until an admin asks for the archive. An application whose APPID is on no sheet at all is soft-deleted through OpenRegister's trash, from which it can be restored. An application that returns to the export is unarchived, or restored from the trash, and then updated as usual, so no duplicate is ever created. The import option `missingRecords` accepts `keep` and `archive`, and `archive` is the default; the settings section shows the choice. Stackiq's Applications and Applications in use pages get an "Archived" quick filter. The report gets the outcomes `archived`, `unarchived`, `deleted` and `restored`. + +## Motivation + +The first version of the import (openspec/changes/archive/2026-10-05-cmdb-export-import) left records that disappeared from a newer export untouched. The municipality delivers a new export every month and asked on 2026-10-01 (Jira WOO-587, parent WOO-586, epic WOO-281): an application that left the CMDB sheets but is on the archive sheet must be marked as archived and be findable in OpenCatalogi only with an explicit filter; an application that is on no sheet any more must be soft-deleted with the existing functionality. Without this, the catalogue keeps showing applications the municipality retired, and Portaliq keeps listing them under "Software we use". + +OpenRegister already has both states. Its archive state (`@self.archived`, `POST /api/objects/{register}/{schema}/{id}/archive`) keeps an object whole but out of every list unless `_archived=true` or `_archived=any` is asked; its soft delete (`@self.deleted`) puts an object in the trash for 30 days, from which it can be restored. OpenCatalogi passes the list parameters through to OpenRegister, and Portaliq lists usages without either lens, so both apps hide archived and deleted records with no change of their own (decided 2026-10-09). + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `cmdb-export-import`: `missingRecords` becomes `keep | archive` with `archive` as the default (REQ-CMDB-012 modified); the reconciliation of missing records against the archive sheet (REQ-CMDB-015), the return of an archived or deleted application (REQ-CMDB-016) and the archive lens in stackiq (REQ-CMDB-017) are added; the settings section shows the choice (REQ-CMDB-014 modified). The `module` and `usage` schemas opt into OpenRegister's archive state through the register fragment (see design.md, Mixed-spec rationale). + +## Affected Projects + +- [ ] Project: `stackiq`: import service, workbook reader and profile, controller option, register fragment, the "CMDB import" settings section, the manifest quick filters, tests, fixtures, administrator docs and translations. + +## Scope + +### In Scope + +- The sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging) is read for its APPID column only, when it is present. +- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet: archived only, with a warning in the report. +- A returning APPID: an archived module or usage is unarchived, a soft-deleted one is restored from the trash, and the row is then updated as before. +- `missingRecords`: `keep` leaves everything as it is (the behaviour so far), `archive` reconciles and is the default; `remove` stays reserved and is refused. +- The settings section offers the choice with help text; the report shows the new outcomes in its summary and rows. +- `x-openregister-archive` on the `module` and `usage` schemas through the register fragment, with a version bump. +- An "Archived" quick filter on the Applications and Applications in use pages. +- PHPUnit, Jest, a Playwright e2e for the re-import with the archive sheet, a fixture variant, docs and Dutch and English strings. + +### Out of Scope + +- Hard deletion of objects; the trash's own retention (30 days) stays OpenRegister's. +- Changes to the municipality's TOPdesk export, and the raw "Invoer gearchiveerde appl" sheet (the derived "Gearchiveerde Applicaties" sheet is the source). +- Contact persons: the owner of an archived or deleted application stays, because it is shared with the municipality's other applications. +- Modules only other organisations use, and organisations (suppliers) that no application names any more. +- An archive widget on the detail pages: `@conduction/nextcloud-vue` 2.65 does not render `@self.archived`; a follow-up once the library does. +- Changes to OpenCatalogi or Portaliq. + +## Approach + +`CmdbWorkbookReader` loads the archive sheet next to the two CMDB sheets when the workbook has it, under the same size and row bounds, and hands the import service the APPIDs it holds. `CmdbExportImportService` widens the match of a module (by import key) and of a usage (by consumer and module) to archived and soft-deleted objects, and revives what it matches before the usual update. After the last row, when the import was not cancelled and `missingRecords` is `archive`, a reconciliation step pages through the municipality's usages with no lens and with OpenRegister's `_archived=true` lens, reads the trash once through `MagicMapper::findDeletedAcrossAllMagicTables()` (a search with `_includeDeleted=true` returns no deleted rows on the current OpenRegister beta; design.md D9), loads their modules in batches, and archives, unarchives, soft-deletes or restores module and usage through OpenRegister's `ArchiveHandler`, `ObjectServiceInterface::deleteObject()` and `MagicMapper::restoreObject()`, each object in its own error boundary. The step runs under the import's register lock, reports its progress in a phase of its own, and adds its outcomes to the report. Details are in design.md. + +## New Dependencies + +None. `ArchiveHandler` and `MagicMapper` come from OpenRegister, which stackiq already requires; both are resolved through the container behind a guard, like the mapping engine. + +## Impact + +- **Backend**: `lib/Service/CmdbExportImportService.php` (match lenses, revival, reconciliation), `lib/Service/Cmdb/CmdbWorkbookReader.php` and `CmdbImportProfile.php` (archive sheet), `CmdbImportReport.php` (outcomes), `lib/Controller/CmdbImportController.php` (option default and accepted values), `lib/Service/ProgressTracker.php` (phase `reconciling`), `lib/Exception/CmdbImportException.php` (`ARCHIVE_UNAVAILABLE`). +- **Configuration**: `lib/Settings/cmdb-import/topdesk-profile.json` (archive sheet, accepted modes). +- **Schema**: `lib/Settings/register.d/topdesk-cmdb-import.json` declares `x-openregister-archive` on `module` (0.3.9) and `usage` (1.5.7). +- **Frontend**: `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `src/manifest.json`, `src/manifest.d/usages.json`, `src/icons.js`. +- **Data**: a re-import with the default option archives or soft-deletes modules and usages the export no longer lists. Nothing is hard-deleted; a soft-deleted object can be restored from OpenRegister's trash for 30 days. + +## Cross-Project Dependencies + +- **openregister** (consumed, not changed): `ObjectServiceInterface::deleteObject()` (contract), `Service\Object\ArchiveHandler` and `Db\MagicMapper::restoreObject()` and `findDeletedAcrossAllMagicTables()` (not public contracts, guarded), the `_archived` list lens, and the schema annotation `x-openregister-archive`. +- **opencatalogi** and **portaliq** (consumers, not changed): both list through OpenRegister's default lens, so archived and deleted applications disappear from them without a change; an OpenCatalogi caller finds archived applications with `_archived=true` on the existing public routes. + +## Risks + +### Risk 1: A re-import with the new default archives what an admin did not expect +**Severity:** Medium — **Mitigation:** only the municipality's own usages and the modules they point at are in scope, never a supplier's or another municipality's module; nothing is hard-deleted; the choice is visible in the section with help text, and the report lists every archived and deleted application. A cancelled or failed import never reconciles. + +### Risk 2: A workbook without the archive sheet soft-deletes everything that moved there +**Severity:** Medium — **Mitigation:** without the sheet, missing applications are archived, never soft-deleted, and the report warns that the sheet was not found. + +### Risk 3: The archive services are OpenRegister internals +**Severity:** Low — **Mitigation:** resolved through the container behind a guard; when missing, an import with `missingRecords=archive` is refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read, and `keep` still works. + +### Risk 4: The schemas on an installation predate the annotation +**Severity:** Low — **Mitigation:** OpenRegister refuses to archive an object of a schema without `x-openregister-archive`; the import reports that object as failed and the docs point to Force Update, which deploys the fragment. + +## Rollback Strategy + +Revert the PR. Objects archived or soft-deleted by an import keep their state; an admin unarchives them through OpenRegister's object actions or restores them from the trash. The schema annotation may stay deployed without harm. + +## Open Questions + +- Should a module that no usage points at any more (every municipality archived it) be archived as well? Out of scope here; a supplier's module stays. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md new file mode 100644 index 000000000..1ac827456 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md @@ -0,0 +1,191 @@ +# cmdb-export-import Specification + +**Status**: in-progress +**Scope**: stackiq +**OpenSpec changes**: +- [cmdb-export-import](../../../archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ +- [cmdb-import-archive-reconciliation](../../) + +## Purpose + +A repeated CMDB import reconciles the applications a municipality uses with the newer export. An application whose APPID left the two CMDB sheets is archived in OpenRegister's archive state when the export's sheet "Gearchiveerde Applicaties" lists it, and soft-deleted into OpenRegister's trash when no sheet lists it any more; one that returns is unarchived or restored and updated as usual, never duplicated. Archived and deleted applications disappear from OpenCatalogi and Portaliq without a change there, and stackiq lists the archive on request (Jira WOO-587). + +OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` (`searchObjects()` with no lens and with the `_archived=true` and `_ids` lenses, `deleteObject()`), `Service\Object\ArchiveHandler` (`archive()`, `unarchive()`) and `Db\MagicMapper` (`findDeletedAcrossAllMagicTables()` for the trash, `restoreObject()`) (both resolved through the container behind a guard; why the trash is read that way: design.md D9), and the schema annotation `x-openregister-archive`. + +## ADDED Requirements + +### Requirement: An application missing from the CMDB sheets SHALL be archived when the archive sheet lists it and soft-deleted when no sheet does (REQ-CMDB-015) + +With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. + +#### Scenario: An application that moved to the archive sheet is archived +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the modules with APPID `1234` and `2` were imported for "Gemeente Voorbeeldstad" with their usages +- **WHEN** an export is imported in which `1234` is only on "Gearchiveerde Applicaties" and `2` is on no sheet +- **THEN** the module and the usage of `1234` SHALL carry OpenRegister's archive state and the report row SHALL be `archived` +- **AND** the module and the usage of `2` SHALL be in OpenRegister's trash and the report row SHALL be `deleted` +- **AND** the Applications page SHALL list `1234` under the quick filter "Archived" and neither under "All" +- **AND** the owner contact person SHALL be unchanged + +#### Scenario: Without the archive sheet a missing application is archived, never deleted +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted imports a workbook without the sheet and asserts the archive state, no delete, and the warning naming the sheet. + +- **GIVEN** the module with APPID `7` and its usage were imported +- **WHEN** a workbook without "Gearchiveerde Applicaties" that no longer lists `7` is imported +- **THEN** the module and the usage of `7` SHALL be archived and SHALL NOT be deleted +- **AND** the report SHALL carry a warning that the sheet "Gearchiveerde Applicaties" was not found + +#### Scenario: Keeping missing records, a cancelled import and another municipality leave everything as it is +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRecordsMissingFromTheExportStayWhenKept, testACancelledImportDoesNotReconcile and testAnotherMunicipalitysUsagesAreNeverReconciled assert no archive, no delete and no reconciliation row. + +- **GIVEN** the modules with APPID `1` and `7` and their usages for "Gemeente Voorbeeldstad", and a usage of `7` for "Gemeente Anders" +- **WHEN** an export with only `1` is imported with `missingRecords` `keep`, or with `archive` but cancelled after row 1 +- **THEN** no module or usage SHALL be archived or deleted +- **AND** when it is imported with `archive` and completes, only the usage of "Gemeente Voorbeeldstad" for `7` and the module SHALL be archived; the usage of "Gemeente Anders" SHALL be untouched + +#### Scenario: A cancel after the last row or during the reconciliation stops it +@e2e exclude Timing-dependent; tests/Unit/Service/CmdbExportImportServiceTest.php testACancelDuringTheLastRowArchivesNothing and testACancelDuringTheReconciliationStopsBeforeTheNextApplication request the cancel at those two moments and assert the transitions and the cancelled report. + +- **GIVEN** the modules with APPID `1`, `7` and `8` and their usages for "Gemeente Voorbeeldstad" +- **WHEN** an export with only `1`, and `7` and `8` on the archive sheet, is imported with `archive` and a cancel arrives while row `1` is processed +- **THEN** no module or usage SHALL be archived or deleted, and the report SHALL be cancelled +- **AND** when the cancel arrives instead while `7` is archived, `7` SHALL stay archived, `8` SHALL be left as it is, and the report SHALL be cancelled + +#### Scenario: One application that cannot be archived does not stop the others +@e2e exclude Fault injection; tests/Unit/Service/CmdbExportImportServiceTest.php testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled makes the archive of one application throw and asserts a failed row naming the step, the other application archived, and the log line without a name. + +- **GIVEN** the modules with APPID `7` and `8` left the export and are on the archive sheet +- **WHEN** archiving `7` throws +- **THEN** the report SHALL list `7` as `failed` with the step `archive` and the exception class +- **AND** `8` SHALL be archived + +#### Scenario: Without the archive services an archiving import is refused before reading +@e2e exclude Environment condition; tests/Unit/Service/CmdbExportImportServiceTest.php testMissingArchiveServicesRefuseAnArchivingImport asserts 503 ARCHIVE_UNAVAILABLE with no read and no save, and that keep still imports; tests/Unit/Controller/CmdbImportControllerTest.php asserts the status and the translated message. + +- **GIVEN** an OpenRegister without `ArchiveHandler` +- **WHEN** an export is imported with `missingRecords` `archive` +- **THEN** the endpoint SHALL answer 503 with error `ARCHIVE_UNAVAILABLE` and nothing SHALL be read or written +- **AND** the same export with `keep` SHALL be imported + +### Requirement: An application that returns to the export SHALL be unarchived or restored, never duplicated (REQ-CMDB-016) + +The import SHALL match a module by its import key, and a usage by consumer and module, among archived and soft-deleted objects as well as working ones. Before it updates a matched module or usage that is archived, it SHALL unarchive it through OpenRegister's archive state; before it updates one that is soft-deleted, it SHALL restore it from the trash. It SHALL then update the row as REQ-CMDB-006 and REQ-CMDB-009 describe, and SHALL NOT create a second module or usage. The row's outcome SHALL be `restored` when a module or usage was restored, else `unarchived` when one was unarchived, else the usual outcome; the summary SHALL count both. With `updateExisting` false the row SHALL be skipped as `exists` and nothing SHALL be revived. When more than one module or usage matches, the import SHALL take a working one before an archived one, and an archived one before one in the trash, so a module deleted by hand next to the one a later import created SHALL NOT be restored. Organisations and contact persons SHALL keep matching among working objects only. + +#### Scenario: An archived application returns to the export +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the module and the usage of APPID `1234` are archived and those of APPID `2` are in the trash +- **WHEN** an export that lists both on the CMDB sheets is imported +- **THEN** the report SHALL show `1234` as `unarchived` and `2` as `restored` +- **AND** there SHALL still be one module and one usage for each +- **AND** both SHALL be listed again without a lens + +#### Scenario: A returning application is not revived when existing records are not updated +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testUpdateExistingFalseLeavesAnArchivedMatchArchived asserts the row is skipped as `exists`, the archive state stays and no object is created. + +- **GIVEN** the module and the usage of APPID `7` are archived +- **WHEN** a row with APPID `7` is imported with `updateExisting` false +- **THEN** the row SHALL be skipped with reason `exists` +- **AND** both SHALL stay archived, and no module or usage SHALL be created + +#### Scenario: A working match wins over a deleted one with the same import key +@e2e exclude Data condition from earlier imports; tests/Unit/Service/CmdbExportImportServiceTest.php testAWorkingMatchWinsOverADeletedOneWithTheSameKey stores a deleted module and usage ahead of the working ones and asserts the working ones are updated and nothing is restored. + +- **GIVEN** a module with APPID `7` in the trash and a working module with the same import key, each with a usage of "Gemeente Voorbeeldstad" +- **WHEN** a row with APPID `7` is imported +- **THEN** the working module and its usage SHALL be updated +- **AND** the module and the usage in the trash SHALL stay there, and nothing SHALL be restored + +### Requirement: Archived applications and usages SHALL be listable in stackiq and SHALL stay out of OpenCatalogi and Portaliq by default (REQ-CMDB-017) + +The `module` and `usage` schemas SHALL declare `x-openregister-archive` with `enabled` true through the register fragment `topdesk-cmdb-import.json`, as `module` 0.3.9 and `usage` 1.5.7, so OpenRegister offers the archive state on both. Stackiq's Applications and Applications in use pages SHALL offer a quick filter "Archived" that lists the archive alone (`_archived=true`). No change SHALL be made to OpenCatalogi or Portaliq: both list through OpenRegister's default lens, so an archived or deleted application is not in OpenCatalogi's search results or in Portaliq's "Software we use", and an OpenCatalogi caller finds archived applications with `_archived=true` on the existing routes. + +#### Scenario: The merged register offers the archive state on both schemas +@e2e exclude Configuration; tests/Unit/Settings/TopdeskCmdbFragmentTest.php testTheMergedSchemasOfferTheArchiveState asserts the versions and the annotation after merging every fragment. + +- **GIVEN** every register fragment merged in filename order +- **THEN** `module` SHALL be version 0.3.9 and `usage` 1.5.7 +- **AND** both SHALL declare `configuration.x-openregister-archive.enabled` true, and the usage lifecycle of 1.5.6 SHALL still be declared + +#### Scenario: The Applications page lists the archive on request +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** the module of APPID `1234` is archived +- **WHEN** an admin opens the Applications page +- **THEN** the module SHALL NOT be listed +- **AND** under the quick filter "Archived" it SHALL be listed + +## MODIFIED Requirements + +### Requirement: Records missing from a newer export SHALL be archived by default or kept on request (REQ-CMDB-012) + +The import SHALL accept `missingRecords` with the values `keep` and `archive`; `archive` is the default. With `keep` it SHALL NOT change, archive, depublish or delete a module, usage, organisation or contact person because its APPID is absent from the upload. With `archive` it SHALL reconcile as REQ-CMDB-015 describes. Any other value, including the reserved `mark` and `remove`, SHALL be refused with 422 `MISSING_RECORDS_UNSUPPORTED`, naming both accepted values in `details.accepted`. The accepted values SHALL be declared in the import profile. + +(Previously: `keep` was the only accepted value and the default; records missing from a newer export were always left untouched.) + +#### Scenario: An application dropped from the export stays when records are kept +@e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testRecordsMissingFromTheExportStayWhenKept imports two rows, then one with `missingRecords` `keep`, and asserts the other module and usage are unchanged. + +- **GIVEN** the modules with APPID `1` and `7` were imported for "Gemeente Voorbeeldstad" +- **WHEN** a newer export that only contains APPID `1` is imported with `missingRecords` `keep` +- **THEN** the module with APPID `7` and its usage SHALL be unchanged + +#### Scenario: Without the option a missing application is reconciled +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testMissingRecordsDefaultsToArchive asserts the service receives `archive` when the field is absent and the sent value otherwise. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it without `missingRecords` +- **THEN** the import SHALL run with `archive` + +#### Scenario: A reserved value is refused +@e2e exclude Validation; tests/Unit/Controller/CmdbImportControllerTest.php testAReservedMissingRecordsValueIsRefused. + +- **GIVEN** a valid export +- **WHEN** a Nextcloud admin posts it with `missingRecords=remove` +- **THEN** the endpoint SHALL answer 422 with error `MISSING_RECORDS_UNSUPPORTED` and `details.accepted` `["keep", "archive"]` +- **AND** no object SHALL be written + +### Requirement: The admin settings SHALL offer a CMDB import section (REQ-CMDB-014) + +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, choose what happens to applications missing from the export (archive, the default, or keep) with help text that says what each does, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary, including the archived, unarchived, deleted and restored counts when they are not zero, and a report table that can be filtered by outcome, including the four new ones; a reconciliation row SHALL show no row number. Every control SHALL have a visible label, and every string SHALL be translatable. + +(Previously: the section had no choice for missing records; the request always sent `keep`.) + +#### Scenario: The admin runs an import from the settings page +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts + +- **GIVEN** a Nextcloud admin on stackiq's admin settings page +- **WHEN** they choose "Gemeente Voorbeeldstad", choose the anonymised export and press "Import" +- **THEN** a progress bar SHALL appear while the import runs +- **AND** afterwards the summary and the report table SHALL be shown +- **AND** filtering the table on `created` SHALL show the two imported rows + +#### Scenario: The admin chooses to keep missing applications +@e2e exclude Covered by the Jest test; src/utils/cmdbImport.spec.js asserts buildImportForm() sends `archive` by default and `keep` when chosen, and src/views/settings/sections/CmdbImport.spec.js asserts the section's default. + +- **GIVEN** the section +- **WHEN** the admin selects "Keep them as they are" and starts the import +- **THEN** the request SHALL carry `missingRecords=keep` +- **AND** without a choice it SHALL carry `missingRecords=archive` + +## Non-Functional Requirements + +- **Performance:** the reconciliation SHALL page the municipality's usages with the import's page size and load their modules in one search per page, so an export of 1,100 rows adds a handful of searches, and SHALL make no write for an application that is already in the state the export asks for. +- **Security:** unchanged from the first change; the archive sheet is read under the same bounds, APPID column only; the scope is the imported municipality's usages; nothing is hard-deleted; every state change is an OpenRegister audit entry. +- **Privacy:** reconciliation rows and log lines carry APPIDs, module names and uuids only; contact persons are never touched. +- **Accessibility:** Target WCAG 2.2 AA. The new choice is a labelled radio group (SC 1.3.1, 3.3.2; gates `form-label-association`, `nc-input-labels`) with help text; the summary tiles and the outcome filter follow the existing pattern. New in 2.2: 2.4.11, 2.5.8 and 3.3.7 apply as before; 2.5.7, 3.2.6 and 3.3.8 do not apply (no dragging, no help mechanism, no authentication). +- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the choice, the outcomes, the warning and the error message. + +## Acceptance Criteria + +- [ ] A re-import with the archive sheet archives the application that moved there and soft-deletes the one on no sheet; the report shows `archived` and `deleted`. +- [ ] Importing the original export again unarchives and restores them; no duplicate is created. +- [ ] With "Keep them as they are" nothing is archived or deleted. +- [ ] The Applications and Applications in use pages list archived records only under "Archived". +- [ ] OpenCatalogi's search and Portaliq's "Software we use" no longer show an archived application; `_archived=true` on OpenCatalogi's route finds it. + +## Notes + +- The decision table per application is in the change's [design.md](../../design.md), D5. +- The library does not render `@self.archived` on detail pages yet; a follow-up for `@conduction/nextcloud-vue`. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md new file mode 100644 index 000000000..4ae74b821 --- /dev/null +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -0,0 +1,89 @@ +# Tasks: cmdb-import-archive-reconciliation + +Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md` (`SPEC` below). Design: `design.md` (the decision table in D5 is authoritative for what happens per application). + +## Implementation Tasks + +### Task 1: Schema opt-in and the import profile +- **spec_ref**: `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017`, `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` +- **files**: `lib/Settings/register.d/topdesk-cmdb-import.json`, `lib/Settings/cmdb-import/topdesk-profile.json`, `lib/Service/Cmdb/CmdbImportProfile.php`, `tests/Unit/Settings/TopdeskCmdbFragmentTest.php`, `tests/Unit/Settings/PublicationFieldRulesTest.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php` +- **acceptance_criteria**: + - GIVEN every register fragment merged in filename order THEN `module` is 0.3.9 and `usage` is 1.5.7, both with `configuration.x-openregister-archive.enabled` true, and the usage lifecycle of 1.5.6 is still there + - GIVEN the profile THEN `archiveSheetName()` is "Gearchiveerde Applicaties", `readSheetNames()` lists the two CMDB sheets plus the archive sheet, and `missingRecordsModes()` is `["keep", "archive"]` without OpenRegister +- [x] Implement +- [x] Test + +### Task 2: The reader reads the archive sheet's APPIDs +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `lib/Service/Cmdb/CmdbWorkbookReader.php`, `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php` +- **acceptance_criteria**: + - GIVEN the anonymised fixture THEN `read()` returns `archive.present` true and `archive.appIds` `["1198"]`, and the rows of the CMDB sheets are unchanged + - GIVEN a workbook without the sheet, or with the sheet but without an APPID column THEN `archive.present` is false (and `keyColumnMissing` says which), and the CMDB rows are still read + - GIVEN an archive sheet beyond the row limit THEN `TOO_MANY_ROWS` names it; only its APPID column is materialised +- [x] Implement +- [x] Test + +### Task 3: Matching revives archived and deleted applications +- **spec_ref**: `SPEC#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016` +- **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/Cmdb/CmdbImportReport.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php`, `tests/Unit/Service/Cmdb/CmdbImportReportTest.php` +- **acceptance_criteria**: + - GIVEN an archived module and usage for an APPID WHEN a row with that APPID is imported THEN both are unarchived before the update, no second module or usage exists, and the row's outcome is `unarchived` + - GIVEN a soft-deleted module and usage WHEN the row returns THEN both are restored from the trash and the outcome is `restored` + - GIVEN only the usage archived or soft-deleted THEN the usage alone is unarchived or restored, with that outcome, and no second usage exists + - GIVEN a module and usage soft-deleted while archived THEN both are restored and unarchived, the outcome is `restored`, and the update starts from the restored object (design D9) + - GIVEN OpenRegister's search results (no archive marker; no soft-deleted row, `_includeDeleted` or not) THEN the state comes from the lens that found the object and from the trash listing, read once per import (design D9) + - GIVEN `updateExisting` false THEN the row is skipped as `exists` and nothing is revived + - GIVEN a deleted and a working module (and usage) sharing one import key THEN the working one is updated and nothing is restored +- [x] Implement +- [x] Test + +### Task 4: Reconciliation after the rows +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/ProgressTracker.php`, `lib/Exception/CmdbImportException.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` +- **acceptance_criteria**: + - GIVEN APPIDs 1 and 7 imported WHEN an export with row 1 and 7 on the archive sheet is imported THEN module and usage of 7 are archived with a reason, the report row is `archived`, and the owner contact person is untouched + - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent THEN they are archived instead and the report warns + - GIVEN APPID 7 soft-deleted and listed on the archive sheet THEN module and usage are restored and archived; GIVEN it already archived or, when on no sheet, already deleted THEN nothing is written + - GIVEN `missingRecords` `keep`, a cancelled import, or a usage of another municipality THEN nothing is archived or deleted + - GIVEN a cancel during the last row THEN nothing is archived or deleted; GIVEN a cancel during the reconciliation THEN it stops before the next application and the report is cancelled + - GIVEN the archive of one application throws THEN that application is a `failed` row and the others are reconciled; GIVEN no ArchiveHandler THEN 503 `ARCHIVE_UNAVAILABLE` before reading + - GIVEN the step runs THEN the operation is in phase `reconciling` and its progress advances per application +- [x] Implement +- [x] Test + +### Task 5: Controller option +- **spec_ref**: `SPEC#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012` +- **files**: `lib/Controller/CmdbImportController.php`, `tests/Unit/Controller/CmdbImportControllerTest.php`, `openapi.json`, `postman/stackiq-tests.json`, `l10n/en.json`, `l10n/nl.json` +- **acceptance_criteria**: + - GIVEN no `missingRecords` THEN the service receives `archive`; GIVEN `keep` or `archive` THEN it is passed through; GIVEN `remove` or `mark` THEN 422 `MISSING_RECORDS_UNSUPPORTED` with `details.accepted` `["keep", "archive"]` and no import + - GIVEN `ARCHIVE_UNAVAILABLE` from the service THEN 503 with its own translated message +- [x] Implement +- [x] Test + +### Task 6: Settings section, report words and the archive quick filter +- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014`, `SPEC#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017` +- **files**: `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `src/utils/cmdbImport.spec.js`, `src/views/settings/sections/CmdbImport.spec.js`, `tests/vitest/cmdbArchiveQuickFilter.spec.js`, `src/manifest.json`, `src/manifest.d/usages.json`, `src/icons.js`, `l10n/en.json`, `l10n/nl.json` +- **acceptance_criteria**: + - GIVEN the section THEN a labelled choice "Applications missing from the export" offers Archive (default) and Keep with help text, and `buildImportForm()` sends the choice as `missingRecords` + - GIVEN a report with the new outcomes THEN the summary shows their tiles, the outcome filter offers them, and a reconciliation row shows no row number + - GIVEN the Applications and Applications in use pages THEN a quick filter "Archived" sends `_archived=true` + - Every new string is in `l10n/en.json` and `l10n/nl.json`, and the generated `l10n/*.js` are current +- [x] Implement +- [x] Test + +### Task 7: Fixture, docs and e2e +- **spec_ref**: `SPEC#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015` +- **files**: `tests/fixtures/cmdb/build-fixtures.py`, `tests/fixtures/cmdb/topdesk-archived-applications.xlsx`, `tests/fixtures/cmdb/README.md`, `tests/Unit/Fixtures/CmdbFixtureHygieneTest.php`, `tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php`, `tests/e2e/spec-coverage/cmdb-import.spec.ts`, `docs/features/cmdb-import.md`, `docs/features/README.md`, `CHANGELOG.md` +- **acceptance_criteria**: + - GIVEN `build-fixtures.py` THEN it derives `topdesk-archived-applications.xlsx`: APPID 1234 on the archive sheet only, APPID 2 on no sheet, both CMDB sheets present and empty; the hygiene test passes on it + - GIVEN the e2e THEN after the first import, importing the variant archives 1234 and soft-deletes 2, the Applications page lists 1234 under "Archived", and importing the original again reports 1234 `unarchived` and 2 `restored` with the object counts back to the first import's + - GIVEN the docs THEN "Repeat imports" describes archive, soft delete, the archive sheet, the option and the return of an application, and the error table has `ARCHIVE_UNAVAILABLE` and the new meaning of `MISSING_RECORDS_UNSUPPORTED` +- [x] Implement +- [ ] Test + - Status: PHPUnit covers the fixture round trip (testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack); the Playwright test is written and not run yet, because no Nextcloud was reachable from the worktree. + +## Quality checklist + +- PHPUnit for every new rule in the service, the reader and the controller; the fixture hygiene test covers the new variant +- Jest for the section's request and the report words; Playwright for the re-import with the archive sheet +- `composer test:unit`, `composer phpcs`, `composer lint`, `npm run test:unit`, `npm run lint`, `npm run format`, `npm run test:l10n`, `npm run check:l10n-js`, `npm run check:manifest`, `npm run check:schema-l10n` and `npm run check:vue-demi` pass diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index d2f7fd4b5..927da9cfb 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -1,15 +1,16 @@ --- capability: cmdb-export-import -status: done +status: in-progress built_by: openspec/changes/archive/2026-10-05-cmdb-export-import --- # cmdb-export-import Specification -**Status**: done +**Status**: in-progress **Scope**: stackiq **OpenSpec changes**: - [cmdb-export-import](../../changes/archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) +- [cmdb-import-archive-reconciliation](../../changes/cmdb-import-archive-reconciliation/) — a re-import archives the applications that moved to the sheet "Gearchiveerde Applicaties" and soft-deletes the ones on no sheet, revives the ones that return, with `missingRecords: keep | archive` (default archive) and an "Archived" quick filter (kind: code) ## Purpose diff --git a/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php index 3da15838b..cc3dbd53a 100644 --- a/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php +++ b/tests/Unit/Fixtures/CmdbFixtureHygieneTest.php @@ -188,7 +188,7 @@ private function parts(string $path): array { */ public function testTheFixturesExist(): void { $names = array_keys(self::fixtures()); - foreach (['topdesk-export-anonymised.xlsx', 'topdesk-missing-appid.xlsx', 'topdesk-shuffled-columns.xlsx', 'topdesk-formula-and-connection.xlsx'] as $expected) { + foreach (['topdesk-export-anonymised.xlsx', 'topdesk-missing-appid.xlsx', 'topdesk-shuffled-columns.xlsx', 'topdesk-formula-and-connection.xlsx', 'topdesk-archived-applications.xlsx'] as $expected) { $this->assertContains($expected, $names); } }//end testTheFixturesExist() diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 2d330258f..17e11067d 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -306,4 +306,28 @@ public function testTheUploadLimitNeedsNoOpenRegister(): void { $profile = new CmdbImportProfile(container: $this->emptyContainer(), directory: '/nonexistent'); $this->assertSame(CmdbImportProfile::DEFAULT_MAX_FILE_BYTES, $profile->maxFileBytes()); }//end testTheUploadLimitNeedsNoOpenRegister() + + /** + * The archive sheet is read after the source sheets, and the accepted missingRecords values need no OpenRegister. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testTheArchiveSheetAndTheMissingRecordsModes(): void { + $profile = new CmdbImportProfile(container: $this->emptyContainer()); + $this->assertSame(['keep', 'archive'], $profile->missingRecordsModes(), 'readable before load(), without the pack validator'); + + CmdbTestSupport::loadMigrationPack(); + $profile->load(); + $this->assertSame('Gearchiveerde Applicaties', $profile->archiveSheetName()); + $this->assertSame( + ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB', 'Gearchiveerde Applicaties'], + $profile->readSheetNames() + ); + $this->assertNotContains('Gearchiveerde Applicaties', $profile->sheetNames(), 'the archive sheet holds no rows to import'); + + $missing = new CmdbImportProfile(container: $this->emptyContainer(), directory: '/nonexistent'); + $this->assertSame(['keep'], $missing->missingRecordsModes(), 'without a profile only keep is safe'); + }//end testTheArchiveSheetAndTheMissingRecordsModes() }//end class diff --git a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php index 2e0e06736..b20bba79b 100644 --- a/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php +++ b/tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php @@ -920,6 +920,110 @@ public function testOneRowOverTheLimitWithinTheSpanIsRefused(): void { } }//end testOneRowOverTheLimitWithinTheSpanIsRefused() + /** + * The archive sheet of the anonymised export yields its APPIDs; the CMDB rows are unchanged. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchiveSheetYieldsItsAppIds(): void { + $result = $this->read(name: 'topdesk-export-anonymised.xlsx'); + + $this->assertSame('Gearchiveerde Applicaties', $result['archive']['sheet']); + $this->assertTrue($result['archive']['present']); + $this->assertFalse($result['archive']['keyColumnMissing']); + $this->assertSame(['1198'], array_map('strval', $result['archive']['appIds'])); + $this->assertSame(['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'], array_column($result['rows'], 'sheet'), 'the archive sheet adds no rows to import'); + }//end testTheArchiveSheetYieldsItsAppIds() + + /** + * The later export moves APPID 1234 to the archive sheet and drops APPID 2 from every sheet. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchivedApplicationsVariantListsOnlyTheArchive(): void { + $result = $this->read(name: 'topdesk-archived-applications.xlsx'); + + $this->assertSame([], $result['rows']); + $this->assertTrue($result['archive']['present']); + $this->assertSame(['1234'], array_map('strval', $result['archive']['appIds'])); + }//end testTheArchivedApplicationsVariantListsOnlyTheArchive() + + /** + * Only the APPID column of the archive sheet is read; without the sheet or its APPID column it is absent. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheArchiveSheetIsOptionalAndReadForItsKeyOnly(): void { + $this->requireSpreadsheet(); + $cmdb = [['APPID', 'Applicatie Naam'], [1, 'Een']]; + $cases = [ + 'with the sheet' => [ + ['Beheerde Applicaties CMDB' => $cmdb, 'Gearchiveerde Applicaties' => [['APPID', 'Applicatie Naam', 'Bron'], [7, 'Zeven', 'P-0007'], [], [8, 'Acht', 'P-0008']]], + ['present' => true, 'keyColumnMissing' => false, 'appIds' => ['7', '8']], + ], + 'without the sheet' => [ + ['Beheerde Applicaties CMDB' => $cmdb], + ['present' => false, 'keyColumnMissing' => false, 'appIds' => []], + ], + 'without its APPID column' => [ + ['Beheerde Applicaties CMDB' => $cmdb, 'Gearchiveerde Applicaties' => [['Applicatie Code', 'Applicatie Naam'], ['APP-7', 'Zeven']]], + ['present' => false, 'keyColumnMissing' => true, 'appIds' => []], + ], + ]; + + foreach ($cases as $case => [$sheets, $expected]) { + $path = CmdbTestSupport::buildWorkbook(sheets: $sheets); + try { + $result = (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile()); + } finally { + unlink($path); + } + + $archive = $result['archive']; + $archive['appIds'] = array_map('strval', $archive['appIds']); + unset($archive['sheet']); + $this->assertSame($expected, $archive, $case); + $this->assertSame([2], array_column($result['rows'], 'row'), $case); + $this->assertStringNotContainsString('P-000', (string)json_encode($result), $case); + $this->assertStringNotContainsString('Zeven', (string)json_encode($result), $case . ': only the key column is read'); + } + }//end testTheArchiveSheetIsOptionalAndReadForItsKeyOnly() + + /** + * An archive sheet with more rows than the profile allows is refused and named, like a source sheet. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnArchiveSheetOverTheRowLimitIsRefused(): void { + $this->requireSpreadsheet(); + $path = CmdbTestSupport::buildWorkbook( + sheets: [ + 'Beheerde Applicaties CMDB' => [['APPID', 'Applicatie Naam'], [1, 'Een']], + 'Gearchiveerde Applicaties' => [['APPID'], [7], [8]], + ] + ); + $directory = CmdbTestSupport::profileDirectory(overrides: ['maxRowsPerSheet' => 1]); + + try { + (new CmdbWorkbookReader())->read(path: $path, profile: $this->profile(directory: $directory)); + $this->fail('TOO_MANY_ROWS expected'); + } catch (CmdbImportException $e) { + $this->assertSame('TOO_MANY_ROWS', $e->getErrorCode()); + $this->assertSame(['sheet' => 'Gearchiveerde Applicaties', 'limit' => 1], $e->getDetails()); + } finally { + unlink($path); + CmdbTestSupport::removeDirectory(directory: $directory); + } + }//end testAnArchiveSheetOverTheRowLimitIsRefused() + /** * A text file named .xlsx, a .xlsm and a CSV are refused before PhpSpreadsheet is touched. * diff --git a/tests/Unit/Settings/PublicationFieldRulesTest.php b/tests/Unit/Settings/PublicationFieldRulesTest.php index f5ee7d5b7..84026c2a6 100644 --- a/tests/Unit/Settings/PublicationFieldRulesTest.php +++ b/tests/Unit/Settings/PublicationFieldRulesTest.php @@ -173,7 +173,7 @@ public function testAFragmentNeverLowersAVersion(): void { $this->assertSame('1.5.5', $merged['components']['schemas']['usage']['version']); $schemas = $this->register()['components']['schemas']; - $this->assertSame('1.5.6', $schemas['usage']['version'], 'topdesk-cmdb-import.json bumps it past publication-field-rules.json; value-assessment.json, sorting last, does not lower it'); + $this->assertSame('1.5.7', $schemas['usage']['version'], 'topdesk-cmdb-import.json bumps it past publication-field-rules.json; value-assessment.json, sorting last, does not lower it'); $this->assertSame('0.3.5', $schemas['connection']['version']); $this->assertSame('0.1.6', $schemas['moduleVersion']['version']); }//end testAFragmentNeverLowersAVersion() diff --git a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php index 7ef15d1f9..df49a4556 100644 --- a/tests/Unit/Settings/TopdeskCmdbFragmentTest.php +++ b/tests/Unit/Settings/TopdeskCmdbFragmentTest.php @@ -58,14 +58,14 @@ private function mergedRegister(): array { }//end mergedRegister() /** - * The merged module is 0.3.8, carries the six optional, titled properties and allows BBN2+. + * The merged module is 0.3.9, carries the six optional, titled properties and allows BBN2+. * * @return void */ - public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { + public function testTheMergedModuleIsVersion039WithTheExternalIds(): void { $module = $this->mergedRegister()['components']['schemas']['module']; - $this->assertSame('0.3.8', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); + $this->assertSame('0.3.9', $module['version'], 'a fragment sorting after topdesk-cmdb-import.json overwrote the bump'); foreach (self::PROPERTIES as $property) { $this->assertArrayHasKey($property, $module['properties']); $this->assertNotEmpty($module['properties'][$property]['title'] ?? '', $property); @@ -91,7 +91,7 @@ public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { $this->assertSame(['BBN1', 'BBN2', 'BBN3', 'BBN2+'], $module['properties']['bbnLevel']['enum'], 'the fragment adds BBN2+ to the BIO levels'); $this->assertArrayHasKey('roadmapStatement', $module['properties'], 'the 0.3.4 fragment still applies'); $this->assertSame(['name'], $module['required']); - }//end testTheMergedModuleIsVersion038WithTheExternalIds() + }//end testTheMergedModuleIsVersion039WithTheExternalIds() /** * A CMDB import sets the usage status TOPdesk records from any state; only an administrator may. @@ -104,7 +104,7 @@ public function testTheMergedModuleIsVersion038WithTheExternalIds(): void { */ public function testAnAdministratorMayMoveAUsageToAnyStateTheSourceRecords(): void { $usage = $this->mergedRegister()['components']['schemas']['usage']; - $this->assertSame('1.5.6', $usage['version'], 'a lifecycle-only edit deploys only with a version bump'); + $this->assertSame('1.5.7', $usage['version'], 'a configuration-only edit deploys only with a version bump'); $lifecycle = $usage['configuration']['x-openregister-lifecycle']; $states = $usage['properties']['status']['enum']; @@ -167,4 +167,24 @@ public function testTheSeedModulesShowTheNewProperties(): void { // The base seeds are still there: the fragment appends, it does not replace. $this->assertGreaterThan(3, count($register['components']['objects'])); }//end testTheSeedModulesShowTheNewProperties() + + /** + * Module and usage offer OpenRegister's archive state, and the usage keeps its lifecycle. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-archived-applications-and-usages-shall-be-listable-in-stackiq-and-shall-stay-out-of-opencatalogi-and-portaliq-by-default-req-cmdb-017 + */ + public function testTheMergedSchemasOfferTheArchiveState(): void { + $schemas = $this->mergedRegister()['components']['schemas']; + + $this->assertSame('0.3.9', $schemas['module']['version']); + $this->assertSame('1.5.7', $schemas['usage']['version']); + foreach (['module', 'usage'] as $slug) { + $this->assertSame(['enabled' => true], $schemas[$slug]['configuration']['x-openregister-archive'] ?? null, $slug); + } + + $this->assertArrayHasKey('importAcquisition', $schemas['usage']['configuration']['x-openregister-lifecycle']['transitions'], 'the 1.5.6 lifecycle is still declared'); + $this->assertArrayNotHasKey('x-openregister-archive', $schemas['contactPerson']['configuration'] ?? [], 'contact persons are never archived by the import'); + }//end testTheMergedSchemasOfferTheArchiveState() }//end class diff --git a/tests/fixtures/cmdb/README.md b/tests/fixtures/cmdb/README.md index 415cef21b..219d5b379 100644 --- a/tests/fixtures/cmdb/README.md +++ b/tests/fixtures/cmdb/README.md @@ -1,6 +1,7 @@ # CMDB import fixtures -Test workbooks for the TOPdesk CMDB import (`openspec/changes/archive/2026-10-05-cmdb-export-import`). +Test workbooks for the TOPdesk CMDB import (`openspec/changes/archive/2026-10-05-cmdb-export-import`, +`openspec/changes/cmdb-import-archive-reconciliation`). They are used by the PHPUnit tests under `tests/Unit/` and by the Playwright test `tests/e2e/spec-coverage/cmdb-import.spec.ts`. @@ -19,6 +20,7 @@ in the script. The "Invoer" sheets are not read and stay as they are. | `topdesk-shuffled-columns.xlsx` | The same rows with the columns of both CMDB sheets in reverse order, and the header "Vendor" written as `Vendor⚡`. Reads to the same rows as the original. | | `topdesk-formula-and-connection.xlsx` | On "Beheerde", "Applicatie Naam" is a formula that would evaluate to `Evaluated` with the cached value `Rekenmodel`, "Roepnaam" is a formula without any cached value, and the package declares a synthetic external web connection to `https://example.invalid/`. | | `topdesk-no-source-sheet.xlsx` | A minimal workbook with only a sheet "Blad1". | +| `topdesk-archived-applications.xlsx` | A later export of the same municipality (`openspec/changes/cmdb-import-archive-reconciliation`): both CMDB sheets lose their data row, so APPID 1234 and APPID 2 are gone from them, and the one row of "Gearchiveerde Applicaties" caches APPID 1234. Importing it after `topdesk-export-anonymised.xlsx` archives 1234 and soft-deletes 2. | ## Placeholder data only diff --git a/tests/fixtures/cmdb/build-fixtures.py b/tests/fixtures/cmdb/build-fixtures.py index 84a4f72aa..74c870ee1 100644 --- a/tests/fixtures/cmdb/build-fixtures.py +++ b/tests/fixtures/cmdb/build-fixtures.py @@ -65,6 +65,7 @@ ONBEH_SHEET = 'xl/worksheets/sheet2.xml' # "Onbeh Applicaties CMDB" (from AIA) BEHEERDE_SHEET = 'xl/worksheets/sheet4.xml' # "Beheerde Applicaties CMDB" (from APP) +ARCHIVE_SHEET_NAME = 'Gearchiveerde Applicaties' # Placeholder cached values for formula cells of the CMDB sheets, per sheet and # cell. A cell that does not exist yet is appended to its row (columns are in @@ -367,6 +368,50 @@ def formula(data): return parts + [('xl/connections.xml', SYNTHETIC_CONNECTION.encode('utf-8'))] +def sheet_part(parts, sheet_name): + """The worksheet part of a named sheet, through xl/workbook.xml and its relationships.""" + parts_by_name = dict(parts) + workbook = text(parts_by_name['xl/workbook.xml']) + sheet = re.search(r']*name="%s"[^>]*r:id="([^"]+)"' % re.escape(sheet_name), workbook) + if sheet is None: + sys.exit('sheet "%s" not found' % sheet_name) + rels = text(parts_by_name['xl/_rels/workbook.xml.rels']) + target = re.search(r']*Id="%s"[^>]*Target="([^"]+)"' % sheet.group(1), rels) + if target is None: + target = re.search(r']*Target="([^"]+)"[^>]*Id="%s"' % sheet.group(1), rels) + return 'xl/' + target.group(1).lstrip('/').replace('xl/', '', 1) + + +def archived_applications(parts): + """A later export of the same municipality (cmdb-import-archive-reconciliation): + APPID 1234 moved from "Onbeh Applicaties CMDB" to "Gearchiveerde Applicaties", + and APPID 2 is on no sheet any more. Both CMDB sheets keep their header and + their empty rows but lose their data row; the archive sheet's one row gets + APPID 1234 as its cached value.""" + def drop_data_row(data): + xml = text(data) + new, count = re.subn(r']*?(?:/>|>.*?)', '', xml, count=1, flags=re.S) + if count != 1: + sys.exit('archived-applications: data row 2 not found') + return new.encode('utf-8') + + def archive_appid(data): + xml = text(data) + new, count = re.subn( + r'(]*>(?:[^<]*)?)[^<]*()', + r'\g<1>1234\g<2>', + xml, + count=1, + ) + if count != 1: + sys.exit('archived-applications: APPID cell B2 not found on %s' % ARCHIVE_SHEET_NAME) + return new.encode('utf-8') + + parts = replace_part(parts, ONBEH_SHEET, drop_data_row) + parts = replace_part(parts, BEHEERDE_SHEET, drop_data_row) + return replace_part(parts, sheet_part(parts, ARCHIVE_SHEET_NAME), archive_appid) + + def no_source_sheet(): """A minimal workbook with one sheet "Blad1" and neither CMDB sheet.""" main = 'http://schemas.openxmlformats.org/spreadsheetml/2006/main' @@ -416,6 +461,7 @@ def main(): 'topdesk-shuffled-columns.xlsx': shuffled_columns(base), 'topdesk-formula-and-connection.xlsx': formula_and_connection(base), 'topdesk-no-source-sheet.xlsx': no_source_sheet(), + 'topdesk-archived-applications.xlsx': archived_applications(base), } for name, parts in variants.items(): write_package(os.path.join(HERE, name), parts) diff --git a/tests/fixtures/cmdb/topdesk-archived-applications.xlsx b/tests/fixtures/cmdb/topdesk-archived-applications.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..68cd6a4d2c6dc17d18804d17268edd5747a8e105 GIT binary patch literal 151786 zcmeFa3piA7+dixinouFBn0BZnZG;LllqB0qsmN562&2e;W=bRqg^c~!Q$i(VGc!p@ zvfmqw{kAc-Gh?>@nNpeG^E~hOKHv9#@B4p$bsRS{>t5%5uJgLCdvmR|W*y5!pO;UZ zhlfXy{kO%ULF%PS$TA)tiFO@4%megj%Y|%M`jbu+b(r3}iOfONU#? zjvU7n?)7qYjBr%#0FTLqX(zglr&{Z6fPw4x>83QM%tuFQ3YFKxPpo^xCEhQ!)Z znIOAKK(SBFUa0bvzFM&Phg*$ngPS6pV)J~H@ih6>{S|)I6GT>Op5f8UDSoBzCYYyU zRAfh1n}AODuERRtt?LoqB)j&;X38TO&*B+Ju*v1-YqbQ|%2hY<4ht_~g0;Hw50*1G z^;&g3Z@yG_e0@ecbGj|So-rX`VHet%p=w?2sj>;pYr+-3yt8|^O|au;u$+en!2X2X zzGm;Nw1@LQE40S0Z11YO#QW&`4d_#yrR$T8l&tD;>pOUFsAjM|FnhFul(}STJa;>^4+o?6i_|pd3MR&jyQhnW#7j&t}TQb(lgzY zg*#l)A-m34G}osN0@oyuxhI`}XoEy-5E4tJuTu2f&ztu+=1ocERLrs#pXa1=N852n zOy;X*58B&C_e_OFmVMk4XM0aw-Zg^o+1~QD4mYeJ-uzOAgnq-zx+HR}ETzI~UhDXyjE!lo#0pR7Oaw7He-j}8GI z9#OVW-0hXzpto;cg+gy~`g7_tb7<#Y@k+cr;mCP-df^i%u`9b$JDzlA+g+fkMGg9% zztjIo#m}zO4X<~1bwZ4Zyw-{e^=9OSh$=5!amW9okXL1jFMOv{`YhIJp$oG}8X@xGz2Jh~s z*Nm<`C+@ae;fU;xH3l&gksBZKcIuqTzKrcXl@w9#Ud6w`p_t&Baqvaug5}foYxBjg zKiG)to3?%zUp&1V9k2Fs&eAm6n(<}!eD?D3BbK7huuO*nWwU4VU#!AbG!t-^Xp%5x z>%sLOh|eB9r~!WVIDQ34j&@3|vAVE)ZpVOmkg#cLN4K7(#N}x0Nt40bo_bfWub08k z^ofU+!(S|`1lnyd%Sij`rRg7TvCgk|%>x0$Y)AQl2Ssfr#g*fi>S?rr1jnPTnh`TTg;MNZ9wJKJ@`dX)v^tUer_XWrZ8aY698 zv4b1u1a;Te~JytpbNdd# zA3G5SSsNxQ!;C-s{9aV*2h!sQ8#@z99^hI$Z%yPIvNpf(y>dG3_Je_KOu$ODrbFI` zo2JJnU0uD>h&A(HN1V^k4|m%u+J79rm{;D3etI0i!#xDF-92~cv8VMi_AI~Q`w;j( zGb|kl-)0GPL^XSsV9)M;%=-D{817@lV`bPNQ6W@CRPer} zw!*%)(VntA@Wq$#=)()z6T6ZRG!Rp_YGoN%j%^hK#vuKpPHehYo2atkz-wLkbJeus z@{O%vc$h+Ft89->u|fDTLUOp#v-Z0aQS0(pZw}@!n@Lu*2G|Mkf7XgNNazVk_P}{@nK@%GPMD{+hy2O#!jD zPet~MJuZbfZ*m5MSz{LG)HF5k8xwVdE=WjqT|wLr%w^+8*EZ zMScI5SjH)>%v-YOzu>E=Zk0np?mJmcpEA=cXC%1;a2Kt0Hdcg($Iyj`XZ^1^$oa;# zYc9@8-#+$zn~dVj!#dBc^TVgxF?q|qHneIMxfaGHcDwd($9y!pDi4i3SG?1C?5=kV z1ho9(`$INR;p*ozsaN#%bsL%66R?NXDa>l`R+?#6YtvAFQb_I2G%DYI$A}BE3Z1>Hz>n0vC9WLSHL|05yIUEG$ z)J3?pBt7riZ8m$WTipT;d`UdlLh&qa^j3M&w|C>0$|fBI(g|}|*2=^AK%A*rbycND zUWVJ2OgokAnCTp7y4!?`gXSSwM@>r@#hu)3eF%GVYmcE#6Lmu6Ek4G$7KMTAHXCru zy^h6hmNNE)oT~xaGK*Xf;6{=aYtqA7Zlg<{*F_J~_Ei=`L_U|gzcCIZvr^V?uUSS> z?v8zbG~C zWOQX#5-rVg?M|Du8QPG%VuHO2Ea2_pC>3MjnEH9wdxoR$7Kmr&Goovvzh0l3z`Dge zty!^x);xg1WodM%e5^i8EGm*h#I}S)4F8~(l;OaIufv@3U=bQU{zhpiKo|N zq%XEXRNGI$Z(sHVM#?OIbYOcK^l|m%z#Eu_Wo*f9IQBLMa;UVSHZJmXSGqVYeCr|o z*+eTNj}DdEsro}|9u8k=lNrNqT=s4^Qsp0b{hC%qo^VcEjT_8|*Ilyd?YEc$Asp{rgnTK(Dyp+4{$xnt)Zx7|&v$&7*@)dF^QM4VwQ^(oQC>kZRSE0eo}=X)(t*F>Th#E- zEm^3xFV#NBKBhoRmm#%k;^~_wfQw@%JM{8!Wu zY9&%0BkX^7bb{4gk|rjv+037^nK@fo2-KLgnAfwJ_ph}BA9cUv-B~ze9CsGTCzKP7 zr3_8ISa^AW=3z6hV>5rvCQs~cg90#qRLElvF4@x z5*Eiw{Uj_-mtGUMxL6uv91}1|=!d^0hocP9SFn8j_K_XUH{a@?h*i9It&!`Ii!h3~gc zlD&S~K#t$k<#9^ni9nNveNV5h-5DGhd{12LgkHvIr0zbC@bJLkTjFB6RqV~<`)tJn zj|bR_haFG65x#U&w_)_^hB&?C=u5J+f1~}DoAX-^YJV?*L>S|-8iL%_jpa4B9?S{yY%5ri+LOND!n<68B z0ZuEY?F)LYXLYh6vO+mcE@-EY)rO|XXyr8VAX@_~yT-`x0LB^-9jh1K53GJv{0A*g zDPP|g6s2c%ydkn&`MO+?l#bQ9rpPGe>*7Hd46LpO$-}k^ZntQ6>$y z_5Bl{+Xe2Cj+s111FO6HuM{cruG*>W_Q-!MUu18}=r@q|`zOm?I3R^(M{9-F?>hay zK7aXx*mZkNOYSBru2^;Gz~zjLWk+MqI=!^3Fhag5d;Q|@5tYo|^DQsqL>vS*WtyD# ztJ$S??x1t}(e7oHFNP;wJ{@82{Te3%^Ph85zL#FsyG25#rf=+t`-w-oP44y~0TU4z> zAdtXoPF^-b1FwN!-U#Xq2kxj9wI36Ro#)k|-6JsWiXjprL9beXPLrbI3j+BJUL)4M ze%AYSci&5aO~g01i>y`?5)9<$Gv{4CvP^5uy~{D+UenDx_N=y-5{wn*(-H6?@ZJ;i zNm#G->e*%|qt)U&1@k5Ni~v6U{H5#ezmL)~Y2MuKxLWOyV4yUgx#;o{0j)K@mp5wn zif`EwyxLw*FjkgNN6d#HbWhALAyWI**)2}ltHmz}<}30UN&55yO4mJjFLl%;d`o-n zYBgKIz=M3|8<&p=YpwCW9CNg{dCQK;)%G5Ou^N0jKp%qWJ+X%g>yN#X*y<#*Mm#_; zUx&|Vn@|6m(scpvqmG%FZf)PQMlDJ((2&nue)))))|$Y}8+Cfax9%`nWB*(*_AH-{ zf)7FBp4g*=NS#+LTb&%&h!+axU*a=T^68f>UHACC)NvE>ZSBEp)T#snuko25SUw`9 zwI=9t%<*2+Z9B5p*mnrVI`HYJ`VcnW6AMmQulwrRHmBM(;$wpOcleBs`1DJct_yh| zrEAi>t$lKh+JaypjL#gjd<3Yq=E>!aCwj%V?+{sQFC-M}$ES1Dhp^?I*wci_6R*y0 zciOX7TuLZEh|fsZr+-`Ny3qGhdM4r9+l|($?Gy@p#%Hd-d_-1jP1xlaz24^SI~>>A z9}o@|A?FQ}9}GJ)!(mKBF@}{X0w7J$oN@(!^A@J$tR%1);z! zK6A6>BMMq;A}(*#?+usTQM=aORwy=)PsiMcpeQt=7|^d6@ZbDJ?Ofukl`&^)&DWNj zuf>_K{ZD@h4PNm7B=qpD)WgGRo41Z^Zj{?H9_{xq_Q8o5|4-MS#Ri;+3CytCv-S4= ztqv+l!DA2ApnBraD)!7Va=vAF`*>vd!`NphVj?oEEVtgiy4B%&QtU7b}&v$H(Nn_asum!KA0Myw!a z+6=VN#;$h9oK?l4?L;Vcg z!kkR%z$Kz%mpx&A4%^}y-8W>{x#}A2V2mWAc<6;?Q2M;9DNSiPi~{05qZMxH-B-!J zqE@c@FVATAbIxe9a1PE%=fnl3$Z=zCPVU$bFicxO$7{VNL!3hat(_!&Ds*q=)qCpm zO=O?tfg{$=Zx7HmS54QUo!}2w%I5$jXG>wBlD7Gq1nk%I-?u(gbA4>)`X+Cx`}F4( z_mdtYbKT<=!){H5i&E33hqr`mmEP0uBK1=2_=5iaHxm;s(Xlp7#X4_xw{7v*AEFOp z_N!cyFzmca4SlB*i$I&Bqpv@D7V_@oi0y)@|BCzNE-oIqUuGzFlOv2z*6Pyg!%R^! zLMJzxI^uQnVoH;`!kdZPqhYF6Jtz#|(t2ZD3S$AgNyFR-H$@Hu4PF-T&rzp9?6 zv$-eFXXE0|)Z~sPkJARQkOc|R!WXC0J4!NG(z_d1J|aG)yk@et#62q*xLcff_N3iS z)70VTcAKvIl%72uE@^uKX+#X%Z|;2obkE^x+dXfwPdKnfxP??TMa$!An0%-Dmf#Hu zU3{Na>La&=J=L`Lwi8S5R4ae*`B?w^g>l39N!N$!cD&Js3AVgZyaE)12i?}~d&)ND ziFL`<-^m?3>L_vMNO37f?TL=zHeH7lWnBlg*3Mr1WA8m_W0za=_MSOfwpCsP=l;_6 zO6$DS!3y5c=*erBXfvaN!Y^$hX$9s(f}$_SAgu@4r;`qtw3w*w__=)nhDUO<;OoF3 zF_C0L(Cm%On<6LeIwRdG-FF5@?SY0&ORw@0KiIRrZ~50zBv8QJ>OK?)4o;{WQr`cr z^IS3T==zIR_uW*!+%jxz@4Y_JLUKZvJS{B{nWMkcz}5GfD}OzeBb}7EC5doqQWxXh z_ORmNWmelqo%_dJPEc+sMJH)Uj)$vVb|b2%Pj4P{1p*H~Cg~gnC}0nUI>A0%K;_0) zUi0=0wGq8vVyX8!FjMuC{r36;=de95+)tm)(H%pap#bryOpKy#AQF5>Kt@-W3ov+PhT?+Uho9VUbZ!*$TYhn77*j@FC@UzBo2zi_@W zl~nLt?#Tr?&y6>iKfm?{q${{5jB!ZYHR@}~#M&3PFCFk@Uha_D$3OJYLvGv}5x6eD zNvtMqrB_vG_-=BTY{bhGRd%2YSE{1JUy{QPM%bUIx(@2NTon;+PYzRz_K^*4mGW`?vHMo|9#D>L`fzA<^cBe(S`uWiJK%oODS$z z{rOZPoAKKw|MyLc#_yXFi<^skZM6?=4o}uTWVl1X_qOSdbfKZUCc!HuloOY)-er>L zBWKE^&A-dE-ghN91MIgl+GJT=W0E$%VFr)yO4E!4A&I*t_x>ZaphSj=p4hHsiEAZx z*=oyfHcir&HO%1iJ!zVeBJ}L8NdR_&{aH^l`mw$8nR4RV)w@n5hRT^P*OuL7+UR={ zoN>(WWVDHp;Ij-99kCZYi7O>u+|ZWTY?`PoVVJSpcb92KlF-?^CVnf=DkrX7{bDNQ z0ekmc-|Aha4Zgd;8QOlkqD}Y(&;CQ}gN7L^eCngp$EQBK^j`o*clXgO0} z?Ss2an|$rS8OQzXqD_PaTPPz1?A=s8Nr@M>+6OnACTstt^+Dyt6|3z`5+BGl^Jok2 zYOeR)0M5|z+YoKSD|j%&}<6aH#a9~D;j3-`(8B7NEOPvYw~Dio^s;4)pn;6 zBjlQwYb!DhcNk{)VF15c>xzx@B#KCk-_Vxc+?=Q_ZJ5F9d)PE1S?JDP6aSTWloLf( zkDDY0%QY|4mfqFe;CmRHan$c{w26S=oeY!jjjFAo|mf)k_tU@3jA zXCF^rin^3CyT7W~{9aG9D^VCve3T6)t|3l38QVNL3+0Zqa^lS6J zZ;NIXq8O1Q7c9T-QE}T~xTnZGB~p_jw{zq1{6gXEKqtCRYTLp4;*)M@I5##gE- zuh7egbAMU-E9tIOF3IE#O57cTT#+pWk6FI|fC>UXO318SM;w@Z7h zwoSbwCwp>0WO>Y1ZJwBo!cQA<#ERzug%zNa21o1z=;zLztI57Nx+C{|2181yBuY@qaA~ax>Qe2MaLU#^mfnGzjWDSN5-18ZhyQZr2&bk4OYc2)Yzaf zBvll%QZffYZYj{mcrzB{A!r&E&m`sIu?#p)62&|;fuRgmBUmALR*o^=dz$8^z?{#m z1JTB@m8A~s~qZekv5Da>^Gs|WSWKUs{H4v8FOa2!J=3M0oX&IW|c6%;^DNG zVz!S?v)jG`uUG`(9P-4Gb% zEz27*-#5#wX0&)9?sQvbiQ?T0(XukwyxU&-kNtB_Izj^$rp^Z7HA#J5n-RBItsspO zdM>Rx1jv}B)lmnIcK58DolcyD`lVNjVz83&GVrg1%vi z_W2*1BtBf2fh)N+i@{hao&hC>hRo2Vqhh)h2d}th|qB)J|YA_+5gP% zARu#+pS+$x&N$ftA9)Q=VWrZH)X5|N9&(P!95Y?>Kbr`N$8{W z%5HLj4vaojf>xA4gf(EX2qhW#$2n$tE!^z%LQ}b2Q<)XgH!D#;_yi)`U>+XoR@2R=JZA zGH@qWzoy1SEU-osq5%qVMcausfs%DcML|zMSlFEq$$otZ6@r<&S~Azqf{m49J=uR4 z#z|05Gh&vI$)eU|ismo~ZdjMP0%%LwEVEl5MI%Byr-7E9(~|QzMCeY@X7k4NY=5 zF;?lKF0<`;rc90q%UKf-DVfU(MWHF9Q+Qg zy>eh41?i$IDPf8%J6ROhdppf4^MR)OK;UtNH=!0QWQQ-NKnE^&yif+)u$$*{*{nScKpwVU>lW~DoI%P z=@tlvGX97tz(|*bFj^kvU?}i;TWVAA@?g3+zGc*4$kdt+R# zhC^L`%}YaT$o8Tz!r+nerm>FrGj*gA4csir9`~j57)dATNQV*Tj5;cwFo9@@SRm`< zS`XLvz4!>J>^)o6M3vC>svN|QDbk)cN}e}A{K4rfGi)kC90kJx>Nw}dlZa@FU=%#_Wh)v!oA$+Wbu0=1fXu4~WB zt-O9&WZjVaNO*eJnMtHkhjARtj@~8aW@pTq`~+!>A5U9oJKf9#@V1Q9ue9t#(!Y1z$4ERm|cy=5S-S`AT1Ro14VV- zHZH_#-oe61&-23G??1Jd*ZEaOB`%%B*7MKkxBYYa@h|_JKDrn-{?88k@76~Y!#d%# zx7_ljp{V~C^&9B4T@=Z4Q_z>AkP8ji+$4mp4y!egnb$H+vMt1=6AFxRTwd+`pQ|5Z z>me2x?d?eg(bb8l|jyDjhi%qSS(aSwUP|QJ=B{PbL4lUO1IggFRBY*KC8+zAl-=*$4igZNQHRk!Ef^ zLN|1!W`3P>>ljnYZNgtq$JWA2VA);Y>|m>2GGSX(;zmAqbL?Q7nQQ!O)wxpwi)Y)| z1gEKU8*j{RvD58aEqmt@>!)4*FK;5&?7ew}R|B|`YSQ2zs@Xi3m{B)P@?e{9Oe4&` zg?HPX%mKbI(yxMe_OZQHRF%g6*`Z4 zfc)dh&Nr4UfFlEk=3kB?}oLnoElbkiP*{_pA(_a^!!`5qtjy;bQyQX{R0 zuLWr;g|8tO^2@z739lE7+dD~_Dk*dx!!1;f_#N-pEO_s`E^zIJM6n8KV7#iE8te#* zQbHT!<34m)%SW>(hTeI=Q+v4^bQpW{FDo~r9pe8>1KD!3C#TsB&T%!+uCAg5`ZSt` zr==*Xr0&}lbk_FENh>pI>MNHS%&zc=w0!0A)Yp50JZ!(}TV37&Xd0ot*iKXkG&plD zdVf=UR7JgVMB2WCL0MyxSDR!#Ag3IoJ)6X1EBck?(qA76YOsB2gnbgXhkcL_O{Vc_ zbISSY`!s{*ZNC^>A*g;lcl3J9!V}U~9Ms8ptsNv~_tnJe;RgL?rIZZ)8BEgpgDn|5 zb%QkQR-Lwbx*@E&Jq5nyWL(ziGZE26&Ejbl`wyl)e{C3a%kJeF?346b((|1rhJJP@ zO|6nQY->6Hyu#q1>kEkBSsVS9@E2+44wk=oeJ-fb?(12rR~sI+j9A~%Q^^Dz%LJHa zUilTi+ref?YjhD9Qv}9t7+x1yuYF2;o6CnCBaiRsxxNJWz66B7TnZPth_r4f0yh?c zo4DX_@95=c0X}8{nzNR|gDsHOLq*_WD!Myfa#CT}C9z#UdQ%u#f-eUkp0gA#0zq0Y z6oIKlU>X-YpR1nAE5NZ=0Ml1X;lWps*22YLkz%mu#--YKyXv{V2Kc@Pguh-27qLcK zZz%?EEe3Dnf}6VP<-Y-Zd;@5HvlJeD9cg{A7_3qZR^@_wyXuj00eo)(;%}G2MQoAQ zdd1+A#bA9dcnVv49^hCW9!K!#!z5*^B>wE}9uO_dVladY{)MeQAK;r02+!wYcSKs> zDhA&!20L@X=h)f{03Qng%>_&0!A?kPzhdx%Vz56Ke3P3V=^cQt5FlQ-6fWY7w2mqU zM;C)*xZua!^i+xf$BF=^MN8qqcaYXu#o+8>a1IyT-c8T77~oqB2rpg=7jZ{gmllIR z6obpS;Nfn1`6YmlC4lCVrSM=+q;*R%xV0GE#sx27Yexh4FaUAPQn&~VX+2g99xnz@ zaKQ)J+OdFRSb!;ZDLfc~v}P28nZ;lh7krhwp6h#n?|VS_`=xLZAEfn~67bp*u-K-h zT};_sFTWJnC1BSQup1Yg&emQD;QIs+|Fje?@)T(uPy!At0YBn`Z}-qs zsRA6U0+?1Ug$IWrt>a3-@g-ms7u>)@&-F9F_cI{;^HR7-1k(C-3HVJ3IF}3V;-Qyc z4ft3MXs%ug4~|4ymzRJmO2Cy|@K_H$(iZ?<4M4nRDO@B5Y28r*?koZ0xZoez+G_#F zY5}ITOX0zBNNYj~c(Mdc~T1%k8>(F3HF1Wg<9;pGq*9Z`ATnZOSMOyDfgLk393S4k&Pd$|;z_BKP zY12}8a5~aj0}a+hgF#&I$DVqw%>dtKKzQ>~xX25n^(i#?G#Y${3;vd^y#?^G1<>5G z6dwE%X?+b1zK#alaKXFS+FJp9Z2Lo1}XY02m2eru?4u<4=^2utM<);bIEJ)s=z?*u2idRe~9$-oJXr7K2xs3J+vNn2c$Y)^mFIBoAy&~=QVB!?sl*9# z3L9s_Ns9)8MeAZpauV=8iFlD;_XFpgv1oZ3L7K`bzitN}2a$1JZ#dZB2+r(rtOJlG z=5g$&Bo=SReskn+HY7Y-$4TGQ-9R4Z@-_R7LN6V8Inaf#cjzY z0aVPSMu!Wro>RE8h2IIT8EQyg#wklYV-*`Sxvj$mSkEa-JX29P_IHFQZzpCh=Uia2 zL;bv7^dm7ETU`DRc5^NN(UZKks_j^vrLy z{QsMo&VddWR6VCG>0-m|#E6|?3USKP9vL4q`FDi>2N#&4C;!#*pSUzJ`od0(EsDOMq}iJ> zxYmEVo<=ZxgNJKj89qyYJ$8%zW2nDL{r9Kj#tXQflAFURas4W;tq>2Fr4c0mzV(aU z2hQDn{=Q-V)ty<0Yi_TcZ5=mVr<~*KzUzH%d(bEFQ+~jf%hp${%ewjO^n?}0wK9V? z7g8VVW*wqZKo(A_vk69dFZeqw(D>D9P7&;z-4-MqfJ)$JxPDPQf8&Zv-=FF~KKauj z{)~qIMN@$#`QD~_47 z0XQ_0$azeaN{IZ~r(UujM@6%ruKZK|$0y$=nLi!k&uI8FL;P5B|NBM*P1?Rx8*$>> zsamUZ6Z(=$_r~Z+VMDiHXK?wyO*Q&INPi5;-_swvBChnO37t5$B>i!9EGno@^E&!_ zS1+bN(UV;1j|=oh(USCseG!iEW~`_F?^eB-{-klGKgyg@^=qlOA4&5nwl7lsQ~k#$ z-_oBy9pcYu_%lQNNPqrEMgyESf!f16iAPM8P=Z)aZ{(1B-zguyV-j{)wOa}6Wd~!G z`d5QA^OXyIOH&)3-;Zu)-+Wu>PM)BNYFaR-J5sA;SM3Qhl5S3^H6&LP;~{LOzL^lw zy@q?PIN7?4C&=A#EP}wvt7**pqDX{ypzUp;`})ae1P(rTZeq+7%Xg50-tXuG$p&Hi z>n%3K9nR2WFOf*RMlFWbTQ&ql*+g4>=xrQr)-oLLa>3Fkqeu(^;2svmq-k*N3gq#0 z5xdhPX|y3X(R?C^;Cjy6X7{XVl|6fB1p0QABt_Ra2>Lqmh({-487VP_^bkJp1j|P&NLc=RZxt z^wmE@49wm}_k~BKyZ-PDoA}Biu{9TQ)*^8s7qRKccVfR0E@IRoal#McOK_C*8 zICqiwEf;aeB5@ZNaS-jh;wZ00#ke8lYZ?*MuB1C@$=z|-UPdo6W4M@?vA&0o54ALW ziuCWyhrE~i!gP93@-&y^%0=Q&T*NyUiFb1m*DMlua1l4pE;@ORi@0HtxbX)uJABRG z!nb|?AHwIr_%7K4#wD4wD4E2?j9X;x`oa7yd>)IEy}2X{Grto{!Iu&@Qx+vJa1jqJ z5)X3`CoU4Fa1m!M5*Pj;W{0nV`7M0;kbejtaglk7i@9o%`3o1bA?v&33kWXBj77;z zF3Agv#8fWg=|$o>E@HwW@d6jIH-GQ9PSoVjld)u6Pifr!E(67vusEIzJaV?B1gE^<3o06^rEd+~k`U$vb|KvqK34QaCe>Hyr%eSWZd%EV6Iq zVwV(JWDn8tK>h4k1W`2{{M z>C%1Qd!xWwF7i0vMd>HG$R))V$(6aujTXrlevorQdg^;fk8AxyNHzS}?5w>DNBY*T zV5KZB%&dubu=YM@0Mwg#XY@+C(jJo|KXM3meHpvnGTT26g9a0REaaps{>FPtracN} zT&XL0^L``*IDcu)im8f`AR!fn+WWs`VQT-(zu%RxUAgx!DUH!TdcBxV z`X=KaqxA2vVCf{`^bpmOx#<-Kf5``b7>|%=#~LNRxEvs)3q+5=8u}!^4TMUi!zY#u zgt12=O0b5%I7aj9oQ^*dGs@k?*d3mlRHhGcc|S<>)n88!OIjQuIRm7;^g8vDxhR39 ziY~!vQT|BH*{MbwnZJ1FrZLLHee8fJcc1*HEbK|Bv&lxrlKSN@0|7>06DYxk|55w5 zqyD4zFJ(&X#YyxCd{~P!lf^Iz6?po*F-j9w4a6{}*e|#9XVFPR5D;qtm!5;AQpZGD zcRqna=pH0v7P-8AGAUztIE7kShgg`%VbaG$L99#6xgIKYAp}9Cfe^HrfjZV)6XGaqqPhSIVUqDI zf-wqCVWj7<=Aj^lC`lB8@utzR^kJwM`_+8~Ec&z=h&k1SA`ww6d^-EM=`0pwXkrA# z+(_%gjVK`&2ssG)%sj3QsEMF0fLd4!GxH!g6GuU^M7dvEoC?3Tmwt(dr_7M|U%k0d z`P*xYyUoL5E1#mK*>5g9^8Q^%rmV@xt`${5`(zVuniWToBPWgsZVD89x_gyU~SMIOo_+*(7;qjQ)Ngt1pRJd zcqkL(y+FY;=T#9nAjs2;MsHEA!(u(|Fop)OmRK0w4Zl8cx3;!|kYF?S zc~W&HzPks*q*sL%C}E13eG9lcM@tCyFf$CwMo!U5lmaMoYLGCY+E#}xr0IaXS=%*X z7g6k8Lz$IC%f4Av(9l6g41+ZEXadq*LTM6h%W=e`KgCQCQFxg;7O5DSgO7(0{3Ss{ zLvVNIa0tSaQJ~}qcZZdHBtqcq7trOnDN!j6qDq=5g!`S@;RPZVG|ANeR0mO~R>usm zJ`S+bW(x`@aBkGTp+~B!P$&k?`074Hv+1b=8drpkD9KpbVp;NNgHM$krv~&H!W3p| zf1P*M%+$qc#-mi!!qxe{^2P!7%e01vrp*mlHOZ=Nc=!Tkmp6RAcw(ilVSpOj?K zswY5LMlS?Zg!W+ICZJF(hRLK2OdzCXK#jm6Jid#5iZ!rMgA75*pR7rSW@BN@bRwH{feEsNJJY(6 zAZ&#z+hZl?`F`72^iTQOP|Fac^b~1aXH!DxbFUP;a++JxcbzHOs&i zv&d{;&69HmD#sTflS~!}>osi(a-S|JK(wIp>SaJw#yk#(FDXIO8YK~)US6|hj3!?i z5=Qo!X7Ue|0Uk^_TgrE1jo|Kan=~JxeNPVz-G>e8d}cffCyUO`yPLv#5w*{`m_dZNLg zL^0B_gJpFD0nod?+$X4PBF-3AI$P&(0_9#Bf%{X|1`n!p>(-IzimK&Bw;rP*QD z=mRpSIFS(w40B@k2@}f9=9qI2W<#Lvtl5|#(yy4UbE5a!s4Y(EAwek75y$4}tVOy{w}b)mdGY zWDxoFEROLSd5BeCNiD6mpf4M-!cytHG8>uW#@@u)x;tN4*Fe4_s0j*l=#77G#)`s&XpRwxv@^9&W5;ovIeq1DWGXH6e7Dx z0I_+p7ItC;SZNCKs4|8**MgGA?xPw|2Um~LXQPYene=A1^^s}Qkhy(3Q3!+CCKSs* z&6`>J1xUZV0Vn_>?PoXM1fuIv_>V?6l_{q)OGmnWh*3=P12Q z^4Z;@b4qjC)zDXn)SL_OM~scInwgldcp`9zH|2hjdL82Z#N#0^A$BdADTXm~FDhwb zjW8cImP;x^=UXH&%`?{_eZ&l7q}$B+O{$h>5q$3vNuh>#Z1b3pp4-Ncp_ z1~HdfSuTpP^C-xUZ*U%(uMmT`cn*Bbxrmw|j;Z#~9GPkX)o*+aZ-98xN@ zM{?^U+D4*rOow)5y#e-pyLv19=R^eW!F)$B3BSp#VS~l zJ5^EI7+WCr)}vt$N|PF|IcbcQEi{%i#-&`cGp*LqaT1$Rv&E?km zJ$xNqonLXC9ME>Qw#D$V4VZRt=wKUgn&Z-z12^PvxC~_ikK`# zj-t;$7405wU59Ut;3&HGQ_(u-n~$J3t2v6^{8aR)Ft5BIkDvKMo>Cp(o8y&)oeE%n zmZ$2Y{wsJn2iM~m@!it$uFi2>XA%40DV&a)X;W`hfBj%VJ~cm|y)FOzXH;^d&COxv z=A5WB|FX0uD+`mA#j$k!r=>54ThHNJTRE2A`Dy7n=bM?(n-d&MIS2bbbTSL`*fEu0 zp0uLq2leF~OvAQEGVQFW)j|KCd20&t95|NZe)hlXjt;X94xH`AUnAl`LHYT)Na<)@` zbyP=32eN|$XF6>BHPTcI@@e_`oRK#6%hDKg9LyZYiRs)gOKY+Sm@ERvQtD4jzYe#$ z;#)}^OJP4Pb#=bERMGdC78mAq7vxDZgD+Cd?fQt%Q?bt@nAsPpIJNL-L9vVakEP0qy@8)8V}Zu9SRN zHb`FTr=`x3=108Ek8oo8_@||ZzGO*bvZOhdR{XSd({SrjMc-p8;(XH|db5(_dyazd zmVTGXD-O84`~B<9pM16hl6Grey+ElTb+xw|dLj0ZX`?)?ZoCb8WD9E-eF_(Amp5d%jrk9?Hg zYw^`EXY<(d{*uSl5}18(^8AFv`%CKl#H&zG{s76Vpo z-dL-wJth`6B*AwTtbXd7jDV}^{1R5s+m1-H)sL4=AG>8)p>zATqpF_iu~ScV{MJj@ znSMG2`;yGA3N$@-`l+7Z4v810pH6q+q+@Z0`z0Dot!d%M}1OFFI+{bw`Gwm|0BU_S%SKfSEuDb!7iQiFM{m!DV%W-Yi!?#KJLceK;7Kg)+; z&M)sI0ef%Mh(5_OS!7EAtT&u{`H^mI0zUK7HGra{0SY3%fU8 zVH2)fr;PT=k5PMh+3S3%&{h7`uM_lm!~oHjS~$bq=U!cUxjGIfeWjTEElduoAH~YQ z#SSN|y^ehp>+eUYTA3dv5o>aeXD1+<@70@xokRLKLyL2-&@X%Wa+g26z&0_8-!LbE zAg7|x?s(4W!-ZvoyV=TXzbi*c*nyZ(=>XrWNHl1LlHP*b8TQz0!Asm2;wWM7gs+muL=v{KoMq-JDiVldgO z$&zU-jU{`7vCr6M`F}n$<@9Lz_MG#a&iVb%^LptX?z!f=-}klO*XJgaB3f2}(0kt1 zcDrx8V)Fsq#~>k~{ifVsM@CxL4oWnCWfO3vU=B>!CZh)tQ z03CoRwPS8VU{1&b62ms0gey!C7fk z#Gyw{Xq(G9wwZRq=S0z>03N|2&^8xyY||*sEbU`p z(cD)j0GLbWA7V*c0s6eeLt4P6yITH&;QT{#m!IUwzBqsq$Ri^xx{DWBDVej9%gG{I zae&ZbzSWn3m8>``HKd)uo-A4v%yXjzSV@($lCM!3Bkd!1>D+~S+@lc$g)dM3_rqbm$LLjX&$P()4Af80}=`0NJNA$M@}?4&4|l*F(CUgjzmIv zNMd}tC*&_|263L|;4BZIT;q}1Cb~-;#JR<0DtzcMWrLG^LyOVbfYWnI!KEaCr2uDF z6?O2GS7}S<8Ua@d=d2{fmje@x<}&8G2&}aE6=$UzJS1s8-7E4Jw1Ab2EIG1&6+pSk zBePF**E(P&IFv2>bju=-*8xHs_*UNrB(i-4TlQWVC#Efn7Tw_yd;>@%0g%W=R+c5b zVm*mO34Akm-x)w82#!cndFBaKtD;5sd2YM~mcny{A8nkbp81ixa_$u?U?p45N_FxL zDaL1ut>##61y%StnF&iWWWM zxlseam&EOxmRoem{&P4MIj9eCTELn=2&h6VD9C>3>S5ff~Z`~ z;@Tq)z-P0$CYfB6(KHpP_MOLZNHX-7L7?U;Wk|IK3Ou6{%Tdpv>NSY?B?dq+lT z1I+e=T+Qc@4!P_lAwqQxAs{o5>lkl?Q^?j2e}MDE@TJGm=yx8(cmg66V0IhiS}->! z>Q^9AUIZLh`j%sj$8EIeY>YP8*ND*ByUuFvL}||k`N8QNF@E;89hGzKW}^fW)bZ@u z`vwwgiU}ae<7C%%(dXN2}0K&uRIp!a?&wZ8_?XVrEB#T$^R4a3-_UKM5jj0yZq z7-gTD1{Qv&=9y9Vmu-J#K)*#hV0j7ZhErZTDbE-$xJSrrTl<$R0MDKRCeFN5VFH8 zn;Xbg`)3loCD!gW8MPja_;bG`_zfh)K_I~cA3Owd;3W9Q6q9p#c8h>&;nqt8P!@p+ z{!xXVm>=+g};bZ_y4%ykM5&v}k6Gm!0JPQp@1fZr?b>pSFyzM8q!YQ%wF+ zm|$Dv*9^Eqq!Xq8jpKetlmA(mU=!`fe)g3#e~ZC!qTugQy2tU|fS-O72!HCRe+bsU z2PW9i__g5rw+qcme*YDzs*#L31IA6@p!k8fHAE(4D=EdqBhPLHn5%%k04KpWdJr=R z2s{|^7k^3cZ-LwiG#S7XrsYh6&oRsHcOyc}`V)_sndRpz~$S-ZvP^07;%Z>*RTX(!zdBlGXg>*$L-x%NE*q0DX=I{~kbJRLkkg z13j0|UHKUk_^B`wT(N5%##=a@hs&QI8@Xctq3+My{;~?7^WUN!&JzjB{AtVhN<^QF`o9z==Op(l2Fy9hedD;_YVtn|lXFD+k)M4f&EH~h zoGAEvl>WIH@*AVpSJM1 zkQvC8o!~QphZ-(tQxaFhf6|4HC?f2D7&fmJD*UQGzi-WcI$nk|BhGy8DCs2QxzZ4j@i6_(p1f4WKWo z{Z22QV*)=FhE|V++A_Wt5$9a|m%`+nvU}Z#AhX9aHyO|r)Yov>O$K8}!{@+F29VjanHuC5AaYJW-*5yI!)%OfJ44g{tkoVM8Q8F z4!(l<%hLSZ4Edc=>z{%7xy$eiF#mR;zcL{Gx~Ko#h3o;PgI5Q{fjpfOs}DRjuJj;2 zARz3)1V8^vf_KP4*8wN^c|d~SHk05p&9b}Qh~NhwgB;jb6{9F9r6hD6*ko7&lnv|z z-w2$xftw5;~j%GxY@Eu`!l{at9JT*icwCli-1~ zF>pKCq}jS`i)9ruMFsSXy<#EEIGytCb_=Zw)U@B#^>7YU$=~}M8qlT|5BKr zPjbHi^YcmWM+W@g0`v0`=@(%B4uj)F!9O1k{wZm4njzl=!k;{1g;Lu0n+mdAWvhx_X3ZN`ALYCx`vM6HG%nG61;0e%7?RgcJqJ) z51l!|mv|5_6A&xG1P^jxCwPe*^!DhK4_rXCu=q=pp(o8OyUmRVGP9hy$q-CIElEOe z2b&BaS9X(OhlAnQa5%pYc@k?^LPjkCn7szMvTKlEfXF%he8Uk;3@c}21QWxO*}Dca z^yS$gLFb#jZFb2p8zty`A7<|xEEyaDNiM9S@c*C}&k@>p0CA$hH&Xj+0DV^N_j>Uh z6Zn}h_PRNF27dpNB}d&~w*8d>{TA(f=aMDIc)@aI`)phLmn?%*yZ!5y@s)@;Mg3n3 zlXH^$1(=+Z+&7N<9Zmj6VRDX0KlHP&r1@J6jvWR6d^q?D=C4bW(+v405dO?j{}8Ny z6HHDs6bnI=PqR8)f-40Ko)ha67G`r_gYkjwl>l;>N!$x@4MEY z#g}7GTY{=w7?|t^{*q7PU zoY`FV?8KMZ1kP+DdzSuXwu3V}3C<49;p$t|cf8j-Y*6zhL$M*H5jJ0~v`c(q^#y96 zyMlXe`J(J*RsdD};Rng~E~UWilj?`8;5{~&ChA;iuQw#^W}`9gScR~!#bR=_29 zK8$bIKR^!8Q1*h9UE(ry4;EjY{RMxYE0Ohr%**U;7L-pLb8)MLj0i^B4`p&~Jj>Q$ zejLxPfP~LHZ_Ez{ zLEQcJjkjMow|96z+QntzPSf708v|u?O0TsyYqq!FIGyvJD%~79Z|{mSY;b9r^b-dF zd7jvUh0qXBfvcgKyp2cdgWVMQ7oHX#lw2kjd?OtIKk4tXhd!psq%jR8ESj&=D2B+b z77Q;}M)y17p#S)%s2)r;s~btbqfkxM*9;%@Y6$NbdmVDe&i0dugUx_KPeV>*JVvJ)Lh3#|*TrId0niE+Vf`yM!Sc zriarm@ej2@7iqGY3_zfTgl5w~Q$5;;BaxU-^ZY0>h1%BKl(o+Kes+c04>R=NV?G^i z-PgLNPnB${mBMyR2=4$!)^`fR3oI6Qbl15*u!?it7Y2w)RD0Z17{(CQo`%Hh?SEP7 z#~?cca96aF2O!0I##*(2XhUyJLk`6Ghu&g{zO&x8`BNctOgYad^cs6#)@maIAnHbj zXcvIG(LeN)&>AjJ3*^+w<^t_?f z-yNy|;lyOb_0j(P3pjuMD;m`C-KHvFcc&d zr#czP5u#Ws4tw6+Z+fI`utjZ()=i|)yYUqjQWZ|Dsi`S^MXFO_;dyEYH43BXY)&5c zV$cQZSrhb1Df2?_=}*&r&V!PpDx;Nnn6D?Z9;2K($RxTD2A!Rpo&8l<*uj)Iq$^`D zt9f{&wWX;kmQ~5ktW?;{cj(Ba2ZIx_g0*gAgG!>p_ zj?e}gV6FL+9nH@y7|1{V#dvXrBSR63xVW^GxPJ5(iXfpvWixW9d2IP z^qg+gP5eaix{9kYp)o3vu2oTUGoJIVHngA9io-7L z9;=q>BYxT_ht6+!TVV-Z8JqAX_Y^I_6TiQzp1$^v_IcSEhY0%f#(q`bXMK4p}=Ggw52XoW1!Al|y%w7c}I?&-1%lCq{`*zgT72cP*@1H!@cvz5Dgkw*%cB zSEAR|1ApIFwK;4e+PYP)Z}_P~j&lZf=W7G>q)P6Zj(64ZMy(392wKXHU5rilZB174 z<3!6aRyAGcG-AomP!4w?;%oLkV^Z%>Vxncd4-t``nabhR_XH1(>GZ1|rxv_9w8O;8 zliYz~9`;4v?HU)zjy3s2vAxc;ob-)F@KwggdhgNhaMw>pHM0`jTdo|)yu)l@Fk=<_ zf+z1)ZHNxpwbYOC$=-k{cD^7}Q4*bJglx3X<}aj~e=6d0EZ7HqbU;vAJwNS|Se>`D z@3I@luL7956Ge*#qqz-a5sW6;%kv9Xg*Oycq^Z1Sp78RX5Tf;A! zF@C~*dGd#%w@9CoOSjWpmracBv6_!Hu86sJnQMeUn2x^1g%MMu=hdl&EX&c!YDsvPTTUcK|BK&gk=^N#jjOa=zBZBruF*8@zczA%L9V4 z=wtN>n-}u+6)f+AH9eHu={AoQy>4+{@qw&2rd72McPAIOt-yOMdYz(E`Dj{FW=l$nqvUH zR5!ZeI>YQKEq-%gef~t5QlOyMyCC&2!J3KM?eiClTU_K?|A4;CXr3@rBrE=&xu#rD zopMb`ytlonW>&1{iKn|%{g)(5^Y+e<a3izf1xmfd!4lYv3EO`hgPljo)c~^ zJ<7AKXbDYkpdLWj$CHyY@o4_s^x3ohHeY`6Bl$B_&7 zt=T1^wQJcNNj@!?&J`hZr9_YMXvoR(g?Ms*YPTudC8ndp<6g8S?(p(uQIDXTaa#`0 zFJC?X&EZcHr;X&7e@L5ADl00u-LwQ>7~OGI?H#vNUV|(4lb?~?0>v`nblm=jh9spK2%dNF$(CsC>rsy zwy36fnbFQY5<7$!R#cr7E33&_m*#Oc-^oZ-WoHdHk)QkMTzj1>I+`~HCB%gn6;(aj zcdW-3rAp3;d16l4#zWawUY~QMsb@ouh}7^7rI>T4yH3?LrCZ&qoSWF%w13+>J~#KL zt+l-6>z8=K7GB`xCDa9S`*&lL-~|0vXa9?R>l^QXc+%ig@W zp<cAhmoh)+yh#9+qjmmx$vC_Lt-81eJ zOiE!*%0%JI^M{3zx$!L08G6N;Yip8}G#v2y3Rf=p4w(`^HIb=71n;)2LN^nO9nW`3 z8)>`~Fd_EXmXCHhnK1$=b_Mnd$PuE4|9#IE=r&|}bRQ#oAuXA5k z>&gl8IGtXbB)cYQ)0!mt_Jtu!#}7Tt`dHkN3`svKu|-hlimuxH_J>=WV5-EWz?a3& z=Z3oIqpOBjzA$|e7MM7%b@i2)sVinvSDcS0ZucK_>z5ZBeomNrXY1Z8?a?dEY|q+B z6;uVkZm8h3mZ?71RayspRUNsha!|WnSo05G@*-C=kFC?)xUQa6kvEu{RQJ4ZEL}li z1QBz$Zd}9zlZz23+khS|JG3KDZoB(v;=Ja-`f8GcMvL(N^c}u2PjYFyaV9Avm6$i) z`JN39htJ>Yd)L))0@@p zG_J_@<{3p<3mE6Uebaqk;|ezZY2|Hl%b=qH%t?oItlA@r_0|JvVsR&z|IB#7qe(B* zJMQluOt^=A>e=i$^2n0 zw;inIVwxSDSN7a58y@HD_dHB}0!h2+71Q^ittRcbiu;k3HUx)Q(P_Kg2k%sdtwXJ2 z-0+`TrQUdJ!1aX&&n=_QYYP0YCrRlOB~5y)v0=KyAmdblaf-rqTE~eD9Oi|Jd5pYb ziv)#5Ctxg=O#4Jp=1>QmC#+`BURr4Th6M)2Im&y&OaXQ8urdMj&Twwe~qbV(qJ^U z^`G^p+a8&w<80vXC+6l`m=|1((`mDaCi*=HsxBF^kf=k7M@(18#>!WC_V<>zc)NIK zvD9MoQH8BAU)tD|slTy;hQokgfv0;wf%$X)g1q70P8%e&szJU=w8GGt+Tu&^VzgI> zW@aWZ3_H@Dm@ew>^m7tk6Si*{se@0DJ`5%j*P?11UDX=nh&1x^$~7uHk6l&Ga`LS+ z9Q<_qcB{Gk_KJ>;+ck#f5 z`p`-msf*^_H;k?HaA#U5PnS|2N3D1fb5U>WsdF`lWLA{l$X|6U?(!YjW2fNvNz!JX zo88Q)C1dy`H_Xwz>eT%of@@p($uOL3$(+jyJ9`piT6FLGKY&WWPeTuOkQZaNUar=G zW(U7@3Vu5vFj-vRIn*7E)o`?Uiy5`nt49m?^z{`eM~GkBy+o36Rn+@+A>P0 z8SEY%9UhX5fm&#gCr3&9gU72kb?hs|sot^>?GwPW)K_5CcP<(J2>0=oG>_5hc($#; z`*A_XBqFrhe`-3khs6lS8sfawG<+A2<3VFdHJy()apBN?`@;sD~=9q z<{#wmXQ~TyM{LDB$W83_sA@{NPkNSHuodInO|u(#e_z$HTbI~WbYFFw>jEXu?zb^Y zuGV%=@aUp?A6PW~V@hyB46MrBe2`(|_Uc{L;H2U_ip{Hc2It>B5z8n@YY)vSyfGO> zw8)xsS$=aecY@<)%fR8>;x@j$m=%}%H_x|sGd=Z6__S4+)9J;=Yd2KcWGU^byOOpS z1H3FWN+5|#+iRI{UW_>})?rTefyqk?l=7Z0&Dd+C6#vajd6wPVobo$LPY|bbl%6DS zc`ly$e9M_ITjIS=8aalox0gqk+u6%AFk0yHHAuZXN5{~getiN3Ye=(; zz7@XRGVCb2+}5n^%l~>{m(B+SwyWCLXHwe=-?O1Iukt`mZ zQ@*=S#Yl#Pfw7;na{#3Npo zo}MGrK6iuoe7M?*&C=UW#G5WjMOe9eoLrTPc;oJdZp(gCTE(9sWR6};=|8Lr{Mb~* z{{r~&+CYCA_~Td=f8pt=Rr9?(7E0b8+FX)3Z}?tF@D@pmSMepa-Rtyh-U!8ZEebfl z$3AvQMNhC=NPH|%#yj9fY|RrHwf90=)Iclsw%pj&X+3Rz;ZJ;?FCz(wk8|f6>^2E< zBF@v;9jIs1E)+YwD8P4*T-?w-J;5F!@u@%=zktJWHFYv-{X$yPf!!Ky=i^p?l*yV{ z6i(;f0(c?%i64ZI%hgF5(Rc*n@sJ zWOP_iOjvw=kj$lknuj&pWz{5vwH5?*?{1rXxO$gtmLPBVLY^&q9^E|bbwGBZ5HBUL z_|l%;j}O~y5{?z&4G7v}|7hsNVL?S<@x?(hmjiA*s6(v*4|{pbE|lV>1Q*ZK+Rd$Rvqv~qnm6Fe9=Z6T<@$m;!s5$_RzSN=We~?cJ98HYUQc5Z-{$J$>;* zUiyL-!s5$=WUd7WCe)P6s#ywatq8K(+qNTNb&YJ+Hr{Y?p7gzs()GPQ$SzdorGypF z)7f3BZ*xvKb_Z|3^*!i>p$2_HPhoM%Aer!hnuHpfteUT|mQ+x;PTOR{YOal0yLiJ_ z^K9AoXz39z-i-@2c_}xFFYVjC?ubo@aI7|Oz|B4OkB3x_2!;!buM3j76>#Hm&4!I? zcZ9Xp2U+cJ%YD3h^TsSVZ}eY~qDu^?3vC?2$_x zx_3k{NmzVSkj&kH!-+Kp8`YA9wd8`j544?6Tzz6=mJx5bJkOSckKP^evfH@Ol$R3m zF?YQnhWo$tHE>nE6jX6nY09pwj%8ZiN>>`RuXvaRy~HUg$C_%$#TSXeWtcl3Ac;K@w_YT^ciO0qxl;;B^f*Y)OX^C0(yxT%?A{fJ%nl%2yxrDjxaskrHB$BH%f-J9$k?q{#+ zOTN$dJpFE+hkUkGN_M%ZDJlKKgWM)@Q)kDD0Q>F`dxzIb|Bs01|2Mk+Z`sH*HO#S6 z$X$wnr>>3IDkzYX*#kL zbL<1;ZcjwWi^N)||3((%&!roPQ{;2$_W$BO{>v1i-BjDwO*xhBx23fzPCW)kN+~pp zbrkC>ptg_Mu+FQU7cs<|SGCGBw_D(h%^he}qDl4PN|$rZ3(xzEQ$5_zGl*XFR6`P_ zmp)KODx+Jb7v?DyLXyx#+CVv_NlJ|58R-)_iZ7e0=z@vqOm`eLeID7d^?`FFIy8ZS zCwgOJ6)_!8hndKf@+H=a3Xzy8urX#P*^2Gyz3*rA99iIRUd^2K|8?tZxk2by!o}rq z=7$|SWlqP=*!6?W+@QAUXAf8}&l0wZty+9NCeguj)&47fBGapi&cU;{EWY9B&sdnh zj+}43p&7A%u~eRgGzVui!<-WJ;u_V4UsHTSrU~t-t72bz_oGlI2KQ*_~Xu-%Jz$5$aaN3=`D}P z8zIVl6uZLe6sJ}(k0{5TIE|F7Jh?l%x>bDW$q^0K<`NR6`fYk)uCYR`O~FLkfx&Ts z+YSWmqy~xAOgWe7^T08C;y&46lrku_9j8!-KD50I+tqxI)um#qficN{UVyW3?in1& z=wKfA-C@zG;hYIe>>nf(tgd&!9Bi|XJJe&Tl}zpJqa7M7uL);fXWr4WQ8-MJrFZD4 z@BQXEQjx(z6Yi*5i|D@o^C`9UuXS1NmOa(CW4conQX(5ys2+dqNQlDchd2`-#CG!; zid7)?;>QRBQTW_>X`X%5!C<-UK0JN@_6mi9gJQu)W1_4rG@hLF_KZA4g>Ic13^ulq z+#s)3JwoY9YV7Ek9CUK6Cs)!-nhSBvV}~LKoyYI9GW(wjSdUStk+RJY=&8ob7I&)n zGZpGQu^qTv;>eKE>nm?zQaFDT^Fe3D(WFNI{DYp;Ub!%SMU8%1A913syuCwMKrT|! zP(eNQxG#es7pIZpJLTS$DQ2Cjs~g&QotY{UDwA3rN1O5*wgn!g7f{+?M;TThl~B+) zr<8X&>yhA>v zWTY{pu^_fN1AiH2E)v=!78D-rsHPySrr^e`+#^#RDx3jxgFpcWzJ0;qQWtAr_3!AHOol=s#yU9a`WxG=6`(@vDlG(@_ja*$h5KD1?K(KPf z9Jg$Zt~FC|&@NJB9nP6J$zrQFq5FP*+}UKNv!(#Nl-^eWoP-jMb~v*+%6|96Ih`3*0Iy>p7p9=-O_?ZQh>F7-Bkm*a{^_ACd6N}l3p&IY;lRXJpiUWxWTNgN{xSRZ zr&Fk6)TZQ_q1%|-0C5r_@@EEw0pbket6{WL7>&w6ZTj^g9VL!NeLXxlUv(QnG(v>^ z;t+!%8Y5;6&1!e6P`$~hUmC=dQAMb4hm-!ACEuC;9_ee;gJjhA23!B(iXXlAHN8nP zstomm;f#OW+aJC6j}KL%J|v@lI5?U=dHGXrgpAz6vdC91s5BBEA z%6?4cos6nL{hx!iIr@iG)u^dt)GrLq=5ybCmvouI-857nVtw+alZZv=14f8q)c2?E z8H#3pL#T8DG0)(ahs}yc*0bp~4U{Yqe`46SiW@DSfs&pXmZHVeQGaT<`P$xycm)ym zdANrNGbaAo;ZSxNsyx~5EjSdTHvBWgsZ+ZQlujaq43y3wj{cJa;a&!6Bl2XODm!7neY~Qf-;`F3L2CNZ;<*XXvMe|h zLV)A72vA}cd^?122p>p*c4fhLLI}F}QUa8k1=oNO^zhv8p*-2}JrKfSyw-cDNH%;g zgrJWPd=Fip4L<-O9Kn~qhbm>mbs>bKcy1z8I~#rkLO6!kB0>?_a03YNKM({Gp=Yw; zCJ=%FzLW@c&xV^r2!?p>I%rTf{1k*>gx9Kr-pz(vK?uh9z&dDRHrx(EFu|ABL9?^r zP7s1Ap1U4emJL4#AsokR)kBHda1RK<3?Enz?aGFGLkQ;h(t0R08}1JwSm3!EpgcM7 z00`j(UaJ8rk^{dCA)LeqHbB?sz(XK}Q~1&bs8SC6I)rc<&)o>s&Vk>85G?UpjZj1m zJOV;EgAZ(kp2>kffDo+krHxSc9C$2*V2$Vg01e84$3qAW+c8KnR|A?iOee2Hp-Kc;U5Lpm#Cw zZV2H#KClIvh=KP)2;TV87HBpGJ^&&3;JI6&Wf=Gfgy4(UYK0Opa2kZ*hYxIpc46RC z5Q0Cxv=vIlz?l#N63_h+%JTv~SAlQ=uk{fs@&Z0zfp8HY_z}AP1)Nub5YSh#oQ_ql zf-!D8Mz1PRd7a1TVJup8ST{69Qxvw3j@#N7 z`r-0hy9R2bIwbSo^yCOgjh@ZVw{X55$scE!)-qU+PO`_4={uVxQ_f(|WttDhrKnX! zJ)RbbYO*Tak9j=;BcIVw{Ydq=zp7{W=stzfD5?i8Msd7CK-G&GeFx2&`^T#sw>s7@ ziIm{tic^^LqieY;oP#0B>A0Z>a4_ULzaYTywYF(?{qa%DeRe9(m)ox19pEOjTfz33 z<Yi-NF+tAD(N=)y8i-WffcO(=9j&afZfc`IMPC#ijnS)$ROIim zuE12MnjbG@&|Xg>8Iygj#L31=N*PH^EH$-7GA^3so4QBT%O*j96I~ zuu3vLp?a))WRgm(B;rYmb+2)1-QjWmuwEUpYGl;d@F**J3;wvp1u2EW!V7~sVK|ov z)^_`RDJx`0+Jxi17?>Zu{Tw5txxm%K&v&wuj_$%+;~z2T?+6u1;gf^-j&S8DO!dGr zS|yR_={ZOrF1tNiva_>$VrqguVDIOV&KxQk7O1FCjf+MWG$oEHCgplMr>>P^Y`31K zrHv_Ha9s?8-V%zar?G+3%XJ+h+B@J26!{SoqYUaALA?xO?(J2K6X%i>|=f zB(X1wU{T$2yoGq?(Caciy{h{9-U_@JCN(yil#j+&dtNmjq4!61R8s0AF-cL?I9zUl z_b6<9Ji_8hb9K5!cZ0L*ROlJWQ;f$^wxsZTINwq_WvwXd6o0jeq<>A11~==>$7Kei zsp{8s##2kkiM}2#KEq?fRARVweNwf{=s^2`;iWEo?)c7>IO@QFp=acvRL9g-<-TD7 zJ(3h`s?CY1=*isQ^lW&jO6q;ImsijI7n$-@Qld^@2aB?Q@yNPzk9fa{p^8g-K0XEJ zKIgqhiDlH`vPnN7U5tX>B-75%NpWzmA2m1=SwP2+M=D2RiZk>yy`kStUIOh zq1FDmM@I-PjQ-Kk>D0!qW&u4GeXzQ4suej=VIf(3E&coosr)^A&81bl`p3Nb>nlb- zUAlOCarEGVs)CEU^QWdS7AkLk_0kKrnKpmJihF|#x?5hZnb*g7`C(;)6B5!%2^YP$ZU1uSV}&UNq3D9K5tvS@>+Xilq1!;dJO!X>D$l+{O?C3$-F9lV9H>b&n0 z8pdQje1xCsq?7ggX|jjIPZ1ItDEbd2%bw@}V%?=4awz;1%g3!h%8fKwOZgE6M(3*c zohZpJr|@M|S!MM%&64YT^_{%$VAT)2PiPvG_4K*(R413LKR}b!4?jmpXrk!HOLjfc z$z@TTXp8}ha{hNH9!V0P=wy=;_Nb!{h1+3tg1nDKNfMpHd8tpgtD~AFMSArey^mql zH@#2zF!t2L=h#!7Y_dLu_VjSL9UeYAlevehRd!NuU_SDno{ZpL+vi>0LseZUOA)$q$pCCznqEkRh;4qLn zdB50c1x<*bPnveIbl_PiWXB2oLN>IQ+n&v4jnzK0F0V5s3aK92n z(Ud0zT{(kXd+MazQoL81X;v#VMqK*;(;W=gMKD%f4T9{KCqb^BK^{7F(q|doLmwc$W7L2GNbefaBG-%N zKme}mBN%=FDO`ShrI6IM3)9{k_NLLA1~q**2Pk#oVJDNtPE?pBNV6af* zbl|?8OJNtQmrK`9p4k0y>)tDF8Vw63u7Ph)uu{X%-mF^ifwf|-)=sI9OzUfcWZx5$ zaa;C09-Y&SEMIt2*P871-J$9PdGXCZH&~M4H>>{CaEdH{v+7R`R%C~pRlh#;hZA=2 z%f@sJx$LZ%ljtg;q1&VK(CvCj1zBRjohp|Jm6(nTwLAEwl3XPY@2qI+aJxS5K#z)V zbd}34vPAIb53gq4%>bvbxGBiTgzoDpuFd_mZ$~U%ogp9B+PUChk04n>K1TDgYnmJX z&8ni>9slPf!D#29$6tmZktL_e!Uo%}B_1_;w>d)hq;`qX&Tl7AXSngoiQfi^?#ag# zhm>1_AKGv*DJ{Mz*=C`f_}h@ne}DT6a)4TCZ%BT1Pvm~un$rfB#Ua;>Z!VJiOD9KQ zJ73ps{P(W$@4)>($kPTY_nb^*B+8S$v z&f<`0%Z(y|YercW&l`9v;U9I~8=fRuPi*MFx-ZPTk-zN=EPJ2uh++bi+>%CA&PR&+5@Ty3BmSk}$ z?@<%}#dmeb%LMNkwH*KnZzO_*4QxU;Hen%~@Fbh>7Lf0r`yk&oIR4^p0sv;NkX$mG z@B^FBnN6s|1`rJrKGFsGo=pPzz9N7C`fYb?_TAYmrg`F25RA?7AxL=gFi3d#DM+aD z9wc062O<^C29bub`RYIx%s&Fk0iL}18A$k)O?aG52xSvSu?cnA0Fu}eI0o|FjOu!M zqu0%1@x`kmx@^K{Y(fMZ0Ghoe1V|`|2KnA2f&kvL`8s7wEI$rv29RkAdrO8O;Sif} zVI4@g*d9b0!sc7S=4;6YkP$JjB$?p1g?od67n@LvP56XONMsY*vjOC=C148jt$7Xt zn5a8<;-x!BsU2*>b8L>e>@ArAg#C4SlFj)$C7t8Un#|Swe1{nnVtr()b9lXVeMO|F zA&%a2hVepY5ZO-d@Dyv6WKr6uNO+90b!xbx9-%(k+`@mlw;to1+7i=ZB2}ksh`|^- z`}++Nhuh1Brv?hB**@pp-KYCh!v}hpWUpxkHq%pV@OnKmkB;lOJ*wnkSRdKv`r<`F zzV~z|y_ra*P|La~wK#)#Y-c7iL5Bj$9w855E=uxUVL0)s0J8ufiYi zJ2*U9C?5^E9TK`}x7SDMV8pXmUfBm~D3x@1t?RW}Z)~JDMLb;aq-JuA66Gb+SzxmD z$*R@IHtn{5-gmVR#IZ}83{z0+6Yjfj;*0cZX@G1@J*;`})zRf^ zs?Wbs(8}9roVfCeB@gii&SccYJqxPNV|e5|?pgH_{kjWABLRI2__THJ>!ujoz2RoO ztJ$ZY@V@cjgx~}}b~A1A_P+IciHFzJ&s&Yz0fWHKt&M!5PPS#poJ;sW9G z>+RoXzM8Y$cfCVX$z>aHYl)p!!a5mhySJ(t3wG+B?mJu2sJrU*^%bd!g5!I0-z@8# zvpYQ?@yR>>ARJ`6G-S>0NUJ?alMPq6+*xue19^Iai;J&Yd&fMq`FLAj@YH6{2sn9% z^1jZ;D-*RR6SfpXP!}g4mhUz9ZCjWow&W5)dN*H(#B_)}yvG-D#ei_nBPOHtc_jcJNzdCpTGrZ}jqTj?N!MlNfD$3^7Pevo+pO}E!Emt;aoV~ z+aplTBH(*JnyPBZ&(U6evrZv~Qp!7)=)krT{Rsh0M*LdF0oj@4JM%%Eou;+TH4 zYUE^FMJkhw6@U%0)S_`yQ<*IKa0yZhMz6M=!qa={cMDmxJ}Kk`IT*&GV+983>X7yS+Zbgu$7tdvbCjm0_K~Fr<+%_w)Bs|d^3|KD|O>o%~H%U zN^lrsL{VdUI10uZ9{o5py;?=0D!3XqfrZhhE=@V(Cd1=e7^Btx%(m1?Wh8S_s&H}u z3-dRZ(qQ#RMolM4`A^ham{&5#Dynhxp&XdsXtS~fD}w<%!yXap4@0 z4_%LI=~ThJ9v)I=eIz7F`944nb&X+5ChSy+N1Q){e;u;9WfMt}*>1t3<$S zYHVI?x769G5zp#K{3Q8&MX(2BOrgG2BWrZ9ssc})OzW^_(#dAAJ)S%A`&(M^Ny_KX zvo=)p;M+2+7X>ROnNE#h;%W&@f$q(%oz6OwZWEM`tUAp7l#!tUc^qAep7ms&I^E+D zU>;0h+G%WvbW<|iM}V<;RbLy%Uj4YIB7JkrP#b2YhQ&B5`pHU-62&SOFuMG-`Z7@>d)*QWDa!*u!bAMak{Lr`c(f()B2J@n_0(T|q}89^t4Sa7EWlN%IZM44lS*;bc-IjtwB$OjaB#Z9 zRDG&dHQoCK^6aVX?wYH_l(jM`XyvP`jGMLEOM5pTkuj*h>8SRq_h8XgVoXzv(Q~!d ziNl6*KIvOBrat1sZW4nH9Iqv7OnINSq!w*oxfZRL)4bf+FRQuyRd1#r`0Welm23MH zpN6m18x%N4B{MsKcL;tCz=xVo0v{q@U%9r2DmGqmKXQC)Bt>z6kOKTm9rW>K6{2J4 zjEb9t?s{sufCWXaX}K{)?!n%|dxh7P{omSMyt+1tIF|rm7_|l#=4sKm9u4!4i7U)c zMp~1+LWu#Rzlegz#lW!sX|vLk^qcTsjNz!Xm=v4J7wujxMiVc@9v?!Ib=C|;lBKl6Hme9mT&esF~s7hnwJ$e>7Li7^v zHODoxE_*F`Encf+Y5yVy25||P_FiM5nSS})5~6so(l&dW`kLo$?u^II_MfWbBGigt zDQ)SOPcG>Y570wTpB1%8R`{6FH&>yL??tlW3H7BZ(k7bumo1l&#WR$efLSBGPJ7w9 z!zqtUGz%_UEuo0#DUJPq?0tDyjo;dT$&`?<5E&BMl#+@(u|U5py3E*XhbNa zG~1}CXi`)%Rfca!(j-liDT+&CU^;brIMejcFrrJJ0^iK@z$ zy%nCRU%D+Sz_*X=JpOV_s)X5!+eaNnWqmo2`^C?FRG!$=g@!w?R1_!2Y+jLdwBl!Q zPD)JfRk*K@qX!RNCdEY#e2ydp4JV)Kd{M{5e`gI%JH#a*OHu@ zK6>B34v!EF8txY~cummATOMcJ9^cam7^mfDsC8k&<`#7wrA2+HwVE!xO?}m@ zh_Ksxr{3vl8FK&ay}eUz2K>}HuBml$mX`mvC(XfkbHU&J2zT#IuJ;-F@w;lYQA2diRXhcHLQ2U_K`|Bg?0(;;iGnWSPPYT*%{`uwJrk&Zx z_S$xLKJ@!yT69F~$lqFrVzrLG8a?}}^|IiZLkwn&H<&r9bjH;0PZe_|y5IhdkE^~^ ze~j=fO&g6_8e3;AH<-1;X_&zb#l;5GB@HANmQI%~mFV`_cAPm{<;#!lz7GZJzo{?T zM$w!xYt`6UD-}MtPM1(X2Pzdm?)OAsQs7^L@xG3Zo|Xnni^mCTu0)4;hYyju^vC2< zajhVxRGic1$2rsYM{7;fI`a%!QS2HUmsoY}SdAUA%gtj~&;B8FynqkT1sJy zP|z^#4?g4a5#_6&*+yACB34s6c9mG{$|bRCfzyJ9DNLU^R!69&x+{lof3*5Etw3xD zEo=yj{U4Qqlq6GXZSBo~_c==YV|Az52P0wu8TiZ?Vx<(e*zEuh$M0IyrKOZea(}0S zXME7|3DNR>p>;Sz>lkhI?U~=zhi=n6WN)>RTGz4qNMiu_NKs;}g#4N5QfDNxX4VEp zPLT>6n-C~wA2^{gut#fy=Cmi?C$;9wI_&(>dG=jU{JvNz1(6wljh!iYX2v8JmC|vM zeGQ0|+WTcm+rqq=x~*Cp&`@}%rxCqmDWCx-{Sa1#@iY_Jl zo2J~-5+QUc4Zhe?{#%AHpN_6#kY2_;5m{^%6R=eDXezVlvSYia`FRFw z=B~Y1M=5CuZcx^ysYudO#)8GU#`b{qq4u<{exwiT<93pShQf} z2PFM%SXwp?e1zl(+p~C%=#GqM#UGf47Un7Hw(4L}D;|$prBU|&$TBVeEkhuh^e}1i z$o{4wcefa|^39x9F3MqoavAr8Fcb8}O;F1#W(ev?vfng>?iLS19BH$i59UaWn822T zF`?7^jqikgF+&hrx&xkpriMnj40Yy?Jh-8hn>xh8&IimDF;_TE%v?2uP;>p0A@-=E zL;Tp>;4ZpE93A5PX^tU2tw0jok~-h#ZZSXP$pm+&xFZ5z#XyH3tdU$h0mh($!AFQS zLQPy}8zm8{hNzYdiUL+eVy_GR*;ub8J6!fJmlOqZ$JA0DvHYEBp(?hqEy}Rl!?=KnHj>?Y#Y7 zq_))$13K_8G@JHny0IO>`bPx-6xwcgF+j1I15hwrG)@;u+nv=PLY4N%>7p(Q;38_= z4EV2|%NmSOdcoZ8@7lSD-36e?<*r3X@ChxiI6LM@#y9s0b^3n{y|_Ql4jsv0L4PBa zpd*bhoV)Ueb_p_)D|iOFPot^@TRFMJ78WaW@Jb>xa}9p-<*Ye%JQH zhS203;v_o6_PYj?QVmz3LrBlWh9GJSODU|j7`s1C(?Ti;vG1!P{Q&=5gBd7-hf}+2 zFawRe;2D^M3i1mtR#j2O-nV&#( zi=6OF2a9g-ul^+n==vWc!4VH5pSSB^@euxnuj4HmK}WSj9&Oioh(u7#zi36Y#mtm; zK&h?kdTLR2@h6Lh-H2(yzfdEg$+1OiUJLVNj5(H*aZ76y>W zTjdN=NOf8?~aX&gKllZ|A|gKza(L|37_(ah2dMpu(2i#e`x z<5Up8f`6gWl@TjrJ~Nd?pp3-VLPjmViI0(@5dMY4N(M5P3sFPCU&q0iu5A4tgmRgQ zT6?No*X7eeJ!Hmx`%<=Prw;&57(U@J;FvDv%m@RnZ}I)!uF>(^ZIolfh66X+?xz08 zF7B{sa7TCXSNZ{L8;KC!t1*=wBHu3=ZTYp}u=pp@cKMP!0=uL`W=MsNl?qj8R4Tc% zVwZIIh=dU7gfOv$&?N~`4YiB4gzW=YNkxd>+Aq@i;O*m%a^1ku=SRe8`NwH$$LZYG z&A2I;c+pTR&TviU3$2tFYf@FasW0?0ZcdSoU7&So+_NvQ@1FCJ`u@J_v&-?Ib-o6( z@(gCg8Bl5|=9@;JH;|4sNQ^b#5o`Q8*7IVXuD(p%aCesd2x z!*`T95e5>W29o6lGcyflXVYoU25KemT&_fxPCsQZKlR`bmo~}5(?605Bz7zruN^dQ ztzU?CEpA=FnMHj0*Pjx!{sUQQXU?&k-r$b|<#~Vl) z8OSa)n7bFQieh(>Se2a14@#SF30M_%`YC)>B8D6KTa}?PxhjJ%#Hu!OtST{yb5$wK zRi#iDH8mv-18=;}d5W&8UHQJ@JaUGmNqW7MpCZ}0QyL^jmChORM#t*!4eQ!8O( z(|B}c^B#{7>LVu$KmF(Aj1jq{WtBUGZaO2;`JY>KmQCCci93q5^#7+}g2~lHt`W8=JscUrv6^7c)hJ3aS99C% z{d1iI3xcN=#_AyMd%#OXryg-dXbFN$OPHbTyw#R(nV9Fca1N+WdFmidKG40|L+k_z!y$&rSSluadevHTn7#&ZGK@5URRxzy zzT0KsLx>K!B+0om=A_CdfdPQn84i`1bE@o;BxwL}5HK?i1Hgu1N6In^J5rv9>tA|S z36WcS3z=G~i0t!ix*!NCUUOwku%=M4%1q_-%jcIegRu!(Zb?4RskmxHam`E3f!I1y zam4;q3v%pF@_FX|e4(sU+4(8IdjEA-1v0XRyCCt0?)O@pj}|!8KIG8;oz!7zijEg_ zsI$u`8)Cq3WQ1rXgsgFs!t#sVEZRC?;k&5*p`eV1OW5xUY;(ZDWxUR#9e0;j}g=GKPNi(=To}$To{9e9K z=LK6GG?r4w{LTiuHCu%F4rmeaF|$yhGIC52YK-yq(_GW{M!tSL@269u*f7j!eco#e$ zyYJdy3mG3IlE-brKv?@QzGblxguSl*$loO6Bf9oY-qbQuCC3oRCdF~u zJ>HMc<2M}?!{zag7u&r;jwj%A%F~@Vj1VG#;mO0I@89Gt`rVukoNwrkk~~cJn_#(NP&5V5@6xH@?+d=p1%YL^Ii`nK~ORV5GG#J&2 z+nxOsm74d#&8_)uTz4Y|8Kn5vHamA7VbzWdtBMO$Cy7OkJcSqs#k^}-eDhz3Kbb`J z=z?wE6yQ6Su&TYz7SEq+bZm=pzpnqg)d0&Kwgt;Ws%CSdWI~XEID8gZ;T)zFzU1HJ zHsNWK@An-s7{=+WQ;EPm7XY~86{M-#?O$kaaVm`4AQNRtQ$HDIa;d)D?$38{U*<9k z$J5B;fwHb{2{=NQn~mj{y@J8w!>yS)zLpxEt_UY(g64SOw&B)~`;~jl*c~XBZf(&V zVWKhs=usZ9Ka)H3U_Krqo4${S(ZsD%o->*-_Z^mf&o%aE>RZU7`$#c=b);l$fA(T8 zLfT8l{)LVfx>Q7WefB>>iM{_zTXd*J@rAZ1?1Fw-idAm_dv{X3r?KlxfW2+YKBn8_ z7?PUi$_a9J&L%-_=8vlC4u@vM(B|PKf$ZpeOvw3CNqCEGv-=HK)r~6La_4*|?uMCv zl5Eteup$)O1BmKr0=`0`3)<(h+%*6X7sxXOzZMo$^%ca5g3>PlKezT)JWID1Vd5Bw zq9c`}bb;n{F+}X?1AsO;98u_TSAh~J+<74H|Ee0W)etoT70NR3czNDy1zB(rg0L9T7r`qZphpO#N{PyIM=N_B>}fm2YYpHz>R$o^6&M zQ-1)`kgp*EAr1S)y<&NSFeI?*mk2-@f*}#Yu#X*kPmv<118B#V0bPr7m+-m7k~Y(~ zJk3~5Fln$n?W}$em^A+4X(*Gn$J0wBV>MO-Pf%WN#8`F^Rry+~G+1`+V51Ul%YGM4 zE_>HExa{MHxh;FlFy^vf@=tMnbpPL0ZkuOv}h(!2YK`_j0;Nq}3hx^r5x|1j=#C-d<{_5ZJZeB<7Wd393XR!ht_Kj+89 zlC@43@}IZuOItS00YxHxtcn>Q2&-~9J*?*aebm?cRzMhv+&6#0Gz=B?c|0DshOKyi zzxdBU>ePF?Z0(kciJd;WirhU;FYOdyrgehwJo@NTOe?rvLQ-mdeBz$PaJEm_$}O!H zI^Cx6`^UfIYw{)Ow6+JAvcK8M{r`OYSY-BsOU(zJ4pZkgdl?^c_F?xfNq(~vL}ss4 z(y8Wj52d*YlJewXCJ6>e%18Gv?fPbix*N>+O}+@%#y*xv_9x`vv4pE}-X)_}J0oPU zzB`e~T+5BoZ!&n&HVxhU>Wp%S=|1_1`>}fG$5Q06mTRKThkDE2U z;~RO3RM^o+!7;7dxLXtR@kFTM{&;}HT6&uW0H#_bvU?U00q@i<4L-m4`?i3kktn{v zI8cE01m>?MDVE5YdLN$zFGlED4!D!Ch>!Vrf`8)icxM7*u4Yz}M84nsA%Dt#h1FaO zHZ1V*CfwM84-#%htqw*@H;zXD6~}^c!p&@ak#SzS(f=gEZ8W)ElZF*Nx}Uy_D4L-bC|Mp^04142Ux=ZAOC-Z+iyW3V5qqBd6WaatDp6S`nwn;AW`Qo+9yBS;^g7q z)vFbt7`+v9B6|$8_JYT#RYG!Y4^CurHHw20Y>;$5``cEzyOlz}GYW7_;<+A>$mH3z zipaaXBm3kfxcV@YE+mdF`p|`au^~d5>w)J73%2GT`CTH1tYpgwJ>!+=5g8=s0Oj_( zRlq=(CQW({LC%3ka7%B@j6>vpNY1?r<{l@K5Mth_H6JpDTr<=WB-?7ZD@dqDaycge z(-$SDUnLaPG;Vlipzowgw;-k+m3>VuHNR0v`s8Mu5DJPi%L8NFN9mshZN^5Wv;taOy>AwJMc6 zG+IEq93ZGiohlj#zO!BNK0SsjvRE;b48t+4RmHy?27m|Ml@_Q@7ULH1=KuNhA&+OC z77rMm$cUK?DyaEmWYe>tBDf8|J5GhW4!$2%7{qkeTBq~*1<$8+Y=ykm?;OJdcsE5K zkfw^v?QU`Xq2mgfBftgyPG7Voz_MNv_*RI~8$jUOKjGUn5WWraW`GUjrnNMEa?{Nq zZzeZEn_HTY2_Zo}q@?Ok_;SV1%;^*O`9EO#4yVcK^VkEZeLi-UTjX69-kJFBaYr7% z4(%)7IkG>86LF7LM~=8Fgg}?qXm5)KF1U$LQlFsg2{LW$FG!|^sQ-pcgO$&rYlLAO zh+!8#;9drxesmgqxlupGLI~sKOE6w?Tp2HmIqd4*QjTD}4#aQ^=QItH@j3t-{`iZ9 z5XOr`PbUPTMHzi#kg@+GJ!K<$vLN(C)A)^^2yVN~CA-mi``C5|9kTG<_aV>ostleQVduo#R$z(P9l%X5CW_-aLQh z@mIYXukQ`lSbr=yVp^BLpj{F*mZ3>7b(GF3ES)jLKJh{OCfXX93mD3x{$I>QeHHZO^N=bl->zwOa|aFTvvS6~vm3G)uoy+g4Nn;@l? zl(ABK=$0u*TpI)Zf0WM~7C32`<;lZxkCtC$w^4p22;{S|X6CWeQ%WaDP4eC~raEqg z^mCItQ`BvyxXX{unK$&7`u({(Q-RDdCO)OzJKTBuMYio z7R>(C;Z!cOZ*1XxW19T~g=u*%AVSl3WczkTe9dL6)*UsbT8e4B@|N@=5%!fTI+S-IqSt z#fa68xhEX=E3aXE-P(8dXytY62p4!FGZ6)w%XMBW9DGa7{#Ix~ZkK_0V^QzUsw8-VxnNCthSqZPIPhATnT|m{ z^M=twiEwpGES`cz6pbrPGp{~t^61n($;gk-W0B&ml z$eO)|k580$f1oZt^P4r$PKI=y`y~gmOeAO(A*5;wBDbw7P`*@8EkUGI4Gb(_k^h_AS3RS9Dj#ibbE(AmPf43cPX`+}S*&iiv$4Od1iR7(5Uj<+?g_7* z&TTql$?2pI_B0)&Cgbx*(*cISb2gli?3m=YhXY6*|63+M>Wv~$z4;(D+25$`*O3kgA!!evG`>#*Oej4E$pyTA1na=gC@VwJN22xukCVO z*SX9dhgnU&mTeE@2BOOstbfq? z&FFvtV-4keIf_Z|vW!}55`UjB@YI_w?YVJ$Zi+uQA4r-(`at3@N29JAe>NT913YH~ zz`8W)z{&n~#(B<3zsp`QN{Ell(LK&oknsLne$S5la@6QjlHjHwJfCI>w$=> zBqO9pY*sFH`T|G-Ap?OA7z2eg}+(Y{6EDEI+^qgGfr&5&CSUMX5cc@kv7Mu9(r&gkmTfrJ^!fanI!j3cn= zlLFlj^-sLod#eU7E+rwYs=;-^5phUuAxnHfSTMAgyBZxWJ!;fw{R3>By;!ZSv^)c{ zP9$>zKOjmG2|48;QQ6+wURx2(wW)0CgszoOIUa`7Kkx(Sh|mVUU#!pV>~%O~*T}}e z&y6SKBb&{pT%Z+9H|N|ObLQrlQ1|7Z43mQmqmt<6 z!#t;gq6u2&g8)d_BbEt1U|VK7oD^6;86%-#f@X2tFu?~b z!`ucZ1=b&o+ocM;DafjMp=@wktKmQcTE2NVS_q_@)$WK16yz^6GHD)ko z0hydZEE5T0&3DaIoL{cPBPDSS7o|(U2kb}*N7$e4$`Ov6;W0jYCqaB}W`<+n`C(=% zb;;*BM&af8nLi4vJ8YUc0v~a~0}XuDNZ@nLAqK)d-&BPo@PQB5f$!yWpfGACN8lr_ zT(ANkgX}v~U#ZCu`&v-HO{qkg+CbF}!Q$l<>KqQMq0S-WC53Jy;*svhp*0a-4y}p! zl1WUn&f%i7(t3Pr94>M&S@_maq1v$huR6kg|J~D}+BlLJaornDVoW0o1W63;Bv9LI zJYc+B2iiW+BnCcUB(WVLEs>c&YO6b(u+0QXeiMFN)Z;XWNvJ{;1GX~dd<$30yx`e= z8XOyiSu?c;dDtq=I>(DG99nb1w8j=&%uqHIGh0_Dd$}cS1Ey>jnado?2G3#p0kz?d zF0GdG6hll$bjKmttSq%R2#^HuIfPhyy^BIY zt$`2NTGORHq|mKIu;n{Epl~i9i&CDZDr(8+(bb>P2l_eGAv91*mn~>Zw8!u?O-P74$G}{S5I>lwUmKrO%D#lEw=a7ntgLu#y_Do zdy(!8Zdzk2yFiLmwhP^Uad*Z!EG2|lFXmJ>GfELLah;m%;rM=2PTxcXbhIZtz*{>F+@Ce&X~WiDHWmC4~@ zilZ822A$jjA5TUh4fAhXxm@SxaF0+t|8~Mx;;_;-+}ciQ(jju_+bz4Vr(()hva&>! z&EBKAGUOhO=-lqxefb`f8u!r-ZxQi&Y3Ash!R z-&NLQh=m1oOxKY>2ks7-9tY4VfN$$T70tA2g5YwYZ(rPf9k*&FtKV8RfQRSJ*5%4x zuEWim6CN^BsaLoFlOKX`R|W``IpgNDZ84VR*eE%bM9ED_b#3L&ozhsChe-_wFnDsT zaiKeMCdUZQFt5YPa!xZ8gFUHhatmWQ!u&c;Fv?Umpog}>wKQgND4X%3Csf(ih*d|> z9d){u;<=gHvsfbX5~ilum=#H@(IK;FUsD5ju-OL`$>-}{K6l2E-UpK*e*ll2sSR1U zwZS~YVKr=ce9f=Dw|-%XS9?BXMt9W_bYIk{-#EA-Tai+RuMzZEHG>{=<)Fu!Iq9*- zBt2FkPtap~bZ~mCA%cS*10S%k-uERR33@D`5d(XJr^45)v8W3c*t>F`AqT+T_Z^>r zoXoLmOwMd{W`A)`V(6+k`x=AYkJHw`5`W5dJGW*EepEfIqV|TwNCU- z%Zb`>ESm}5kwe^t6`NUNFbVc6HsxO_24+o65{~xrt>~9;C-z2QlbrHpJ~By|a_`8I zDfesV)^?9;on&w#c-!{=QZ7@Bz@Yt!g;}FwpypGuDq&X2kXba_V<~$(b;4#8;QUbn z0YbGVC+49MXWQZln>jMU4RaFkr?izDckW2y0Q|&D0#RCG1jCRB==vbX+{!W~4YLSc zQ!B(G*!E|P*k5^0^$ttNOiZl4at9&?4GBDlZAiwICUcvT#)n~hBsf=wV~=oHw8PRP z^N(N=Q;gW79C}}|Fl!V|5qreMpb)xNnOHU_rE*|U3Wh=ZhpzpIxnN?jyZ|3CVs$yi z36!KZmf2M%Se&RG%sWn`zHUlV<_ZhgXyO5hCX&;X=Qb6ND8M3h!1xJB#ARg$hfI^qb!L>q0DOsZbrejrHhCyHN2L&t<2NTf#I_Y+m-D$CD~?s zP$hx`y#`Lsp`4)z$O}1SD#Jx#il{o0(w`z^Zj*{Q@wM>c!Sn_WMU)_lAWSI9ahaG3c>Pl6eb4^lN=$GQ-$RmK)IH4 zs)1<_1U;7KY+R{4R|bnIBF3g1F@?#Y5QiKdKMQl{HaLd@DHER5OJA{ms23L!l%}7{ z0YsO~Wh|!X0>F-X+YS@npRVH=KNiBFR@(O>59Ypn4Z!`g+ayPCK0?=b3_}2EN$MCJi z=`p}`q~_GX`H_3nx4Tf0!6Fddg*tR6iidJA=|6ad3ad>fYe|#V;O#q|BK4c9Us?TB0@w#N_u>Vn;(cx6h!Q5vcN0w|R5j+P*9D+&Yr`7V&o%qPi zieD#KHd9iu)ww6PMw595f#@lP3@qIV=ij=#ulaF0D2u-atOFph#a})b#>*NG*1`Tn zpEGEy$6aa2$r7&R3qd_SE`wk%DtI>*k&j{39bYJ1_VJiK!a()C@xVN_hBqFnxUT~R*0_qC@t6ZS4E@R$Y_+I;G`QUS0912{0= zgD~HbOw&5lsQ5y?GLG|J%*a?UN1ho7AlEW7I%;UK9v%p`Eym)&pNnCXI!o*tlNn?% zJ5ga}dMLBQNpeN{^?q)+>@l8sV&o`id!{eVi5rh1=(p?r+I4yS1eQ>$&G%3t8RoV<~w9s z-l&$_D2pP4+}xykoZtY5C0Oy`NlcK;2C$g}D<1k*0djMj!qF-Kn;Sq=L$|n)E*A;u zOG}Cgxqi?qBRnK{E(e^#X{YTv-3kcpL{xS<2xeoGsZ24VYKWls6$`UQ#R#ZJA)#L6 znq0_t8eo(AO@_x#^Kv1z%yF7`GJ(Tonba7(1dyZHhZ_SoKX@HV_<7c7<~m-)Hz#5Z z5=Pl*oJkErW}#ndu)cq>2nKi~#Ns_CZ^%8BfQ?o^aRV|MGzIVhyC8h@!rA(BmOR3f z_ca9_(iA4a$JaeF#I!9ugm~3Fi-BYX`G`#<*rQZ92Y<=RM}`4yZBuwy5C?JjdkYGl zOO`($=#DPkN(i+M#KNpy`ZK(LQtTm%{J(bYbZ@kh{&kf_7P3tLfwj0YWZc?sj16D;FR_7LRhxXn0uI_EY*p0etA z@F4c}D2Ci(4mlT04i@s@16CIO?*Ek_T`zt?OY5q^D@~Bn8j_hzl91?0Od`n4;vE<% zNsTT?NJ-e48vTH57)?qe8b+m)(JkI4VrYsqjIweRU^f}fHYNr8^OMu$j4MU)N-wwI zZ9k@gpI$+srW0s@hVFvVG%POzbPt<>yHPE=vH1jA_F~RVEt~cqYNM$Q?DqLLp^jI!6Wp2Y)Qy$wD#sOH%&N@(<+c3hxY}7@s4y!IC4U z!w^maadrm5Qb9jr^_3=w7=a3~!zGh~#v~Mo!`5T0j)(j$`%DJ3s#EAYL_9L_V{Ij? zKFt7F%aWzz#Z->w8_FLv-&o3CPUTG3*s(T@4S4fSk}j?MpJ={Gs;8ABXRV;im!tWH z1+FjY(~lyEsLTZ?o~uHnT#}`5&EFK#ETp1&SE@ z1+F*p-UkLSAnqgHdaZL+Ie0D^zxsZi*?$ASp5BM&?E7`5tr1U;e|<7R*%~=5?fZ2m z#Y8;)+-sT3%D+YK#N7FixqH7J8Lf<#6@8!1Y@Q*W?)n<`I3m4geZOwU=m;d{ZvOgY zm@?XJ-uLTm$PKGyuN_V+*KoZ`d4%=#QEy}wC2}Ii)ae{&#=Pz*1C^?j$tIWP)3qxK zcPWmGK5Af=EvxC?JM7kB?<0|Me(^%R>U_%U3@jzJvT0E?Nye4blu}A_soTZ6>@iEm zc3%*rtzHs0$EX8TG%o3)In_dz;K~={2dIW5g!csJFi#wfs$(}V89N@n^y>1sIO$w) zhs&w$fyu$QN!fu(j32q0{QIGGmYS`o!zYF@8R3gn`^BKYv~DLj#P1O*jm_^~-6^JRNXJRcd9zQ+UsZ8q-={{A&le zk4*e_my}laf?3iwTMr&#v;Z302fH@bfabRV%{&)y)fS3Dq(b%mUarxxBL))-18-nQ z#Iy_@z)$X+Hwa1W0(9!lNwSuw!cteQy~_A0twS%?uDG}rcE}UWruLyq_TgX<*xL^W zfzclH+A`adk+~^ZRdWuKK@JkBWT{I{jn~Bo0UeCwAUWWOuu;oU<}qaiF*i`9-m21M zhC6g_i){=LF*r9+W-#WK>G!x$hdw`8^;s;rqenG!K@WU573f&Nl(5(r(R59!+Su`m z^6?D&X|C6Cdi3I${7)+#Ovq-d4k`q<$AzH+xIJd;#NkLGh{av0gndH99T>K~QRk5M zCgM@4DGqW3sx%S62#~slN&~^3y`j1^D=37mUKfy49IN*uSFcoZ3<_ZjfvF2wJzNNL zFX2L}z?^^($3j3iC%%xKv&n@tyPK-O&Npl!n9;z60D_0L`&mSHF45FX?g~9y@G9=p z-LD5TV*o(hyTI$XBbDlOsHnx{LHdaRaNsp705AiG1-#=1e}hnXo}gUzk!t^=FeBfH z@#AiR2khZuGX#-vlLcO30L1D+-ysx(1T4`BO>_?IviHL?!6YSFqb~HQ+^b24DsR@rx(5Mh}#T4XbLJYL_ z?goT9ApAkQDJlk!5J@qpK!Bdu3WP@&;7kUa8)WT~CBeBd2O;Lx-yrZ<@;8Hkrf6_L zatC(LqNC6bifr;&!|1I_%2;z}&n#db4W~()v-HS<=K~zBV8ZDNsiZ5w6ALg%C=PG8~6!28s z>_FAUw(a0A$KXJy)q&vffz!6-84g55pm@)T8`MrF;Ddk_zg*}M%XOKu9MsN%g z!8u6%IK;U@ueKUEnnb-6cW&sVAm_0Mfd>n;hj;{m$%(^Db&)~*JS#QT94_L6fK)@S zgdIVcb0Z=M>?7x#8*J()#fS*P2#Q1qhG!+9HJn2MSWtu!BvXW;0MNt&uGBDFoDmK5 z0}W2Ym@tJI696yx$p-cwdsgG=5nfjz(<8kN^m+q#ox*fiLJamE!+u@VUF&&Z3O(2t z%#kCpWZ!=%(d+zt8IM5ygpy8aGL%?$XQX;o4=l`a4&qgcfhs?LwRK&#SE48;Ox@L3 zJpW+h(!Jo8=h;n|5~9E_bRuwUHNcyX3>!i&8rB1f>F>e_y1h>K0ziHfKgKyO)D4}7UlyRD-F3! zLhS2_`n$a?7B1ZfEoEkxq}K04M(Z{YFtKFtH_Y_!>8L@U$%f3L8+u=Z8_-)5$?|imHRDDe?`vLQYL$ed{H4zEDJl6&#<$5|e{5^!GCwC3}jqDRNhQa!cEMv&# zqqp~fJJ9Ocs2731iBz_zSUw2gncgQdu zOjJh#5UftXsU$2^iRxr%@ocynj!44KiMGoqWJsuXS8!B~RKf=V@Rd|FQ3CxNR`RbH zLoC3GeIpWmgBkVZU6j1~RqM*)s4cz+Xuzyz_G8q2fApcHvCF$OM#eQc1 znLJ8bz|CotJYrOHo^r#OSpec)GZo9^^Ku;J>0+!rWy55g({Q*b$5Eay=24#KQRrI- zR%~1L9fTJL@V|5r@Yam%{)nbJ@YW2B)&o|d?iI^M$2&hIW&S7!m+X-G+Tj6K%|iq_ zm5D%MeGT6n0Nl&{~H9sVgF)vf%lODwCASs@y*1$Zt z1r9VYwD%t|2(1~shyYY#0L>6c6m;r82QLe8YIe^8cejxc249Xs!dVb}4&xstR9*-(a7!Jg(3KP%m(K3AYHxnR&E=F+Z1qd|G z^BW2V5Iwu-GuO+DC^EQVf@h(0&ky$GJir1iZ+OiE?m4qdM4M7!#l~Zi9FvPgOT>R_ za@d&+AbB%?JcKVxz>Xy0+iMU)0-l~!lc$Ad@bwLS4t_u@EPjH;v;pEs5^g5wxkjm7 zU{rklQNm1eyvQT-ToGWP%XH}W7(33B5)Sq}yG9LMJjSv=cq(pgK=*8@sMvTJ4uld5 zV8uEpk;*d?cqyXqPwWVJxi89~v8X!mAlC5*oFJg;VjhrG9Uw^Iy3))8Fl(}dqe z`y7$+Ud^+DDbh9?6vl!;fB36{Sg@c-!N?crf_^nl59U+?AE^YPXhI46m`vqhenUhF zEP7ocSepox?d2{PoNP*GD>epWk+PxPg*!QS1L z@{DAW3mX^uQNfj8B9E{T23-k@MXHf&qVP@(qE~}koD7bn<#Wc#J^>Fqa8j+(j6x5b z@CF|GI>WBy%OQaoEz->Q4H>9ZhFVC{1H z4AK|>U=+EnKtnoLaK?%TOixe)(ZB+@f~aESiGf7}KKfQZ4|#AO42uTdoF>3WMuU;j zazl$nLEj!51jA?mu7ifPFN~JR65c!=Qn8;s#5O zvvC(5XX7X2f9CN#8^;dd`x%6S4bo?B;GXLNc6Tx2a=GB1D|#-v`eGf`yGM%+yC1zU zvHqL{u8fc;4IpmHbM}2$BZsv*9t&X`Gwwj38vyrGa6&rJLGb31%yT&$r`5bdfeE!C ziEAieIS{yd#z?22@g=N`*(i-Dz!bipkL;I$p%;%JnSMDO2k`wIvSAT~=iCN92>X8rygzvN?9kryT{tu21`f2RzM-BKEnF65=cwY{9ixOEDu@4+_80|bn z6QzlGTlukpbq9D<0wE)OAaLutpMgdQvAZ?Rti+8gkAZ6!*v5hvHQ*bIwIT2v2v(Dz z%L{^EqR{jBX=Mq7a9;ow2I&jzU79PyJ`T3=^)LmqGsRZ47`!upToF;WrjY}UZ?l3z z_&}g0AqOI|7ON2q&`O}Ci+LxXQ~=LIL>D^qQa+j)K%XfVv>9Yfn-R&Xz}Fdgq=VOr zAY!n+>|ZBlY)Xzcq7x`E9lWFs8|^`VqYX@qpniPGte4Dsw*%=+AQXU`J7m^lW8fr) z7{tI1nc#&Lu%90ln*v@~@n05$IrfG*p$Q_#vR`w7Xo8wnj%b2d5_4`(!1-dGLuSj3 zYBy|hea;s_J2HjYj!am2N?WaQrxSPy4nk=uStX%#2INeb&PVhKTh*ncwQyKCd@BZN z;q2gm`D{O8eH{b$K;|14u(?@s&JF8fu4BCW6>BK`gTTdniv(M&cMaZ_(u(!%AJsmxEQ5WqQ`+3INA+3!+|cXj6%0ZYW4*T${e9kSws3OUf}I1hSkLWx;!{> zJ%NVHT+1%3^8WKd97+5OU%>G3GI%ivnTY|&win@mB3S!V*8)e+vmEVDpy{6@3j7xa zad`K`tG)DNUFTL2KTpR8$lMw#D0ewpjdqHZlC4JB86B_-E>H>Yf(umYuM4g;o(+2} zs=HtpoE8;hC0)P^6iE(-RoH-cAF@08Q~i3fdu@KQo7ox#We;vLw{dLIFAkd0F#~|du)tj%OWx4IWfJOX_ z5VQoV*bg1dq$gp~+D_1uVJmQY68w$QlPo_PAmPdmj~V8~(qMSZHtqzwRuAqNHe_yGkQ36L(F-e^%`>ww zk4=x=t^WH*A%*TB;&B5zHwE{Wu)bu5>jDJ9gpS1J2whqg6eWW`$QAGn<4CT6&=I?t z1z5u(j*hgx&0_3(R>YZJRdt8Mz|QodQqC7~bjtzy3c;s+4-Wt9gY<(K8TYsd#8~z* z-Tp@vevhkb4v(O33#A5>S=^NzRjGfRS;{FEPVt4JWgkyqqwuUG?K-w*xXYX4W;x%S z)A=*Vc2xFnMPasoNfL2>X#1YneZKg?Tae?p-;?k4s_d`d1DD%0=+A zG^vcA3nEA9Q!X;jI{UPvrkqdZaIimWM`gTG0L%1Pw*mA^{I+O0d~*R7VR+sB?B^aY z9${E;0vvCXgwi_ZYYVWsVRa*%8x!&pg)@Bl0DT7nc?qP9-O7d04=Yal83cfQj==P9 zAB3#{7Ig>Sfbyy-Nx1Vf{8bL}02>@GalgU1*yE2`LQL0v?->u2j_fDcGMXTm<@rN+ zHyevjY9s#OTA{d2DMy=%X+WQoI{Z{CVvRW0`-{+Q*tf=si-;W8zq@DSr~T4Hw=j#i z%mV&bknn9lFP}B4HDay8>*e{FRT6kh!W(dMI8hzGbdPPveKiHHC&0D^_!hl>S`#^+ zU(&+QY(*wi3xAONdm(ro9?MKPzYpe9Gm`+A#C#6Ar!UK?nnvQwewJ&;$P;k6@ORh2 z@14&?Fj7fNufhksJsMy$<`~4$VxKYKTMsIyanw`b7cAL_7RGtXN$;peL1bZs-!KEY zh>xO4M3};`BsuHT}v8>8&_IEsIFP0NJU3tFQ zCt(#9Xc?kUGH2A({=CY@uMlRYGQgc?ut+6*Y z4>@CCY#!=g8lN%ODDM5?qx^PuV&g{;x!%^}kp8RPhBP)srq3C%g#J@mdJX4}QPZDCy!RGs(*zLJt`jo(-{y zTPY>AJVAFJXRy?L#_sX|DLj}50BND4f4Ikbw&R(NugjFq*=dAoq`zo~JKjzs_lg0orUFiG zV2;OoF?}3wbJDvz19ZGQ#(8cj{>Z-e7ZJ~nAx)L?x0~+qma}69WU4K|R5J%?s+ogx zc4DU5-^Pq(MZ}@szFk(UUF(zUHD4#XMtQQgiqOYn2yG5~bP}n_% z$`Qu*l+vaqADUYGXuFpFV3qq~5x1rVhwn<2?&-PTUJ!V7xZ8whKKs-7Cfui_iQir( zyff(Ia1OytEf1z(M1_moawxbnXt*FIcxq>mx?mr-aG&h1)Hk8o4qu1RB+@2^W6YIF7RhwZQC zp3C+6)~yh`>%sXUySGe}@m`rG=rDBSy|Jc4H{KsnIrPS1qqIvkH%BI1*fQHxK(WI5 zyTaGFqB^x-o|DF^y?uUY@qFP4_hwWMRyr)ZMNsB&%9bGx0yX!gD@z7w*EM1rUu`^{ z4@xC+w>@u9X}0Y-sd;#N()qd#xtpH)-0^JF%L-LhpX=dmER7gYKZXz;>d!WW8g21t zC>6@x4vMw&*mmF;CtM7ro^m$>ySS}vGq_PY%nnL%-Ih&5GYqb=tiWw!qd*3f;;}7a zSD4#!^W@_@qiBDLHOyR~&vjqIB#@v`JCr^gN?qZxFGG=i5%4$yrMT>i$5*AU=TvCe zV^~Wn*}ohtX=X<0YOYJ#owaLuT@4gmV8qY`6r_HhxuAUY08R9-vp{NKCOTaA=U+Tc zG#*jK29G20^x?WjJWO;0VxkGE8c>Rti6$T>YA7B9rFfaBA=5L5 zLp+s$`DqmJ)8zy6Q@vtsAjMS&H%H=rI@>M|Nb%4C%c!9aHa3c6lmaO(I)LHRJb?_K zDKtTIAZ5VSO~d(949W0mOdkxT5_q_&G0QgRz3*_E>v7bWJ~D-ipH8hky8XBZKN>$* zKq)T9sp>Y_KZ5XVmLV`4RB=JMEmuS6tAt7Bjfw+Q!pIcbq(4`J-1|S$!K3XQIfR!E zAeg{&NK{Oy>WYEcW)ZN>n1R`*tz8_D`cvE7H7V-rjZ$FDYCOEgYSLEAdq{QzF5`KK zn+*`__dx=by4rGq?!nwL0tAWb0!SG2@jNt?F32tKbiQ)@)jj?igg7i0z4YVIpDk(E zBs^jN*^;nu1GLsX7;_%AJ(GKKKYVb)fTilg)b~XIYwt2DHe=Io2f0g zH;1K%d-~$qjQCKq(&k!iKJ(q8LRRF9xKzBiC^LQOLdw zrH_D814FL9P>O3tUUH zpjes5zR0X-5+b3>DB377AT_Xk849J)eerBi>S}j%$-?-FsJ9iZwy=VgZn(@?*O+}q zZk>&jX=Ta-^yyJ_AT=OlM)U@`#D08(82^yWxBekwO411TPUy2qI5r@JP#xT$ACH#kE|&@uV|SCVeUPYkM|5dI;zszEL!OEO_^!h({HS z=^Vt~9wz}0ZzwgOq^b(Qj`zTls(7>ykmBX5EU-f)RpDZFAT=Og9cC8|qA zEFD*W8UjeC)5Vah;-&FK1jayG4@mLSIFk%ykWT+c2L3r22p!=Mw}r)5zf-sXs_Z`? z!tis1$qgb4if0ed69mP8)WFc?;88TLya7DJ;_w7OH4KoQP+czIacE!xDhdRs) zdd2EMikG`!s2?a8NbzzPtQZ^Ew%k1V2~=Z~#2OTN>*wX%uJ^%@d!3%Cd4+lPwx+7ZiFt)$_e5vdemDZckT5u18mEg=J4qksAH6T2EJvoMmp$_gpnk8hChDlv+2v z*NaAbtkzW@ZkglR^`>3Txx4X;S9iBCcxsDUx4BnOu3C42cyFud4NK4N4sq+Q)-Of5 zU5|6ST3d3v=szpOy?W?HQMBHU`ikD4MN!{#d(`yT3Je~{$HzAu{FB)9;8@M~$3yt| zDkk&siGWY`Imqp?+wQoX@xT1MeU1*zW_qV5FC3RUvvq3nt*qcRHm>POHc7fl_A#}t znbZkC8=fsvbXxj8sJTf=`Ng!Lz%fI1Er0ijpFh;xZ*9i{p_Tfvo%Ol zy-v|IH*tII)iUC>@n3#|gXo%0@59DRg*w&Tpjn9(6fBh#OdfUX__aeZ!<*ILh`gDs zc*i2iu~I(X&v)z;fts3!5h^UIJ_8gs}FIPHb z=IqWOkt(Y}Q>|ZKSNFW9NB`h+eau6H`x*s__2LDuZ*46)9$~&bspRK(y)`Cd2VXj{ zZuW!wCyN_zZKNJl?D=`9SK(?+v+M2WCoMUyzxD_v-Nu*xz>iyiYf%ds0xa?Tb zI`N)8^oEkghvK7-RO}l2`G{O@cT-WS+A6Q5-5=}U%RSqd@ZPKYZLpvA>hIa6@%ry` zU!E*e{j9lY-KiyIvr4atKl#gH{mIcc6{PPcO*(T*V)P=jNnh`TT=^0AbE?>?kE70{ zob;8F9G?2|{gF}fYo3~0CcKc>eJ)*dQlr+>#yZ1T3}nB)QB)4>9R z%v!H% z?7p?s?L$ICo<{|}`8+f6;-%r2rXK{;9FR zLF}1UU`O_r`J)Pak7=!1G=2NYDVHSHS?gvkv}{?ZTW$QlFg8VG*5-3}G|sv4uTPX+ znQyUfb(rDmhQS%%KVR^5KX)Tpf@b+`;^ljKHrs{L=i8g*NbBx*RFD&McpbLRG4B5O zzY+p>2p3$5ei|O1YJcWd&)%ZKu7Ai6otRj1zOc`sL5)mm}uu#=l zbIQv5f?{dc7X2NawXx}{xAr2X&t*Q|Bli}ZGAZm*5Sm~)EVRnUOziD-%EHV_(QjS* z<5#GbzDPG|=uy_S-xMwNx@M>LjomR-j)OZEjx-}^_8}t8Gc0>@WcLLB`1062ngJU}o^^bBb$Ml{u$$^v|L32Y+P2JlvP@6z-Hw%( zDkE;Z*74mvEpu;r-d^YCOG>_Tg`Va_|6{{jMTIZUon2!tp0=4PdFa%>1he=$2N%92 z$_w8s8236%UMReL_sY9+*E?REt2KSwl%Ch^VS8|> zzGt_brOS4Q-EzbqiZBJtgMSwWUE3ibKk{h1M%{h8mX0$qA!96;WcZHUtWD>8a5-Jq zp{jDnn%6n;`gYS-JYJ{KP;pr}qxQUkVU*#3ctJo#pygQjoniXAfwm_j z^Y2B9HO1FZn2d1>_ z?Jo3gJe1Iu?sGe;B6{>E4KJ%t+XcfeT<_NU+ok?U+j&FlH`d3t?wvJs?9n;)&_SE(7o9pewI}mh&c0W)A-G~Tp33yo1qsnjl8Z{+|;j=2M^tFAQOX)IN-mKB4r>s9j z%xh}wX-^MUpQ51^T|6jM>6+q?LS@CazuV$_x1U;3KdN+MkX`gy;iHN7p3GGB|0wXY zB_V0Ap`^|9@5{AjEYeq=Z=5^Qt2aeFU3Ykt=okgx6`HN;o9EwXYCO;+JAG)YfZ2oN zYJ$V_yjv;fzqFpwc=K38@YwC`6CI}tj7yNx5I(4@K0i7&DemK6cL$Fw)IVNHxo1R&iOpW_89}FwvM=jm%cq1V$x$&h|>Bj4agI2H68Ps-f*WR&%pB&zPdvkV@fZXbL zPd-UR1~{1xsjHyr%-VSQ+bBO;!b8uD2lH&My$dq>duU?j7lYCQ<)}|L9h0=hisf7P zpL6eB`EX*c9raYAxZywiM@;S!k(LuPIX`d0vXcjQb$uW6Si`bNYh)+CpUXUoxY?Sf zgQJE zu^Yw-Z=k$p|*V^_qr4>SDvMm)SbJmW=%ZRMvt>TWkgunCJBnf_aP@bOkF)3j;rx2CLM@g8=;#b`8?!g{jS%FVLi_m8oxbjb#mn8 zEu&VdZdzec-I4UB`o_Gmk2a*#^XEioj%%}iXL@6A;UZ)6I}TIa-d<0hyIOwkxr%i! zB@_$4hiuxn_uOBff8@2_P*qK=xoV|$g#Z2LafPDqo<1=!l{nd&_a?1n=KR@z4fpj) zIhvd>>&p+HrxUbfMwTfE8=u^k`v0ms>!_-}ZI9DZQUW5~AqWR)=@9Aeu7h;f0i==c zlm=-5X{4p3Lr_v0se`0+y(8Y=kL!6p?;ZEA`!)j@W3%Vy%(dp)d(ZEjYYdomk%8+q zl_4r3*=q6qR+{5|`9{yrV!J>_GNhSl;-m>278eg0KY`T8%{E_qh1c^9QIDP}G9%_$ zjus}BqRnWkfMYh1I=cg%!>VC|InyH&D!=xqhms%7a%wO?A-CAB^GZnIXAa#60&*T-KjYTDT(W=*71jzi&6>%>FU=B@~HKmmr- z3>R4e!cjDIWaa|E9mqdIWOm~;~p3005TSQiDl4M&(_H)xX3l&3AC&wX{qZ_#jt&> zjwx1`;pHOZm;>MS%5W}2JE$OBmlYTqkSa^Qc15djK28@MU)+tBU+m~zv^ghzS>8z3 zoQ9DUb;)k}tnI#dO$nn<-&LKXLY2?-S?(D_qQ{~6n_TO8I_tGnGcsoTe1<9;_0R8K zi0AbhW%}!3$GzmloX{M5vOO}0t96dS!+YFbpEIE6%-83b1uq zi0>F5R89mRRKSjW8TZH;FGtt6ao$6zx$J?JwAgE&j!-|C&Bval-HFwpNpyEHp-NZA zlPhv*4n4sf@;&P|MKKS~QSUvNLJdTJYM%AIa>#L?13jyyQ)+aIS7J%!!rAL(JRflB zJf4y#aS&r?Qa?>cmm_+dnr zjc(^VL3?x*hDNSnf(Izp?I8Hx8RtFstFBMG-oHL~DT)^11tFS5l@tc%hdhA@5l1sa z2cU_vlY_a<^P9(Sf~Ky+GAEALeDN39yr6hxJgM5{fn3_!=QLCM^p@J8dEsx4?oy`W zGT|)1WUWk1$8F8%8L^aeO~vI3OI79~6;17j4cfp&T%Dd-p6oAMCRWmzQN=e|q}Rx6 zqni{`QpCL(CmR#ql@5E*5A0N?Su8%plL0Rux_Z~Me|W6JIH+oAV||r9#=1!S*k=0S zosd|Kjd1M?Wjc~L%Y~R%@RLm6(lRqzz$Q99_-LvW-&Db~8|V0pxhG0Ot!EBiULPA; z*q6>m^;{45@F?owL=wK5y{t|m@pv(1zf{xb#@9qaRtMJ5e>y_$wZg}XE8nN>NWxJf zN}`=C8tBy*fo-3s+&WF4*Zb54h+D}0?mla|FokHCJ*-4XiiA*(ebBszk0MssgV0s- z;0;l8ijVDnsMTaH@+I~x2tR#PPUAtsn|Hu?GRk(ia19I(3>?}O(gQRR2 zqXY6lb=G@D6D7RK8zM{q;YUmyV&pGs`Gxf+A5&@(U>6g<%^!NBjJ&3lm1+?$6SGE; z?A|;~PW9AgkG?%{EP*484&I6>}#*f{h|7gd(%D-6Df`J{1aH72zrKGnuJ=V~bes{wsQ zZRx0fZw5~Qm{XkUJCoiO1y1#>o37XEb~vK4LlQ4Wla;7@sD4(nYm$Wck%1#=!R340 zZ0z~TYXFt^!CgqgQKNeg&2pH{Y=zoKC|yaA-WD?YevXdb^QV&4NX{$&c98f#yLgJzXWXpQKIGNc?o z7Bri(&F42UqvoWM`%t~YuF6?SamJt$tGUBHK*6oH-twN}LvuTfALuYwgAIg-UFoAj zQ{%O~NJ;O2Z+W5=|7tupa6L#DF}mXcOV<|oZM|y9D(jiZSK7P|K}DkLx#ust=Nyl5 zEBYnkJf>U5n$#t~V$O3f;gY&FEV<9>2|ZuOR1@ss;v1+Rw6P~}Zt;u%EZ9=!*gudE zic&g7i~5!*ffF@5Q_}h?mdPZ)MVI9h9mZkY{gwF7PCKM>pBYuEhKyPE{hwtI+vlrI z$8qx?p@gsx*|8h(bB;em7%ShomrPkh9H&Q+c`>EvHNqU}6F&%lhEbx6~rn%Z6SNiwjl)_2vLjrd&n^bAVw?KbXom zXxYNqOj$rIGGmJWhP#A%>GfBVi*AS1tcfF3qnq0y9m*Kw7sV0Kv$Hr(Y z1_7<}r7WU0qu_Z2v!17RReRxVEZnx4?!4@j>vu&0CfwF$=h3(Zywf$9bgw%wY%27; z?de^v1K7!)q$g+uA4Q(?s@gp{5ms--DXSxj8Ro#AdO`jkS1f<=WlAhL6a8jpizDsT z#r7pjMvs~=O?7uO2t-_UAg-LzxOC?^n~usT?$H>JaM@`1age@#YKDl5KBcB?@i)R6 zx@`jEx7P^2CLDJ_Y{;XKL_;3-Unt<4BIu0*ju>f^LSRLQ8EIS)jQ^584VUzvLEJKb!G6?NbTr z65?kfQs|b>zMcf>`sWo`3F1^-REV52ex_F8i-C#YFnQrk1+G_)I0)T#|>g+;|HywnNy0Ij5yA8PtDuV)fS zkcIPXo>}@39g=Of{D=Ot|9ARd06Jwlg3P@Y{J)rgj=%l?9M>iV&x-YO$8QH~b=y?? zX|7Wpf8mJ#{US_&$#d5Z+Blk!+85^-%pz#~>%t@2`nYN@tdEsBO&}8t4>D!81TC`eXJDBo5&9E|L+ZAUGgLKT&W#mbfvBiyE zRa3YRI7FnAeng*)$w>Oz99e)~IGk)Trh>&a_^9xkVeU@3=!}+VR9lnBlIJ4IEE>a; zet#iWmv}lWob4fu+V!3|l_G-zlCLs~(pUw?XAheL#62^PVp*SzBz3K|GHrisAjXf= z_RYWkB-|=++BqEl{CenQTb>A9MN$;?ABLUt?`ix0xoo-qpZm}K-|s&))`I8|Pu}GS7@$$0G=PL5hjd9`V*XtQI{4ZR*D z+3@p?H=HbAhxZHC?lfWNeG5i271Dh2n8^jMrJSJ4KTIwk#q}i*170@^-V5=stw_mRhFb z%*)vqZ@^nejZGk~DrI6#-Sdb=eogez9Gw0(yf^acBt;Q%O%nn3x9>^!oIQG1s{rPaI_AaLxqPQtCZ_Q*Dvx&Z8EY6vLKkp71gi55@t(_?L$R2edf?aHE zr^}WbYuAWHXkLiW%dFD$?g^^~g7@BIC#?5$IIXaS!qZ85fACNxHs0O}=mz1OSg4Mr zSVRl>I8T@1q?V`y``Sz%<)3V%0uBc*(;Um?4TQE8CkT(dscPDpwEZ+kq7kvH45abJ zWc+1iaj26Koi(vo(bRy#olF{WKmMePFE-$J_s2<2ps?TR%c3M+>-c;;kB8JA*JQ-x z=-3~S?*%ZlqZ>w+3u#c6V9#N5B5RMrG4$og#z--ZhCG$35Eqk>$7k5QAkghTbFLJn z*}$&IU!j)|4i7;OL0_j6&JC&*b`#a0D#6mj*2Buc&cI4UMhoD9+lcriu0=!gEhZ0#IbjBOo&KO&m0)MnETKyQ|x z7F4NEswU4a%6U3XD<_(QFqS`C*(oHRDi)Hxyz_io$zX|{;f`SI+Wqc5lQZRnkEEU> z<10)g)J$+xrK6_lRdrvPybKhD4CCobebY$D%gyb*xnh{(#yj5BVenZXWAnUVUA46j5$sXnd_hjPZ@Dq_<`cmk08&;*Aom9_A!WAZSvq)Mf-&WTr_6G_T} zJ{!*8x4=I>vj({Zl?#nv(_RoO;kxM0lUK+d`xtEqXB|HJ2v2x`k5O~@5X&)8BA;C1Ey?Iwjk7w6 zQWE7tnwMjqFh8kQP%DoVXc9b{o=TV&BEmq`J-Mq#O(%@IT>9Whds1k@3cc__t|Yo zV9F7G-f4da#?a30CN3(vwle^9PqOlBgZ1W^F^nz+dglF#Qbc@qv(fEWG3MHZlqj(l z)aQqF z6=&9=fL-Eca05fFOX(PA{9FzZMu~0e9Oh=LgV~(=n#a0o5qgeS z9eP306=ydM0euC7lbMYx2Kpnw&WI~euRU&N5>bq=^Z>vxj^{S8f#?LDqv<5E4xH%B z{20pqh9kTYFtD*tvq~3o9Jp#NK;|tpb>becmuG%XEzi?7@5M32q96?mhYNqZqvzv; z63!^d>d-;`xuXYi=iVGXkaI9F+M>2LPCy$ceHD-$&{6lta)Acar~;_KkV|DCZ)3C@ z)1N2$av0i>K*^4L9AY#Dc?0}-jsJD{zxx!v2nOBYdDKvCA}flZatpGMQna=B1e6x0D z?=O-M>~m_k9}@$;HxC*By{+RGp#Sg&@Oxo%yU7C@2)!NR7U;%!1H9cF0c{H1(YrO> z1^hIH?(abppi6YO1Vf%bjOMmr2MvTSaNPo(p8NrLTkiVzf$Y400N&Q0pn=dOpj!{W z{|R_o5P}B&-$Q2P|AU9OrILRi2>uiBws!IF1K$Yz!Nc2P!k>UY^!#6k53Hr&AAtX) z?*BymA+Y~Cd|;V`{y@By+t5I0#d{0f`U(7J{R?djt+Z~9D@1O*`9*g@TSKSyx7L1Q qaQ{2Whc<^!K5xy5#Q(|sAE~H5w-&mjvWd3-dp%rvT~z literal 0 HcmV?d00001 From 33585a1d5205818abc4bc12fdcb7659057daf478 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:14 +0200 Subject: [PATCH 02/87] feat(cmdb-import): archive or soft-delete applications that left the export With missingRecords archive, the default, a completed import reconciles the municipality's usages and the modules its import created: an APPID on the archive sheet is archived, one on no sheet is soft-deleted, and without the archive sheet missing applications are archived only, with a report warning. A returning application is unarchived or restored before it is updated, never duplicated. Object state is read per lens (working, then _archived=true) and from the trash, because OpenRegister's list search hydrates no archive marker and drops soft-deleted rows. The endpoint accepts missingRecords keep or archive and translates ARCHIVE_UNAVAILABLE; OpenAPI and Postman describe the new values and outcomes. Second of three slices of #1281, on top of the archive-sheet slice. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 16 +- l10n/en.json | 16 +- l10n/nl.js | 16 +- l10n/nl.json | 16 +- lib/Controller/CmdbImportController.php | 22 +- lib/Exception/CmdbImportException.php | 2 + lib/Service/Cmdb/CmdbImportReport.php | 32 +- lib/Service/CmdbExportImportService.php | 937 ++++++++++++++++-- lib/Service/ProgressTracker.php | 5 + openapi.json | 40 +- postman/stackiq-tests.json | 3 +- .../Controller/CmdbImportControllerTest.php | 32 +- .../Service/Cmdb/CmdbImportReportTest.php | 23 + .../Service/CmdbExportImportServiceTest.php | 908 ++++++++++++++++- 14 files changed, 1964 insertions(+), 104 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 8dd84a2f4..8e95ef029 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1038,7 +1038,21 @@ OC.L10N.register( "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", "No running CMDB import has this id.": "No running CMDB import has this id.", - "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", + "Unarchived": "Unarchived", + "Restored": "Restored", + "Archived": "Archived", + "Deleted": "Deleted", + "Applications missing from the export": "Applications missing from the export", + "Archive or delete them": "Archive or delete them", + "Keep them as they are": "Keep them as they are", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", + "Checking the applications missing from the export: {processed} of {total}": "Checking the applications missing from the export: {processed} of {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", diff --git a/l10n/en.json b/l10n/en.json index 313f968ea..0d85cfbaf 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1037,7 +1037,21 @@ "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.", "Choose a municipality or enter the name of a new one.": "Choose a municipality or enter the name of a new one.", "No running CMDB import has this id.": "No running CMDB import has this id.", - "Only keeping records that are missing from the export is supported.": "Only keeping records that are missing from the export is supported.", + "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", + "Unarchived": "Unarchived", + "Restored": "Restored", + "Archived": "Archived", + "Deleted": "Deleted", + "Applications missing from the export": "Applications missing from the export", + "Archive or delete them": "Archive or delete them", + "Keep them as they are": "Keep them as they are", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", + "Checking the applications missing from the export: {processed} of {total}": "Checking the applications missing from the export: {processed} of {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Sheet \"%1$s\" has more than %2$s rows.", "Sheet \"%1$s\" has no column \"%2$s\".": "Sheet \"%1$s\" has no column \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is not configured: the register or its schemas cannot be found.", diff --git a/l10n/nl.js b/l10n/nl.js index 3151813d6..b03118333 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1106,7 +1106,21 @@ OC.L10N.register( "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", - "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Unarchived": "Uit archief gehaald", + "Restored": "Hersteld", + "Archived": "Gearchiveerd", + "Deleted": "Verwijderd", + "Applications missing from the export": "Applicaties die in de export ontbreken", + "Archive or delete them": "Archiveren of verwijderen", + "Keep them as they are": "Laat ze zoals ze zijn", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", + "Checking the applications missing from the export: {processed} of {total}": "Applicaties controleren die in de export ontbreken: {processed} van {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", diff --git a/l10n/nl.json b/l10n/nl.json index 680f6f853..92a16d73e 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1105,7 +1105,21 @@ "The stackiq register or its schemas cannot be found. Run Auto Configure at the top of this page, then try again.": "Het stackiq-register of de schema's ervan zijn niet gevonden. Voer bovenaan deze pagina Auto Configure uit en probeer het daarna opnieuw.", "Choose a municipality or enter the name of a new one.": "Kies een gemeente of voer de naam van een nieuwe in.", "No running CMDB import has this id.": "Er loopt geen CMDB-import met deze id.", - "Only keeping records that are missing from the export is supported.": "Alleen het behouden van records die in de export ontbreken, wordt ondersteund.", + "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", + "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", + "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Unarchived": "Uit archief gehaald", + "Restored": "Hersteld", + "Archived": "Gearchiveerd", + "Deleted": "Verwijderd", + "Applications missing from the export": "Applicaties die in de export ontbreken", + "Archive or delete them": "Archiveren of verwijderen", + "Keep them as they are": "Laat ze zoals ze zijn", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", + "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", + "Checking the applications missing from the export: {processed} of {total}": "Applicaties controleren die in de export ontbreken: {processed} van {total}", "Sheet \"%1$s\" has more than %2$s rows.": "Tabblad \"%1$s\" heeft meer dan %2$s rijen.", "Sheet \"%1$s\" has no column \"%2$s\".": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\".", "Stackiq is not configured: the register or its schemas cannot be found.": "Stackiq is niet ingesteld: het register of de schema's ervan zijn niet gevonden.", diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 7d1c00a8f..bc7869cbe 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -86,7 +86,8 @@ public function __construct( * Import a TOPdesk CMDB export for one municipality. * * Multipart fields: `cmdbFile`, `municipalityUuid` or `municipalityName`, - * `updateExisting` (default true), `publish` (default true), `missingRecords` (only `keep`) and + * `updateExisting` (default true), `publish` (default true), `missingRecords` (`keep` or `archive`, + * default `archive`) and * `operationId` (pattern `cmdb-` plus 8 to 64 letters, digits or hyphens). * * @return JSONResponse The report (200), or an error envelope with the contract code. @@ -94,6 +95,7 @@ public function __construct( * @auth admin-only the import writes with RBAC and multitenancy off, across tenants, so only a full Nextcloud admin may run it. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ public function import(): JSONResponse { try { @@ -247,15 +249,21 @@ protected function iniBytes(string $name): ?int { * @return array{path: string, options: array}|JSONResponse The import input, or the first error. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ private function readOptions(string $path, string $fileName): array|JSONResponse { - $missingRecords = $this->stringParam(name: 'missingRecords', default: 'keep'); + $missingRecords = $this->stringParam(name: 'missingRecords', default: CmdbExportImportService::DEFAULT_MISSING_RECORDS); if ($missingRecords === null) { return $this->invalidField(field: 'missingRecords'); } - if ($this->importService->supportsMissingRecords(mode: $missingRecords) === false) { - return $this->error(code: 'MISSING_RECORDS_UNSUPPORTED', status: Http::STATUS_UNPROCESSABLE_ENTITY, details: ['accepted' => ['keep']]); + $accepted = $this->importService->missingRecordsModes(); + if (in_array($missingRecords, $accepted, true) === false) { + return $this->error( + code: 'MISSING_RECORDS_UNSUPPORTED', + status: Http::STATUS_UNPROCESSABLE_ENTITY, + details: ['accepted' => $accepted] + ); } // An unrecognised value is refused rather than read as true: true is the mode that overwrites. @@ -293,6 +301,7 @@ private function readOptions(string $path, string $fileName): array|JSONResponse 'municipalityName' => $municipalityName, 'updateExisting' => $updateExisting, 'publish' => $publish, + 'missingRecords' => $missingRecords, 'operationId' => $this->request->getParam('operationId'), 'fileName' => basename(str_replace('\\', '/', $fileName)), ], @@ -394,7 +403,7 @@ private function message(string $code, array $details): string { 'NO_FILE_UPLOADED' => $this->l10n->t('No file was uploaded.'), 'NOT_XLSX' => $this->l10n->t('The file is not an Excel workbook (.xlsx).'), 'FILE_TOO_LARGE' => $this->l10n->t('The file is larger than the maximum of %s MB.', [$megabytes]), - 'MISSING_RECORDS_UNSUPPORTED' => $this->l10n->t('Only keeping records that are missing from the export is supported.'), + 'MISSING_RECORDS_UNSUPPORTED' => $this->l10n->t('Applications missing from the export can only be archived or kept.'), 'FIELD_INVALID' => $this->fieldMessage(field: (string)($details['field'] ?? ''), accepted: $accepted), 'MUNICIPALITY_REQUIRED' => $this->l10n->t('Choose a municipality or enter the name of a new one.'), 'MUNICIPALITY_INVALID' => $this->l10n->t('The chosen organisation is not a municipality.'), @@ -410,6 +419,9 @@ private function message(string $code, array $details): string { 'SCHEMA_OUTDATED' => $this->l10n->t('The stackiq register is out of date; import its configuration again.'), 'IMPORT_IN_PROGRESS' => $this->l10n->t('Another CMDB import is running; try again when it has finished.'), 'MUNICIPALITY_AMBIGUOUS' => $this->l10n->t('Several municipalities have this name; choose one from the list.'), + 'ARCHIVE_UNAVAILABLE' => $this->l10n->t( + 'Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.' + ), default => $this->l10n->t('The import failed. The details are in the Nextcloud log.'), }; }//end message() diff --git a/lib/Exception/CmdbImportException.php b/lib/Exception/CmdbImportException.php index 31404c505..65127a9fe 100644 --- a/lib/Exception/CmdbImportException.php +++ b/lib/Exception/CmdbImportException.php @@ -50,6 +50,7 @@ class CmdbImportException extends RuntimeException { public const SCHEMA_OUTDATED = 'SCHEMA_OUTDATED'; public const IMPORT_IN_PROGRESS = 'IMPORT_IN_PROGRESS'; public const MUNICIPALITY_AMBIGUOUS = 'MUNICIPALITY_AMBIGUOUS'; + public const ARCHIVE_UNAVAILABLE = 'ARCHIVE_UNAVAILABLE'; /** * HTTP status per error code. @@ -70,6 +71,7 @@ class CmdbImportException extends RuntimeException { self::SCHEMA_OUTDATED => 503, self::IMPORT_IN_PROGRESS => 409, self::MUNICIPALITY_AMBIGUOUS => 422, + self::ARCHIVE_UNAVAILABLE => 503, ]; /** diff --git a/lib/Service/Cmdb/CmdbImportReport.php b/lib/Service/Cmdb/CmdbImportReport.php index 730365fde..2615c8b07 100644 --- a/lib/Service/Cmdb/CmdbImportReport.php +++ b/lib/Service/Cmdb/CmdbImportReport.php @@ -36,6 +36,10 @@ class CmdbImportReport { public const UNCHANGED = 'unchanged'; public const SKIPPED = 'skipped'; public const FAILED = 'failed'; + public const ARCHIVED = 'archived'; + public const UNARCHIVED = 'unarchived'; + public const DELETED = 'deleted'; + public const RESTORED = 'restored'; /** * The row entries, in processing order. @@ -90,7 +94,7 @@ public function __construct( * Add one row outcome. * * @param string $sheet The sheet name. - * @param int $row The 1-based sheet row number. + * @param int $row The 1-based sheet row number, or 0 for an application the reconciliation archived or deleted. * @param string $appId The APPID ('' when missing). * @param string $name The application name ('' when missing). * @param string $outcome One of the outcome constants. @@ -180,14 +184,25 @@ public function markCancelled(): void { }//end markCancelled() /** - * The number of rows processed so far. + * Whether the run stopped on a cancel. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function isCancelled(): bool { + return $this->cancelled; + }//end isCancelled() + + /** + * The number of sheet rows processed so far; reconciliation entries (row 0) are not sheet rows. * * @return int * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 */ public function processed(): int { - return count($this->rows); + return count(array_filter($this->rows, static fn (array $row): bool => $row['row'] > 0)); }//end processed() /** @@ -195,21 +210,28 @@ public function processed(): int { * * `unpublished` counts the modules created without a publication date; * a row whose module was created but whose usage then failed counts too. + * `processed` counts sheet rows only; every outcome, also of a + * reconciliation entry, is counted under its own key. * * @return array{rowsRead: int, processed: int, created: int, updated: int, unchanged: int, skipped: int, failed: int, - * warnings: int, unpublished: int} + * archived: int, unarchived: int, deleted: int, restored: int, warnings: int, unpublished: int} * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function summary(): array { $summary = [ 'rowsRead' => $this->rowsRead, - 'processed' => count($this->rows), + 'processed' => $this->processed(), self::CREATED => 0, self::UPDATED => 0, self::UNCHANGED => 0, self::SKIPPED => 0, self::FAILED => 0, + self::ARCHIVED => 0, + self::UNARCHIVED => 0, + self::DELETED => 0, + self::RESTORED => 0, 'warnings' => 0, 'unpublished' => $this->unpublished, ]; diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 1efc049d7..3c98a3f1e 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -30,7 +30,14 @@ * - `publicationDate` is set to the import's start on create, unless the * admin chose not to publish (`publish` false), and never written on * update; neither is `depublicationDate`. - * - Records missing from a newer export are left untouched. + * - Records missing from a newer export are left untouched with + * `missingRecords` `keep`. With `archive` (the default) the municipality's + * usages and their modules are reconciled after the last row: archived + * when the APPID is on the archive sheet, soft-deleted when it is on no + * sheet, archived only when the workbook has no archive sheet + * (openspec/changes/cmdb-import-archive-reconciliation, design D5). A + * module or usage matched among archived or soft-deleted objects is + * unarchived or restored before it is updated, never duplicated. * - Owners become contact persons, never Nextcloud user accounts, and no * report entry or log line carries an owner name or e-mail address. * - One import runs per register at a time: every match is find-then-create, @@ -55,6 +62,7 @@ use DateTimeImmutable; use DateTimeZone; +use JsonSerializable; use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\Stackiq\Exception\CmdbImportException; use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; @@ -68,6 +76,7 @@ use OCP\Lock\LockedException; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; +use RuntimeException; use Throwable; /** @@ -111,6 +120,58 @@ class CmdbExportImportService { */ public const SCHEMA_MAPPER_CLASS = 'OCA\OpenRegister\Db\SchemaMapper'; + /** + * OpenRegister's archive handler, behind POST and DELETE .../archive (not a public contract). + */ + public const ARCHIVE_HANDLER_CLASS = 'OCA\OpenRegister\Service\Object\ArchiveHandler'; + + /** + * OpenRegister's object mapper, whose restoreObject() empties the trash of one object (not a public contract). + */ + public const OBJECT_MAPPER_CLASS = 'OCA\OpenRegister\Db\MagicMapper'; + + /** + * What happens to records missing from the export when the request does not say. + */ + public const DEFAULT_MISSING_RECORDS = 'archive'; + + /** + * The start of every archive reason the import writes, so the audit trail says who archived and why. + */ + private const REASON_PREFIX = 'cmdb-import: '; + + /** + * The search lens that lists the archive alone (OpenRegister's `_archived=true`). + * + * The state of a stored object is read from the lens that found it, never + * from the archive marker of a search result: OpenRegister's list search + * does not hydrate that marker (design D9). + * + * @var array + */ + private const ARCHIVED_LENS = ['_archived' => 'true']; + + /** + * The state of an object found without a lens. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_WORKING = ['archived' => false, 'deleted' => false]; + + /** + * The state of an object found with ARCHIVED_LENS. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_ARCHIVED = ['archived' => true, 'deleted' => false]; + + /** + * The state of an object found in the trash; whether it was archived too is read when it is restored. + * + * @var array{archived: bool, deleted: bool} + */ + private const STATE_DELETED = ['archived' => false, 'deleted' => true]; + /** * The properties every match search and the module key rely on, per schema. * @@ -222,6 +283,20 @@ class CmdbExportImportService { */ private array $winningSheets = []; + /** + * OpenRegister's archive handler and object mapper, once resolved. + * + * @var array{handler: object, mapper: object}|null + */ + private ?array $archiveServices = null; + + /** + * The register's soft-deleted modules and usages, read once per run: schema key => uuid => entity. + * + * @var array{module: array, usage: array}|null + */ + private ?array $trash = null; + /** * Contact UIDs by e-mail or display name, per run. * @@ -300,17 +375,16 @@ public function assertXlsx(string $path, string $fileName): void { }//end assertXlsx() /** - * Whether a `missingRecords` value is supported (only `keep`). - * - * @param string $mode The requested value. + * The accepted `missingRecords` values: the profile's modes (`keep`, `archive`); any other is refused. * - * @return bool + * @return array * * @spec openspec/changes/cmdb-export-import/tasks.md#task-8 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ - public function supportsMissingRecords(string $mode): bool { - return $mode === 'keep'; - }//end supportsMissingRecords() + public function missingRecordsModes(): array { + return $this->profile->missingRecordsModes(); + }//end missingRecordsModes() /** * Ask a running import to stop between rows. @@ -349,19 +423,21 @@ public function requestCancel(string $operationId): bool { * * @param string $path The xlsx file, already checked by assertXlsx(). * @param array $options municipalityUuid, municipalityName, updateExisting, publish, - * operationId, and fileName (the upload's name, for the audit log line). + * missingRecords (keep or archive, default archive), operationId, + * and fileName (the upload's name, for the audit log line). * * @return array The report (contract.md). * * @throws CmdbImportException MAPPING_UNAVAILABLE, NOT_CONFIGURED, SCHEMA_OUTDATED, IMPORT_IN_PROGRESS, * WORKBOOK_TOO_LARGE, READER_UNAVAILABLE, NOT_XLSX, NO_SOURCE_SHEET, * MISSING_COLUMN, TOO_MANY_ROWS, MUNICIPALITY_REQUIRED, - * MUNICIPALITY_INVALID or MUNICIPALITY_AMBIGUOUS. + * MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS or ARCHIVE_UNAVAILABLE. * @throws \Exception An unexpected OpenRegister error outside a row, such as * creating the municipality; rows catch their own. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ public function import(string $path, array $options): array { $this->resetRun(); @@ -371,6 +447,10 @@ public function import(string $path, array $options): array { $this->profile->load(); $this->engine = $this->resolveEngine(); $this->coordinates = $this->resolveCoordinates(); + if (self::missingRecordsFrom(options: $options) === 'archive') { + // Refused before the lock and the file: an import that cannot reconcile must not half-run. + $this->archiveServices(); + } $lock = $this->acquireImportLock(register: $this->coordinates['register']); try { @@ -442,29 +522,20 @@ private function acquireImportLock(int $register): string { * @return array The report. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ private function runImport(string $path, array $options, string $startedAt): array { $workbook = $this->reader->read(path: $path, profile: $this->profile); $municipality = $this->resolveMunicipality(options: $options); $operationId = $this->operationIdFrom(options: $options); + $missingRecords = self::missingRecordsFrom(options: $options); $rows = array_values($workbook['rows']); $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); - if ($municipality['created'] === true) { - $report->addImportWarnings( - warnings: [ - [ - 'sheet' => '', - 'message' => $this->l10n->t( - 'No municipality named "%s" was found, so it was created. Check the name if you meant an existing one.', - [$municipality['name']] - ), - ], - ] - ); - } + $archive = $this->archiveSheet(workbook: $workbook); + $this->addRunWarnings(report: $report, municipality: $municipality, archive: $archive, missingRecords: $missingRecords); $this->progressTracker->startOperation( operationType: self::OPERATION_TYPE, @@ -488,6 +559,7 @@ private function runImport(string $path, array $options, string $startedAt): arr 'municipalityCreated' => $municipality['created'], 'updateExisting' => $updateExisting, 'publish' => $publish, + 'missingRecords' => $missingRecords, ]; $this->logger->info('CmdbExportImportService: import started', array_merge($audit, ['rows' => count($rows)])); try { @@ -507,6 +579,16 @@ private function runImport(string $path, array $options, string $startedAt): arr $this->progressTracker->updateProgress(processedItems: ($index + 1)); } + if ($missingRecords === 'archive' && $report->isCancelled() === false) { + $this->reconcile( + municipalityUuid: $municipality['uuid'], + archive: $archive, + operationId: $operationId, + processedItems: count($rows), + report: $report + ); + } + $result = $report->toArray(); $this->finishOperation(report: $report->toStoredArray(maxRows: self::STORED_REPORT_ROWS)); } catch (Throwable $e) { @@ -528,6 +610,39 @@ private function runImport(string $path, array $options, string $startedAt): arr return $result; }//end runImport() + /** + * Add the run's own warnings: a missing archive sheet on an archiving import, and a created municipality. + * + * @param CmdbImportReport $report The report. + * @param array{uuid: string, name: string, created: bool} $municipality The consumer, as resolveMunicipality() found it. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $missingRecords The import's `missingRecords` mode. + * + * @return void + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function addRunWarnings(CmdbImportReport $report, array $municipality, array $archive, string $missingRecords): void { + if ($missingRecords === 'archive' && $archive['present'] === false) { + $report->addImportWarnings(warnings: [['sheet' => $archive['sheet'], 'message' => $this->archiveSheetWarning(archive: $archive)]]); + } + + if ($municipality['created'] === true) { + $report->addImportWarnings( + warnings: [ + [ + 'sheet' => '', + 'message' => $this->l10n->t( + 'No municipality named "%s" was found, so it was created. Check the name if you meant an existing one.', + [$municipality['name']] + ), + ], + ] + ); + } + }//end addRunWarnings() + /** * Process one row in its own error boundary and add its outcome. * @@ -748,11 +863,20 @@ private function translateImportWarnings(array $warnings): array { * @param string $module The module outcome. * @param string $usage The usage outcome. * - * @return string created when the module was created, updated when anything was saved, else unchanged. + * @return string restored when either was restored from the trash, else unarchived when either was + * unarchived, else created when the module was created, updated when anything was saved, + * else unchanged. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private static function rowOutcome(string $module, string $usage): string { + foreach ([CmdbImportReport::RESTORED, CmdbImportReport::UNARCHIVED] as $revived) { + if ($module === $revived || $usage === $revived) { + return $revived; + } + } + if ($module === CmdbImportReport::CREATED) { return CmdbImportReport::CREATED; } @@ -781,7 +905,7 @@ private function finishOperation(array $report): void { } $this->progressTracker->completeOperation(finalStatistics: ['report' => $report]); - // A cancel that came in after the last row's check has nothing left to stop. + // A cancel that came in after the last check has nothing left to stop. $this->progressTracker->clearCancelRequested(operationId: (string)$report['operationId']); }//end finishOperation() @@ -1028,19 +1152,19 @@ private function resolveManufacturer(array $values, int $rowNumber): ?string { * @param string $matchKey The APPID's match key. * @param array{updateExisting: bool} $options Whether a match is updated. * - * @return array{existing: object|null, uuid: string|null, skipReason: string|null} The stored module, the uuid - * to report for a skipped row, - * and why the row is skipped. + * @return array{existing: array{entity: object, state: array{archived: bool, deleted: bool}}|null, uuid: string|null, skipReason: string|null} + * The stored module with its state, the uuid to report for a skipped row, and why the row is skipped. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function matchModule(string $externalKey, string $municipalityUuid, string $matchKey, array $options): array { - $existing = $this->findOne(schemaKey: 'module', filters: ['externalKey' => $externalKey]); + $existing = $this->findInAnyState(schemaKey: 'module', filters: ['externalKey' => $externalKey]); if ($existing === null) { return ['existing' => null, 'uuid' => null, 'skipReason' => null]; } - $uuid = (string)$existing->getUuid(); + $uuid = (string)$existing['entity']->getUuid(); if ($this->moduleBelongsTo(moduleUuid: $uuid, municipalityUuid: $municipalityUuid) === false) { $this->logger->warning( 'CmdbExportImportService: import key belongs to a module another organisation uses; row not imported', @@ -1062,7 +1186,8 @@ private function matchModule(string $externalKey, string $municipalityUuid, stri * * @param array $data The mapped module fields. * @param string $externalKey The module's import key. - * @param object|null $existing The module matchModule() found, or null to create one. + * @param array{entity: object, state: array{archived: bool, deleted: bool}}|null $existing The module matchModule() + * found, or null to create one. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. @@ -1075,7 +1200,7 @@ private function matchModule(string $externalKey, string $municipalityUuid, stri private function importModule( array $data, string $externalKey, - ?object $existing, + ?array $existing, ?string $providerUuid, ?string $publicationDate, CmdbImportReport $report, @@ -1102,6 +1227,8 @@ private function importModule( * together with the usages: the module must already have a usage whose * consumer is this municipality, or, before the first import, no usage * at all. A module only another organisation uses is never taken over. + * This municipality's usage counts in every state: an archived or + * soft-deleted usage still says the module is the municipality's. * * @param string $moduleUuid The module found by its import key. * @param string $municipalityUuid The consumer of this import. @@ -1109,9 +1236,10 @@ private function importModule( * @return bool * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): bool { - if ($this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]) !== null) { + if ($this->findInAnyState(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]) !== null) { return true; } @@ -1123,19 +1251,22 @@ private function moduleBelongsTo(string $moduleUuid, string $municipalityUuid): * * @param array $data The mapped module fields. * @param string $externalKey The match key. - * @param object|null $existing The stored module, or null to create one. + * @param array{entity: object, state: array{archived: bool, deleted: bool}}|null $existing The stored module and its + * state, or null to create one. * @param string|null $providerUuid The supplier, when there is one. * @param string|null $publicationDate ISO start time of the import for a module that is published * when created, or null to create it unpublished. * - * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged. + * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged, or restored or + * unarchived when the stored module was revived first. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function upsertModule( array $data, string $externalKey, - ?object $existing, + ?array $existing, ?string $providerUuid, ?string $publicationDate, ): array { @@ -1154,14 +1285,15 @@ private function upsertModule( return ['uuid' => $this->save(schemaKey: 'module', data: $create, uuid: null), 'outcome' => CmdbImportReport::CREATED]; } - $uuid = (string)$existing->getUuid(); - $merged = $this->merge(target: 'module', stored: $existing->getObject(), mapped: $data); + $uuid = (string)$existing['entity']->getUuid(); + $revived = $this->revive(found: $existing); + $merged = $this->merge(target: 'module', stored: $revived['entity']->getObject(), mapped: $data); if ($merged === null) { - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UNCHANGED)]; } $this->save(schemaKey: 'module', data: $merged, uuid: $uuid); - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UPDATED)]; }//end upsertModule() /** @@ -1173,9 +1305,11 @@ private function upsertModule( * @param string|null $providerUuid The supplier, when there is one. * @param array{businessOwner: string|null} $owners The owner contact person. * - * @return array{uuid: string, outcome: string} + * @return array{uuid: string, outcome: string} Outcome created, updated or unchanged, or restored or + * unarchived when the stored usage was revived first. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 */ private function upsertUsage(array $data, string $municipalityUuid, string $moduleUuid, ?string $providerUuid, array $owners): array { if (isset($data['interneAnnotation']) === true && is_string($data['interneAnnotation']) === true) { @@ -1199,19 +1333,20 @@ private function upsertUsage(array $data, string $municipalityUuid, string $modu } } - $existing = $this->findOne(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]); + $existing = $this->findInAnyState(schemaKey: 'usage', filters: ['consumer' => $municipalityUuid, 'module' => $moduleUuid]); if ($existing === null) { return ['uuid' => $this->save(schemaKey: 'usage', data: $data, uuid: null), 'outcome' => CmdbImportReport::CREATED]; } - $uuid = (string)$existing->getUuid(); - $merged = $this->merge(target: 'usage', stored: $existing->getObject(), mapped: $data); + $uuid = (string)$existing['entity']->getUuid(); + $revived = $this->revive(found: $existing); + $merged = $this->merge(target: 'usage', stored: $revived['entity']->getObject(), mapped: $data); if ($merged === null) { - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UNCHANGED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UNCHANGED)]; } $this->save(schemaKey: 'usage', data: $merged, uuid: $uuid); - return ['uuid' => $uuid, 'outcome' => CmdbImportReport::UPDATED]; + return ['uuid' => $uuid, 'outcome' => ($revived['outcome'] ?? CmdbImportReport::UPDATED)]; }//end upsertUsage() /** @@ -1633,7 +1768,7 @@ private function organisationsOfType(string $type): array { }//end organisationsOfType() /** - * The one object matching every filter, or null. + * The working object matching every filter, or null. * * @param string $schemaKey The coordinates key of the schema. * @param array $filters Field => exact value. @@ -1643,10 +1778,67 @@ private function organisationsOfType(string $type): array { * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 */ private function findOne(string $schemaKey, array $filters): ?object { + return $this->search(schemaKey: $schemaKey, filters: $filters, lens: []); + }//end findOne() + + /** + * The object matching every filter in any state, with that state, or null. + * + * A working object wins over an archived one, and an archived one over one + * in the trash: an object deleted by hand may share its import key with the + * one a later import created, and bringing the deleted one back would + * duplicate it. The state is the lens that found the object (design D9): + * a search without a lens finds only working objects, the archive lens only + * archived ones, and the trash is read from OpenRegister's trash listing, + * because its list search cannot return a soft-deleted object. + * + * @param string $schemaKey module or usage. + * @param array $filters Field => exact value. + * + * @return array{entity: object, state: array{archived: bool, deleted: bool}}|null + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function findInAnyState(string $schemaKey, array $filters): ?array { + $entity = $this->search(schemaKey: $schemaKey, filters: $filters, lens: []); + if ($entity !== null) { + return ['entity' => $entity, 'state' => self::STATE_WORKING]; + } + + $entity = $this->search(schemaKey: $schemaKey, filters: $filters, lens: self::ARCHIVED_LENS); + if ($entity !== null) { + return ['entity' => $entity, 'state' => self::STATE_ARCHIVED]; + } + + foreach (($this->trash()[$schemaKey] ?? []) as $entity) { + if (self::matchesFilters(entity: $entity, filters: $filters) === true) { + return ['entity' => $entity, 'state' => self::STATE_DELETED]; + } + } + + return null; + }//end findInAnyState() + + /** + * The first object a search with one lens returns that matches every filter, or null. + * + * The filters are checked again on the result: a filter OpenRegister cannot + * apply must not turn into a match. + * + * @param string $schemaKey The coordinates key of the schema. + * @param array $filters Field => exact value. + * @param array $lens No lens (working objects) or ARCHIVED_LENS. + * + * @return object|null The entity (ObjectEntityInterface). + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function search(string $schemaKey, array $filters, array $lens): ?object { $coordinates = $this->coordinates(); $results = $coordinates['objectService']->searchObjects( query: array_merge( ['@self' => ['register' => $coordinates['register'], 'schema' => $coordinates[$schemaKey]], '_limit' => 10], + $lens, $filters ), _rbac: false, @@ -1657,22 +1849,34 @@ private function findOne(string $schemaKey, array $filters): ?object { } foreach ($results as $entity) { - $data = $entity->getObject(); - $matches = true; - foreach ($filters as $field => $value) { - if (self::relationUuid(value: ($data[$field] ?? null)) !== $value) { - $matches = false; - break; - } - } - - if ($matches === true) { + if (self::matchesFilters(entity: $entity, filters: $filters) === true) { return $entity; } } return null; - }//end findOne() + }//end search() + + /** + * Whether an object's data has every filter's value. + * + * @param object $entity The entity (ObjectEntityInterface). + * @param array $filters Field => exact value. + * + * @return bool + * + * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + */ + private static function matchesFilters(object $entity, array $filters): bool { + $data = $entity->getObject(); + foreach ($filters as $field => $value) { + if (self::relationUuid(value: ($data[$field] ?? null)) !== $value) { + return false; + } + } + + return true; + }//end matchesFilters() /** * Save an object through OpenRegister and return its uuid. @@ -1699,6 +1903,621 @@ private function save(string $schemaKey, array $data, ?string $uuid): string { return (string)$entity->getUuid(); }//end save() + /** + * The `missingRecords` mode of an import: the option, or DEFAULT_MISSING_RECORDS. + * + * @param array $options The import options. + * + * @return string + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + private static function missingRecordsFrom(array $options): string { + $mode = ($options['missingRecords'] ?? null); + if (is_string($mode) === false || $mode === '') { + return self::DEFAULT_MISSING_RECORDS; + } + + return $mode; + }//end missingRecordsFrom() + + /** + * The archive sheet as the reader found it, with its APPIDs as match keys. + * + * @param array $workbook What the reader returned. + * + * @return array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveSheet(array $workbook): array { + $archive = ($workbook['archive'] ?? []); + $key = $this->profile->keyColumn(); + $keys = []; + foreach (($archive['appIds'] ?? []) as $appId) { + $values = $this->normaliser->normalise(cells: [$key => $appId], dateColumns: [], idColumns: $this->profile->idColumns()); + $matchKey = self::matchKey(appId: ($values[$key] ?? '')); + if ($matchKey !== '') { + $keys[$matchKey] = true; + } + } + + return [ + 'sheet' => (string)($archive['sheet'] ?? ($this->profile->archiveSheetName() ?? '')), + 'present' => (($archive['present'] ?? false) === true), + 'keyColumnMissing' => (($archive['keyColumnMissing'] ?? false) === true), + 'keys' => $keys, + ]; + }//end archiveSheet() + + /** + * The report warning of an archiving import whose workbook lacks the archive sheet or its key column. + * + * @param array{sheet: string, keyColumnMissing: bool} $archive The archive sheet. + * + * @return string + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveSheetWarning(array $archive): string { + if ($archive['keyColumnMissing'] === true) { + return $this->l10n->t( + 'Sheet "%1$s" has no column "%2$s", so applications missing from the export are archived, not deleted.', + [$archive['sheet'], $this->profile->keyColumn()] + ); + } + + return $this->l10n->t('Sheet "%s" was not found, so applications missing from the export are archived, not deleted.', [$archive['sheet']]); + }//end archiveSheetWarning() + + /** + * Resolve OpenRegister's archive handler and object mapper, failing closed. + * + * @return array{handler: object, mapper: object} The handler, with `archive()` and `unarchive()`, and the + * mapper, with `restoreObject()` and `findDeletedAcrossAllMagicTables()`. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function archiveServices(): array { + if ($this->archiveServices !== null) { + return $this->archiveServices; + } + + try { + $handler = $this->container->get(static::ARCHIVE_HANDLER_CLASS); + $mapper = $this->container->get(static::OBJECT_MAPPER_CLASS); + } catch (Throwable $e) { + $handler = null; + $mapper = null; + } + + if (is_object($handler) === false || method_exists($handler, 'archive') === false || method_exists($handler, 'unarchive') === false + || is_object($mapper) === false || method_exists($mapper, 'restoreObject') === false + || method_exists($mapper, 'findDeletedAcrossAllMagicTables') === false + ) { + throw new CmdbImportException( + errorCode: CmdbImportException::ARCHIVE_UNAVAILABLE, + message: 'OpenRegister ArchiveHandler or MagicMapper is not available' + ); + } + + $this->archiveServices = ['handler' => $handler, 'mapper' => $mapper]; + return $this->archiveServices; + }//end archiveServices() + + /** + * Whether an object as OpenRegister's mapper reads it by uuid is archived and whether it is in the trash. + * + * Read from its `@self` metadata. Only for an object the mapper read + * itself (the one restoreObject() answers): a list search result carries + * no archive marker (design D9). + * + * @param object $entity The entity (ObjectEntityInterface). + * + * @return array{archived: bool, deleted: bool} + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private static function objectState(object $entity): array { + $self = []; + if ($entity instanceof JsonSerializable) { + $serialised = $entity->jsonSerialize(); + if (is_array($serialised) === true && is_array($serialised['@self'] ?? null) === true) { + $self = $serialised['@self']; + } + } + + return [ + 'archived' => (self::isEmptyValue(value: ($self['archived'] ?? null)) === false), + 'deleted' => (self::isEmptyValue(value: ($self['deleted'] ?? null)) === false), + ]; + }//end objectState() + + /** + * Bring a matched module or usage back before it is written: out of the trash, then out of the archive. + * + * OpenRegister refuses a data write to an archived object, and the + * archive handler cannot reach one in the trash, hence the order. The + * trash listing does not say whether an object was archived before it was + * deleted; the restored object, read by the mapper, does. The update then + * starts from that object: the trash listing decodes every string that + * parses as JSON, so its copy of the data must not be written back. + * + * @param array{entity: object, state: array{archived: bool, deleted: bool}} $found The matched entity and its state. + * + * @return array{outcome: string|null, entity: object} restored, unarchived, or null when it was neither, and the + * entity whose data the update starts from. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE when the archive services are missing. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function revive(array $found): array { + $entity = $found['entity']; + $archived = $found['state']['archived']; + if ($found['state']['deleted'] === false && $archived === false) { + return ['outcome' => null, 'entity' => $entity]; + } + + $uuid = (string)$entity->getUuid(); + $outcome = null; + if ($found['state']['deleted'] === true) { + $restored = $this->restore(uuid: $uuid); + $outcome = CmdbImportReport::RESTORED; + if (is_object($restored) === true && method_exists($restored, 'getObject') === true) { + $entity = $restored; + $archived = self::objectState(entity: $restored)['archived']; + } + } + + if ($archived === true) { + $this->archiveServices()['handler']->unarchive(identifier: $uuid, reason: self::REASON_PREFIX . 'back in the export'); + $outcome = ($outcome ?? CmdbImportReport::UNARCHIVED); + } + + return ['outcome' => $outcome, 'entity' => $entity]; + }//end revive() + + /** + * Take one object out of the trash, and out of this run's copy of the trash. + * + * @param string $uuid The object. + * + * @return mixed What MagicMapper::restoreObject() answers: the object as it reads it. + * + * @throws CmdbImportException ARCHIVE_UNAVAILABLE when the archive services are missing. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function restore(string $uuid): mixed { + $restored = $this->archiveServices()['mapper']->restoreObject(uuid: $uuid); + if ($this->trash !== null) { + unset($this->trash['module'][$uuid], $this->trash['usage'][$uuid]); + } + + return $restored; + }//end restore() + + /** + * The register's soft-deleted modules and usages, read once per run. + * + * OpenRegister's list search cannot return a soft-deleted object, whatever + * `_includeDeleted` says, so the trash is read from the trash listing that + * the Deleted page uses (design D9). That listing scans every magic table + * of the instance, so it is read once, lazily, and only this register's + * modules and usages are kept. Its objects carry no archive or trash + * marker; everything in it is deleted. Without the archive services (a + * `keep` import on an OpenRegister without them) the trash cannot be read: + * nothing in it is matched, and the log says so. + * + * @return array{module: array, usage: array} Schema key => uuid => entity. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private function trash(): array { + if ($this->trash !== null) { + return $this->trash; + } + + $this->trash = ['module' => [], 'usage' => []]; + try { + $services = $this->archiveServices(); + } catch (CmdbImportException $e) { + $this->logger->warning('CmdbExportImportService: OpenRegister trash is not readable; soft-deleted modules and usages are not matched'); + return $this->trash; + } + + $coordinates = $this->coordinates(); + $deleted = $services['mapper']->findDeletedAcrossAllMagicTables(limit: null, offset: null, _rbac: false, _multitenancy: false); + $this->trash = self::registerTrash(deleted: $deleted, coordinates: $coordinates); + + return $this->trash; + }//end trash() + + /** + * The modules and usages of this register among the instance's soft-deleted objects. + * + * @param mixed $deleted What MagicMapper::findDeletedAcrossAllMagicTables() answered. + * @param array $coordinates The register and schema ids, from coordinates(). + * + * @return array{module: array, usage: array} Schema key => uuid => entity. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + private static function registerTrash(mixed $deleted, array $coordinates): array { + $trash = ['module' => [], 'usage' => []]; + if (is_array($deleted) === false) { + return $trash; + } + + foreach ($deleted as $entity) { + if (is_object($entity) === false || (string)$entity->getRegister() !== (string)$coordinates['register'] || $entity->getUuid() === null) { + continue; + } + + foreach (['module', 'usage'] as $schemaKey) { + if ((string)$entity->getSchema() === (string)$coordinates[$schemaKey]) { + $trash[$schemaKey][(string)$entity->getUuid()] = $entity; + } + } + } + + return $trash; + }//end registerTrash() + + /** + * Archive or soft-delete every application of the municipality whose APPID left the CMDB sheets. + * + * Runs after the last row of an import that was not cancelled, under the + * same register lock. The whole scope is loaded before anything is + * written, so the paging is not shifted by the writes. Every application + * has its own error boundary. A cancel is honoured before the scope is + * loaded and before each application: what was archived or deleted so far + * stays, and the report says the run was cancelled. + * + * @param string $municipalityUuid The consumer of this import. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $operationId The operation id, for the archive reason. + * @param int $processedItems The rows already counted as processed. + * @param CmdbImportReport $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconcile(string $municipalityUuid, array $archive, string $operationId, int $processedItems, CmdbImportReport $report): void { + // A cancel that came in during the last row stops the import before anything is archived or deleted. + if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { + $report->markCancelled(); + return; + } + + $scope = $this->reconciliationScope(municipalityUuid: $municipalityUuid); + $this->progressTracker->setPhase(phase: 'reconciling', data: ['total_items' => ($processedItems + count($scope))]); + + foreach ($scope as $index => $application) { + if ($this->progressTracker->isCancelRequested(operationId: $operationId) === true) { + $report->markCancelled(); + return; + } + + // Every APPID on a CMDB sheet was handled by its row, whatever the row's outcome. + if (isset($this->winningSheets[$application['key']]) === false) { + $this->reconcileApplication(application: $application, archive: $archive, operationId: $operationId, report: $report); + } + + $this->progressTracker->updateProgress(processedItems: ($processedItems + $index + 1)); + } + }//end reconcile() + + /** + * The municipality's usages, in every state, each with its module when that module carries this municipality's import key. + * + * A usage of a hand-made module, or of a module another organisation + * imported, is left out: only what this municipality's import created is + * reconciled. Working and archived usages are paged with one lens each, + * soft-deleted ones come from the trash (design D9); each object keeps the + * state its lens or the trash gave it. + * + * @param string $municipalityUuid The consumer of this import. + * + * @return array}> + * Per application, the APPID's match key, the usage, the module and the state of both by schema key. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconciliationScope(string $municipalityUuid): array { + $usages = $this->usagesInEveryState(municipalityUuid: $municipalityUuid); + + $owned = []; + foreach ($usages as $usage) { + $data = $usage['entity']->getObject(); + $moduleUuid = self::relationUuid(value: ($data['module'] ?? null)); + // The filter is checked again: a filter OpenRegister cannot apply must not widen the scope. + if (self::relationUuid(value: ($data['consumer'] ?? null)) === $municipalityUuid && $moduleUuid !== null && $usage['entity']->getUuid() !== null) { + $owned[] = $usage + ['module' => $moduleUuid]; + } + } + + $prefix = $this->profile->externalKeyPrefix() . ':' . $municipalityUuid . ':'; + $modules = $this->modulesByUuid(uuids: array_values(array_unique(array_column($owned, 'module')))); + $scope = []; + foreach ($owned as $usage) { + $module = ($modules[$usage['module']] ?? null); + if ($module === null) { + continue; + } + + $externalKey = (string)($module['entity']->getObject()['externalKey'] ?? ''); + if (str_starts_with($externalKey, $prefix) === true && strlen($externalKey) > strlen($prefix)) { + $scope[] = [ + 'key' => substr($externalKey, strlen($prefix)), + 'usage' => $usage['entity'], + 'module' => $module['entity'], + 'states' => ['usage' => $usage['state'], 'module' => $module['state']], + ]; + } + } + + return $scope; + }//end reconciliationScope() + + /** + * The usages that name this municipality as consumer, in every state, each with that state. + * + * Working and archived usages are paged with one lens each; soft-deleted + * ones come from the trash, which is not filtered by consumer, so the + * caller checks the consumer of every usage. + * + * @param string $municipalityUuid The consumer of this import. + * + * @return array + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function usagesInEveryState(string $municipalityUuid): array { + $coordinates = $this->coordinates(); + $usages = []; + foreach ([[[], self::STATE_WORKING], [self::ARCHIVED_LENS, self::STATE_ARCHIVED]] as [$lens, $state]) { + $offset = 0; + do { + $page = $coordinates['objectService']->searchObjects( + query: array_merge( + [ + '@self' => ['register' => $coordinates['register'], 'schema' => $coordinates['usage']], + 'consumer' => $municipalityUuid, + '_limit' => self::PAGE_SIZE, + '_offset' => $offset, + ], + $lens + ), + _rbac: false, + _multitenancy: false + ); + if (is_array($page) === false) { + $page = []; + } + + foreach ($page as $usage) { + $usages[] = ['entity' => $usage, 'state' => $state]; + } + + $offset += self::PAGE_SIZE; + $pageSize = count($page); + } while ($pageSize === self::PAGE_SIZE); + }//end foreach + + foreach ($this->trash()['usage'] as $usage) { + $usages[] = ['entity' => $usage, 'state' => self::STATE_DELETED]; + } + + return $usages; + }//end usagesInEveryState() + + /** + * The modules with these uuids, in every state, each with its state. + * + * Per chunk of PAGE_SIZE uuids one search without a lens and one with + * the archive lens; the rest is looked up in the trash. + * + * @param array $uuids The module uuids. + * + * @return array Uuid => entity and state. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function modulesByUuid(array $uuids): array { + $coordinates = $this->coordinates(); + $modules = []; + foreach (array_chunk($uuids, self::PAGE_SIZE) as $chunk) { + // The chunks are disjoint, so a uuid found in an earlier chunk is never overwritten. + $modules += $this->modulesInChunk(chunk: $chunk, coordinates: $coordinates); + } + + foreach ($uuids as $uuid) { + $trashed = ($this->trash()['module'][$uuid] ?? null); + if (isset($modules[$uuid]) === false && $trashed !== null) { + $modules[$uuid] = ['entity' => $trashed, 'state' => self::STATE_DELETED]; + } + } + + return $modules; + }//end modulesByUuid() + + /** + * The working and archived modules of one chunk of uuids; a module both lenses return keeps the first state. + * + * @param array $chunk At most PAGE_SIZE module uuids. + * @param array $coordinates The object service and register and schema ids, from coordinates(). + * + * @return array Uuid => entity and state. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function modulesInChunk(array $chunk, array $coordinates): array { + $modules = []; + foreach ([[[], self::STATE_WORKING], [self::ARCHIVED_LENS, self::STATE_ARCHIVED]] as [$lens, $state]) { + $results = $coordinates['objectService']->searchObjects( + query: array_merge( + ['@self' => ['register' => $coordinates['register'], 'schema' => $coordinates['module']], '_ids' => $chunk, '_limit' => count($chunk)], + $lens + ), + _rbac: false, + _multitenancy: false + ); + if (is_array($results) === false) { + continue; + } + + foreach ($results as $entity) { + $uuid = (string)$entity->getUuid(); + if (in_array($uuid, $chunk, true) === true && isset($modules[$uuid]) === false) { + $modules[$uuid] = ['entity' => $entity, 'state' => $state]; + } + } + } + + return $modules; + }//end modulesInChunk() + + /** + * Archive or soft-delete one application that left the CMDB sheets, usage first, in its own error boundary. + * + * The usage is written before the module, so a failure between the two + * leaves the municipality's view right and the module at worst visible. + * An application already in the state the export asks for is not reported. + * + * @param array{key: string, usage: object, module: object, states: array} $application Application. + * @param array{sheet: string, present: bool, keyColumnMissing: bool, keys: array} $archive The archive sheet. + * @param string $operationId The operation id, for the archive reason. + * @param CmdbImportReport $report The report. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function reconcileApplication(array $application, array $archive, string $operationId, CmdbImportReport $report): void { + $moduleData = $application['module']->getObject(); + $onArchiveSheet = ($archive['present'] === true && isset($archive['keys'][$application['key']]) === true); + $entry = [ + 'sheet' => '', + 'row' => 0, + 'appId' => (string)($moduleData['externalNumber'] ?? $application['key']), + 'name' => (string)($moduleData['name'] ?? ''), + ]; + + $target = CmdbImportReport::ARCHIVED; + $reason = self::REASON_PREFIX . 'not in the export (' . $operationId . ')'; + if ($onArchiveSheet === true) { + $entry['sheet'] = $archive['sheet']; + $reason = self::REASON_PREFIX . 'on sheet "' . $archive['sheet'] . '" (' . $operationId . ')'; + } elseif ($archive['present'] === true) { + $target = CmdbImportReport::DELETED; + } + + $uuids = ['module' => (string)$application['module']->getUuid(), 'usage' => (string)$application['usage']->getUuid()]; + $step = 'archive'; + $changed = false; + try { + foreach (['usage', 'module'] as $schemaKey) { + foreach (self::transitions(state: $application['states'][$schemaKey], target: $target, restoreDeleted: $onArchiveSheet) as $step) { + $this->transition(step: $step, schemaKey: $schemaKey, uuid: $uuids[$schemaKey], reason: $reason); + $changed = true; + } + } + } catch (Throwable $e) { + // The module's stored values are taken out of the log line, as a row's cells are. + $values = array_map('strval', array_filter($moduleData, static fn (mixed $value): bool => is_scalar($value))); + $this->failRow(report: $report, entry: $entry, step: $step, e: $e, warnings: [], uuids: [$uuids['module'], $uuids['usage']], values: $values); + return; + } + + if ($changed === true) { + $this->addRow(report: $report, entry: $entry, outcome: $target, moduleUuid: $uuids['module'], usageUuid: $uuids['usage']); + } + }//end reconcileApplication() + + /** + * The steps that bring one module or usage to the state the export asks for (design D5). + * + * Deleted: soft-delete what is not in the trash yet. Archived: archive + * what is working; an object in the trash is restored first only when the + * archive sheet lists it, so a workbook without the sheet never takes + * anything out of the trash. + * + * @param array{archived: bool, deleted: bool} $state The object's state. + * @param string $target archived or deleted. + * @param bool $restoreDeleted Whether an object in the trash is restored to be archived. + * + * @return array delete, restore and archive, in order; empty when nothing changes. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private static function transitions(array $state, string $target, bool $restoreDeleted): array { + if ($target === CmdbImportReport::DELETED) { + if ($state['deleted'] === true) { + return []; + } + + return ['delete']; + } + + if ($state['deleted'] === true) { + if ($restoreDeleted === true) { + return ['restore', 'archive']; + } + + return []; + } + + if ($state['archived'] === true) { + return []; + } + + return ['archive']; + }//end transitions() + + /** + * Run one step through OpenRegister: soft delete, restore from the trash, or archive. + * + * @param string $step delete, restore or archive. + * @param string $schemaKey The coordinates key of the schema. + * @param string $uuid The object. + * @param string $reason The archive reason. + * + * @return void + * + * @throws RuntimeException When OpenRegister does not delete the object. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private function transition(string $step, string $schemaKey, string $uuid, string $reason): void { + if ($step === 'delete') { + $coordinates = $this->coordinates(); + $deleted = $coordinates['objectService']->deleteObject( + uuid: $uuid, + register: $coordinates['register'], + schema: $coordinates[$schemaKey], + _rbac: false, + _multitenancy: false + ); + if ($deleted !== true) { + throw new RuntimeException('OpenRegister did not delete the object'); + } + + return; + } + + if ($step === 'restore') { + $this->restore(uuid: $uuid); + return; + } + + $this->archiveServices()['handler']->archive(identifier: $uuid, reason: $reason); + }//end transition() + /** * Resolve OpenRegister's mapping engine. * @@ -1915,6 +2734,8 @@ private function resetRun(): void { $this->winningSheets = []; $this->contactUids = []; $this->contactPersons = []; + $this->archiveServices = null; + $this->trash = null; }//end resetRun() /** diff --git a/lib/Service/ProgressTracker.php b/lib/Service/ProgressTracker.php index b0c9ce229..5a9706ada 100644 --- a/lib/Service/ProgressTracker.php +++ b/lib/Service/ProgressTracker.php @@ -62,6 +62,11 @@ class ProgressTracker { 'processing_relationships' => ['weight' => 15, 'description' => 'Processing relationships'], 'processing_organizations' => ['weight' => 10, 'description' => 'Processing organizations'], 'processing_views' => ['weight' => 10, 'description' => 'Processing views'], + // The CMDB import's reconciliation of records missing from the source + // (openspec/changes/cmdb-import-archive-reconciliation). Weight 0, so the + // weighted percentage of the ArchiMate import's phases is unchanged; the + // CMDB section computes its own percentage from the item counts. + 'reconciling' => ['weight' => 0, 'description' => 'Reconciling records missing from the source'], 'finalizing' => ['weight' => 5, 'description' => 'Finalizing import'], 'completed' => ['weight' => 0, 'description' => 'Completed'], ]; diff --git a/openapi.json b/openapi.json index 7ee628b87..6469cf208 100644 --- a/openapi.json +++ b/openapi.json @@ -62,10 +62,11 @@ "missingRecords": { "type": "string", "enum": [ - "keep" + "keep", + "archive" ], - "default": "keep", - "description": "Only keep is accepted; mark and remove are reserved" + "default": "archive", + "description": "What happens to the municipality's applications whose APPID is no longer on either CMDB sheet. archive: after the last row, archive module and usage when the sheet \"Gearchiveerde Applicaties\" lists the APPID, soft-delete them when no sheet does, and archive only when the workbook has no archive sheet; an archived or deleted application that returns is unarchived or restored. keep: leave them as they are. mark and remove are reserved and refused with 422 MISSING_RECORDS_UNSUPPORTED" }, "operationId": { "type": "string", @@ -128,7 +129,7 @@ } }, "422": { - "description": "MISSING_RECORDS_UNSUPPORTED, MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS (more than one live Municipality has the typed name; details.matches lists their uuids), NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", + "description": "MISSING_RECORDS_UNSUPPORTED (details.accepted lists keep and archive), MUNICIPALITY_REQUIRED, MUNICIPALITY_INVALID, MUNICIPALITY_AMBIGUOUS (more than one live Municipality has the typed name; details.matches lists their uuids), NO_SOURCE_SHEET, MISSING_COLUMN or TOO_MANY_ROWS", "content": { "application/json": { "schema": { @@ -148,7 +149,7 @@ } }, "503": { - "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on, or the module schema predates 0.3.8; details.schema names it and details.missing lists the properties, and externalKey.authorization.update for a missing admin-only write rule)", + "description": "MAPPING_UNAVAILABLE, READER_UNAVAILABLE, NOT_CONFIGURED, or SCHEMA_OUTDATED (a stackiq schema lacks a property the import matches on, or the module schema predates 0.3.8; details.schema names it and details.missing lists the properties, and externalKey.authorization.update for a missing admin-only write rule), or ARCHIVE_UNAVAILABLE (missingRecords=archive, but OpenRegister's archive handler or object mapper is missing; nothing is read or written)", "content": { "application/json": { "schema": { @@ -499,6 +500,10 @@ "unchanged", "skipped", "failed", + "archived", + "unarchived", + "deleted", + "restored", "warnings", "unpublished" ], @@ -524,6 +529,22 @@ "failed": { "type": "integer" }, + "archived": { + "type": "integer", + "description": "Applications whose module and usage this import archived because they left the CMDB sheets" + }, + "unarchived": { + "type": "integer", + "description": "Rows whose archived module or usage this import unarchived before updating it" + }, + "deleted": { + "type": "integer", + "description": "Applications whose module and usage this import soft-deleted because no sheet lists them any more" + }, + "restored": { + "type": "integer", + "description": "Rows whose soft-deleted module or usage this import restored from the trash before updating it" + }, "warnings": { "type": "integer" }, @@ -571,7 +592,8 @@ "type": "string" }, "row": { - "type": "integer" + "type": "integer", + "description": "The sheet row, or 0 for an application the reconciliation archived or deleted" }, "appId": { "type": "string" @@ -586,7 +608,11 @@ "updated", "unchanged", "skipped", - "failed" + "failed", + "archived", + "unarchived", + "deleted", + "restored" ] }, "reasons": { diff --git a/postman/stackiq-tests.json b/postman/stackiq-tests.json index 385214b7c..0da0499b8 100644 --- a/postman/stackiq-tests.json +++ b/postman/stackiq-tests.json @@ -23073,9 +23073,10 @@ "listen": "test", "script": { "exec": [ - "pm.test(\"Only missingRecords=keep is accepted\", function () {", + "pm.test(\"Only missingRecords=keep or archive is accepted\", function () {", " pm.response.to.have.status(422);", " pm.expect(pm.response.json().error).to.eql(\"MISSING_RECORDS_UNSUPPORTED\");", + " pm.expect(pm.response.json().details.accepted).to.eql([\"keep\", \"archive\"]);", "});" ], "type": "text/javascript" diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index a27b7cfbf..75af4b05f 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -114,7 +114,7 @@ private function controller( if ($service === null) { $service = $this->createMock(CmdbExportImportService::class); $service->method('maxFileBytes')->willReturn(10485760); - $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + $service->method('missingRecordsModes')->willReturn(['keep', 'archive']); } $logger = ($logger ?? $this->createMock(LoggerInterface::class)); @@ -157,7 +157,7 @@ protected function iniBytes(string $name): ?int { private function service(): CmdbExportImportService|MockObject { $service = $this->createMock(CmdbExportImportService::class); $service->method('maxFileBytes')->willReturn(10485760); - $service->method('supportsMissingRecords')->willReturnCallback(fn (string $mode): bool => $mode === 'keep'); + $service->method('missingRecordsModes')->willReturn(['keep', 'archive']); return $service; }//end service() @@ -371,9 +371,30 @@ public function testAReservedMissingRecordsValueIsRefused(): void { $response = $this->controller(file: $this->file(path: $this->upload()), params: ['missingRecords' => $mode], service: $service)->import(); $this->assertSame(422, $response->getStatus(), $mode); $this->assertSame('MISSING_RECORDS_UNSUPPORTED', $response->getData()['error'], $mode); + $this->assertEquals((object)['accepted' => ['keep', 'archive']], $response->getData()['details'], $mode); + $this->assertSame('Applications missing from the export can only be archived or kept.', $response->getData()['message'], $mode); } }//end testAReservedMissingRecordsValueIsRefused() + /** + * Without missingRecords the import archives; keep and archive are passed through as sent. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testMissingRecordsDefaultsToArchive(): void { + foreach ([[[], 'archive'], [['missingRecords' => 'keep'], 'keep'], [['missingRecords' => 'archive'], 'archive']] as [$params, $expected]) { + $service = $this->service(); + $service->expects($this->once())->method('import') + ->with($this->anything(), $this->callback(static fn (array $options): bool => $options['missingRecords'] === $expected)) + ->willReturn(['success' => true]); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: array_merge(['municipalityName' => 'Gemeente Voorbeeldstad'], $params), service: $service)->import(); + $this->assertSame(200, $response->getStatus(), $expected); + } + }//end testMissingRecordsDefaultsToArchive() + /** * Without a municipality the answer is 422 MUNICIPALITY_REQUIRED and nothing is imported. * @@ -407,6 +428,7 @@ public static function serviceErrors(): array { 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 503, ['schema' => 'module', 'missing' => ['externalKey']]], 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 409, []], 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 422, ['matches' => ['00000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000002']]], + 'archive' => [CmdbImportException::ARCHIVE_UNAVAILABLE, 503, []], ]; }//end serviceErrors() @@ -421,6 +443,10 @@ public static function engineMessages(): array { 'schema' => [CmdbImportException::SCHEMA_OUTDATED, 'The stackiq register is out of date; import its configuration again.'], 'running' => [CmdbImportException::IMPORT_IN_PROGRESS, 'Another CMDB import is running; try again when it has finished.'], 'ambiguous' => [CmdbImportException::MUNICIPALITY_AMBIGUOUS, 'Several municipalities have this name; choose one from the list.'], + 'archive' => [ + CmdbImportException::ARCHIVE_UNAVAILABLE, + 'Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.', + ], ]; }//end engineMessages() @@ -533,6 +559,7 @@ public function testAValidUploadReturnsTheReport(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => false, 'publish' => true, + 'missingRecords' => 'keep', 'operationId' => 'cmdb-00000000-0000-0000-0000-000000000000', 'fileName' => 'export.xlsx', ] @@ -567,6 +594,7 @@ public function testTheFieldsReachTheServiceAsSent(): void { 'municipalityName' => 'Gemeente Voorbeeldstad', 'updateExisting' => true, 'publish' => true, + 'missingRecords' => 'archive', 'operationId' => 'not-a-cmdb-id', 'fileName' => 'export.xlsx', ] diff --git a/tests/Unit/Service/Cmdb/CmdbImportReportTest.php b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php index a3d7a576a..ef208a36f 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportReportTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportReportTest.php @@ -69,4 +69,27 @@ public function testWithinTheLimitEveryRowIsStored(): void { $this->assertSame(array_merge($report->toArray(), ['rowsStored' => 5, 'rowsTruncated' => false]), $stored); }//end testWithinTheLimitEveryRowIsStored() + + /** + * The reconciliation outcomes are counted, and their entries (row 0) are not counted as processed sheet rows. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheReconciliationOutcomesAreCounted(): void { + $report = new CmdbImportReport(operationId: 'cmdb-report-02', rowsRead: 2); + $report->addRow(sheet: 'S', row: 2, appId: '1', name: 'Een', outcome: CmdbImportReport::UNARCHIVED); + $report->addRow(sheet: 'S', row: 3, appId: '2', name: 'Twee', outcome: CmdbImportReport::RESTORED); + $report->addRow(sheet: 'Gearchiveerde Applicaties', row: 0, appId: '7', name: 'Zeven', outcome: CmdbImportReport::ARCHIVED); + $report->addRow(sheet: '', row: 0, appId: '8', name: 'Acht', outcome: CmdbImportReport::DELETED); + + $summary = $report->summary(); + $this->assertSame(2, $summary['processed']); + $this->assertSame(2, $report->processed()); + $this->assertSame([1, 1, 1, 1], [$summary['archived'], $summary['unarchived'], $summary['deleted'], $summary['restored']]); + $this->assertFalse($report->isCancelled()); + $report->markCancelled(); + $this->assertTrue($report->isCancelled()); + }//end testTheReconciliationOutcomesAreCounted() }//end class diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index d81417ea7..7ac56ddc6 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -91,6 +91,41 @@ class CmdbExportImportServiceTest extends TestCase { */ private $beforeSave = null; + /** + * The archive and trash markers per schema id and uuid, as OpenRegister keeps them in `@self`. + * + * @var array|null, deleted?: array|null}>> + */ + private array $states = []; + + /** + * Every archive, unarchive, restore and delete: verb, schema id, uuid, reason. + * + * @var array + */ + private array $transitions = []; + + /** + * Called before every archive, unarchive, restore and delete with the verb and the uuid; may throw. + * + * @var callable|null + */ + private $beforeTransition = null; + + /** + * How often the trash listing was read. + * + * @var int + */ + private int $trashReads = 0; + + /** + * Whether the container knows OpenRegister's archive handler and object mapper. + * + * @var bool + */ + private bool $archiveServicesAvailable = true; + /** * Contacts in the fake address book: uid => name, email. * @@ -215,6 +250,11 @@ protected function setUp(): void { $this->store = [self::MODULE => [], self::ORGANIZATION => [], self::USAGE => [], self::CONTACT_PERSON => []]; $this->saves = []; $this->beforeSave = null; + $this->states = []; + $this->transitions = []; + $this->beforeTransition = null; + $this->trashReads = 0; + $this->archiveServicesAvailable = true; $this->contacts = []; $this->addressBooks = []; $this->contactsEnabled = true; @@ -341,20 +381,26 @@ private function noteScope(string $method, bool $rbac, bool $multitenancy): void * * @param string $uuid The uuid. * @param array $data The object data. + * @param array $state The archive and trash markers, serialised under `@self`. + * @param string|null $schema The schema id, as OpenRegister's own reads set it. * * @return ObjectEntityInterface */ - private function entity(string $uuid, array $data): ObjectEntityInterface { - return new class($uuid, $data) implements ObjectEntityInterface { + private function entity(string $uuid, array $data, array $state = [], ?string $schema = null): ObjectEntityInterface { + return new class($uuid, $data, $state, $schema) implements ObjectEntityInterface { /** * Constructor. * * @param string $uuid The uuid. * @param array $data The data. + * @param array $state The markers. + * @param string|null $schema The schema id. */ public function __construct( private string $uuid, private array $data, + private array $state, + private ?string $schema, ) { } @@ -371,7 +417,7 @@ public function getRegister(): ?string { } public function getSchema(): ?string { - return null; + return $this->schema; } public function getOrganisation(): ?string { @@ -383,7 +429,7 @@ public function getOwner(): ?string { } public function jsonSerialize(): array { - return $this->data; + return array_merge($this->data, ['@self' => array_merge(['archived' => null, 'deleted' => null], $this->state)]); } }; }//end entity() @@ -403,6 +449,11 @@ function (array $object, ?array $extend = [], $register = null, $schema = null, ($this->beforeSave)($schema, $object); } + // OpenRegister refuses a data write to an archived object (ObjectStateWriteException). + if ($uuid !== null && empty($this->states[$schema][$uuid]['archived']) === false) { + throw new RuntimeException('write to an archived object'); + } + $create = ($uuid === null); if ($create === true) { $uuid = sprintf('00000000-0000-4000-8000-%012d', count($this->saves) + 1); @@ -435,12 +486,32 @@ function (array $query = [], bool $_rbac = true, bool $_multitenancy = true): ar } } + $state = ($this->states[$schema][$uuid] ?? []); + // The list search loses every soft-deleted row, `_includeDeleted` or not: + // MagicSearchHandler::convertRowToObjectEntity() parses the JSON `_deleted` + // marker as a date, fails, and skips the row (OpenRegister beta ff4dad5c). + if (empty($state['deleted']) === false) { + continue; + } + + if (self::inLens(query: $query, uuid: (string)$uuid, state: $state) === false) { + continue; + } + + // The same converter hydrates no `_archived`: a search result never carries the archive marker. $found[] = $this->entity(uuid: $uuid, data: $data); } return array_slice($found, $offset, $limit); } ); + $service->method('deleteObject')->willReturnCallback( + function (string $uuid, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): bool { + $this->noteScope(method: 'deleteObject', rbac: $_rbac, multitenancy: $_multitenancy); + $this->changeState(verb: 'delete', uuid: $uuid, reason: null); + return true; + } + ); $service->method('find')->willReturnCallback( function ($id, ?array $_extend = [], bool $files = false, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): ?ObjectEntityInterface { $this->noteScope(method: 'find', rbac: $_rbac, multitenancy: $_multitenancy); @@ -459,6 +530,222 @@ function ($id, ?array $_extend = [], bool $files = false, $register = null, $sch return $service; }//end objectService() + /** + * Whether a working or archived object is in the list a query asks for, as OpenRegister's lenses decide. + * + * Archived objects are left out unless `_archived` asks for them (`true` + * only the archive, `any` both); `_ids` narrows to those uuids. + * + * @param array $query The search query. + * @param string $uuid The object. + * @param array $state Its markers. + * + * @return bool + */ + private static function inLens(array $query, string $uuid, array $state): bool { + if (isset($query['_ids']) === true && in_array($uuid, (array)$query['_ids'], true) === false) { + return false; + } + + $archived = (empty($state['archived']) === false); + $lens = ($query['_archived'] ?? null); + if ($lens === 'any') { + return true; + } + + if (filter_var($lens, FILTER_VALIDATE_BOOLEAN) === true) { + return $archived; + } + + return $archived === false; + }//end inLens() + + /** + * Archive, unarchive, restore or soft-delete a stored object, as OpenRegister's handlers do. + * + * @param string $verb archive, unarchive, restore or delete. + * @param string $uuid The object. + * @param string|null $reason The archive reason. + * + * @return array{uuid: string, archived: array|null} + */ + public function changeState(string $verb, string $uuid, ?string $reason): array { + if ($this->beforeTransition !== null) { + ($this->beforeTransition)($verb, $uuid); + } + + $schema = 0; + foreach ($this->store as $id => $objects) { + if (isset($objects[$uuid]) === true) { + $schema = $id; + } + } + + $this->assertNotSame(0, $schema, 'no stored object ' . $uuid); + $this->transitions[] = [$verb, $schema, $uuid, $reason]; + $state = ($this->states[$schema][$uuid] ?? []); + if ($verb !== 'restore' && empty($state['deleted']) === false) { + // The archive handler and the delete resolve the object without the trash. + throw new \OCP\AppFramework\Db\DoesNotExistException('object is in the trash'); + } + + match ($verb) { + 'archive' => $state['archived'] = ($state['archived'] ?? null) ?: ['by' => 'admin', 'at' => '2026-10-09T00:00:00+00:00', 'reason' => $reason], + 'unarchive' => $state['archived'] = null, + 'restore' => $state['deleted'] = null, + 'delete' => $state['deleted'] = ['deletedBy' => 'admin', 'deletedAt' => '2026-10-09T00:00:00+00:00'], + }; + $this->states[$schema][$uuid] = $state; + + return ['uuid' => $uuid, 'archived' => ($state['archived'] ?? null)]; + }//end changeState() + + /** + * OpenRegister's archive handler and object mapper over the in-memory store. + * + * @return array{0: object, 1: object} + */ + private function archiveDoubles(): array { + $test = $this; + $handler = new class($test) { + /** + * Constructor. + * + * @param CmdbExportImportServiceTest $test The test, which holds the markers. + */ + public function __construct( + private CmdbExportImportServiceTest $test, + ) { + } + + /** + * Archive an object. + * + * @param string $identifier The uuid. + * @param string|null $reason The reason. + * @param string|null $register Ignored. + * @param string|null $schema Ignored. + * + * @return array + */ + public function archive(string $identifier, ?string $reason = null, ?string $register = null, ?string $schema = null): array { + return $this->test->changeState(verb: 'archive', uuid: $identifier, reason: $reason); + } + + /** + * Unarchive an object. + * + * @param string $identifier The uuid. + * @param string|null $reason The reason. + * @param string|null $register Ignored. + * @param string|null $schema Ignored. + * + * @return array + */ + public function unarchive(string $identifier, ?string $reason = null, ?string $register = null, ?string $schema = null): array { + return $this->test->changeState(verb: 'unarchive', uuid: $identifier, reason: $reason); + } + }; + $mapper = new class($test) { + /** + * Constructor. + * + * @param CmdbExportImportServiceTest $test The test, which holds the markers. + */ + public function __construct( + private CmdbExportImportServiceTest $test, + ) { + } + + /** + * Take an object out of the trash and answer it as the mapper reads it, archive marker included. + * + * @param string $uuid The uuid. + * + * @return object + */ + public function restoreObject(string $uuid): object { + $this->test->changeState(verb: 'restore', uuid: $uuid, reason: null); + return $this->test->storedEntity(uuid: $uuid); + } + + /** + * The trash: every soft-deleted object, in every register and schema. + * + * @param int|null $limit Ignored. + * @param int|null $offset Ignored. + * @param bool $_rbac Must be false. + * @param bool $_multitenancy Must be false. + * + * @return array + */ + public function findDeletedAcrossAllMagicTables(?int $limit = null, ?int $offset = null, bool $_rbac = true, bool $_multitenancy = true): array { + return $this->test->trashListing(rbac: $_rbac, multitenancy: $_multitenancy); + } + }; + + return [$handler, $mapper]; + }//end archiveDoubles() + + /** + * A stored object as OpenRegister's mapper reads it by uuid: data plus both markers. + * + * @param string $uuid The object. + * + * @return ObjectEntityInterface + */ + public function storedEntity(string $uuid): ObjectEntityInterface { + foreach ($this->store as $schema => $objects) { + if (isset($objects[$uuid]) === true) { + return $this->entity(uuid: $uuid, data: $objects[$uuid], state: ($this->states[$schema][$uuid] ?? []), schema: (string)$schema); + } + } + + $this->fail('no stored object ' . $uuid); + }//end storedEntity() + + /** + * The trash as MagicMapper::findDeletedAcrossAllMagicTables() lists it. + * + * Its rows go through MagicMapper::rowToObjectEntity(): uuid, register, + * schema and the data, with every string that parses as JSON decoded + * ("2" becomes 2), and no archive or trash marker. + * + * @param bool $rbac The `_rbac` argument. + * @param bool $multitenancy The `_multitenancy` argument. + * + * @return array + */ + public function trashListing(bool $rbac, bool $multitenancy): array { + $this->noteScope(method: 'findDeletedAcrossAllMagicTables', rbac: $rbac, multitenancy: $multitenancy); + $this->trashReads++; + $listed = []; + foreach ($this->states as $schema => $states) { + foreach ($states as $uuid => $state) { + if (empty($state['deleted']) === true || isset($this->store[$schema][$uuid]) === false) { + continue; + } + + $data = array_map( + static function (mixed $value): mixed { + if (is_string($value) === true) { + $decoded = json_decode($value, true); + if (json_last_error() === JSON_ERROR_NONE) { + return $decoded; + } + } + + return $value; + }, + $this->store[$schema][$uuid] + ); + $listed[] = $this->entity(uuid: (string)$uuid, data: $data, schema: (string)$schema); + } + } + + return $listed; + }//end trashListing() + /** * OpenRegister's schema mapper over the declared properties. * @@ -674,19 +961,24 @@ function (&$value): void { /** * A reader that hands out given rows, for tests that do not need the fixture. * + * The workbook has the archive sheet, listing $archived; null is a workbook without the sheet. + * * @param array}> $rows The rows. + * @param array|null $archived The APPIDs on the archive sheet, or null when there is no such sheet. * * @return CmdbWorkbookReader */ - private function rowsReader(array $rows): CmdbWorkbookReader { - return new class($rows) extends CmdbWorkbookReader { + private function rowsReader(array $rows, ?array $archived = []): CmdbWorkbookReader { + return new class($rows, $archived) extends CmdbWorkbookReader { /** * Constructor. * * @param array> $rows The rows. + * @param array|null $archived The archive sheet's APPIDs, or null. */ public function __construct( private array $rows, + private ?array $archived, ) { } @@ -699,7 +991,8 @@ public function __construct( * @return array */ public function read(string $path, CmdbImportProfile $profile): array { - return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false]; + $archive = ['sheet' => 'Gearchiveerde Applicaties', 'present' => ($this->archived !== null), 'keyColumnMissing' => false, 'appIds' => ($this->archived ?? [])]; + return ['rows' => $this->rows, 'importWarnings' => [], 'date1904' => false, 'archive' => $archive]; } }; }//end rowsReader() @@ -716,10 +1009,11 @@ public function read(string $path, CmdbImportProfile $profile): array { private function service(?CmdbWorkbookReader $reader = null, ?string $profileDir = null, array $config = ['register' => '20']): CmdbExportImportService { $objectService = $this->objectService(); $schemaMapper = $this->schemaMapper(); + [$archiveHandler, $objectMapper] = $this->archiveDoubles(); $container = $this->createMock(ContainerInterface::class); $container->method('has')->willReturn(false); $container->method('get')->willReturnCallback( - function (string $id) use ($objectService, $schemaMapper) { + function (string $id) use ($objectService, $schemaMapper, $archiveHandler, $objectMapper) { if ($id === ObjectServiceInterface::class) { return $objectService; } @@ -728,6 +1022,14 @@ function (string $id) use ($objectService, $schemaMapper) { return $schemaMapper; } + if ($this->archiveServicesAvailable === true && $id === CmdbExportImportService::ARCHIVE_HANDLER_CLASS) { + return $archiveHandler; + } + + if ($this->archiveServicesAvailable === true && $id === CmdbExportImportService::OBJECT_MAPPER_CLASS) { + return $objectMapper; + } + throw new RuntimeException('not in this container: ' . $id); } ); @@ -846,7 +1148,7 @@ public function testTheFixtureCreatesModulesUsagesAndSuppliers(): void { $this->assertTrue($report['success']); $this->assertFalse($report['cancelled']); $this->assertSame('cmdb-test-0001', $report['operationId']); - $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 2, 'processed' => 2, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 0, 'archived' => 0, 'unarchived' => 0, 'deleted' => 0, 'restored' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertSame('Gemeente Voorbeeldstad', $report['municipality']['name']); $this->assertTrue($report['municipality']['created']); $this->assertSame(['No municipality named "Gemeente Voorbeeldstad" was found, so it was created. Check the name if you meant an existing one.'], array_column($report['importWarnings'], 'message'), 'only the warning that the municipality was created'); @@ -1448,29 +1750,591 @@ public function testUpdateExistingFalseSkipsMatches(): void { }//end testUpdateExistingFalseSkipsMatches() /** - * A module missing from a newer export, and its usage, are left as they are. + * The uuid of the stored module with this APPID. + * + * @param string $appId The APPID. + * + * @return string + */ + private function moduleOf(string $appId): string { + foreach ($this->store[self::MODULE] as $uuid => $module) { + if (($module['externalNumber'] ?? null) === $appId) { + return (string)$uuid; + } + } + + $this->fail('no module with APPID ' . $appId); + }//end moduleOf() + + /** + * The uuid of the stored usage of a module by a consumer. + * + * @param string $moduleUuid The module. + * @param string $consumer The consumer. + * + * @return string + */ + private function usageOf(string $moduleUuid, string $consumer = 'muni-1'): string { + foreach ($this->store[self::USAGE] as $uuid => $usage) { + if (($usage['module'] ?? null) === $moduleUuid && ($usage['consumer'] ?? null) === $consumer) { + return (string)$uuid; + } + } + + $this->fail('no usage of ' . $moduleUuid . ' by ' . $consumer); + }//end usageOf() + + /** + * Whether a stored object carries the archive marker, and whether the trash marker. + * + * @param int $schema The schema id. + * @param string $uuid The object. + * + * @return array{0: bool, 1: bool} Archived, deleted. + */ + private function stateOf(int $schema, string $uuid): array { + $state = ($this->states[$schema][$uuid] ?? []); + return [empty($state['archived']) === false, empty($state['deleted']) === false]; + }//end stateOf() + + /** + * Import APPIDs 1, 7 and 8 for muni-1, as a first import would. * * @return void */ - public function testRecordsMissingFromTheExportStay(): void { + private function importOneSevenAndEight(): void { $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2), $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB')])) - ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $rows = [ + $this->row(appId: '1', row: 2), + $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB'), + $this->row(appId: '8', row: 4, cells: ['Applicatie Eigenaar (Persoon)' => 'Achternaam, Voornaam']), + ]; + $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + $this->transitions = []; + }//end importOneSevenAndEight() + + /** + * A module missing from a newer export, and its usage, are left as they are when records are kept. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 + */ + public function testRecordsMissingFromTheExportStayWhenKept(): void { + $this->importOneSevenAndEight(); $modules = $this->store[self::MODULE]; $usages = $this->store[self::USAGE]; - $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])])) - ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); foreach ($modules as $uuid => $module) { - if ($module['externalNumber'] === '7') { + if ($module['externalNumber'] !== '1') { $this->assertSame($module, $this->store[self::MODULE][$uuid]); } } $this->assertSame($usages, $this->store[self::USAGE]); - $this->assertCount(2, $this->store[self::MODULE]); - }//end testRecordsMissingFromTheExportStay() + $this->assertSame([], $this->transitions, 'nothing is archived or deleted'); + $this->assertSame([], $this->states); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame([], $report['importWarnings'], 'keep does not need the archive sheet'); + }//end testRecordsMissingFromTheExportStayWhenKept() + + /** + * By default an application that moved to the archive sheet is archived and one on no sheet is soft-deleted, + * usage before module, and the owner contact person stays. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testMissingApplicationsAreArchivedOrDeletedByDefault(): void { + $this->importOneSevenAndEight(); + $contactPersons = $this->store[self::CONTACT_PERSON]; + $this->assertNotSame([], $contactPersons, 'APPID 8 has an owner'); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '1198'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-archive-01']); + + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven))); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight)); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '1'))); + $this->assertSame( + [ + ['archive', self::USAGE, $this->usageOf(moduleUuid: $seven)], + ['archive', self::MODULE, $seven], + ['delete', self::USAGE, $this->usageOf(moduleUuid: $eight)], + ['delete', self::MODULE, $eight], + ], + array_map(static fn (array $t): array => array_slice($t, 0, 3), $this->transitions), + 'the usage is written before its module' + ); + $this->assertSame('cmdb-import: on sheet "Gearchiveerde Applicaties" (cmdb-archive-01)', $this->transitions[0][3]); + $this->assertSame($contactPersons, $this->store[self::CONTACT_PERSON], 'contact persons are never touched'); + + $reconciled = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame( + [ + ['sheet' => 'Gearchiveerde Applicaties', 'row' => 0, 'appId' => '7', 'name' => 'Applicatie 7', 'outcome' => 'archived', 'reasons' => [], 'warnings' => [], 'moduleUuid' => $seven, 'usageUuid' => $this->usageOf(moduleUuid: $seven)], + ['sheet' => '', 'row' => 0, 'appId' => '8', 'name' => 'Applicatie 8', 'outcome' => 'deleted', 'reasons' => [], 'warnings' => [], 'moduleUuid' => $eight, 'usageUuid' => $this->usageOf(moduleUuid: $eight)], + ], + $reconciled + ); + $this->assertSame(1, $report['summary']['processed'], 'reconciliation entries are not sheet rows'); + $this->assertSame([1, 1], [$report['summary']['archived'], $report['summary']['deleted']]); + $this->assertStringNotContainsString('Achternaam', (string)json_encode($report)); + $this->assertSame([], $report['importWarnings']); + + $this->transitions = []; + $again = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame([], $this->transitions, 'an application already in the state the export asks for is left alone'); + $this->assertSame(['unchanged'], array_column($again['rows'], 'outcome')); + }//end testMissingApplicationsAreArchivedOrDeletedByDefault() + + /** + * Without the archive sheet a missing application is archived, never deleted, and the report warns. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted(): void { + $this->importOneSevenAndEight(); + $eight = $this->moduleOf(appId: '8'); + $this->states[self::MODULE][$eight] = ['deleted' => ['deletedBy' => 'admin']]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: null)) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '7'))); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $this->moduleOf(appId: '7')))); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight), 'what is in the trash stays there'); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertNotContains('delete', array_column($this->transitions, 0)); + $this->assertNotContains('restore', array_column($this->transitions, 0)); + $this->assertSame('cmdb-import: not in the export (' . $report['operationId'] . ')', $this->transitions[0][3]); + $this->assertSame(['unchanged', 'archived', 'archived'], array_column($report['rows'], 'outcome')); + $this->assertSame( + [['sheet' => 'Gearchiveerde Applicaties', 'message' => 'Sheet "Gearchiveerde Applicaties" was not found, so applications missing from the export are archived, not deleted.']], + $report['importWarnings'] + ); + }//end testWithoutTheArchiveSheetMissingApplicationsAreArchivedNotDeleted() + + /** + * A cancelled import reconciles nothing. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelledImportDoesNotReconcile(): void { + $this->importOneSevenAndEight(); + $service = null; + $this->beforeSave = function (int $schema) use (&$service): void { + if ($schema === self::MODULE) { + $service->requestCancel(operationId: 'cmdb-cancel-02'); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw']), $this->row(appId: '2', row: 3)])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-02']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([], $this->transitions); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + }//end testACancelledImportDoesNotReconcile() + + /** + * A cancel that comes in while the last row is processed stops the import before anything is archived or deleted. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelDuringTheLastRowArchivesNothing(): void { + $this->importOneSevenAndEight(); + $service = null; + $this->beforeSave = function (int $schema) use (&$service): void { + if ($schema === self::MODULE) { + $this->assertTrue($service->requestCancel(operationId: 'cmdb-cancel-03')); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])], archived: ['7', '8'])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-03']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([], $this->transitions); + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-03']['status']); + $this->assertArrayNotHasKey('cancel_cmdb-cancel-03', $this->cache); + }//end testACancelDuringTheLastRowArchivesNothing() + + /** + * A cancel during the reconciliation stops it before the next application and keeps what was archived. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testACancelDuringTheReconciliationStopsBeforeTheNextApplication(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $service = null; + $this->beforeTransition = function (string $verb, string $uuid) use (&$service, $seven): void { + if ($uuid === $seven) { + $this->assertTrue($service->requestCancel(operationId: 'cmdb-cancel-04')); + } + }; + $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])); + + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-04']); + + $this->assertTrue($report['cancelled']); + $this->assertSame([['archive', $this->usageOf(moduleUuid: $seven)], ['archive', $seven]], array_map(static fn (array $step): array => [$step[0], $step[2]], $this->transitions)); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'what was archived stays archived'); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $eight)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); + $this->assertSame(['unchanged', 'archived'], array_column($report['rows'], 'outcome')); + $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-04']['status']); + }//end testACancelDuringTheReconciliationStopsBeforeTheNextApplication() + + /** + * Only this municipality's usages and the modules its import created are reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAnotherMunicipalitysUsagesAreNeverReconciled(): void { + $this->importOneSevenAndEight(); + $this->seedOrganisation(uuid: 'muni-2', name: 'Gemeente Anders', type: 'Municipality'); + $seven = $this->moduleOf(appId: '7'); + // Another municipality uses the same module (added by hand), and muni-1 uses a hand-made module. + $this->store[self::USAGE]['usage-other'] = ['id' => 'usage-other', 'consumer' => 'muni-2', 'module' => $seven]; + $this->store[self::MODULE]['module-manual'] = ['id' => 'module-manual', 'name' => 'Handmatig', 'externalKey' => 'topdesk:muni-2:9']; + $this->store[self::USAGE]['usage-manual'] = ['id' => 'usage-manual', 'consumer' => 'muni-1', 'module' => 'module-manual']; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven))); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the module carries this municipality\'s import key'); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-other')); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-manual')); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: 'module-manual')); + $this->assertSame(2, count($this->transitions)); + }//end testAnotherMunicipalitysUsagesAreNeverReconciled() + + /** + * One application that cannot be archived is a failed row; the others are still reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $this->beforeTransition = function (string $verb, string $uuid) use ($seven): void { + if ($uuid === $seven) { + throw new RuntimeException('Schema "Applicatie 7" does not declare x-openregister-archive'); + } + }; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $rows = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame(['failed', 'archived'], array_column($rows, 'outcome')); + $this->assertSame(['step "archive" failed (RuntimeException)'], $rows[0]['reasons']); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $seven)), 'the usage went first'); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $this->moduleOf(appId: '8'))); + $this->assertSame(1, $report['summary']['failed']); + $failed = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: row failed'))); + $this->assertCount(1, $failed); + $this->assertStringNotContainsString('Applicatie 7', $failed[0]); + }//end testAFailingArchiveIsAFailedRowAndTheOthersAreReconciled() + + /** + * Without OpenRegister's archive services an archiving import is refused before the file is read; keep still imports. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testMissingArchiveServicesRefuseAnArchivingImport(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $this->archiveServicesAvailable = false; + $reader = new class extends CmdbWorkbookReader { + /** + * Constructor. + */ + public function __construct() { + } + + /** + * Never reached. + * + * @param string $path Ignored. + * @param CmdbImportProfile $profile Ignored. + * + * @return array + */ + public function read(string $path, CmdbImportProfile $profile): array { + throw new RuntimeException('the file was read'); + } + }; + + try { + $this->service(reader: $reader)->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->fail('ARCHIVE_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('ARCHIVE_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + } + + $this->assertSame([], $this->saves); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + $this->assertSame(['created'], array_column($report['rows'], 'outcome')); + }//end testMissingArchiveServicesRefuseAnArchivingImport() + + /** + * An archived application that returns is unarchived and a soft-deleted one restored, before the write; no duplicates. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAReturningApplicationIsUnarchivedOrRestoredNotDuplicated(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7']))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $modules = count($this->store[self::MODULE]); + $usages = count($this->store[self::USAGE]); + $this->transitions = []; + + $rows = [ + $this->row(appId: '1'), + $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug']), + $this->row(appId: '8', row: 4), + ]; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(['unchanged', 'unarchived', 'restored'], array_column($report['rows'], 'outcome')); + $this->assertSame([1, 1], [$report['summary']['unarchived'], $report['summary']['restored']]); + $this->assertSame([$modules, $usages], [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no second module or usage'); + foreach ([$seven, $eight] as $module) { + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $module)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $module))); + } + + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name'], 'the row is updated after the unarchive'); + $this->assertSame(['unarchive', 'unarchive', 'restore', 'restore'], array_column($this->transitions, 0)); + }//end testAReturningApplicationIsUnarchivedOrRestoredNotDuplicated() + + /** + * A usage archived or soft-deleted on its own, next to a working module, is unarchived or restored when its row returns. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAReturningUsageIsUnarchivedOrRestoredOnItsOwn(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + [$usageSeven, $usageEight] = [$this->usageOf(moduleUuid: $seven), $this->usageOf(moduleUuid: $eight)]; + $this->states[self::USAGE][$usageSeven] = ['archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'by hand']]; + $this->states[self::USAGE][$usageEight] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $usages = array_keys($this->store[self::USAGE]); + + $rows = [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB'), $this->row(appId: '8', row: 3)]; + $report = $this->service(reader: $this->rowsReader(rows: $rows))->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['unarchived', 'restored'], array_column($report['rows'], 'outcome')); + $this->assertSame([$usageSeven, $usageEight], array_column($report['rows'], 'usageUuid')); + $this->assertSame($usages, array_keys($this->store[self::USAGE]), 'no second usage'); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usageSeven)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usageEight)); + $this->assertSame([['unarchive', $usageSeven], ['restore', $usageEight]], array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions)); + }//end testAReturningUsageIsUnarchivedOrRestoredOnItsOwn() + + /** + * An application soft-deleted while archived is restored and unarchived when it returns, and updated from the restored object. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAnApplicationDeletedWhileArchivedIsRestoredAndUnarchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + $marker = [ + 'archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'cmdb-import: on sheet'], + 'deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00'], + ]; + $this->states[self::MODULE][$seven] = $marker; + $this->states[self::USAGE][$usage] = $marker; + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['restored'], array_column($report['rows'], 'outcome')); + $this->assertSame( + [['restore', $seven], ['unarchive', $seven], ['restore', $usage], ['unarchive', $usage]], + array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions) + ); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: $usage)); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no second module or usage'); + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name']); + $this->assertSame('7', $this->store[self::MODULE][$seven]['externalNumber'], 'the update starts from the restored object, not from the trash listing'); + }//end testAnApplicationDeletedWhileArchivedIsRestoredAndUnarchived() + + /** + * A soft-deleted application the archive sheet lists is restored and archived, and the trash is read once. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testADeletedApplicationOnTheArchiveSheetIsRestoredAndArchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + $this->states[self::MODULE][$seven] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00']]; + $this->states[self::USAGE][$usage] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-02T00:00:00+00:00']]; + $this->trashReads = 0; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame( + [['restore', $usage], ['archive', $usage], ['restore', $seven], ['archive', $seven]], + array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions) + ); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $usage)); + $reconciled = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame([['7', 'archived']], array_map(static fn (array $row): array => [$row['appId'], $row['outcome']], $reconciled)); + $this->assertSame(1, $this->trashReads, 'the trash is read once per import, not per row'); + }//end testADeletedApplicationOnTheArchiveSheetIsRestoredAndArchived() + + /** + * With updateExisting false a returning application is skipped as `exists` and stays archived. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testUpdateExistingFalseLeavesAnArchivedMatchArchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '8', row: 3)], archived: ['7']))->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $modules = $this->store[self::MODULE]; + $this->transitions = []; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB')])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'missingRecords' => 'keep']); + + $this->assertSame(['skipped'], array_column($report['rows'], 'outcome')); + $this->assertSame(['exists'], $report['rows'][0]['reasons']); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([], $this->transitions); + $this->assertSame($modules, $this->store[self::MODULE]); + }//end testUpdateExistingFalseLeavesAnArchivedMatchArchived() + + /** + * A module and usage deleted by hand next to the working ones with the same import key stay in the trash. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testAWorkingMatchWinsOverADeletedOneWithTheSameKey(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $usage = $this->usageOf(moduleUuid: $seven); + // Stored ahead of the working ones, so a first-hit match would find them first. + $this->store[self::MODULE] = ['module-deleted' => array_merge($this->store[self::MODULE][$seven], ['id' => 'module-deleted'])] + $this->store[self::MODULE]; + $this->store[self::USAGE] = ['usage-deleted' => array_merge($this->store[self::USAGE][$usage], ['id' => 'usage-deleted'])] + $this->store[self::USAGE]; + $this->states[self::MODULE]['module-deleted'] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $this->states[self::USAGE]['usage-deleted'] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', sheet: 'Onbeh Applicaties CMDB', cells: ['Applicatie Naam' => 'Terug'])])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); + $this->assertSame($seven, $report['rows'][0]['moduleUuid']); + $this->assertSame('Terug', $this->store[self::MODULE][$seven]['name']); + $this->assertSame([], $this->transitions, 'nothing is restored'); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'module-deleted')); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: 'usage-deleted')); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no module or usage is created'); + }//end testAWorkingMatchWinsOverADeletedOneWithTheSameKey() + + /** + * The fixture round trip of the e2e test: the later export archives 1234 and deletes 2, the original brings both back. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack(): void { + $path = $this->fixture(); + $options = ['municipalityName' => 'Gemeente Voorbeeldstad']; + $this->service()->import(path: $path, options: $options); + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE]), count($this->store[self::CONTACT_PERSON])]; + [$archived, $deleted] = [$this->moduleOf(appId: '1234'), $this->moduleOf(appId: '2')]; + + $report = $this->service()->import(path: CmdbTestSupport::fixtures() . '/topdesk-archived-applications.xlsx', options: $options); + + $this->assertSame(['archived', 'deleted'], array_column($report['rows'], 'outcome')); + $this->assertSame(['1234', '2'], array_column($report['rows'], 'appId')); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $archived)); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $deleted)); + $this->assertSame([], $report['importWarnings']); + + $back = $this->service()->import(path: $path, options: $options); + + $this->assertSame(['unarchived', 'restored'], array_column($back['rows'], 'outcome')); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE]), count($this->store[self::CONTACT_PERSON])]); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $archived)); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $deleted)); + }//end testTheArchivedApplicationsFixtureArchivesDeletesAndBringsBack() + + /** + * The reconciliation reports its own progress phase and advances per application. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testTheReconciliationHasItsOwnProgressPhase(): void { + $this->importOneSevenAndEight(); + $seen = []; + $this->beforeTransition = function () use (&$seen): void { + $progress = $this->tracker->getProgress(operationId: 'cmdb-reconcile-1'); + $seen[] = [$progress['phase'], $progress['processed_items'], $progress['total_items']]; + }; + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-reconcile-1']); + + // One row, then three usages in scope: APPID 1 (on a CMDB sheet), 7 and 8. + $this->assertSame([['reconciling', 2, 4], ['reconciling', 2, 4], ['reconciling', 3, 4], ['reconciling', 3, 4]], $seen); + $this->assertSame('completed', $this->cache['progress_cmdb-reconcile-1']['status']); + $this->assertSame(4, $this->cache['progress_cmdb-reconcile-1']['processed_items']); + }//end testTheReconciliationHasItsOwnProgressPhase() /** * An unknown Applicatie Status drops only that field and warns with column and value. @@ -1806,7 +2670,7 @@ public function testAnImportLeavesAnAuditRecord(): void { options: ['municipalityUuid' => 'muni-1', 'updateExisting' => false, 'publish' => false, 'operationId' => 'cmdb-audit-01', 'fileName' => 'C:\\Users\\beheer\\export.xlsx'] ); - $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false,"publish":false'; + $audit = '"operationId":"cmdb-audit-01","uid":"admin","fileName":"export.xlsx","municipality":"muni-1","municipalityCreated":false,"updateExisting":false,"publish":false,"missingRecords":"archive"'; $started = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import started'))); $finished = array_values(array_filter($this->logLines, static fn (string $line): bool => str_starts_with($line, 'CmdbExportImportService: import finished'))); $this->assertCount(1, $started); @@ -1879,7 +2743,7 @@ public function testOneBadRowDoesNotStopTheOthers(): void { $this->assertSame(['created', 'failed', 'created'], array_column($report['rows'], 'outcome')); $this->assertStringStartsWith('step "module" failed', $report['rows'][1]['reasons'][0]); - $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'warnings' => 0, 'unpublished' => 0], $report['summary']); + $this->assertSame(['rowsRead' => 3, 'processed' => 3, 'created' => 2, 'updated' => 0, 'unchanged' => 0, 'skipped' => 0, 'failed' => 1, 'archived' => 0, 'unarchived' => 0, 'deleted' => 0, 'restored' => 0, 'warnings' => 0, 'unpublished' => 0], $report['summary']); $this->assertCount(2, $this->store[self::MODULE]); }//end testOneBadRowDoesNotStopTheOthers() @@ -2151,7 +3015,7 @@ public function testCancelNeedsARunningImport(): void { }//end testCancelNeedsARunningImport() /** - * A cancel that arrives after the last row's check leaves no flag that a later run with the same id would inherit. + * A cancel that arrives after the last row's check, with records kept, leaves no flag that a later run with the same id would inherit. * * @return void */ @@ -2165,7 +3029,7 @@ public function testACancelAfterTheLastCheckLeavesNoFlag(): void { }; $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', row: 2)])); - $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-late-cancel-1']); + $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-late-cancel-1', 'missingRecords' => 'keep']); $this->assertFalse($report['cancelled'], 'the only row was already done'); $this->assertSame('completed', $this->cache['progress_cmdb-late-cancel-1']['status']); From 7184a4a22df83553bdd36dacacb7f3d1eee1ffe7 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 12:09:15 +0200 Subject: [PATCH 03/87] feat(cmdb-import): offer the missing-records choice and an Archived quick filter The CMDB import section lets the admin archive or delete applications missing from the export, or keep them, and shows the archived, unarchived, deleted and restored outcomes. The Applications and Applications in use pages get an Archived quick filter. Vitest and the Playwright spec cover it; the feature docs and the changelog describe it. Third of three slices of #1281, on top of the reconciliation slice. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 9 ++ docs/features/README.md | 2 +- docs/features/cmdb-import.md | 90 +++++++++++-- src/icons.js | 2 + src/manifest.d/usages.json | 3 +- src/manifest.json | 7 + src/utils/cmdbImport.js | 72 +++++++++- src/utils/cmdbImport.spec.js | 58 +++++++- .../settings/sections/CmdbImport.spec.js | 58 ++++++++ src/views/settings/sections/CmdbImport.vue | 77 ++++++++++- tests/e2e/spec-coverage/cmdb-import.spec.ts | 125 +++++++++++++++++- tests/vitest/cmdbArchiveQuickFilter.spec.js | 62 +++++++++ 12 files changed, 540 insertions(+), 25 deletions(-) create mode 100644 tests/vitest/cmdbArchiveQuickFilter.spec.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 87505881a..69d2c85b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ sheets becomes or updates a module, its vendor, a usage for the municipality and a contact person for its owner; a repeat import matches on APPID, so it updates instead of duplicating. See `docs/features/cmdb-import.md`. +- CMDB import, repeat imports: applications that left the CMDB sheets are + archived when the sheet "Gearchiveerde Applicaties" lists them and moved to + OpenRegister's trash when no sheet does; one that returns is unarchived or + restored, never duplicated. The new option `missingRecords` (`archive`, the + default, or `keep`) is a choice in the CMDB import section, and the report + counts archived, unarchived, deleted and restored applications. The module + (0.3.9) and usage (1.5.7) schemas opt into OpenRegister's archive state, and + the Applications and Applications in use pages get an "Archived" quick + filter. (cmdb-import-archive-reconciliation) ### Changed - EOL feed: the register and schema slugs the feature reads from are corrected. diff --git a/docs/features/README.md b/docs/features/README.md index 7fa964f0a..c57ff8486 100644 --- a/docs/features/README.md +++ b/docs/features/README.md @@ -158,7 +158,7 @@ The ArchiMate integration maps GEMMA Softwarecatalogus objects to ArchiMate appl ## CMDB Import -Import a TOPdesk CMDB export (`.xlsx`) for one municipality from the **CMDB import** section of the admin settings. Every application row of the two CMDB sheets ("Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB") becomes or updates a module, its vendor as a Supplier organisation, a usage that links it to the municipality and records whether maintenance is arranged, and a contact person for its owner (identity in Nextcloud Contacts, never public). A repeat import matches on APPID per municipality, so it updates instead of duplicating, and leaves applications missing from the newer export as they are. The column mapping is declarative JSON executed by OpenRegister's mapping engine. +Import a TOPdesk CMDB export (`.xlsx`) for one municipality from the **CMDB import** section of the admin settings. Every application row of the two CMDB sheets ("Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB") becomes or updates a module, its vendor as a Supplier organisation, a usage that links it to the municipality and records whether maintenance is arranged, and a contact person for its owner (identity in Nextcloud Contacts, never public). A repeat import matches on APPID per municipality, so it updates instead of duplicating. By default it archives the applications that moved to the sheet "Gearchiveerde Applicaties" and moves those on no sheet any more to OpenRegister's trash, and brings either back when it returns; the admin can choose to keep them instead. The Applications and Applications in use pages list the archive under the quick filter "Archived". The column mapping is declarative JSON executed by OpenRegister's mapping engine. See [CMDB import](cmdb-import.md) for the steps, the expected file structure, the error codes and how to adjust the mapping. diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index b9d505abc..d4077f959 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -17,9 +17,11 @@ updates: in Nextcloud Contacts. Owners are never readable by the public. All of it is stored as OpenRegister objects in the stackiq register. Import a -newer export later and the same applications are updated, not duplicated. +newer export later and the same applications are updated, not duplicated; +applications that left the export are archived or moved to the trash (see +[Repeat imports](#repeat-imports)). -Specification: [`openspec/specs/cmdb-export-import/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/cmdb-export-import/spec.md) (change archived in [`openspec/changes/archive/2026-10-05-cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/archive/2026-10-05-cmdb-export-import)). +Specification: [`openspec/specs/cmdb-export-import/spec.md`](https://github.com/ConductionNL/stackiq/blob/development/openspec/specs/cmdb-export-import/spec.md) (change archived in [`openspec/changes/archive/2026-10-05-cmdb-export-import/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/archive/2026-10-05-cmdb-export-import); archiving and deleting on a repeat import in [`openspec/changes/cmdb-import-archive-reconciliation/`](https://github.com/ConductionNL/stackiq/tree/development/openspec/changes/cmdb-import-archive-reconciliation)). ## Who can import @@ -77,9 +79,15 @@ page in stackiq. and the summary counts them as *created unpublished*. Applications that were imported before keep their publication as it is, whichever you choose. -6. Press **Import**. A progress bar shows how many rows have been processed. - **Cancel import** stops the import before the next row; rows that were - already processed stay imported. The section says whether the server +6. **Applications missing from the export.** **Archive or delete them** is + the default: an application of this municipality that is on neither CMDB + sheet any more is archived or moved to the trash after the last row (see + [Repeat imports](#repeat-imports)). Choose **Keep them as they are** to + leave such applications untouched. +7. Press **Import**. A progress bar shows how many rows have been processed. + **Cancel import** stops the import before the next row, or, while the + missing applications are compared, before the next application; what + was already processed stays imported. The section says whether the server accepted the cancel; one pressed before the server has started on the rows cannot take effect yet, and the section says so. @@ -92,11 +100,15 @@ When the import finishes, the section shows: - the **summary**: rows read, created, updated, unchanged, skipped, failed and warnings, and, when publishing was off, how many applications were - created unpublished; + created unpublished; after a repeat import also how many applications + were archived, unarchived, deleted (moved to the trash) or restored, when + there are any; - **warnings for the whole file**, for example an optional column that is missing; - the **rows** table: sheet, row number, APPID, application, outcome, - and the reasons and warnings for that row. Filter it with **Show rows with + and the reasons and warnings for that row. An application archived or + deleted because it left the export has no row number; its sheet is + `Gearchiveerde Applicaties` when that sheet listed it. Filter it with **Show rows with outcome**, and sort it by clicking a column header. It shows 100 rows at a time; **Show 100 more rows** adds the next ones. The application name links to the module in stackiq. @@ -106,8 +118,10 @@ the same organisation. ## The file -The import reads the two CMDB sheets of the export and ignores all others, -including the `Invoer` sheets they are derived from: +The import reads the two CMDB sheets of the export, and of the sheet +`Gearchiveerde Applicaties` only the `APPID` column (see +[Repeat imports](#repeat-imports)). It ignores all other sheets, including +the `Invoer` sheets they are derived from: | Sheet | What it holds | Recorded on the usage | |---|---|---| @@ -213,9 +227,9 @@ colliding. - **Known APPID, nothing changed**: nothing is saved; the row is reported as *unchanged*. Importing the same export twice creates nothing the second time. -- **APPID missing from a newer export**: the application, its usage and - its contact persons are left as they are. They are not changed, depublished - or deleted. +- **APPID missing from a newer export**: see + [Applications that left the export](#applications-that-left-the-export) + below. - Each application keeps exactly one usage for the municipality. - **Known APPID, but the application belongs to another organisation**: an application found by its match key is only updated when the municipality @@ -243,6 +257,55 @@ OpenRegister refuses to save it, that row is reported as *failed* with the step that failed, and the other rows are imported. Importing again completes the failed row. +### Applications that left the export + +With **Archive or delete them** (the default; `missingRecords=archive` in +the API), the import compares the municipality's applications with the +export after the last row. It looks only at the municipality's own usages +and the applications its import created, never at an application only +another organisation uses or one added by hand. For an application whose +APPID is on neither CMDB sheet: + +| The APPID is | The application and its usage | +|---|---| +| on the sheet `Gearchiveerde Applicaties` | are **archived**: OpenRegister's archive state, with the reason `cmdb-import: on sheet "Gearchiveerde Applicaties"`. Reported as *archived*. | +| on no sheet at all | are **deleted**: moved to OpenRegister's trash, where they stay restorable for 30 days. Reported as *deleted*. | +| on no sheet, and the workbook has no sheet `Gearchiveerde Applicaties` (or it has no `APPID` column) | are **archived**, never deleted, and the result warns that the sheet was not found. A workbook saved without that sheet therefore never moves anything to the trash. | + +An archived or deleted application disappears from the stackiq lists, from +OpenCatalogi and from Portaliq's **Software we use**. In stackiq, the quick +filter **Archived** on the **Applications** and **Applications in use** pages +lists the archive. An OpenCatalogi caller finds archived applications by +adding `_archived=true` to its request. Contact persons are never archived +or deleted: an owner is shared with the municipality's other applications. +An application that is already in the state the export asks for is left +alone and not reported again. + +When an archived or deleted application **returns** in a later export, it is +taken out of the archive (*unarchived*) or out of the trash (*restored*), +and then updated as usual. No second application is created. With **Update +existing records** off, it is skipped as `exists` and stays archived or in +the trash. When an application was deleted by hand and a later import +created it again, the working one is updated and the deleted one stays in +the trash. + +The comparison runs only when the import went through all rows: an import +cancelled or failed before the comparison archives or deletes nothing. +**Cancel import** during the comparison stops it before the next +application; what was archived or deleted until then stays so, and the +result lists it. The progress bar shows the +step as *Checking the applications missing from the export*. Each +application is handled on its own; one that cannot be archived is reported +as *failed* with the step (`archive`, `restore` or `delete`) and the others +continue. Archiving needs module schema 0.3.9 and usage schema 1.5.7 or +later, which declare that their objects can be archived; on an older register configuration each application fails +with the step `archive`, and **Force Update** at the top of the stackiq admin +settings fixes it. + +With **Keep them as they are** (`missingRecords=keep`), the application, its +usage and its contact persons are left as they are: not changed, archived, +depublished or deleted. + ## Owners The owner becomes a **contact person of the municipality**, never a @@ -293,7 +356,8 @@ and the section shows the reason and the error code. | `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | | `UPLOAD_FAILED` | The file reached the server but could not be stored there. | Try again; the Nextcloud log has the details. | -| `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | +| `MISSING_RECORDS_UNSUPPORTED` | The request asked for something other than `keep` or `archive` for records missing from the export, for example `mark` or `remove`. The details list the accepted values. | Not reachable from the section; reported for API callers. | +| `ARCHIVE_UNAVAILABLE` | **Archive or delete them** was chosen, but the OpenRegister on this server cannot archive or restore objects (it is missing or too old). Nothing was read or written. | Choose **Keep them as they are** and import again, or update OpenRegister. | | `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | | `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | | `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | diff --git a/src/icons.js b/src/icons.js index 865c514c0..34ffbfdfb 100644 --- a/src/icons.js +++ b/src/icons.js @@ -17,6 +17,7 @@ import AccountGroup from 'vue-material-design-icons/AccountGroup.vue' import AccountMultiple from 'vue-material-design-icons/AccountMultiple.vue' import AlertCircle from 'vue-material-design-icons/AlertCircle.vue' import ApplicationOutline from 'vue-material-design-icons/ApplicationOutline.vue' +import ArchiveOutline from 'vue-material-design-icons/ArchiveOutline.vue' import ArrowRight from 'vue-material-design-icons/ArrowRight.vue' import BookOpenVariant from 'vue-material-design-icons/BookOpenVariant.vue' import BookOpenVariantOutline from 'vue-material-design-icons/BookOpenVariantOutline.vue' @@ -79,6 +80,7 @@ export default { AccountMultiple, AlertCircle, ApplicationOutline, + ArchiveOutline, ArrowRight, BookOpenVariant, BookOpenVariantOutline, diff --git a/src/manifest.d/usages.json b/src/manifest.d/usages.json index 66990ad7d..eec9d1242 100644 --- a/src/manifest.d/usages.json +++ b/src/manifest.d/usages.json @@ -35,7 +35,8 @@ { "label": "Planned", "filter": { "status": "Planned" } }, { "label": "In production", "filter": { "status": "In production" }, "icon": "CheckCircle" }, { "label": "To be phased out", "filter": { "status": "To be phased out" }, "icon": "AlertCircle" }, - { "label": "Phased out", "filter": { "status": "Phased out" } } + { "label": "Phased out", "filter": { "status": "Phased out" } }, + { "label": "Archived", "filter": { "_archived": "true" }, "icon": "ArchiveOutline" } ], "sidebar": { "enabled": true, "showMetadata": true }, "documentationUrl": "https://stackiq.conduction.nl" diff --git a/src/manifest.json b/src/manifest.json index 22b321684..7df63875f 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -662,6 +662,13 @@ "dpiaStatus": ["not required", "required"] }, "icon": "AlertCircle" + }, + { + "label": "Archived", + "filter": { + "_archived": "true" + }, + "icon": "ArchiveOutline" } ] } diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index ab7fa7080..64d7df36d 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -42,8 +42,27 @@ export const PROFILE_DEFAULTS = Object.freeze({ */ export const AS_TEXT = Object.freeze({ escape: false, sanitize: false }) -/** Every row outcome the report can carry, in display order. */ -export const OUTCOMES = ['created', 'updated', 'unchanged', 'skipped', 'failed'] +/** + * Every row outcome the report can carry, in display order: the outcomes of a + * sheet row, then those of an application that left the export or came back. + */ +export const OUTCOMES = [ + 'created', + 'updated', + 'unchanged', + 'skipped', + 'failed', + 'unarchived', + 'restored', + 'archived', + 'deleted', +] + +/** + * What the import may do with applications missing from the export (the + * profile's `missingRecords`), the default first. + */ +export const MISSING_RECORDS_MODES = Object.freeze(['archive', 'keep']) /** * The words for one row outcome. @@ -51,6 +70,7 @@ export const OUTCOMES = ['created', 'updated', 'unchanged', 'skipped', 'failed'] * @param {string} outcome The outcome key from the report * @return {string} The label * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome-req-cmdb-011 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ export function outcomeLabel(outcome) { switch (outcome) { @@ -64,6 +84,14 @@ export function outcomeLabel(outcome) { return t('stackiq', 'Skipped') case 'failed': return t('stackiq', 'Failed') + case 'unarchived': + return t('stackiq', 'Unarchived') + case 'restored': + return t('stackiq', 'Restored') + case 'archived': + return t('stackiq', 'Archived') + case 'deleted': + return t('stackiq', 'Deleted') default: return String(outcome ?? '') } @@ -141,16 +169,19 @@ export function checkFile(file) { * @param {{uuid: string|null, name: string}} options.municipality The chosen municipality: an existing one has a uuid, a new one only a name * @param {boolean} options.updateExisting Whether matched rows are updated * @param {boolean} [options.publish] Whether the modules the import creates are published; true when left out + * @param {string} [options.missingRecords] What happens to applications missing from the export: 'archive' (when left out) or 'keep' * @param {string} options.operationId The progress operation id * @return {FormData} The body * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-every-import-shall-have-exactly-one-consuming-municipality-chosen-by-the-admin-req-cmdb-004 * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-a-newly-created-module-shall-get-a-publicationdate-when-the-admin-publishes-and-an-existing-one-shall-keep-its-own-req-cmdb-007 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-records-missing-from-a-newer-export-shall-be-archived-by-default-or-kept-on-request-req-cmdb-012 */ export function buildImportForm({ file, municipality, updateExisting, publish = true, + missingRecords = MISSING_RECORDS_MODES[0], operationId, }) { const form = new FormData() @@ -162,7 +193,7 @@ export function buildImportForm({ } form.append('updateExisting', updateExisting ? 'true' : 'false') form.append('publish', publish ? 'true' : 'false') - form.append('missingRecords', 'keep') + form.append('missingRecords', missingRecords) form.append('operationId', operationId) return form } @@ -465,6 +496,7 @@ const KNOWN_ERRORS = new Set([ 'SCHEMA_OUTDATED', 'IMPORT_IN_PROGRESS', 'MUNICIPALITY_AMBIGUOUS', + 'ARCHIVE_UNAVAILABLE', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -626,7 +658,7 @@ export function errorText(error) { return { title: t( 'stackiq', - 'Records missing from the export can only be kept.', + 'Applications missing from the export can only be archived or kept.', ), hint: '', } @@ -807,6 +839,17 @@ export function errorText(error) { 'Choose the municipality from the list instead of typing its name. Nothing was imported.', ), } + case 'ARCHIVE_UNAVAILABLE': + return { + title: t( + 'stackiq', + 'Applications missing from the export cannot be archived.', + ), + hint: t( + 'stackiq', + 'OpenRegister is missing or too old to archive objects. Choose "Keep them as they are" and import again, or update OpenRegister. Nothing was imported.', + ), + } case 'OPERATION_NOT_FOUND': return { title: t('stackiq', 'This import is no longer running.'), @@ -886,9 +929,13 @@ export function cancelFailureText(error) { * server has set them, because the tracker's own percentage is weighted by * the phases of the ArchiMate import. * + * In the phase `reconciling` the counts also hold the applications the + * import checks against the export after its rows. + * * @param {object|null} progress The snapshot from `GET /api/progress/{operationId}` * @return {{percentage: number, detail: string}|null} The view, or null before any progress * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-a-running-import-shall-report-its-progress-and-shall-stop-when-cancelled-req-cmdb-013 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ export function cmdbProgressView(progress) { if (!progress) { @@ -900,6 +947,17 @@ export function cmdbProgressView(progress) { total > 0 ? Math.min(100, Math.round((processed / total) * 100)) : Number(progress.percentage) || 0 + if (total > 0 && progress.phase === 'reconciling') { + return { + percentage, + detail: t( + 'stackiq', + 'Checking the applications missing from the export: {processed} of {total}', + { processed, total }, + AS_TEXT, + ), + } + } return { percentage, detail: @@ -960,9 +1018,13 @@ export function sortReportRows(rows, key, order) { /** * The rows of the report as the table shows them. * + * An application the import archived or deleted after its rows has row 0 + * in the report and no row number in the table. + * * @param {Array} rows The report's `rows` * @return {Array} The table rows * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-each-row-shall-be-processed-in-isolation-and-reported-with-its-outcome-req-cmdb-011 + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ export function reportRows(rows) { if (!Array.isArray(rows)) { @@ -971,7 +1033,7 @@ export function reportRows(rows) { return rows.map((row, index) => ({ key: `${row.sheet ?? ''}:${row.row ?? index}:${index}`, sheet: String(row.sheet ?? ''), - row: row.row ?? '', + row: Number(row.row) > 0 ? row.row : '', appId: String(row.appId ?? ''), name: String(row.name ?? ''), outcome: String(row.outcome ?? ''), diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 8f566d6cb..9377f3b59 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -22,6 +22,7 @@ import { interruptedImportError, isKnownError, makeCmdbOperationId, + MISSING_RECORDS_MODES, municipalityOptions, normaliseError, outcomeLabel, @@ -58,6 +59,7 @@ const SERVER_CODES = [ 'SCHEMA_OUTDATED', 'IMPORT_IN_PROGRESS', 'MUNICIPALITY_AMBIGUOUS', + 'ARCHIVE_UNAVAILABLE', 'OPERATION_NOT_FOUND', 'IMPORT_INTERRUPTED', 'NOT_SIGNED_IN', @@ -198,10 +200,26 @@ describe('buildImportForm', () => { expect(form.get('municipalityUuid')).toBe('uuid-1') expect(form.has('municipalityName')).toBe(false) expect(form.get('updateExisting')).toBe('true') - expect(form.get('missingRecords')).toBe('keep') + expect(form.get('missingRecords')).toBe('archive') expect(form.get('operationId')).toBe('cmdb-abcdefgh') }) + it('sends the choice for applications missing from the export, archive by default', () => { + const options = { + file, + municipality: { uuid: 'uuid-1', name: 'Tilburg' }, + updateExisting: true, + operationId: 'cmdb-abcdefgh', + } + expect(MISSING_RECORDS_MODES).toEqual(['archive', 'keep']) + expect(buildImportForm(options).get('missingRecords')).toBe('archive') + expect( + buildImportForm({ ...options, missingRecords: 'keep' }).get( + 'missingRecords', + ), + ).toBe('keep') + }) + it('sends a typed new municipality by name, and updateExisting as an explicit false', () => { const form = buildImportForm({ file, @@ -642,6 +660,19 @@ describe('cmdbProgressView', () => { ).toEqual({ percentage: 25, detail: '1 of 4 rows processed' }) }) + it('says when the import checks the applications missing from the export', () => { + expect( + cmdbProgressView({ + phase: 'reconciling', + processed_items: 3, + total_items: 4, + }), + ).toEqual({ + percentage: 75, + detail: 'Checking the applications missing from the export: 3 of 4', + }) + }) + it('shows nothing before any progress', () => { expect(cmdbProgressView(null)).toBeNull() }) @@ -683,6 +714,31 @@ describe('the report table', () => { expect(reportRows(null)).toEqual([]) }) + it('shows no row number for an application archived or deleted after the rows', () => { + const [row] = reportRows([ + { + sheet: 'Gearchiveerde Applicaties', + row: 0, + appId: '7', + name: 'Zeven', + outcome: 'archived', + reasons: [], + warnings: [], + moduleUuid: 'm-7', + }, + ]) + expect(row.row).toBe('') + expect(outcomeLabel('archived')).toBe('Archived') + expect(OUTCOMES).toEqual( + expect.arrayContaining([ + 'archived', + 'unarchived', + 'deleted', + 'restored', + ]), + ) + }) + it('sorts numbers by value and keeps the report order for ties', () => { const rows = [ { key: 'a', row: 10, sheet: 'B' }, diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index 931f7edbc..7a16b9858 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -147,3 +147,61 @@ describe('CmdbImport municipality chooser', () => { expect(wrapper.vm.report).toBe(null) }) }) + +/** + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +describe('CmdbImport applications missing from the export', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + it('archives by default and posts the choice the admin made', async () => { + const wrapper = await mountSection() + axios.post.mockResolvedValue({ + data: { success: true, summary: {}, rows: [] }, + }) + wrapper.vm.municipality = wrapper.vm.municipalityOptions[0] + wrapper.vm.selectedFile = new File(['x'], 'export.xlsx') + + expect(wrapper.vm.missingRecords).toBe('archive') + await wrapper.vm.startImport() + wrapper.vm.missingRecords = 'keep' + await wrapper.vm.startImport() + + expect( + axios.post.mock.calls.map(([, form]) => form.get('missingRecords')), + ).toEqual(['archive', 'keep']) + }) + + it('shows the archived, unarchived, deleted and restored counts only when there are any', async () => { + const wrapper = await mountSection() + const tileKeys = () => wrapper.vm.summaryTiles.map((tile) => tile.key) + + wrapper.vm.report = { summary: { rowsRead: 2, created: 2 }, rows: [] } + expect(tileKeys()).not.toContain('archived') + + wrapper.vm.report = { + summary: { + rowsRead: 1, + archived: 1, + deleted: 1, + unarchived: 0, + restored: 2, + }, + rows: [], + } + expect(tileKeys()).toEqual( + expect.arrayContaining(['archived', 'deleted', 'restored']), + ) + expect(tileKeys()).not.toContain('unarchived') + expect(wrapper.vm.outcomeFilterOptions.map((option) => option.id)).toEqual( + expect.arrayContaining([ + 'archived', + 'unarchived', + 'deleted', + 'restored', + ]), + ) + }) +}) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index f8bbc1e4b..00af68892 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -143,6 +143,40 @@ }}

+
+ + {{ t('stackiq', 'Applications missing from the export') }} + + + {{ t('stackiq', 'Archive or delete them') }} + + + {{ t('stackiq', 'Keep them as they are') }} + +

+ {{ + t( + 'stackiq', + 'An application of this municipality that is on neither CMDB sheet any more is archived when the sheet "Gearchiveerde Applicaties" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.', + ) + }} +

+
@@ -304,7 +338,7 @@
@@ -149,6 +162,7 @@ import { loadCmdbMapping, mappingRows, normaliseError, + packRules, packTargetLabel, } from '../../../utils/cmdbImport.js' @@ -260,6 +274,7 @@ export default { { key: 'required', label: t('stackiq', 'Required') }, { key: 'transform', label: t('stackiq', 'Transformation') }, { key: 'details', label: t('stackiq', 'Details') }, + { key: 'written', label: t('stackiq', 'Written') }, ] }, }, @@ -277,6 +292,7 @@ export default { methods: { t, mappingRows, + packRules, packTargetLabel, /** @@ -350,6 +366,13 @@ export default { overflow-wrap: anywhere; } +.cmdb-import-mapping__rules { + margin: 0.5rem 0 0; + padding-inline-start: 1.25rem; + list-style: disc; + font-size: 0.875rem; +} + .cmdb-import-mapping__details { margin: 0; padding-inline-start: 1.25rem; diff --git a/tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts b/tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts index df1164397..a4c41c44b 100644 --- a/tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts +++ b/tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts @@ -109,6 +109,20 @@ test.describe('CMDB import mapping (read-only)', () => { expect(status.target).toBe('status') expect(status.transform.type).toBe('lookup') expect(status.transform.map['In productie']).toBe('In production') + + // The write rules the import applies besides the mapping. + expect(usage.createOnlyFields).toEqual([ + 'interneAnnotation', + 'timeClassification', + ]) + const moduleRules = body.packs.find( + (pack: { target: string }) => pack.target === 'module', + ) + expect(moduleRules.createOnlyDefaults).toEqual({ type: 'Application' }) + expect(moduleRules.neverWritten).toEqual([ + 'publicationDate', + 'depublicationDate', + ]) } finally { await ctx.dispose() } diff --git a/tests/vitest/cmdbImportMapping.spec.js b/tests/vitest/cmdbImportMapping.spec.js index b7ad8715f..ee8e70f16 100644 --- a/tests/vitest/cmdbImportMapping.spec.js +++ b/tests/vitest/cmdbImportMapping.spec.js @@ -70,6 +70,7 @@ vi.mock('@conduction/nextcloud-vue', () => ({ const { mappingRows, mappingSheetNames, + packRules, packTargetLabel, PROFILE_DEFAULTS, transformDetails, @@ -85,7 +86,7 @@ const CmdbImportMapping = ( * @return {{profile: object, packs: Array}} The answer */ function answer() { - const pack = (target, fieldMappings = []) => ({ + const pack = (target, fieldMappings = [], rules = {}) => ({ target, file: `topdesk-${target}.json`, id: `stackiq-topdesk-${target}`, @@ -93,6 +94,11 @@ function answer() { version: '1.0.0', description: '', fieldMappings, + defaults: {}, + createOnlyDefaults: {}, + createOnlyFields: [], + neverWritten: [], + ...rules, }) return { profile: { @@ -115,8 +121,14 @@ function answer() { required: true, transform: { type: 'trim' }, }, - ]), - pack('manufacturer'), + ], { + createOnlyDefaults: { type: 'Application' }, + createOnlyFields: ['type'], + neverWritten: ['publicationDate', 'depublicationDate'], + }), + pack('manufacturer', [], { + defaults: { type: 'Supplier', status: 'Active' }, + }), pack('municipality'), pack('usage', [ { @@ -129,7 +141,13 @@ function answer() { default: null, }, }, - ]), + { + source: 'Classificatie', + target: 'timeClassification', + required: false, + transform: { type: 'trim' }, + }, + ], { createOnlyFields: ['timeClassification'] }), pack('businessOwner'), ], } @@ -178,6 +196,30 @@ describe('CmdbImportMapping', () => { expect(usage.text()).toContain('status') expect(usage.text()).toContain('Lookup') expect(usage.text()).toContain('In productie → In production') + // The write rules the import applies besides the mapping (Review #1288 f2). + const writtenOf = (table) => + table + .findAll('[data-testid="cmdb-import-mapping-written"]') + .map((cell) => cell.text()) + expect(writtenOf(usage)).toEqual([ + 'On create and update', + 'Only on create, or when empty', + ]) + const rules = (target) => + wrapper + .find(`[data-testid="cmdb-import-mapping-${target}"]`) + .element.parentElement.querySelectorAll( + '[data-testid="cmdb-import-mapping-rules"] li', + ) + expect(Array.from(rules('module'), (li) => li.textContent.trim())).toEqual([ + 'Set only when the record is created: type = Application', + 'Never written on an existing record: publicationDate, depublicationDate', + ]) + expect(Array.from(rules('manufacturer'), (li) => li.textContent.trim())).toEqual([ + 'Pack defaults: type = Supplier, status = Active', + ]) + expect(rules('usage')).toHaveLength(1) + expect(rules('municipality')).toHaveLength(0) expect( wrapper.find('[data-testid="cmdb-import-mapping-profile"]').text(), ).toContain('Sheet A, Sheet B') @@ -233,6 +275,23 @@ describe('cmdbImport mapping helpers', () => { ).toEqual(PROFILE_DEFAULTS.sheets) }) + it('words the write rules of a pack, and tolerates an answer without them', () => { + expect(packRules({})).toEqual([]) + expect(packRules({ defaults: [], createOnlyDefaults: [], createOnlyFields: [] })).toEqual([]) + expect( + packRules({ + createOnlyDefaults: { type: 'Application' }, + createOnlyFields: ['type', 'note'], + }), + ).toEqual([ + 'Set only when the record is created: type = Application', + 'Written on create, or on update only when the stored value is empty: note', + ]) + expect(mappingRows({ fieldMappings: [{ source: 'A', target: 'x' }], neverWritten: ['x'] })[0].written).toBe( + 'Never on an existing record', + ) + }) + it('describes every transformation the packs use', () => { expect(transformLabel(undefined)).toBe('As is') expect(transformLabel('bool-map')).toBe('Yes/no lookup') @@ -262,7 +321,7 @@ describe('cmdbImport mapping helpers', () => { it('builds one row per field mapping and names each target', () => { const rows = mappingRows(answer().packs[3]) - expect(rows).toHaveLength(1) + expect(rows).toHaveLength(2) expect(rows[0]).toMatchObject({ source: 'Applicatie Status', target: 'status', @@ -273,6 +332,8 @@ describe('cmdbImport mapping helpers', () => { 'In productie → In production', 'Any other value: —', ]) + expect(rows[0].written).toBe('On create and update') + expect(rows[1].written).toBe('Only on create, or when empty') expect(mappingRows({})).toEqual([]) expect(packTargetLabel('businessOwner')).toBe( 'Business owner (contact person)', From 2e381f7b195f46b9ba58f8176e3034a328349585 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:22:32 +0200 Subject: [PATCH 23/87] fix(cmdb-import): name every sheet the profile reads in the help texts mappingSheetNames() fell back to the shipped names unless the profile had at least two sheets, and the file help and the info panel only read the first two entries and tied "(applications without arranged maintenance)" to position 0. A profile with one sheet showed two sheets the import does not read, one with three lost the third, and a reordered profile mislabelled both. The profile's list is now used whenever it names a sheet, and both texts are built from the whole list, with a singular and a plural form and no position-bound description. The two replaced strings are removed from en and nl; the vitest pins the 0, 1, 2 and 3 sheet cases. Review #1289 f1 Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 8 +- l10n/en.json | 8 +- l10n/nl.js | 8 +- l10n/nl.json | 8 +- .../cmdb-import-mapping-view/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 3 +- .../changes/cmdb-import-mapping-view/tasks.md | 1 + src/utils/cmdbImport.js | 78 +++++++++++++++++-- src/views/settings/sections/CmdbImport.vue | 51 ++++++------ tests/vitest/cmdbImportMapping.spec.js | 42 ++++++++-- 10 files changed, 159 insertions(+), 50 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 8c8fc1d29..ad4e755b9 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1088,8 +1088,6 @@ OC.L10N.register( "The file is larger than the server accepts.": "The file is larger than the server accepts.", "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.", "Split the export and import the parts one after the other.": "Split the export and import the parts one after the other.", - "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", "The page got no answer from the import.": "The page got no answer from the import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", @@ -1173,7 +1171,11 @@ OC.L10N.register( "Pack defaults: {values}": "Pack defaults: {values}", "Set only when the record is created: {values}": "Set only when the record is created: {values}", "Written on create, or on update only when the stored value is empty: {fields}": "Written on create, or on update only when the stored value is empty: {fields}", - "Never written on an existing record: {fields}": "Never written on an existing record: {fields}" + "Never written on an existing record: {fields}": "Never written on an existing record: {fields}", + "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.", + "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.", + "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 14cf19581..c18324251 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1087,8 +1087,6 @@ "The file is larger than the server accepts.": "The file is larger than the server accepts.", "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.", "Split the export and import the parts one after the other.": "Split the export and import the parts one after the other.", - "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", "The page got no answer from the import.": "The page got no answer from the import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", @@ -1172,6 +1170,10 @@ "Pack defaults: {values}": "Pack defaults: {values}", "Set only when the record is created: {values}": "Set only when the record is created: {values}", "Written on create, or on update only when the stored value is empty: {fields}": "Written on create, or on update only when the stored value is empty: {fields}", - "Never written on an existing record: {fields}": "Never written on an existing record: {fields}" + "Never written on an existing record: {fields}": "Never written on an existing record: {fields}", + "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.", + "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.", + "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored." } } diff --git a/l10n/nl.js b/l10n/nl.js index 7f0a389f7..dbd4d8771 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1156,8 +1156,6 @@ OC.L10N.register( "The file is larger than the server accepts.": "Het bestand is groter dan de server accepteert.", "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar.", "Split the export and import the parts one after the other.": "Splits de export en importeer de delen na elkaar.", - "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", @@ -1241,7 +1239,11 @@ OC.L10N.register( "Pack defaults: {values}": "Standaardwaarden van het pack: {values}", "Set only when the record is created: {values}": "Alleen ingesteld bij het aanmaken van het record: {values}", "Written on create, or on update only when the stored value is empty: {fields}": "Geschreven bij aanmaken, of bij bijwerken alleen als de opgeslagen waarde leeg is: {fields}", - "Never written on an existing record: {fields}": "Nooit geschreven op een bestaand record: {fields}" + "Never written on an existing record: {fields}": "Nooit geschreven op een bestaand record: {fields}", + "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad {sheets}. Standaard mag het bestand hoogstens {size} zijn.", + "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met een van de tabbladen {sheets}. Standaard mag het bestand hoogstens {size} zijn.", + "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "Het tabblad {sheets} wordt gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 6d9121ab5..4f89dd5b9 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1155,8 +1155,6 @@ "The file is larger than the server accepts.": "Het bestand is groter dan de server accepteert.", "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other.": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar.", "Split the export and import the parts one after the other.": "Splits de export en importeer de delen na elkaar.", - "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", @@ -1240,6 +1238,10 @@ "Pack defaults: {values}": "Standaardwaarden van het pack: {values}", "Set only when the record is created: {values}": "Alleen ingesteld bij het aanmaken van het record: {values}", "Written on create, or on update only when the stored value is empty: {fields}": "Geschreven bij aanmaken, of bij bijwerken alleen als de opgeslagen waarde leeg is: {fields}", - "Never written on an existing record: {fields}": "Nooit geschreven op een bestaand record: {fields}" + "Never written on an existing record: {fields}": "Nooit geschreven op een bestaand record: {fields}", + "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad {sheets}. Standaard mag het bestand hoogstens {size} zijn.", + "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met een van de tabbladen {sheets}. Standaard mag het bestand hoogstens {size} zijn.", + "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "Het tabblad {sheets} wordt gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd." } } diff --git a/openspec/changes/cmdb-import-mapping-view/design.md b/openspec/changes/cmdb-import-mapping-view/design.md index 225ede564..23b3357b9 100644 --- a/openspec/changes/cmdb-import-mapping-view/design.md +++ b/openspec/changes/cmdb-import-mapping-view/design.md @@ -50,7 +50,7 @@ The write rules the import applies besides the mapping are answered per pack, bu ### D4. The section takes the sheet names from the endpoint, with the constant as fallback -`CmdbImportMapping.vue` fetches once on creation and emits `loaded` with the answer; `CmdbImport.vue` computes `sheetNames` from it, or from `PROFILE_DEFAULTS.sheets` until it arrives or when it fails. The constant stays, as the design of cmdb-export-import intended: a fallback, never the source. +`CmdbImportMapping.vue` fetches once on creation and emits `loaded` with the answer; `CmdbImport.vue` computes `sheetNames` from it, or from `PROFILE_DEFAULTS.sheets` until it arrives or when it fails. The constant stays, as the design of cmdb-export-import intended: a fallback, never the source. The help text and the info text name every sheet the profile lists, whatever their number: they are built from the list (`quotedSheetList`), not from its first two entries, and say nothing that depends on a sheet's position. ### D5. One component, fetched once, collapsed by default diff --git a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md index 7ff22e728..68351e7cc 100644 --- a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md @@ -59,7 +59,7 @@ Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\Mi ### Requirement: The admin settings SHALL offer a CMDB import section (REQ-CMDB-014) -Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. The section SHALL also show the mapping the import uses in a collapsible block "Mapping (read-only)" (REQ-CMDB-020), and the sheet names in its help text SHALL come from that mapping, with the shipped names as the fallback until the mapping has loaded or when it cannot be loaded. Every control SHALL have a visible label, and every string SHALL be translatable. +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. The section SHALL also show the mapping the import uses in a collapsible block "Mapping (read-only)" (REQ-CMDB-020), and the sheet names in its help text SHALL come from that mapping, every sheet it names and in its order, whatever their number, with the shipped names as the fallback until the mapping has loaded or when it cannot be loaded. Every control SHALL have a visible label, and every string SHALL be translatable. (Previously: the section had no mapping block, and the sheet names in its help text were a constant in the page, which could drift from the profile on the server.) @@ -78,6 +78,7 @@ Stackiq's admin settings page SHALL show a section "CMDB import", rendered by th - **GIVEN** a Nextcloud admin on stackiq's admin settings page - **WHEN** the mapping endpoint has answered - **THEN** the help text under the file control SHALL name the sheets the endpoint returned +- **AND** a profile with one, two or three sheets SHALL give a help text and an info text that name exactly those sheets, with no sheet the profile does not name and no description tied to a sheet's position - **AND** when the endpoint fails, it SHALL name the shipped sheets and the mapping block SHALL show the error ## Non-Functional Requirements diff --git a/openspec/changes/cmdb-import-mapping-view/tasks.md b/openspec/changes/cmdb-import-mapping-view/tasks.md index 36fe883c7..4e5d964bc 100644 --- a/openspec/changes/cmdb-import-mapping-view/tasks.md +++ b/openspec/changes/cmdb-import-mapping-view/tasks.md @@ -31,6 +31,7 @@ Spec: `openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.m - GIVEN the endpoint's answer WHEN the block is expanded THEN one table per pack renders with `data-testid="cmdb-import-mapping-"`, the usage table holding "Applicatie Status" → `status` with the lookup pairs - GIVEN a 503 WHEN the block loads THEN it shows the error and the code and no table - GIVEN the answer WHEN the section renders its help text THEN the sheet names are the endpoint's; before and on failure they are `PROFILE_DEFAULTS.sheets` + - GIVEN a profile with one, two or three sheets WHEN the section renders its help text and its info text THEN both name exactly those sheets in the profile's order - [x] Implement - [x] Test diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 07f494e7d..883d15e56 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -1009,12 +1009,13 @@ export async function loadCmdbMapping({ http }) { } /** - * The names of the source sheets: the profile's when the mapping has loaded, - * the shipped defaults until then, when it could not be loaded, or when it - * names fewer than the two sheets the help texts name. + * The names of the source sheets: the profile's when the mapping has loaded + * and names at least one sheet, the shipped defaults until then or when it + * could not be loaded. Every sheet the profile names is returned, in the + * profile's order, so the help texts and the block show the same list. * * @param {object|null} mapping The mapping endpoint's answer, or null - * @return {Array} At least the two default names + * @return {Array} The sheet names, never empty * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ export function mappingSheetNames(mapping) { @@ -1024,7 +1025,74 @@ export function mappingSheetNames(mapping) { typeof sheet === 'string' ? sheet : String(sheet?.name ?? ''), ) .filter((name) => name !== '') - return names.length >= 2 ? names : [...PROFILE_DEFAULTS.sheets] + return names.length >= 1 ? names : [...PROFILE_DEFAULTS.sheets] +} + +/** + * The sheet names as one quoted list: "A", "B", "C". + * + * @param {Array} names The sheet names + * @return {string} The list + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +export function quotedSheetList(names) { + return names.map((name) => `"${name}"`).join(', ') +} + +/** + * The help text under the file chooser: which sheets the workbook needs and + * the default size limit. It names every sheet the profile reads, whatever + * their number, and does not depend on their order. + * + * @param {Array} names The sheet names, from mappingSheetNames() + * @param {number} maxFileBytes The default size limit in bytes + * @return {string} The text + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +export function fileHelpText(names, maxFileBytes) { + const values = { + sheets: quotedSheetList(names), + size: formatMegabytes(maxFileBytes), + } + if (names.length === 1) { + return t( + 'stackiq', + 'Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.', + values, + AS_TEXT, + ) + } + return t( + 'stackiq', + 'Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.', + values, + AS_TEXT, + ) +} + +/** + * The sentence of the info panel that says which sheets are read. + * + * @param {Array} names The sheet names, from mappingSheetNames() + * @return {string} The text + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +export function sheetsReadText(names) { + const values = { sheets: quotedSheetList(names) } + if (names.length === 1) { + return t( + 'stackiq', + 'The sheet {sheets} is read; other sheets, including the "Invoer" sheets, are ignored.', + values, + AS_TEXT, + ) + } + return t( + 'stackiq', + 'The sheets {sheets} are read; other sheets, including the "Invoer" sheets, are ignored.', + values, + AS_TEXT, + ) } /** diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 8b5a6ff62..8fb27fdc5 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -84,18 +84,7 @@ }}

- {{ - t( - 'stackiq', - 'Excel workbook (.xlsx) with the sheet "{first}" or "{second}". By default the file may be at most {size}.', - { - first: sheetNames[0], - second: sheetNames[1], - size: formatMegabytes(profileDefaults.maxFileBytes), - }, - asText, - ) - }} + {{ fileHelp }}

@@ -380,17 +369,7 @@

{{ t('stackiq', 'The file') }}

  • - {{ - t( - 'stackiq', - 'The sheets "{first}" (applications without arranged maintenance) and "{second}" (with arranged maintenance) are read; other sheets, including the "Invoer" sheets, are ignored.', - { - first: sheetNames[0], - second: sheetNames[1], - }, - asText, - ) - }} + {{ sheetsReadInfo }}
  • {{ @@ -449,8 +428,8 @@ import { checkFile, cmdbProgressView, errorText, + fileHelpText, followInterruptedImport, - formatMegabytes, importUrl, interruptedImportError, isKnownError, @@ -464,6 +443,7 @@ import { PROFILE_DEFAULTS, REPORT_PAGE_SIZE, reportRows, + sheetsReadText, sortReportRows, typedMunicipalityOption, } from '../../../utils/cmdbImport.js' @@ -539,13 +519,33 @@ export default { * profile's once the mapping block has loaded it, the shipped names * until then or when it could not be loaded. * - * @return {Array} At least two names + * @return {Array} Every sheet the profile names, at least one * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 */ sheetNames() { return mappingSheetNames(this.mapping) }, + /** + * The help text under the file chooser, naming every sheet the import reads. + * + * @return {string} The text + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ + fileHelp() { + return fileHelpText(this.sheetNames, this.profileDefaults.maxFileBytes) + }, + + /** + * The info-panel sentence naming the sheets the import reads. + * + * @return {string} The text + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ + sheetsReadInfo() { + return sheetsReadText(this.sheetNames) + }, + /** * Whether everything the import needs has been chosen. * @@ -786,7 +786,6 @@ export default { methods: { t, - formatMegabytes, moduleUrl, outcomeLabel, diff --git a/tests/vitest/cmdbImportMapping.spec.js b/tests/vitest/cmdbImportMapping.spec.js index ee8e70f16..e7c21781a 100644 --- a/tests/vitest/cmdbImportMapping.spec.js +++ b/tests/vitest/cmdbImportMapping.spec.js @@ -68,11 +68,13 @@ vi.mock('@conduction/nextcloud-vue', () => ({ })) const { + fileHelpText, mappingRows, mappingSheetNames, packRules, packTargetLabel, PROFILE_DEFAULTS, + sheetsReadText, transformDetails, transformLabel, } = await import('../../src/utils/cmdbImport.js') @@ -269,11 +271,41 @@ describe('cmdbImport mapping helpers', () => { ) }) - it('keeps the shipped sheet names when the answer names only one sheet', () => { - expect( - mappingSheetNames({ profile: { sheets: [{ name: 'Sheet A' }] } }), - ).toEqual(PROFILE_DEFAULTS.sheets) - }) + it.each([ + [0, PROFILE_DEFAULTS.sheets], + [1, ['Sheet A']], + [2, ['Sheet A', 'Sheet B']], + [3, ['Sheet A', 'Sheet B', 'Sheet C']], + ])( + 'names every sheet of a profile with %i sheets in both help texts, in its order', + (count, expected) => { + const names = ['Sheet A', 'Sheet B', 'Sheet C'].slice(0, count) + const mapping = { + profile: { sheets: names.map((name) => ({ name })) }, + } + const sheets = mappingSheetNames(mapping) + expect(sheets).toEqual(expected) + + const quoted = expected.map((name) => `"${name}"`).join(', ') + const help = fileHelpText(sheets, 10 * 1024 * 1024) + expect(help).toContain(quoted) + expect(help).toContain('at most 10 MB') + expect(help).toContain( + expected.length === 1 ? 'with the sheet ' : 'with one of the sheets ', + ) + const info = sheetsReadText(sheets) + expect(info).toContain(quoted) + expect(info).toContain(expected.length === 1 ? 'is read' : 'are read') + // No sheet the profile does not name, and no position-bound description. + for (const shipped of PROFILE_DEFAULTS.sheets) { + if (!expected.includes(shipped)) { + expect(help).not.toContain(shipped) + expect(info).not.toContain(shipped) + } + } + expect(info).not.toContain('maintenance') + }, + ) it('words the write rules of a pack, and tolerates an answer without them', () => { expect(packRules({})).toEqual([]) From b1d249a606b89ad561339ee60e06468fe3aa427b Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:22:39 +0200 Subject: [PATCH 24/87] test(cmdb-import): make the missing-applications choice and the deleted-row rule testable The section's tests set `missingRecords` on the component and stubbed the radio away, so `v-model`, `value`, `name` and `:disabled` on the two radios were never exercised, no test clicked "Keep them as they are", and the rule that a deleted application has no link to its module had no test at all. The jest spec now mounts the section with radio and table stubs that render what the page does, clicks Keep and reads the posted form, checks the default, the shared name and the disabled state, and asserts the report table links an archived or failed row and not a deleted one, and shows no row number for a reconciliation row. Each test goes red when the wiring or the rule is mutated. The e2e adds a Keep import that must archive and delete nothing and a check that the archived application is absent from the Applications page before the Archived tab; the spec scenario for the choice now points at it. Review #1287 f8 Co-Authored-By: Claude Opus 5.5 --- .../specs/cmdb-export-import/spec.md | 2 +- .../settings/sections/CmdbImport.spec.js | 144 ++++++++++++++++++ tests/e2e/spec-coverage/cmdb-import.spec.ts | 62 ++++++++ 3 files changed, 207 insertions(+), 1 deletion(-) diff --git a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md index 1ac827456..fd915d1f0 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md @@ -162,7 +162,7 @@ Stackiq's admin settings page SHALL show a section "CMDB import", rendered by th - **AND** filtering the table on `created` SHALL show the two imported rows #### Scenario: The admin chooses to keep missing applications -@e2e exclude Covered by the Jest test; src/utils/cmdbImport.spec.js asserts buildImportForm() sends `archive` by default and `keep` when chosen, and src/views/settings/sections/CmdbImport.spec.js asserts the section's default. +@e2e tests/e2e/spec-coverage/cmdb-import.spec.ts - **GIVEN** the section - **WHEN** the admin selects "Keep them as they are" and starts the import diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index 6896c2c33..ca01b9517 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -96,6 +96,11 @@ jest.mock( 'div', this.rows.map((row) => h('div', { 'data-testid': 'table-row' }, [ + h( + 'span', + { 'data-testid': 'table-row-number' }, + this.$slots['column-row']?.({ row }), + ), h('span', this.$slots['column-name']?.({ row })), ]), ), @@ -380,3 +385,142 @@ describe('CmdbImport finished announcement', () => { ) }) }) + +/** + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +describe('CmdbImport the choice for missing applications, as the admin makes it', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + const radio = (wrapper, which) => + wrapper.find(`[data-testid="cmdb-import-missing-records-${which}"] input`) + + it('starts on "Archive or delete them" and the radios share one name', async () => { + const wrapper = await mountFull() + + expect(radio(wrapper, 'archive').element.checked).toBe(true) + expect(radio(wrapper, 'keep').element.checked).toBe(false) + expect(radio(wrapper, 'archive').attributes('name')).toBe( + radio(wrapper, 'keep').attributes('name'), + ) + expect(radio(wrapper, 'archive').attributes('value')).toBe('archive') + expect(radio(wrapper, 'keep').attributes('value')).toBe('keep') + }) + + it('posts missingRecords=keep after the admin clicks "Keep them as they are", and only that field changes', async () => { + const wrapper = await mountFull() + axios.post.mockResolvedValue({ + data: { success: true, summary: {}, rows: [] }, + }) + wrapper.vm.municipality = wrapper.vm.municipalityOptions[0] + wrapper.vm.selectedFile = new File(['x'], 'export.xlsx') + + await radio(wrapper, 'keep').setValue(true) + await flushPromises() + + expect(wrapper.vm.missingRecords).toBe('keep') + expect(radio(wrapper, 'keep').element.checked).toBe(true) + expect(radio(wrapper, 'archive').element.checked).toBe(false) + // The choice is not wired to another option: they keep their defaults. + expect(wrapper.vm.publish).toBe(true) + expect(wrapper.vm.updateExisting).toBe(true) + + await wrapper.vm.startImport() + const form = axios.post.mock.calls[0][1] + expect(form.get('missingRecords')).toBe('keep') + + await radio(wrapper, 'archive').setValue(true) + expect(wrapper.vm.missingRecords).toBe('archive') + }) + + it('disables both radios while an import runs', async () => { + const wrapper = await mountFull() + + await wrapper.setData({ importing: true }) + + expect(radio(wrapper, 'archive').attributes('disabled')).toBeDefined() + expect(radio(wrapper, 'keep').attributes('disabled')).toBeDefined() + }) +}) + +/** + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014 + */ +describe('CmdbImport report rows of archived and deleted applications', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + const reconciliationRow = (outcome, extra = {}) => ({ + sheet: 'Gearchiveerde Applicaties', + row: 0, + appId: outcome === 'deleted' ? '2' : '1234', + name: outcome === 'deleted' ? 'Naamtest' : 'Mailen', + outcome, + moduleUuid: 'module-uuid-' + outcome, + ...extra, + }) + + it('links an archived application to its module, and a deleted one to nothing', async () => { + const wrapper = await mountFull() + + await wrapper.setData({ + report: { + summary: { rowsRead: 0, archived: 1, deleted: 1 }, + rows: [ + reconciliationRow('archived'), + reconciliationRow('deleted'), + ], + }, + }) + + const rows = wrapper.findAll('[data-testid="table-row"]') + expect(rows).toHaveLength(2) + const [archived, deleted] = rows + expect( + archived.find('[data-testid="cmdb-import-module-link"]').attributes('href'), + ).toBe('/apps/stackiq/modules/module-uuid-archived') + expect(deleted.find('[data-testid="cmdb-import-module-link"]').exists()).toBe( + false, + ) + // The name of the deleted application is still shown, as text. + expect(deleted.text()).toContain('Naamtest') + }) + + it('shows no row number for a reconciliation row, and links one that failed', async () => { + const wrapper = await mountFull() + + await wrapper.setData({ + report: { + summary: { rowsRead: 1, failed: 1, archived: 1 }, + rows: [ + reconciliationRow('failed', { + reasons: ['archive: RuntimeException'], + }), + reconciliationRow('archived'), + { + sheet: 'Beheerde Applicaties CMDB', + row: 2, + appId: '5', + name: 'Gewoon', + outcome: 'created', + moduleUuid: 'module-uuid-created', + }, + ], + }, + }) + + const rows = wrapper.findAll('[data-testid="table-row"]') + expect(rows.map((row) => row.get('[data-testid="table-row-number"]').text())).toEqual( + ['', '', '2'], + ) + expect( + rows[0].find('[data-testid="cmdb-import-module-link"]').exists(), + ).toBe(true) + expect( + rows[2].find('[data-testid="cmdb-import-module-link"]').exists(), + ).toBe(true) + }) +}) diff --git a/tests/e2e/spec-coverage/cmdb-import.spec.ts b/tests/e2e/spec-coverage/cmdb-import.spec.ts index 82ae8e695..c0bd8d3e3 100644 --- a/tests/e2e/spec-coverage/cmdb-import.spec.ts +++ b/tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -560,6 +560,9 @@ test.describe.serial('CMDB import section', () => { // 2. The Applications page lists 1234 only under the quick filter "Archived". await gotoAppRoute(page, '/modules') await dismissSupportDialog(page) + // Not under the default tab: wait for the list to settle, then look. + await page.waitForLoadState('networkidle') + await expect(page.getByText(ARCHIVED_APPLICATION_NAME)).toHaveCount(0) await page.getByRole('tab', { name: 'Archived', exact: true }).click() await expect(page.getByText(ARCHIVED_APPLICATION_NAME).first()).toBeVisible({ timeout: 30000, @@ -581,6 +584,65 @@ test.describe.serial('CMDB import section', () => { await ctx.dispose() }) + // @e2e cmdb-export-import::the-admin-chooses-to-keep-missing-applications + test('choosing "Keep them as they are" leaves the applications a later export no longer lists as they are', async ({ + page, + }) => { + requireFixture(EXPORT_FIXTURE) + requireFixture(ARCHIVED_FIXTURE) + const ctx = await newApiContext() + const before = await countWritten(ctx) + expect( + before.modules, + 'the first import must have written the modules this test keeps', + ).toBeGreaterThan(0) + + // The same later export as the archive test, which archives 1234 and deletes 2. + const section = await gotoCmdbSection(page) + await chooseMunicipality(page) + const keep = section.locator( + '[data-testid="cmdb-import-missing-records-keep"] input', + ) + await keep.check() + await expect(keep).toBeChecked() + await expect( + section.locator( + '[data-testid="cmdb-import-missing-records-archive"] input', + ), + ).not.toBeChecked() + const response = await runImport(page, ARCHIVED_FIXTURE) + expect(response.status()).toBe(200) + // The request carries the choice, not the default. + expect(response.request().postData() ?? '').toMatch( + /name="missingRecords"\s+keep/, + ) + + await expect(summaryValue(page, 'rowsRead')).toHaveText('0') + // The archived and deleted tiles only exist when the import did either. + await expect( + section.locator('[data-testid="cmdb-import-summary-archived"]'), + ).toHaveCount(0) + await expect( + section.locator('[data-testid="cmdb-import-summary-deleted"]'), + ).toHaveCount(0) + await expect( + reportRows(page).locator('[data-outcome="archived"]'), + ).toHaveCount(0) + await expect( + reportRows(page).locator('[data-outcome="deleted"]'), + ).toHaveCount(0) + + // Both applications are still in the working set; nothing is archived. + expect(await countWritten(ctx)).toEqual(before) + expect( + await objectsInLens(ctx, config.module_schema, { + archived: 'true', + includeDeleted: false, + }), + ).toEqual([]) + await ctx.dispose() + }) + // @e2e cmdb-export-import::a-missing-required-column-is-named-in-the-422-response test('an export without a required column names the column and the sheet', async ({ page, From 8a1562aee83819584ff028ccf7ca0ed033d460ab Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:24:37 +0200 Subject: [PATCH 25/87] fix(cmdb-import): name the archive sheet where the section says which sheets the import reads The WORKBOOK_TOO_LARGE hint said the import "reads only" the two CMDB sheets and told the admin to delete the sheets it does not read; the section's file help said other sheets are ignored. The import also reads the APPID column of "Gearchiveerde Applicaties", and an admin who followed the hint would delete that sheet, so the next import archives and never deletes. Both strings now name the archive sheet and its APPID column (replacing the l10n keys in place), and the spec pins the new hint. Review #1287 f9 Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 4 ++-- l10n/en.json | 4 ++-- l10n/nl.js | 4 ++-- l10n/nl.json | 4 ++-- src/utils/cmdbImport.js | 5 ++++- src/utils/cmdbImport.spec.js | 2 +- src/views/settings/sections/CmdbImport.vue | 3 ++- 7 files changed, 15 insertions(+), 11 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 3e3c7c0d7..c56d7c8d4 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1103,7 +1103,7 @@ OC.L10N.register( "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".", "Split the export and import the parts one after the other with \"Keep them as they are\".": "Split the export and import the parts one after the other with \"Keep them as they are\".", "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.", "The page got no answer from the import.": "The page got no answer from the import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", @@ -1142,7 +1142,7 @@ OC.L10N.register( "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.": "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.", + "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.", "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.", "In development since": "In development since", diff --git a/l10n/en.json b/l10n/en.json index e2d6fd9e0..5eea76d88 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1102,7 +1102,7 @@ "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".": "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".", "Split the export and import the parts one after the other with \"Keep them as they are\".": "Split the export and import the parts one after the other with \"Keep them as they are\".", "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.", "The page got no answer from the import.": "The page got no answer from the import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.", @@ -1141,7 +1141,7 @@ "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.": "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.", + "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.", "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.", "In development since": "In development since", diff --git a/l10n/nl.js b/l10n/nl.js index 135559111..011cf1668 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1171,7 +1171,7 @@ OC.L10N.register( "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar met \"Laat ze zoals ze zijn\".", "Split the export and import the parts one after the other with \"Keep them as they are\".": "Splits de export en importeer de delen na elkaar met \"Laat ze zoals ze zijn\".", "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen, en van het tabblad \"{archive}\" alleen de kolom APPID; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", @@ -1210,7 +1210,7 @@ OC.L10N.register( "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.": "Verdeel de rijen over twee kopieën van de export en importeer beide met \"Laat ze zoals ze zijn\": een import die archiveert, ziet de applicaties van de andere kopie als ontbrekend.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "Het tabblad \"{sheet}\" is te groot om te importeren: uitgepakt is het {size}, en de import leest van één tabblad hooguit {limit}.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "Het onderdeel {part} van de werkmap is te groot om te importeren: uitgepakt is het {size}, en de import leest van één onderdeel hooguit {limit}.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen \"{first}\" en \"{second}\"), en importeer die kopie.", + "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest \"{first}\" en \"{second}\", en de kolom APPID van \"{archive}\"), en importeer die kopie.", "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest.", "In development since": "In ontwikkeling sinds", diff --git a/l10n/nl.json b/l10n/nl.json index 98cdc370d..f327c032e 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1170,7 +1170,7 @@ "A source sheet may hold at most {limit} rows. Split the export and import the parts one after the other with \"Keep them as they are\".": "Een brontabblad mag hoogstens {limit} rijen bevatten. Splits de export en importeer de delen na elkaar met \"Laat ze zoals ze zijn\".", "Split the export and import the parts one after the other with \"Keep them as they are\".": "Splits de export en importeer de delen na elkaar met \"Laat ze zoals ze zijn\".", "Excel workbook (.xlsx) with the sheet \"{first}\" or \"{second}\". By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad \"{first}\" of \"{second}\". Standaard mag het bestand hoogstens {size} zijn.", - "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "The sheets \"{first}\" (applications without arranged maintenance) and \"{second}\" (with arranged maintenance) are read, and of the sheet \"{archive}\" only the APPID column; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen \"{first}\" (applicaties zonder geregeld beheer) en \"{second}\" (met geregeld beheer) worden gelezen, en van het tabblad \"{archive}\" alleen de kolom APPID; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", "The page got no answer from the import.": "De pagina kreeg geen antwoord van de import.", "The connection was cut off before the import answered, so it may still be running or may have finished. Wait a few minutes and check the applications of the municipality before you import again. Importing the same file again creates no duplicates.": "De verbinding werd verbroken voordat de import antwoordde, dus de import kan nog bezig zijn of al klaar zijn. Wacht een paar minuten en controleer de applicaties van de gemeente voordat u opnieuw importeert. Hetzelfde bestand opnieuw importeren maakt geen dubbele records aan.", "The import cannot be cancelled now: the server has not started its rows yet, or has already finished them. If the import keeps running, press Cancel import again in a moment.": "De import kan nu niet worden geannuleerd: de server is nog niet aan de rijen begonnen, of is er al mee klaar. Loopt de import door, druk dan zo dadelijk opnieuw op Import annuleren.", @@ -1209,7 +1209,7 @@ "Split the rows over two copies of the export and import both with \"Keep them as they are\": an import that archives treats the applications of the other copy as missing.": "Verdeel de rijen over twee kopieën van de export en importeer beide met \"Laat ze zoals ze zijn\": een import die archiveert, ziet de applicaties van de andere kopie als ontbrekend.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "Het tabblad \"{sheet}\" is te groot om te importeren: uitgepakt is het {size}, en de import leest van één tabblad hooguit {limit}.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "Het onderdeel {part} van de werkmap is te groot om te importeren: uitgepakt is het {size}, en de import leest van één onderdeel hooguit {limit}.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen \"{first}\" en \"{second}\"), en importeer die kopie.", + "Delete the sheets the import does not read from a copy of the export (it reads \"{first}\" and \"{second}\", and the APPID column of \"{archive}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest \"{first}\" en \"{second}\", en de kolom APPID van \"{archive}\"), en importeer die kopie.", "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest.", "In development since": "In ontwikkeling sinds", diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 0b834dd2f..2c83018a0 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -31,6 +31,8 @@ export const PROFILE_DEFAULTS = Object.freeze({ maxFileBytes: 10 * 1024 * 1024, maxRowsPerSheet: 10000, sheets: Object.freeze(['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB']), + // Read for its APPID column only, in every import. + archiveSheet: 'Gearchiveerde Applicaties', }) /** @@ -564,10 +566,11 @@ function workbookTooLarge(details) { + ' ' + t( 'stackiq', - 'Delete the sheets the import does not read from a copy of the export (it reads only "{first}" and "{second}"), and import that copy.', + 'Delete the sheets the import does not read from a copy of the export (it reads "{first}" and "{second}", and the APPID column of "{archive}"), and import that copy.', { first: PROFILE_DEFAULTS.sheets[0], second: PROFILE_DEFAULTS.sheets[1], + archive: PROFILE_DEFAULTS.archiveSheet, }, AS_TEXT, ) diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 3282e3458..a5579b985 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -426,7 +426,7 @@ describe('errorText', () => { expect( errorText({ error: 'WORKBOOK_TOO_LARGE', details }).hint, ).toContain( - '(it reads only "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB")', + '(it reads "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB", and the APPID column of "Gearchiveerde Applicaties")', ) } }) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 0f92b67ae..3590c47f7 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -414,10 +414,11 @@ {{ t( 'stackiq', - 'The sheets "{first}" (applications without arranged maintenance) and "{second}" (with arranged maintenance) are read; other sheets, including the "Invoer" sheets, are ignored.', + 'The sheets "{first}" (applications without arranged maintenance) and "{second}" (with arranged maintenance) are read, and of the sheet "{archive}" only the APPID column; other sheets, including the "Invoer" sheets, are ignored.', { first: profileDefaults.sheets[0], second: profileDefaults.sheets[1], + archive: profileDefaults.archiveSheet, }, asText, ) From a4570e79e324ec5b782aeaebfb265e29d73d5408 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:24:37 +0200 Subject: [PATCH 26/87] test(cmdb-import): pin the totality promises of the reconciliation REQ-CMDB-015 promises that only this municipality's usages and what its import created are reconciled, that each application has its own error boundary, that a cancel before the scope is loaded reads nothing, and that the match never duplicates (working before archived before deleted). Most of these branches had no test that a mutation could turn red; the first one, the consumer re-check on a trash listing that is not filtered by consumer, guarded the only thing between another municipality's module and the trash. Adds tests for: a trashed usage of another consumer; modules with no external key, a bare prefix or another origin; a delete that OpenRegister refuses and a restore that throws, each with a following application still reconciled; more than one page of usages and one chunk of modules; archived beating deleted and working beating archived in the match; and an archived match failing its row under keep when the archive services are missing (design D7). The cancel test now also asserts the trash was not read and the usages were not paged. Each test was checked by mutating the service and goes red. Review #1286 f3 Co-Authored-By: Claude Opus 5.5 --- .../Service/CmdbExportImportServiceTest.php | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 3a7a0e9b6..b21e88ae7 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -119,6 +119,13 @@ class CmdbExportImportServiceTest extends TestCase { */ private int $trashReads = 0; + /** + * The uuid whose delete OpenRegister refuses (deleteObject() answers false), when set. + * + * @var string|null + */ + private ?string $deleteRefusedFor = null; + /** * How many of the next trash reads throw. * @@ -261,6 +268,7 @@ protected function setUp(): void { $this->transitions = []; $this->beforeTransition = null; $this->trashReads = 0; + $this->deleteRefusedFor = null; $this->trashFailures = 0; $this->archiveServicesAvailable = true; $this->contacts = []; @@ -516,6 +524,10 @@ function (array $query = [], bool $_rbac = true, bool $_multitenancy = true): ar $service->method('deleteObject')->willReturnCallback( function (string $uuid, $register = null, $schema = null, bool $_rbac = true, bool $_multitenancy = true): bool { $this->noteScope(method: 'deleteObject', rbac: $_rbac, multitenancy: $_multitenancy); + if ($this->deleteRefusedFor === $uuid) { + return false; + } + $this->changeState(verb: 'delete', uuid: $uuid, reason: null); return true; } @@ -2020,11 +2032,16 @@ public function testACancelDuringTheLastRowArchivesNothing(): void { } }; $service = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Applicatie Naam' => 'Nieuw'])], archived: ['7', '8'])); + $this->trashReads = 0; + $this->searches = []; $report = $service->import(path: '', options: ['municipalityUuid' => 'muni-1', 'operationId' => 'cmdb-cancel-03']); $this->assertTrue($report['cancelled']); $this->assertSame([], $this->transitions); + $this->assertSame(0, $this->trashReads, 'the scope was not loaded: the trash was not read'); + $usageScans = array_filter($this->searches, static fn (array $query): bool => ($query['consumer'] ?? null) === 'muni-1' && isset($query['_offset']) === true); + $this->assertSame([], $usageScans, 'the municipality\'s usages were not paged'); $this->assertSame(['updated'], array_column($report['rows'], 'outcome')); $this->assertSame('cancelled', $this->cache['progress_cmdb-cancel-03']['status']); $this->assertArrayNotHasKey('cancel_cmdb-cancel-03', $this->cache); @@ -2152,6 +2169,196 @@ public function testTheModulesStateDecidesWhichDuplicateIsReconciled(): void { $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-stale'), 'the working usage of the stale module is not touched'); }//end testTheModulesStateDecidesWhichDuplicateIsReconciled() + /** + * A usage in the trash that names another consumer is never reconciled, though the trash is not filtered by consumer. + * + * The usage points at a working module that carries this municipality's import key; only the consumer + * check keeps that module out of the scope, and with it out of the trash. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testATrashedUsageOfAnotherConsumerNeverReconcilesItsModule(): void { + $this->importOneSevenAndEight(); + $this->seedOrganisation(uuid: 'muni-2', name: 'Gemeente Anders', type: 'Municipality'); + $seven = $this->moduleOf(appId: '7'); + $this->store[self::MODULE]['module-nine'] = ['id' => 'module-nine', 'name' => 'Negen', 'externalKey' => 'topdesk:muni-1:9', 'externalNumber' => '9']; + $this->store[self::USAGE]['usage-other'] = ['id' => 'usage-other', 'consumer' => 'muni-2', 'module' => 'module-nine']; + $this->states[self::USAGE]['usage-other'] = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + + // APPID 9 is on no sheet, so a scope that took the trashed usage in would soft-delete its module. + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '7', row: 3), $this->row(appId: '8', row: 4)], archived: [])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: 'module-nine'), 'the module stays working'); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: 'usage-other'), 'and the other consumer\'s usage stays in the trash'); + $this->assertSame([], $this->transitions); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + }//end testATrashedUsageOfAnotherConsumerNeverReconcilesItsModule() + + /** + * Only a module whose key is exactly this municipality's import key plus an APPID is reconciled. + * + * A module without an external key, one whose key is the bare prefix, and one whose key has another origin + * are used by this municipality but are not what its import created. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testOnlyWhatTheImportCreatedIsReconciled(): void { + $this->importOneSevenAndEight(); + $others = [ + 'module-nokey' => ['id' => 'module-nokey', 'name' => 'Zonder sleutel'], + 'module-bare' => ['id' => 'module-bare', 'name' => 'Kale sleutel', 'externalKey' => 'topdesk:muni-1:'], + 'module-origin' => ['id' => 'module-origin', 'name' => 'Andere herkomst', 'externalKey' => 'xxxxxxx:muni-1:9'], + ]; + foreach ($others as $uuid => $module) { + $this->store[self::MODULE][$uuid] = $module; + $this->store[self::USAGE]['usage-' . $uuid] = ['id' => 'usage-' . $uuid, 'consumer' => 'muni-1', 'module' => $uuid]; + } + + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '7', row: 3), $this->row(appId: '8', row: 4)], archived: [])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([], $this->transitions, 'nothing the import did not create is archived or deleted'); + $this->assertSame([], $this->states); + }//end testOnlyWhatTheImportCreatedIsReconciled() + + /** + * A delete step that OpenRegister refuses, and a restore step that throws, each fail only their own application. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testAFailingDeleteOrRestoreStepFailsOnlyItsApplication(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + + // APPID 7 and 8 are on no sheet: OpenRegister refuses to delete the usage of 7, then deletes 8. + $this->deleteRefusedFor = $this->usageOf(moduleUuid: $seven); + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: [])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $rows = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame([['7', 'failed'], ['8', 'deleted']], array_map(static fn (array $row): array => [$row['appId'], $row['outcome']], $rows)); + $this->assertSame(['step "delete" failed (RuntimeException)'], $rows[0]['reasons']); + $this->assertSame([false, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the module of the failed application is untouched'); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $eight)); + + // APPID 7 and 8 are in the trash and on the archive sheet: the restore of 7 throws, 8 is restored and archived. + $this->deleteRefusedFor = null; + $this->states = []; + foreach ([$seven, $eight] as $uuid) { + $this->states[self::MODULE][$uuid] = ['deleted' => ['deletedBy' => 'admin']]; + $this->states[self::USAGE][$this->usageOf(moduleUuid: $uuid)] = ['deleted' => ['deletedBy' => 'admin']]; + } + + $this->beforeTransition = function (string $verb, string $uuid) use ($seven): void { + if ($verb === 'restore' && $uuid === $this->usageOf(moduleUuid: $seven)) { + throw new RuntimeException('restore failed'); + } + }; + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7', '8'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $rows = array_values(array_filter($report['rows'], static fn (array $row): bool => $row['row'] === 0)); + $this->assertSame([['7', 'failed'], ['8', 'archived']], array_map(static fn (array $row): array => [$row['appId'], $row['outcome']], $rows)); + $this->assertSame(['step "restore" failed (RuntimeException)'], $rows[0]['reasons']); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the failed application stays in the trash'); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $eight)); + }//end testAFailingDeleteOrRestoreStepFailsOnlyItsApplication() + + /** + * More usages than one page, and more modules than one chunk, are all reconciled. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testMoreApplicationsThanOnePageAreAllReconciled(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Voorbeeldstad', type: 'Municipality'); + $count = 501; + for ($index = 1; $index <= $count; $index++) { + $this->store[self::MODULE]['module-' . $index] = ['id' => 'module-' . $index, 'name' => 'Applicatie ' . $index, 'externalKey' => 'topdesk:muni-1:' . $index, 'externalNumber' => (string)$index]; + $this->store[self::USAGE]['usage-' . $index] = ['id' => 'usage-' . $index, 'consumer' => 'muni-1', 'module' => 'module-' . $index]; + } + + // Every application is on no sheet: every one is soft-deleted, the last of both lists included. + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1000')], archived: [])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame($count, $report['summary']['deleted']); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'module-' . $count)); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: 'usage-' . $count)); + $this->assertCount(($count * 2), $this->transitions); + }//end testMoreApplicationsThanOnePageAreAllReconciled() + + /** + * An archived match wins over a deleted one with the same import key, and a working match over an archived one. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testTheMatchPrefersWorkingThenArchivedThenDeleted(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $sevenUsage = $this->usageOf(moduleUuid: $seven); + $eightUsage = $this->usageOf(moduleUuid: $eight); + foreach ([['7', $seven, $sevenUsage], ['8', $eight, $eightUsage]] as [$appId, $module, $usage]) { + $this->store[self::MODULE]['stale-module-' . $appId] = array_merge($this->store[self::MODULE][$module], ['id' => 'stale-module-' . $appId]); + $this->store[self::USAGE]['stale-usage-' . $appId] = array_merge($this->store[self::USAGE][$usage], ['id' => 'stale-usage-' . $appId, 'module' => 'stale-module-' . $appId]); + } + + // APPID 7: the real pair is archived, the stale one deleted. APPID 8: the real pair is working, the stale one archived. + $gone = ['deleted' => ['deletedBy' => 'admin', 'deletedAt' => '2026-10-01T00:00:00+00:00']]; + $shelved = ['archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'earlier']]; + $this->states[self::MODULE][$seven] = $shelved; + $this->states[self::USAGE][$sevenUsage] = $shelved; + $this->states[self::MODULE]['stale-module-7'] = $gone; + $this->states[self::USAGE]['stale-usage-7'] = $gone; + $this->states[self::MODULE]['stale-module-8'] = $shelved; + $this->states[self::USAGE]['stale-usage-8'] = $shelved; + $counts = [count($this->store[self::MODULE]), count($this->store[self::USAGE])]; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '7', row: 2), $this->row(appId: '8', row: 3, sheet: 'Onbeh Applicaties CMDB')])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['unarchived', 'unchanged'], array_column($report['rows'], 'outcome')); + $this->assertSame([$seven, $eight], array_column($report['rows'], 'moduleUuid')); + $this->assertSame( + [['unarchive', $seven], ['unarchive', $sevenUsage]], + array_map(static fn (array $t): array => [$t[0], $t[2]], $this->transitions), + 'only the preferred pair is revived; nothing is restored from the trash' + ); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'stale-module-7')); + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: 'stale-module-8')); + $this->assertSame($counts, [count($this->store[self::MODULE]), count($this->store[self::USAGE])], 'no module or usage is created'); + }//end testTheMatchPrefersWorkingThenArchivedThenDeleted() + + /** + * With keep and without OpenRegister's archive handler a row whose match is archived fails at the module step. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-that-returns-to-the-export-shall-be-unarchived-or-restored-never-duplicated-req-cmdb-016 + */ + public function testKeepWithoutTheArchiveServicesFailsARowWhoseMatchIsArchived(): void { + $this->importOneSevenAndEight(); + $seven = $this->moduleOf(appId: '7'); + $this->states[self::MODULE][$seven] = ['archived' => ['by' => 'admin', 'at' => '2026-10-01T00:00:00+00:00', 'reason' => 'earlier']]; + $this->archiveServicesAvailable = false; + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1'), $this->row(appId: '7', row: 3, sheet: 'Onbeh Applicaties CMDB')])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'keep']); + + $this->assertSame(['unchanged', 'failed'], array_column($report['rows'], 'outcome')); + $this->assertSame(['step "module" failed (CmdbImportException)'], $report['rows'][1]['reasons']); + $this->assertSame([], $this->transitions); + $this->assertSame(0, $report['summary']['archived'] + $report['summary']['deleted']); + }//end testKeepWithoutTheArchiveServicesFailsARowWhoseMatchIsArchived() + /** * One application that cannot be archived is a failed row; the others are still reconciled. * From cdbec3d3ac09a986bb82a49c16edc5da90dc4a42 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:24:48 +0200 Subject: [PATCH 27/87] docs(cmdb-import): say the archive sheet is read, and bounded, in every import The docs said the import "ignores all other sheets" and that a sheet it does not read may exceed the part size limit. The archive sheet is read (its APPID column only) in every import, also with Keep, so it counts against the row span and part-size limits like a CMDB sheet and can refuse a keep import. The file section, the row limit, the part-size limit and the WORKBOOK_TOO_LARGE row now say so, and that deleting the sheet from a copy makes the import archive and never delete. Review #1285 f2 Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index f66fd6626..1e33207d5 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -123,7 +123,12 @@ the same organisation. The import reads the two CMDB sheets of the export, and of the sheet `Gearchiveerde Applicaties` only the `APPID` column (see [Repeat imports](#repeat-imports)). It ignores all other sheets, including -the `Invoer` sheets they are derived from: +the `Invoer` sheets they are derived from. The archive sheet is read in every +import, also with **Keep them as they are**, and it is bounded like the two +CMDB sheets (see [Limits](#limits)): an archive sheet over the limits refuses +the import even when you keep missing applications. Delete the sheet from a +copy of the export to import it without one; the import then archives and +never deletes. | Sheet | What it holds | Recorded on the usage | |---|---|---| @@ -136,8 +141,8 @@ surrounding spaces and a trailing `:` or `⚡` do not matter, and the order of the columns does not matter. A column that one sheet has and the other has not (such as `Nickname`, only on `Beheerde Applicaties CMDB`) is optional on the sheet that lacks it. Empty rows, including formatted rows below the data, -are ignored and not counted. A sheet may hold at most 10,000 rows with data -by default (see [Limits](#limits)). +are ignored and not counted. A sheet, the archive sheet included, may hold at +most 10,000 rows with data by default (see [Limits](#limits)). Two columns are **required** on every CMDB sheet that is present: `APPID` and `Applicatie Naam`. Every other column is optional; when one is missing, @@ -359,7 +364,7 @@ and the section shows the reason and the error code. | `NO_SOURCE_SHEET` | Neither `Onbeh Applicaties CMDB` nor `Beheerde Applicaties CMDB` is in the workbook. | Check the sheet names; they must match exactly. | | `MISSING_COLUMN` | A present CMDB sheet has no `APPID` or `Applicatie Naam` column. The message names the sheet and the column. | Add the column to that sheet. | | `TOO_MANY_ROWS` | A CMDB sheet has more rows with data than the row limit (10,000 by default). The message names the sheet and the limit. | Split the export and import the parts one after the other with **Keep them as they are**: an import that archives treats the applications of the other parts as missing. Applications that left the export are then not archived or deleted. | -| `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads: 50 MB in all, 10 MB for one sheet the import reads or for the table of texts the sheets share, or more than 200,000 different texts, by default. Sheets the import does not read may be larger than 10 MB. An `.xlsx` is a compressed package, so a file of a few MB can unpack to far more. The message names the limit, and for a sheet the sheet and its unpacked size. | For a sheet that is too large: split its rows over two copies of the export and import both with **Keep them as they are**. For a workbook that is too large in all: delete the sheets the import does not read from a copy and import that copy. | +| `WORKBOOK_TOO_LARGE` | Unpacked, the workbook is larger than the import reads: 50 MB in all, 10 MB for one sheet the import reads (the two CMDB sheets and the archive sheet, in every import) or for the table of texts the sheets share, or more than 200,000 different texts, by default. Sheets the import does not read may be larger than 10 MB. An `.xlsx` is a compressed package, so a file of a few MB can unpack to far more. The message names the limit, and for a sheet the sheet and its unpacked size. | For a sheet that is too large: split its rows over two copies of the export and import both with **Keep them as they are**. For a workbook that is too large in all: delete the sheets the import does not read from a copy and import that copy. | | `MUNICIPALITY_AMBIGUOUS` | More than one municipality has the typed name. The import does not guess which one. | Pick the municipality from the list instead of typing its name. | | `IMPORT_IN_PROGRESS` | Another CMDB import is running. Only one import runs at a time. | Wait until it has finished and try again. | | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | @@ -394,9 +399,9 @@ every import: | Setting | Default | What it limits | |---|---|---| | `maxFileBytes` | `10485760` (10 MB) | the size of the uploaded file | -| `maxRowsPerSheet` | `10000` | the rows with data on one CMDB sheet | +| `maxRowsPerSheet` | `10000` | the rows with data on one CMDB sheet, and on the archive sheet | | `maxUncompressedBytes` | `52428800` (50 MB) | the size of the workbook once unpacked, checked before a sheet is parsed | -| `maxPartBytes` | `10485760` (10 MB) | the size of one part the import parses once unpacked, such as a source sheet or the shared-strings table, checked before a sheet is parsed; a sheet the import does not read may be larger, and is read as an empty sheet | +| `maxPartBytes` | `10485760` (10 MB) | the size of one part the import parses once unpacked, such as a source sheet or the shared-strings table, checked before a sheet is parsed; a sheet the import does not read may be larger, and is read as an empty sheet; the archive sheet is read, so it is bounded like a CMDB sheet | | `maxSharedStrings` | `200000` | the number of different texts in the workbook's shared-strings table, each formatted piece of a text with mixed formatting counted as one more, also in text written in a cell itself, counted before a sheet is parsed | | `maxReferencedStringBytes` | `67108864` (64 MB) | the shared text all cells of the workbook reference together, counted once per cell and once per CMDB sheet before a sheet is parsed; a long text that many cells repeat counts as often as it is repeated | From ae929478c2e394c07a1c5352025444737ae1d3b5 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:30:34 +0200 Subject: [PATCH 28/87] test(cmdb-import): run the sheet-name wiring and the mapping error states CmdbImport.spec.js shallow-mounted the section, so the mapping block was stubbed and never emitted `loaded`: no executed test failed when onMappingLoaded, the sheetNames computed or the help-text binding went missing. The section is now mounted with a wrapper that renders its slots; the test emits `loaded` from the mapping block and asserts the help text and the info panel name the emitted sheets, the shipped ones before it, and the one sheet of a one-sheet profile. The vitest suite gains the mapping block's errors without a reason (403 with a message, no answer, 500), which only the 503 path covered. Review #1289 f3 Co-Authored-By: Claude Opus 5.5 --- .../settings/sections/CmdbImport.spec.js | 105 +++++++++++++++++- tests/vitest/cmdbImportMapping.spec.js | 56 ++++++++++ 2 files changed, 159 insertions(+), 2 deletions(-) diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index d0598b407..b20691f00 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -10,7 +10,9 @@ import axios from '@nextcloud/axios' import { flushPromises, shallowMount } from '@vue/test-utils' +import { h } from 'vue' import CmdbImport from './CmdbImport.vue' +import CmdbImportMapping from './CmdbImportMapping.vue' // Virtual: the package's `exports` field declares only the `import` condition, which jest does not resolve. jest.mock( @@ -97,7 +99,7 @@ const ORGANISATIONS = [ * * @return {Promise} The wrapper, after the municipalities loaded */ -async function mountSection() { +async function mountSection(options = {}) { axios.get.mockImplementation((url) => Promise.resolve( url.includes('/voorzieningen/config') @@ -105,7 +107,7 @@ async function mountSection() { : { data: { results: ORGANISATIONS } }, ), ) - const wrapper = shallowMount(CmdbImport) + const wrapper = shallowMount(CmdbImport, options) await flushPromises() return wrapper } @@ -153,3 +155,102 @@ describe('CmdbImport municipality chooser', () => { expect(wrapper.vm.report).toBe(null) }) }) + +// shallowMount stubs the section's wrapper without its slots; this one renders them, so the help text and the info panel are in the DOM. +const SLOT_RENDERING_SECTION = { + render() { + return h('div', [ + this.$slots.default?.(), + this.$slots['info-content']?.(), + ]) + }, +} + +/** + * Mount the section with the wrapper rendering its slots. + * + * @return {Promise} The wrapper + */ +function mountWithSlots() { + return mountSection({ + global: { stubs: { AlwaysVisibleSection: SLOT_RENDERING_SECTION } }, + }) +} + +describe('CmdbImport sheet names in the help texts', () => { + afterEach(() => { + jest.clearAllMocks() + }) + + const SHIPPED = ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'] + + /** + * The mapping endpoint's answer for a profile with the given sheets. + * + * @param {Array} names The sheet names + * @return {object} The answer, reduced to what the section reads + */ + function mappingOf(names) { + return { + profile: { sheets: names.map((name) => ({ name })) }, + packs: [], + } + } + + /** + * The help text under the file chooser and the info panel's sheet sentence. + * + * @param {object} wrapper The mounted section + * @return {{help: string, info: string}} Both texts, as rendered + */ + function renderedTexts(wrapper) { + return { + help: wrapper.find('#cmdb-import-file-help').text(), + info: wrapper + .findAll('.cmdb-import-info li') + .map((li) => li.text()) + .find((text) => /sheets? .* (is|are) read/.test(text)), + } + } + + it('names the shipped sheets until the mapping block has loaded', async () => { + const wrapper = await mountWithSlots() + + const { help, info } = renderedTexts(wrapper) + expect(help).toContain('"' + SHIPPED[0] + '", "' + SHIPPED[1] + '"') + expect(info).toContain('"' + SHIPPED[0] + '", "' + SHIPPED[1] + '"') + }) + + it('names the sheets the mapping block emits, in the help text and the info panel', async () => { + const wrapper = await mountWithSlots() + + wrapper + .findComponent(CmdbImportMapping) + .vm.$emit('loaded', mappingOf(['Alpha', 'Beta', 'Gamma'])) + await flushPromises() + + const { help, info } = renderedTexts(wrapper) + expect(help).toContain('"Alpha", "Beta", "Gamma"') + expect(info).toContain('"Alpha", "Beta", "Gamma"') + for (const shipped of SHIPPED) { + expect(help).not.toContain(shipped) + expect(info).not.toContain(shipped) + } + }) + + it('names only the one sheet of a profile with one sheet', async () => { + const wrapper = await mountWithSlots() + + wrapper + .findComponent(CmdbImportMapping) + .vm.$emit('loaded', mappingOf(['Alpha'])) + await flushPromises() + + const { help, info } = renderedTexts(wrapper) + expect(help).toContain('with the sheet "Alpha".') + expect(info).toContain('The sheet "Alpha" is read') + for (const shipped of SHIPPED) { + expect(help).not.toContain(shipped) + } + }) +}) diff --git a/tests/vitest/cmdbImportMapping.spec.js b/tests/vitest/cmdbImportMapping.spec.js index e7c21781a..b12c1b956 100644 --- a/tests/vitest/cmdbImportMapping.spec.js +++ b/tests/vitest/cmdbImportMapping.spec.js @@ -262,6 +262,62 @@ describe('CmdbImportMapping', () => { }) }) +describe('CmdbImportMapping errors without a reason', () => { + beforeEach(() => { + http.get.mockReset() + }) + + /** + * Load the block against a failing endpoint and return its error card. + * + * @param {object} failure The rejection the endpoint produces + * @return {Promise} The error card + */ + async function errorCard(failure) { + http.get.mockRejectedValue(failure) + const wrapper = mount(CmdbImportMapping) + await flushPromises() + await wrapper + .find('[data-testid="cmdb-import-mapping-toggle"]') + .trigger('click') + expect(wrapper.emitted('loaded')).toBeUndefined() + return wrapper.find('[data-testid="cmdb-import-mapping-error"]') + } + + it('shows the server message and the code on a 403 without a reason', async () => { + const error = await errorCard({ + response: { + status: 403, + data: { success: false, error: 'NOT_ADMIN', message: 'Admins only.' }, + }, + }) + + expect(error.exists()).toBe(true) + expect(error.text()).toContain('Admins only.') + expect(error.text()).toContain('NOT_ADMIN') + }) + + it('shows only the code when the request never got an answer', async () => { + const error = await errorCard(new Error('Network Error')) + + expect(error.exists()).toBe(true) + expect(error.text()).toContain('NETWORK_ERROR') + expect(error.findAll('p')).toHaveLength(1) + }) + + it('shows the code of a 500 whose body carries no reason', async () => { + const error = await errorCard({ + response: { + status: 500, + data: { success: false, error: 'IMPORT_FAILED', details: {} }, + }, + }) + + expect(error.text()).toContain('IMPORT_FAILED') + expect(error.findAll('p')).toHaveLength(1) + }) +}) + describe('cmdbImport mapping helpers', () => { it('takes the sheet names from the answer, and the shipped ones without it', () => { expect(mappingSheetNames(answer())).toEqual(['Sheet A', 'Sheet B']) From 7db0b2a09f0bf439100467d125f310009f0ae918 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:30:57 +0200 Subject: [PATCH 29/87] docs(cmdb-import): narrow the mapping block's claim to what it shows The user docs said the block "shows what the import does", and that it shows which file and entry hold a column. It shows the column-to-field mappings and, since the write-rule rows, when the import writes each field; the sheet constants, absent columns, sheet precedence, id columns, empty-value placeholders, missingRecords modes and limits are returned by the endpoint but not displayed, and a pack's idStrategy is only in its file. The docs now say that and point to the files that hold them. The same overclaim in the change's proposal, design and spec purpose is narrowed the same way. Review #1289 f2 Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 22 +++++++++++++------ .../cmdb-import-mapping-view/design.md | 2 +- .../cmdb-import-mapping-view/proposal.md | 2 +- .../specs/cmdb-export-import/spec.md | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index d6c0d7468..e6f08ed07 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -356,11 +356,18 @@ The section shows the mapping the next import runs, read-only. Open type `Application`) and the fields never written on an existing record (the module's publication and depublication dates). -The block reads the files through the same loader the import uses, so what -it shows is what the import does. When a file is invalid, the block shows -`MAPPING_UNAVAILABLE` with the file and the reason instead of the tables, and -the import refuses to run with the same code. The sheet names in the -section's help text come from the same answer. +The block reads the files through the same loader the import uses, so the +column-to-field mappings and the write rules it shows are the ones the next +import applies. It does not show everything in the files: the constant each +sheet adds to its rows (`Beheer`) and the columns a sheet is known to lack, +the order in which sheets win a duplicate `APPID`, the id columns, the +placeholder values that mean empty (such as `NB` in `BNN Classificatie`), the +`missingRecords` modes and the limits are in the files and in the +endpoint's answer, and a pack's `idStrategy` only in the file; none of them +is in the block. When a file is +invalid, the block shows `MAPPING_UNAVAILABLE` with the file and the reason +instead of the tables, and the import refuses to run with the same code. The +sheet names in the section's help text come from the same answer. The block is only for viewing: nothing in it changes the mapping. The data behind it is also available to Nextcloud administrators as @@ -370,8 +377,9 @@ behind it is also available to Nextcloud administrators as The mapping from columns to fields is not in code. It is a set of JSON files in `lib/Settings/cmdb-import/`, executed by OpenRegister's mapping engine. -It cannot be changed in the section; **Mapping (read-only)** shows which -file and which entry hold a column, so you know what to change: +It cannot be changed in the section; **Mapping (read-only)** shows the pack +file and the column mapping that hold a column, and the files below hold the +settings the block does not show, so you know what to change: | File | What it maps | |---|---| diff --git a/openspec/changes/cmdb-import-mapping-view/design.md b/openspec/changes/cmdb-import-mapping-view/design.md index 23b3357b9..6b3a29e96 100644 --- a/openspec/changes/cmdb-import-mapping-view/design.md +++ b/openspec/changes/cmdb-import-mapping-view/design.md @@ -8,7 +8,7 @@ The CMDB import (archived change `2026-10-05-cmdb-export-import`) maps a TOPdesk **Goals** -- An administrator sees, in the section, exactly the mapping the next import runs. +- An administrator sees, in the section, the column mapping and the write rules the next import applies, read from the files the import loads. The profile's other settings (sheet constants, absent columns, sheet precedence, id columns, empty-value placeholders, missing-records modes, limits) are returned by the endpoint but not shown; the user documentation says so. - The section's sheet names cannot drift from the profile. - Nothing is written; nothing new is parsed. diff --git a/openspec/changes/cmdb-import-mapping-view/proposal.md b/openspec/changes/cmdb-import-mapping-view/proposal.md index e651febe6..b537011dd 100644 --- a/openspec/changes/cmdb-import-mapping-view/proposal.md +++ b/openspec/changes/cmdb-import-mapping-view/proposal.md @@ -7,7 +7,7 @@ depends_on: [] ## Summary -The "CMDB import" section of stackiq's admin settings gets a collapsible, read-only block "Mapping (read-only)" that shows the column mapping the import really uses: the sheets, the key and required columns, and one table per migration pack with the source column, the target field, whether it is required, and the transformation with its lookup values. A new admin endpoint `GET /api/settings/cmdb-import/mapping` loads the import profile and the five packs through the same loader and validator the import uses, so what the administrator sees is what the next import runs. Editing stays in the JSON files on the server; the documentation says where they are and what an administrator can change. +The "CMDB import" section of stackiq's admin settings gets a collapsible, read-only block "Mapping (read-only)" that shows the column mapping the import really uses: the sheets, the key and required columns, and one table per migration pack with the source column, the target field, whether it is required, the transformation with its lookup values and when the import writes the field, plus each pack's defaults, create-only values and never-written fields. A new admin endpoint `GET /api/settings/cmdb-import/mapping` loads the import profile and the five packs through the same loader and validator the import uses, so the column mapping and write rules the administrator sees are the ones the next import applies (the profile's sheet constants, empty-value placeholders and similar settings are in the answer but not in the block). Editing stays in the JSON files on the server; the documentation says where they are and what an administrator can change. ## Motivation diff --git a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md index 68351e7cc..174b1ce02 100644 --- a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md @@ -8,7 +8,7 @@ ## Purpose -The CMDB import maps the columns of a TOPdesk export to stackiq fields through an import profile and five migration packs, JSON files under `lib/Settings/cmdb-import/` that OpenRegister's mapping engine executes (REQ-CMDB-005). Until now an administrator could only learn that mapping by reading the JSON on the server. This delta adds a read-only view of it to the "CMDB import" section of the admin settings, served by an endpoint that loads the files through the same loader the import uses, so what the administrator sees is what the next import runs (Jira WOO-588, sub-task of WOO-586). +The CMDB import maps the columns of a TOPdesk export to stackiq fields through an import profile and five migration packs, JSON files under `lib/Settings/cmdb-import/` that OpenRegister's mapping engine executes (REQ-CMDB-005). Until now an administrator could only learn that mapping by reading the JSON on the server. This delta adds a read-only view of it to the "CMDB import" section of the admin settings, served by an endpoint that loads the files through the same loader the import uses, so the column mapping and the write rules the administrator sees are the ones the next import applies; the profile's other settings (sheet constants, absent columns, sheet precedence, id columns, empty-value placeholders, missing-records modes) are in the answer but not in the block (Jira WOO-588, sub-task of WOO-586). Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\MigrationPack\PackDefinitionValidator` (through the existing `CmdbImportProfile`, guarded). From 2795770c556ae9a526ca57f12b8e33f61fcd4e2d Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:32:07 +0200 Subject: [PATCH 30/87] style(cmdb-import): bring the two reconciliation fixes within phpcs The archive-sheet and per-key fixes lengthened a few docblock lines and the new warning past the 150-character limit, and stateRank() used an inline if. Names the two array shapes once as phpstan types, shortens the warning (en, nl, test) and spells the rank out. No behaviour change. Follows Review #1285 f1 and Review #1286 f4; fold into those commits when squashing. Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 2 +- l10n/en.json | 2 +- l10n/nl.js | 2 +- l10n/nl.json | 2 +- lib/Service/CmdbExportImportService.php | 26 ++++++++++++------- .../Service/CmdbExportImportServiceTest.php | 2 +- 6 files changed, 22 insertions(+), 14 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 39566fb2a..6eba930ad 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1041,7 +1041,7 @@ OC.L10N.register( "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", - "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.", + "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", "Unarchived": "Unarchived", "Restored": "Restored", diff --git a/l10n/en.json b/l10n/en.json index 18f9c061f..274b75bdc 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1040,7 +1040,7 @@ "Applications missing from the export can only be archived or kept.": "Applications missing from the export can only be archived or kept.", "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", - "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.", + "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", "Unarchived": "Unarchived", "Restored": "Restored", diff --git a/l10n/nl.js b/l10n/nl.js index c5c7dd953..ea4be54bb 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1109,7 +1109,7 @@ OC.L10N.register( "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", - "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, die als leeg is gelezen, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Unarchived": "Uit archief gehaald", "Restored": "Hersteld", diff --git a/l10n/nl.json b/l10n/nl.json index 13297a8d9..ccc443ad8 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1108,7 +1108,7 @@ "Applications missing from the export can only be archived or kept.": "Applicaties die in de export ontbreken, kunnen alleen worden gearchiveerd of behouden.", "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", - "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, die als leeg is gelezen, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Unarchived": "Uit archief gehaald", "Restored": "Hersteld", diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 096767971..282219792 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -84,6 +84,10 @@ * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 * + * @phpstan-type ArchiveSheet array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} + * + * @phpstan-type ScopeEntry array{key: string, usage: object, module: object, states: array} + * * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) One import run resolves four linked * object kinds per row (supplier, module, owners, usage), each with its own match rule, * create-only fields and run cache. Splitting them over several classes would hand the @@ -622,7 +626,7 @@ private function runImport(string $path, array $options, string $startedAt): arr * * @param CmdbImportReport $report The report. * @param array{uuid: string, name: string, created: bool} $municipality The consumer, as resolveMunicipality() found it. - * @param array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} $archive The archive sheet. + * @param ArchiveSheet $archive The archive sheet. * @param string $missingRecords The import's `missingRecords` mode. * * @return void @@ -1933,7 +1937,7 @@ private static function missingRecordsFrom(array $options): string { * * @param array $workbook What the reader returned. * - * @return array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} + * @return ArchiveSheet * * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ @@ -1959,7 +1963,7 @@ private function archiveSheet(array $workbook): array { }//end archiveSheet() /** - * The report warning of an archiving import whose workbook lacks the archive sheet or its key column, or whose archive sheet has keys that could not be read. + * The report warning of an archiving import whose archive sheet is absent or has no key column, or has unreadable keys. * * @param array{sheet: string, keyColumnMissing: bool, uncachedRows: array} $archive The archive sheet. * @@ -1976,7 +1980,7 @@ private function archiveSheetWarning(array $archive): string { } return $this->l10n->t( - 'Sheet "%1$s" has an APPID without a cached value in row %2$s, which was read as empty, so applications missing from the export are archived, not deleted.', + 'Sheet "%1$s" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.', [$archive['sheet'], $rows] ); } @@ -2202,7 +2206,7 @@ private static function registerTrash(mixed $deleted, array $coordinates): array * stays, and the report says the run was cancelled. * * @param string $municipalityUuid The consumer of this import. - * @param array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} $archive The archive sheet. + * @param ArchiveSheet $archive The archive sheet. * @param string $operationId The operation id, for the archive reason. * @param int $processedItems The rows already counted as processed. * @param CmdbImportReport $report The report. @@ -2299,9 +2303,9 @@ private function reconciliationScope(string $municipalityUuid): array { * best usage state breaks a tie, and the first one listed breaks the rest. * A stale duplicate is left as it is, never restored into the archive. * - * @param array}> $scope The entries. + * @param array $scope The entries. * - * @return array}> + * @return array * * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 */ @@ -2334,7 +2338,11 @@ private static function stateRank(array $state): int { return 2; } - return ($state['archived'] === true) ? 1 : 0; + if ($state['archived'] === true) { + return 1; + } + + return 0; }//end stateRank() /** @@ -2463,7 +2471,7 @@ private function modulesInChunk(array $chunk, array $coordinates): array { * An application already in the state the export asks for is not reported. * * @param array{key: string, usage: object, module: object, states: array} $application Application. - * @param array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} $archive The archive sheet. + * @param ArchiveSheet $archive The archive sheet. * @param string $operationId The operation id, for the archive reason. * @param CmdbImportReport $report The report. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index b21e88ae7..7f9608fa8 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1975,7 +1975,7 @@ public function testAnUncachedArchiveRowArchivesInsteadOfDeleting(): void { $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $this->usageOf(moduleUuid: $eight))); $this->assertSame('cmdb-import: not in the export (' . $report['operationId'] . ')', $this->transitions[2][3]); $this->assertSame( - [['sheet' => 'Gearchiveerde Applicaties', 'message' => 'Sheet "Gearchiveerde Applicaties" has an APPID without a cached value in row 4, which was read as empty, so applications missing from the export are archived, not deleted.']], + [['sheet' => 'Gearchiveerde Applicaties', 'message' => 'Sheet "Gearchiveerde Applicaties" has an APPID without a cached value in row 4, so applications missing from the export are archived, not deleted.']], $report['importWarnings'] ); $this->assertSame([2, 0], [$report['summary']['archived'], $report['summary']['deleted']]); From aa3b664880428229dce162c086fa217309aebc14 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:45:31 +0200 Subject: [PATCH 31/87] docs(openspec): state the archive-only rule in every copy The rule "archive, never soft-delete" was written as "when the workbook has no archive sheet" in the design risk list, the proposal summary and Risk 2, and in task 4. It also applies when the sheet has no APPID column or holds an APPID the reader cannot see (a formula without a cached value). Review pre-push p1 Co-Authored-By: Claude Opus 5.5 --- .../changes/cmdb-import-archive-reconciliation/design.md | 2 +- .../changes/cmdb-import-archive-reconciliation/proposal.md | 6 +++--- .../changes/cmdb-import-archive-reconciliation/tasks.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md index 85ae87766..5928e26b6 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/design.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -192,7 +192,7 @@ No seed objects change. The fragment's three seed modules stay as they are; the ## Risks / Trade-offs - [The default archives on the first re-import after the upgrade] → The section shows the choice with help text; the report lists every archived and deleted application; nothing is hard-deleted. -- [A workbook saved without the archive sheet] → archive only, never delete, plus a warning (D3). +- [A workbook saved without the archive sheet, or whose archive sheet has no APPID column or an APPID without a cached value] → archive only, never delete, plus a warning (D3). - [A failure between the usage and the module] → the usage is written first, so the municipality's view is right and the module is at worst still visible (D5). - [Reviving before writing adds calls per returning row] → only for rows whose match is archived or deleted; a row whose module is working costs the same searches as before, a new row one more search (the archive lens). - [The trash listing scans every magic table of the instance] → read once per import, not per row (D9). diff --git a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md index 7c27fdd85..dd4017f30 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/proposal.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/proposal.md @@ -34,7 +34,7 @@ None. ### In Scope - The sheet "Gearchiveerde Applicaties" (columns Applicatiecomponent, APPID, Applicatie Code, Applicatie Naam, Roepnaam, Applicatie Status, Bron, Datum Interface, Datum wijziging) is read for its APPID column only, when it is present. -- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet: archived only, with a warning in the report. +- After every row of a completed import: the municipality's usages, with their modules, are compared with the APPIDs on the CMDB sheets and on the archive sheet. Missing and on the archive sheet: module and usage archived. Missing everywhere: module and usage soft-deleted. Without the archive sheet, or when it has no APPID column or an APPID the reader cannot see (a formula without a cached value): archived only, with a warning in the report. - A returning APPID: an archived module or usage is unarchived, a soft-deleted one is restored from the trash, and the row is then updated as before. - `missingRecords`: `keep` leaves everything as it is (the behaviour so far), `archive` reconciles and is the default; `remove` stays reserved and is refused. - The settings section offers the choice with help text; the report shows the new outcomes in its summary and rows. @@ -77,8 +77,8 @@ None. `ArchiveHandler` and `MagicMapper` come from OpenRegister, which stackiq a ### Risk 1: A re-import with the new default archives what an admin did not expect **Severity:** Medium — **Mitigation:** only the municipality's own usages and the modules they point at are in scope, never a supplier's or another municipality's module; nothing is hard-deleted; the choice is visible in the section with help text, and the report lists every archived and deleted application. A cancelled or failed import never reconciles. -### Risk 2: A workbook without the archive sheet soft-deletes everything that moved there -**Severity:** Medium — **Mitigation:** without the sheet, missing applications are archived, never soft-deleted, and the report warns that the sheet was not found. +### Risk 2: A workbook without a usable archive sheet soft-deletes everything that moved there +**Severity:** Medium — **Mitigation:** without the sheet, without its APPID column, or with an APPID the reader cannot see (a formula without a cached value), missing applications are archived, never soft-deleted, and the report warns, naming the sheet (and the rows). ### Risk 3: The archive services are OpenRegister internals **Severity:** Low — **Mitigation:** resolved through the container behind a guard; when missing, an import with `missingRecords=archive` is refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read, and `keep` still works. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md index cc3583dca..1ff7af0d9 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -43,7 +43,7 @@ Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-imp - **files**: `lib/Service/CmdbExportImportService.php`, `lib/Service/ProgressTracker.php`, `lib/Exception/CmdbImportException.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php` - **acceptance_criteria**: - GIVEN APPIDs 1 and 7 imported WHEN an export with row 1 and 7 on the archive sheet is imported THEN module and usage of 7 are archived with a reason, the report row is `archived`, and the owner contact person is untouched - - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent THEN they are archived instead and the report warns + - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent, without an APPID column, or with an APPID that has no cached value THEN they are archived instead and the report warns - GIVEN APPID 7 soft-deleted and listed on the archive sheet THEN module and usage are restored and archived; GIVEN it already archived or, when on no sheet, already deleted THEN nothing is written - GIVEN `missingRecords` `keep`, a cancelled import, or a usage of another municipality THEN nothing is archived or deleted - GIVEN a cancel during the last row THEN nothing is archived or deleted; GIVEN a cancel during the reconciliation THEN it stops before the next application and the report is cancelled From 70d6f9344775505712775ca0f8caf5b616cb81e8 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:45:31 +0200 Subject: [PATCH 32/87] docs(openspec): the Archived tab ignores the facet narrowing D8 said the facet narrowing's _ids composes with the Archived quick filter. The Archived tab of the Applications page overrides _ids with an empty value, so it lists the whole archive and ignores facets and search. Review pre-push p2 Co-Authored-By: Claude Opus 5.5 --- openspec/changes/cmdb-import-archive-reconciliation/design.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md index 5928e26b6..51daa0580 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/design.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -115,7 +115,7 @@ Progress: the operation gets the phase `reconciling` (added to `ProgressTracker: ### D8. Stackiq UI -The Applications (`Modules`, a `FacetedCatalogIndexView` that forwards `quickFilters` to `CnIndexPage`) and Applications in use (`Gebruik`) pages get the quick filter `{"label": "Archived", "filter": {"_archived": "true"}}`. `useSelfFetchList` spreads the active tab's filter into the fetch parameters as they are, so `_archived=true` reaches OpenRegister and the list shows the archive alone; the facet narrowing's `_ids` composes with it (AND). The icon `ArchiveOutline` is registered in `src/icons.js`. `@conduction/nextcloud-vue` 2.65 renders no `@self.archived` on a detail page or in the sidebar metadata, so the detail pages show no archive state yet; that is a library follow-up, not a stackiq widget. +The Applications (`Modules`, a `FacetedCatalogIndexView` that forwards `quickFilters` to `CnIndexPage`) and Applications in use (`Gebruik`) pages get the quick filter `{"label": "Archived", "filter": {"_archived": "true"}}`. `useSelfFetchList` spreads the active tab's filter into the fetch parameters as they are, so `_archived=true` reaches OpenRegister and the list shows the archive alone. The facet narrowing's `_ids` would compose with it (AND) and hide archived applications the facets do not know; the Archived tab of the Applications page therefore overrides `_ids` with an empty value, so it lists the whole archive and ignores the facets and the search. The icon `ArchiveOutline` is registered in `src/icons.js`. `@conduction/nextcloud-vue` 2.65 renders no `@self.archived` on a detail page or in the sidebar metadata, so the detail pages show no archive state yet; that is a library follow-up, not a stackiq widget. ### D9. How the import reads the state of a stored object From df28777e4a8881d8ca48e81b388f097950255afa Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:45:58 +0200 Subject: [PATCH 33/87] fix(cmdb-import): keep create-only defaults out of createOnlyFields createOnlyFields is documented as mapped fields, but for the module pack the accessor returned ['type'], a key of the create-only defaults that merge() never reads on update (it only walks mapped fields). The overview now answers that value under createOnlyDefaults only and subtracts the default keys from createOnlyFields; the import's accessors are unchanged. REQ-CMDB-020 and design.md say so, and the controller and profile tests pin it. Review pre-push p6 Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbImportProfile.php | 20 +++++++++++++++---- .../cmdb-import-mapping-view/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 2 +- ...ettingsControllerCmdbImportMappingTest.php | 9 +++++++-- .../Service/Cmdb/CmdbImportProfileTest.php | 2 +- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index 81a256081..73dc75e5c 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -634,8 +634,9 @@ public function referencedColumns(): array { * hidden from the admin; `required` is normalised to a boolean. Each pack * also carries the write rules the import applies on top of the mapping: * `defaults` (pack values set on every write), `createOnlyDefaults` - * (values set on create only), `createOnlyFields` (mapped fields written on - * create or when the stored value is empty) and `neverWritten` (fields the + * (values set on create only, not mapped columns), `createOnlyFields` (mapped + * fields written on create or, on update, only when the stored value is + * empty; never a key of `createOnlyDefaults`) and `neverWritten` (fields the * import never writes on an existing object). * * @return array{profile: array, packs: array>} @@ -670,6 +671,17 @@ public function mappingOverview(): array { ]; } + // A create-only default (the module type) is a value the import sets on + // create, not a mapped field: merge() never reads it on update, so it is + // listed under createOnlyDefaults only. + $createOnlyDefaults = $this->createOnlyDefaults(target: $target); + $createOnlyFields = array_values( + array_diff( + $this->createOnlyFields(target: $target), + array_map('strval', array_keys($createOnlyDefaults)) + ) + ); + $defaults = $pack['defaults'] ?? []; if (is_array($defaults) === false) { $defaults = []; @@ -684,8 +696,8 @@ public function mappingOverview(): array { 'description' => (string)($pack['description'] ?? ''), 'fieldMappings' => $mappings, 'defaults' => $defaults, - 'createOnlyDefaults' => $this->createOnlyDefaults(target: $target), - 'createOnlyFields' => $this->createOnlyFields(target: $target), + 'createOnlyDefaults' => $createOnlyDefaults, + 'createOnlyFields' => $createOnlyFields, 'neverWritten' => $this->neverWrittenOnUpdate(target: $target), ]; }//end foreach diff --git a/openspec/changes/cmdb-import-mapping-view/design.md b/openspec/changes/cmdb-import-mapping-view/design.md index 225ede564..0bd69a911 100644 --- a/openspec/changes/cmdb-import-mapping-view/design.md +++ b/openspec/changes/cmdb-import-mapping-view/design.md @@ -46,7 +46,7 @@ The method carries no auth attribute (Nextcloud's default: admins with a CSRF to `fieldMappings[].transform` is passed through as the pack stores it (`type` plus `map`/`default`, `fields`/`separator`, `value`, `sourceFormat`/`targetFormat`). Re-shaping per type would hide a key the engine reads. The page renders the known keys and lists any other scalar key as it is. `required` is normalised to a boolean. -The write rules the import applies besides the mapping are answered per pack, built from the accessors the import uses (`createOnlyDefaults()`, `createOnlyFields()`, `neverWrittenOnUpdate()`) and the pack's own `defaults`, so the view does not overclaim for a field the import writes only on create or never. +The write rules the import applies besides the mapping are answered per pack, built from the accessors the import uses (`createOnlyDefaults()`, `createOnlyFields()`, `neverWrittenOnUpdate()`) and the pack's own `defaults`, so the view does not overclaim for a field the import writes only on create or never. A create-only default such as the module `type` is answered in `createOnlyDefaults` only: it is not a mapped field and `merge()` never reads it on update, so `createOnlyFields` leaves it out. ### D4. The section takes the sheet names from the endpoint, with the constant as fallback diff --git a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md index 9d807394b..9675bb693 100644 --- a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md @@ -16,7 +16,7 @@ Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\Mi ### Requirement: The admin settings SHALL show the mapping the import uses (REQ-CMDB-020) -`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack it names through the same loader and validator the import uses, and SHALL answer 200 with a flat envelope of two keys: `profile` (the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns`, `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`) and `packs`, one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings` and the write rules the import applies besides the mapping: `defaults` (the pack's values, field => value), `createOnlyDefaults` (values set on create only, field => value), `createOnlyFields` (mapped fields written on create or, on an existing object, only when the stored value is empty) and `neverWritten` (fields the import never writes on an existing object); the last three come from the profile's `createOnly` and `neverWritten` through the accessors the import itself uses. Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it: the `type`, and for `lookup` and `bool-map` the `map` and `default`, for `concat` the `fields` and `separator`, for `const` the `value`, for `date` the `sourceFormat` and `targetFormat`. The endpoint SHALL read nothing but those files and SHALL write nothing. When the validator is missing, or the profile or a pack is unreadable or invalid, it SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason`, so the administrator who edited a file learns what is wrong without the Nextcloud log. The "CMDB import" section SHALL show the answer in a collapsible block "Mapping (read-only)": the sheets, the key, name, required and date columns, and one table per pack with the source column, the target field, whether it is required, the transformation and its details. The block SHALL show a loading state until the endpoint answers and the error with its code when it fails, and SHALL offer no editing: the mapping is changed in the files on the server, as the documentation describes. +`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack it names through the same loader and validator the import uses, and SHALL answer 200 with a flat envelope of two keys: `profile` (the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns`, `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`) and `packs`, one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings` and the write rules the import applies besides the mapping: `defaults` (the pack's values, field => value), `createOnlyDefaults` (values the import sets on create only, field => value; they are not mapped columns), `createOnlyFields` (mapped fields written on create or, on an existing object, only when the stored value is empty; never a field that is a key of `createOnlyDefaults`) and `neverWritten` (fields the import never writes on an existing object); the last three come from the profile's `createOnly` and `neverWritten` through the accessors the import itself uses. Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it: the `type`, and for `lookup` and `bool-map` the `map` and `default`, for `concat` the `fields` and `separator`, for `const` the `value`, for `date` the `sourceFormat` and `targetFormat`. The endpoint SHALL read nothing but those files and SHALL write nothing. When the validator is missing, or the profile or a pack is unreadable or invalid, it SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason`, so the administrator who edited a file learns what is wrong without the Nextcloud log. The "CMDB import" section SHALL show the answer in a collapsible block "Mapping (read-only)": the sheets, the key, name, required and date columns, and one table per pack with the source column, the target field, whether it is required, the transformation and its details. The block SHALL show a loading state until the endpoint answers and the error with its code when it fails, and SHALL offer no editing: the mapping is changed in the files on the server, as the documentation describes. #### Scenario: An admin reads the mapping the import uses @e2e tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts diff --git a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php index 61ec942cb..46d6badd4 100644 --- a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php +++ b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php @@ -233,7 +233,7 @@ public function testTheShippedMappingIsAnsweredFlat(): void { // The write rules the import applies besides the mapping (Review #1288 f2). $byTarget = array_column($data['packs'], null, 'target'); $this->assertSame(['type' => 'Application'], $byTarget['module']['createOnlyDefaults']); - $this->assertSame(['type'], $byTarget['module']['createOnlyFields']); + $this->assertSame([], $byTarget['module']['createOnlyFields'], 'type is a create-only default, not a mapped field'); $this->assertSame(['publicationDate', 'depublicationDate'], $byTarget['module']['neverWritten']); $this->assertSame(['interneAnnotation', 'timeClassification'], $byTarget['usage']['createOnlyFields']); $this->assertSame([], $byTarget['usage']['createOnlyDefaults']); @@ -242,7 +242,12 @@ public function testTheShippedMappingIsAnsweredFlat(): void { $this->assertSame(['type' => 'Municipality', 'status' => 'Active'], $byTarget['municipality']['defaults']); $this->assertSame([], $byTarget['module']['defaults']); foreach ($data['packs'] as $pack) { - $this->assertSame($profile->createOnlyFields(target: $pack['target']), $pack['createOnlyFields'], $pack['target']); + $this->assertSame( + array_values(array_diff($profile->createOnlyFields(target: $pack['target']), array_keys($pack['createOnlyDefaults']))), + $pack['createOnlyFields'], + $pack['target'] + ); + $this->assertSame([], array_intersect($pack['createOnlyFields'], array_keys($pack['createOnlyDefaults'])), $pack['target']); $this->assertSame($profile->neverWrittenOnUpdate(target: $pack['target']), $pack['neverWritten'], $pack['target']); } diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 5ca02614f..d76e4203f 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -330,7 +330,7 @@ public function testTheOverviewShowsWhatTheImportRuns(): void { } $this->assertSame($profile->createOnlyDefaults(target: $pack['target']), $pack['createOnlyDefaults'], $pack['target']); - $this->assertSame($profile->createOnlyFields(target: $pack['target']), $pack['createOnlyFields'], $pack['target']); + $this->assertSame([], array_intersect($pack['createOnlyFields'], array_keys($pack['createOnlyDefaults'])), $pack['target']); $this->assertSame($profile->neverWrittenOnUpdate(target: $pack['target']), $pack['neverWritten'], $pack['target']); } From 9fb2dfdb0d620cb8b4684935d91cc2cb47779202 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:46:12 +0200 Subject: [PATCH 34/87] docs(cmdb-import): state all three cases where an import archives and never deletes The docs and the choice's help said an import only archives when the archive sheet is missing (the docs added "or has no APPID column" and said the result warns the sheet was not found). The service also archives, with its own warning for each case, when the sheet has no APPID column or an APPID in it has no cached value. The docs table row and the section help now name the three cases and say the result warns; the l10n key is replaced in place and a jest case pins the help. Review pre-push p1 Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 2 +- l10n/en.js | 2 +- l10n/en.json | 2 +- l10n/nl.js | 2 +- l10n/nl.json | 2 +- src/views/settings/sections/CmdbImport.spec.js | 11 +++++++++++ src/views/settings/sections/CmdbImport.vue | 2 +- 7 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 1e33207d5..364bcf5f6 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -277,7 +277,7 @@ APPID is on neither CMDB sheet: |---|---| | on the sheet `Gearchiveerde Applicaties` | are **archived**: OpenRegister's archive state, with the reason `cmdb-import: on sheet "Gearchiveerde Applicaties"`. Reported as *archived*. | | on no sheet at all | are **deleted**: moved to OpenRegister's trash, where they stay restorable for 30 days. Reported as *deleted*. | -| on no sheet, and the workbook has no sheet `Gearchiveerde Applicaties` (or it has no `APPID` column) | are **archived**, never deleted, and the result warns that the sheet was not found. A workbook saved without that sheet therefore never moves anything to the trash. | +| on no sheet, but the archive sheet cannot be relied on: the workbook has no sheet `Gearchiveerde Applicaties`, the sheet has no `APPID` column, or an `APPID` in it has no cached value (a formula whose result the file does not store) | are **archived**, never deleted. Each case has its own warning in the result: *sheet not found*, *no column APPID*, or *an APPID without a cached value in row …* (the first rows are named). Such a workbook therefore never moves anything to the trash. | An archived or deleted application disappears from the stackiq lists, from OpenCatalogi and from Portaliq's **Software we use**. In stackiq, the quick diff --git a/l10n/en.js b/l10n/en.js index 1c20fdbc0..0dd84f275 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1050,7 +1050,7 @@ OC.L10N.register( "Applications missing from the export": "Applications missing from the export", "Archive or delete them": "Archive or delete them", "Keep them as they are": "Keep them as they are", - "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.", "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", "Checking the applications of this municipality against the export": "Checking the applications of this municipality against the export", diff --git a/l10n/en.json b/l10n/en.json index faa7cf8db..e2f6ad6cf 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1049,7 +1049,7 @@ "Applications missing from the export": "Applications missing from the export", "Archive or delete them": "Archive or delete them", "Keep them as they are": "Keep them as they are", - "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.": "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.", "Applications missing from the export cannot be archived.": "Applications missing from the export cannot be archived.", "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.", "Checking the applications of this municipality against the export": "Checking the applications of this municipality against the export", diff --git a/l10n/nl.js b/l10n/nl.js index ce2e71c60..c3ac2b4b1 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1118,7 +1118,7 @@ OC.L10N.register( "Applications missing from the export": "Applicaties die in de export ontbreken", "Archive or delete them": "Archiveren of verwijderen", "Keep them as they are": "Laat ze zoals ze zijn", - "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Ontbreekt dat tabblad, heeft het geen kolom APPID of heeft een APPID geen opgeslagen waarde, dan worden applicaties alleen gearchiveerd en zegt het resultaat dat. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", "Checking the applications of this municipality against the export": "De applicaties van deze gemeente met de export vergelijken", diff --git a/l10n/nl.json b/l10n/nl.json index 059c7ada7..e62e0749a 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1117,7 +1117,7 @@ "Applications missing from the export": "Applicaties die in de export ontbreken", "Archive or delete them": "Archiveren of verwijderen", "Keep them as they are": "Laat ze zoals ze zijn", - "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Staat dat tabblad niet in het bestand, dan worden applicaties alleen gearchiveerd. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", + "An application of this municipality that is on neither CMDB sheet any more is archived when the sheet \"Gearchiveerde Applicaties\" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.": "Een applicatie van deze gemeente die op geen van beide CMDB-tabbladen meer staat, wordt gearchiveerd als het tabblad \"Gearchiveerde Applicaties\" haar noemt, en naar de prullenbak verplaatst als geen enkel tabblad dat doet. Beide verdwijnen uit de lijsten, uit OpenCatalogi en uit Portaliq; een applicatie die in een latere export terugkomt, wordt teruggezet. Ontbreekt dat tabblad, heeft het geen kolom APPID of heeft een APPID geen opgeslagen waarde, dan worden applicaties alleen gearchiveerd en zegt het resultaat dat. Kies Laat ze zoals ze zijn om ze ongemoeid te laten.", "Applications missing from the export cannot be archived.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd.", "OpenRegister is missing or too old to archive objects. Choose \"Keep them as they are\" and import again, or update OpenRegister. Nothing was imported.": "OpenRegister ontbreekt of is te oud om objecten te archiveren. Kies \"Laat ze zoals ze zijn\" en importeer opnieuw, of werk OpenRegister bij. Er is niets geïmporteerd.", "Checking the applications of this municipality against the export": "De applicaties van deze gemeente met de export vergelijken", diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index ca01b9517..5f43536cc 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -435,6 +435,17 @@ describe('CmdbImport the choice for missing applications, as the admin makes it' expect(wrapper.vm.missingRecords).toBe('archive') }) + it('says in the help when applications are only archived, never deleted', async () => { + const wrapper = await mountFull() + + const help = wrapper.get('#cmdb-import-missing-records-help').text() + + expect(help).toContain('is missing') + expect(help).toContain('has no APPID column') + expect(help).toContain('an APPID without a cached value') + expect(help).toContain('only archived') + }) + it('disables both radios while an import runs', async () => { const wrapper = await mountFull() diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 3590c47f7..88eaf5d75 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -172,7 +172,7 @@ {{ t( 'stackiq', - 'An application of this municipality that is on neither CMDB sheet any more is archived when the sheet "Gearchiveerde Applicaties" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. Without that sheet in the file, applications are only archived. Choose Keep to leave them as they are.', + 'An application of this municipality that is on neither CMDB sheet any more is archived when the sheet "Gearchiveerde Applicaties" lists it, and moved to the trash when no sheet does. Both disappear from the lists, from OpenCatalogi and from Portaliq; one that returns in a later export is brought back. When that sheet is missing, has no APPID column, or has an APPID without a cached value, applications are only archived and the result says so. Choose Keep to leave them as they are.', ) }}

    From e7f9125cf467430f43b98adcb5f9e16efbdfc94e Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:46:22 +0200 Subject: [PATCH 35/87] test(cmdb-import): fixture follows the overview's createOnlyFields The endpoint no longer lists the module's create-only default `type` under createOnlyFields; the vitest answer fixture mirrors that, and the block still shows `type = Application` as set on create only. Review pre-push p6 Co-Authored-By: Claude Opus 5.5 --- tests/vitest/cmdbImportMapping.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vitest/cmdbImportMapping.spec.js b/tests/vitest/cmdbImportMapping.spec.js index b12c1b956..f5ddf9062 100644 --- a/tests/vitest/cmdbImportMapping.spec.js +++ b/tests/vitest/cmdbImportMapping.spec.js @@ -125,7 +125,7 @@ function answer() { }, ], { createOnlyDefaults: { type: 'Application' }, - createOnlyFields: ['type'], + createOnlyFields: [], neverWritten: ['publicationDate', 'depublicationDate'], }), pack('manufacturer', [], { From e8772a746819d58572d3761c11b8a0e97d23cd78 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:46:25 +0200 Subject: [PATCH 36/87] docs(cmdb-import): state the full archive-only rule in the service docblocks The class docblock and the addRunWarnings() docblock still named only a missing archive sheet as the reason to archive instead of delete; the rule also covers a sheet without an APPID column and one with an APPID that has no cached value. Review pre-push p1 Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 282219792..fae8bd431 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -34,7 +34,8 @@ * `missingRecords` `keep`. With `archive` (the default) the municipality's * usages and their modules are reconciled after the last row: archived * when the APPID is on the archive sheet, soft-deleted when it is on no - * sheet, archived only when the workbook has no archive sheet + * sheet, archived only when the workbook has no archive sheet, the sheet + * has no APPID column or an APPID in it has no cached value * (openspec/changes/cmdb-import-archive-reconciliation, design D5). A * module or usage matched among archived or soft-deleted objects is * unarchived or restored before it is updated, never duplicated. @@ -622,7 +623,7 @@ private function runImport(string $path, array $options, string $startedAt): arr }//end runImport() /** - * Add the run's own warnings: a missing archive sheet on an archiving import, and a created municipality. + * Add the run's own warnings: an archive sheet that is missing, has no key column or has unreadable keys, and a created municipality. * * @param CmdbImportReport $report The report. * @param array{uuid: string, name: string, created: bool} $municipality The consumer, as resolveMunicipality() found it. From 2d288acd3f113d8a2c6ae31635f8892089dd0081 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:46:29 +0200 Subject: [PATCH 37/87] docs(cmdb-import): correct the Modules page note on how quick filters merge with the facet ids The manifest note and the view's docblock said a quick filter composes with the facet `_ids` as an additive AND that never clobbers. Since the Archived tab carries an empty-string `_ids`, that tab replaces the facet id set and lists the whole archive, ignoring the GEMMA facets and the search term. Both now name that exception; the docs already said so. A vitest pins the note. Review pre-push p2 Co-Authored-By: Claude Opus 5.5 --- src/manifest.json | 2 +- src/views/FacetedCatalogIndexView.vue | 3 ++- tests/vitest/cmdbArchiveQuickFilter.spec.js | 8 ++++++++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/manifest.json b/src/manifest.json index 245f8c802..1541f303e 100644 --- a/src/manifest.json +++ b/src/manifest.json @@ -616,7 +616,7 @@ "type": "custom", "title": "Applications", "component": "FacetedCatalogIndexView", - "_note": "Faceted index page (gemma-faceted-search): CnFacetSidebar (live GEMMA-dimension counts from FacetService) + a standalone CnIndexPage narrowed via { _ids: matchedObjectIds } (OpenRegister's list parameter; the objects endpoint ignores a plain id filter). Columns are objects: `provider` is a reference, so it renders through the fkResolve cell widget (uuid → organisation name) and carries its own label — object-typed properties get no schema-generated column, so a bare key showed the raw uuid under the header 'provider'. Stays custom rather than type:index — see customComponents.js's FacetedCatalogIndexView entry for why CnIndexPage's own embedded facet sidebar cannot drive this narrowing. bio-compliance-assessment: bbnLevel/dpiaStatus columns + BBN/DPIA quickFilters reconciled onto this page after gemma-faceted-search (#382) merged — see FacetedCatalogIndexView.vue's quickFilters prop docblock for how the quick filter's field constraints compose with the facet { _ids: ... } narrowing (disjoint keys, additive AND, no clobber).", + "_note": "Faceted index page (gemma-faceted-search): CnFacetSidebar (live GEMMA-dimension counts from FacetService) + a standalone CnIndexPage narrowed via { _ids: matchedObjectIds } (OpenRegister's list parameter; the objects endpoint ignores a plain id filter). Columns are objects: `provider` is a reference, so it renders through the fkResolve cell widget (uuid → organisation name) and carries its own label — object-typed properties get no schema-generated column, so a bare key showed the raw uuid under the header 'provider'. Stays custom rather than type:index — see customComponents.js's FacetedCatalogIndexView entry for why CnIndexPage's own embedded facet sidebar cannot drive this narrowing. bio-compliance-assessment: bbnLevel/dpiaStatus columns + BBN/DPIA quickFilters reconciled onto this page after gemma-faceted-search (#382) merged — see FacetedCatalogIndexView.vue's quickFilters prop docblock for how the quick filter's field constraints compose with the facet { _ids: ... } narrowing (disjoint keys, additive AND, no clobber). The one exception is the Archived tab: its filter carries an empty-string `_ids` that replaces the facet id set in the merge, because that set is built from the working set, which leaves archived applications out, so ANDing it with `_archived=true` could never match. That tab lists the whole archive and ignores the GEMMA facets and the search term.", "config": { "register": "@resolve:voorzieningen_register", "schema": "module", diff --git a/src/views/FacetedCatalogIndexView.vue b/src/views/FacetedCatalogIndexView.vue index cc29bb2ec..427245c47 100644 --- a/src/views/FacetedCatalogIndexView.vue +++ b/src/views/FacetedCatalogIndexView.vue @@ -232,7 +232,8 @@ export default { * filter's keys (`bbnLevel`, `dpiaStatus`, …) never collide with * `_ids`, so the merge is additive — both the facet-matched id set AND * the active quick filter's field constraints apply together - * (logical AND), never clobbering one another. + * (logical AND), never clobbering one another. The one exception is + * the Archived tab of the Applications page, below. * * The exception is the Archived tab of the Applications page * (cmdb-import-archive-reconciliation): the facet id set is built from diff --git a/tests/vitest/cmdbArchiveQuickFilter.spec.js b/tests/vitest/cmdbArchiveQuickFilter.spec.js index c048abf8f..ff4df73ee 100644 --- a/tests/vitest/cmdbArchiveQuickFilter.spec.js +++ b/tests/vitest/cmdbArchiveQuickFilter.spec.js @@ -111,6 +111,14 @@ describe('the archive quick filter', () => { ) }) + it('the Modules page note names the Archived tab as the exception to the additive merge', () => { + const page = pages().find((candidate) => candidate.id === 'Modules') + + expect(page._note).toContain('additive AND') + expect(page._note).toMatch(/exception is the Archived tab/) + expect(page._note).toMatch(/ignores the GEMMA facets and the search term/) + }) + it('uses an icon the app registers', () => { const icons = fs.readFileSync(path.join(repoRoot, 'src/icons.js'), 'utf8') expect(icons).toMatch(/^\tArchiveOutline,$/m) From 16de6470a4c712b6dafb894d3654bda0855c1c68 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:46:55 +0200 Subject: [PATCH 38/87] fix(cmdb-import): count the rows in the uncached-archive-key warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warning said "in row 2, 3, 4, …" for any number of rows and, once the list was cut after ten, did not say how many rows there were. It now uses a singular form for one row and "in N rows (first ten, …)" otherwise, with the total count; en and nl, and the test covers one, a few and a cut list. Review pre-push p5 Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 1 + l10n/en.json | 1 + l10n/nl.js | 1 + l10n/nl.json | 1 + lib/Service/CmdbExportImportService.php | 16 ++++++++++++---- .../Unit/Service/CmdbExportImportServiceTest.php | 10 +++++++++- 6 files changed, 25 insertions(+), 5 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 6eba930ad..ca33bfe31 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1042,6 +1042,7 @@ OC.L10N.register( "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.", + "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", "Unarchived": "Unarchived", "Restored": "Restored", diff --git a/l10n/en.json b/l10n/en.json index 274b75bdc..f94e81ea8 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1041,6 +1041,7 @@ "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.", "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.", + "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.": "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.", "Unarchived": "Unarchived", "Restored": "Restored", diff --git a/l10n/nl.js b/l10n/nl.js index ea4be54bb..6769c4b32 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1110,6 +1110,7 @@ OC.L10N.register( "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in %2$s rijen (%3$s), dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Unarchived": "Uit archief gehaald", "Restored": "Hersteld", diff --git a/l10n/nl.json b/l10n/nl.json index ccc443ad8..b596fe558 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1109,6 +1109,7 @@ "Applications missing from the export cannot be archived: OpenRegister is missing or too old. Choose to keep them, or update OpenRegister.": "Applicaties die in de export ontbreken, kunnen niet worden gearchiveerd: OpenRegister ontbreekt of is te oud. Kies ervoor ze te behouden, of werk OpenRegister bij.", "Sheet \"%1$s\" has no column \"%2$s\", so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft geen kolom \"%2$s\", dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%1$s\" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in rij %2$s, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", + "Sheet \"%1$s\" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.": "Tabblad \"%1$s\" heeft een APPID zonder opgeslagen waarde in %2$s rijen (%3$s), dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Sheet \"%s\" was not found, so applications missing from the export are archived, not deleted.": "Tabblad \"%s\" is niet gevonden, dus applicaties die in de export ontbreken, worden gearchiveerd en niet verwijderd.", "Unarchived": "Uit archief gehaald", "Restored": "Hersteld", diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index fae8bd431..fc3b15e34 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -1974,15 +1974,23 @@ private function archiveSheet(array $workbook): array { */ private function archiveSheetWarning(array $archive): string { if ($archive['uncachedRows'] !== []) { - // A long list is cut: the warning names where to look, not every row. + // A long list is cut: the warning counts the rows and names where to look, not every one. + $count = count($archive['uncachedRows']); $rows = implode(', ', array_slice($archive['uncachedRows'], 0, self::WARNED_ROWS)); - if (count($archive['uncachedRows']) > self::WARNED_ROWS) { + if ($count > self::WARNED_ROWS) { $rows .= ', …'; } + if ($count === 1) { + return $this->l10n->t( + 'Sheet "%1$s" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.', + [$archive['sheet'], $rows] + ); + } + return $this->l10n->t( - 'Sheet "%1$s" has an APPID without a cached value in row %2$s, so applications missing from the export are archived, not deleted.', - [$archive['sheet'], $rows] + 'Sheet "%1$s" has an APPID without a cached value in %2$s rows (%3$s), so applications missing from the export are archived, not deleted.', + [$archive['sheet'], $count, $rows] ); } diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index 7f9608fa8..da576de5d 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -1984,7 +1984,15 @@ public function testAnUncachedArchiveRowArchivesInsteadOfDeleting(): void { $this->transitions = []; $long = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'], uncachedRows: range(2, 13))) ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'archive']); - $this->assertStringContainsString('in row 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, …,', $long['importWarnings'][0]['message']); + $this->assertSame( + 'Sheet "Gearchiveerde Applicaties" has an APPID without a cached value in 12 rows (2, 3, 4, 5, 6, 7, 8, 9, 10, 11, …), so applications missing from the export are archived, not deleted.', + $long['importWarnings'][0]['message'], + 'the warning counts the rows and cuts the list' + ); + + $few = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'], uncachedRows: [3, 9])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1', 'missingRecords' => 'archive']); + $this->assertStringContainsString('in 2 rows (3, 9), so', $few['importWarnings'][0]['message']); // Keeping missing records needs no sheet and gives no warning. $kept = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'], uncachedRows: [4])) From f04bf8233df4ccc62cf76308f8f590ec22d0dc4c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:47:46 +0200 Subject: [PATCH 39/87] fix(cmdb-import): name every sheet in the missing-sheet and too-large hints The NO_SOURCE_SHEET hint read only the first two expected sheets (or the shipped names) and the WORKBOOK_TOO_LARGE hint always named the shipped two, so a one- or three-sheet profile was misreported there too, unlike the help texts. errorText() now takes the profile's sheet names, the section passes the ones from the mapping block, and both hints list all of them; the server's `expected` still wins for NO_SOURCE_SHEET. The three replaced strings are removed from en and nl, with a singular and a plural form for the missing-sheet hint. The jest cases pin a one- and a three-sheet profile, in the helper and through the section. Review pre-push p7 Co-Authored-By: Claude Opus 5.5 --- l10n/en.js | 7 +-- l10n/en.json | 7 +-- l10n/nl.js | 7 +-- l10n/nl.json | 7 +-- src/utils/cmdbImport.js | 43 +++++++++++-------- src/utils/cmdbImport.spec.js | 31 +++++++++++-- .../settings/sections/CmdbImport.spec.js | 19 ++++++++ src/views/settings/sections/CmdbImport.vue | 2 +- 8 files changed, 88 insertions(+), 35 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index ad4e755b9..f4d72a00c 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -974,7 +974,6 @@ OC.L10N.register( "Error code: {code}": "Error code: {code}", "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".", "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Existing municipalities could not be loaded. You can still type the name of a municipality.", - "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.", "Failed": "Failed", "Import": "Import", "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses", @@ -1126,7 +1125,6 @@ OC.L10N.register( "Split the rows over two copies of the export and import them one after the other.": "Split the rows over two copies of the export and import them one after the other.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.", "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.", "In development since": "In development since", @@ -1175,7 +1173,10 @@ OC.L10N.register( "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.", "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.", "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.", - "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored." + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "Expected the sheet {sheets}. Sheet names must match exactly.": "Expected the sheet {sheets}. Sheet names must match exactly.", + "Expected one of the sheets {sheets}. Sheet names must match exactly.": "Expected one of the sheets {sheets}. Sheet names must match exactly.", + "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index c18324251..5ffcb140a 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -973,7 +973,6 @@ "Error code: {code}": "Error code: {code}", "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".", "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Existing municipalities could not be loaded. You can still type the name of a municipality.", - "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.", "Failed": "Failed", "Import": "Import", "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses", @@ -1125,7 +1124,6 @@ "Split the rows over two copies of the export and import them one after the other.": "Split the rows over two copies of the export and import them one after the other.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.", "The workbook holds more than {count} different texts, the most the import reads.": "The workbook holds more than {count} different texts, the most the import reads.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.", "In development since": "In development since", @@ -1174,6 +1172,9 @@ "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.", "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.", "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.", - "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored." + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.", + "Expected the sheet {sheets}. Sheet names must match exactly.": "Expected the sheet {sheets}. Sheet names must match exactly.", + "Expected one of the sheets {sheets}. Sheet names must match exactly.": "Expected one of the sheets {sheets}. Sheet names must match exactly.", + "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy.": "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy." } } diff --git a/l10n/nl.js b/l10n/nl.js index dbd4d8771..460350e2c 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1042,7 +1042,6 @@ OC.L10N.register( "Error code: {code}": "Foutcode: {code}", "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel-werkmap (.xlsx), hoogstens 10 MB, met het tabblad \"Onbeh Applicaties CMDB\" of \"Beheerde Applicaties CMDB\".", "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Bestaande gemeenten konden niet worden geladen. U kunt nog steeds de naam van een gemeente typen.", - "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Verwacht werd een tabblad met de naam \"{first}\" of \"{second}\". De naam van het tabblad moet precies overeenkomen.", "Failed": "Mislukt", "Import": "Importeren", "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Importeer een TOPdesk CMDB-export (.xlsx) als de applicaties die één gemeente gebruikt", @@ -1194,7 +1193,6 @@ OC.L10N.register( "Split the rows over two copies of the export and import them one after the other.": "Verdeel de rijen over twee kopieën van de export en importeer die na elkaar.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "Het tabblad \"{sheet}\" is te groot om te importeren: uitgepakt is het {size}, en de import leest van één tabblad hooguit {limit}.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "Het onderdeel {part} van de werkmap is te groot om te importeren: uitgepakt is het {size}, en de import leest van één onderdeel hooguit {limit}.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen \"{first}\" en \"{second}\"), en importeer die kopie.", "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest.", "In development since": "In ontwikkeling sinds", @@ -1243,7 +1241,10 @@ OC.L10N.register( "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad {sheets}. Standaard mag het bestand hoogstens {size} zijn.", "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met een van de tabbladen {sheets}. Standaard mag het bestand hoogstens {size} zijn.", "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "Het tabblad {sheets} wordt gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", - "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd." + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "Expected the sheet {sheets}. Sheet names must match exactly.": "Verwacht werd het tabblad {sheets}. De naam van het tabblad moet precies overeenkomen.", + "Expected one of the sheets {sheets}. Sheet names must match exactly.": "Verwacht werd een van de tabbladen {sheets}. De namen van de tabbladen moeten precies overeenkomen.", + "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen {sheets}), en importeer die kopie." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 4f89dd5b9..fdbee7ae0 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1041,7 +1041,6 @@ "Error code: {code}": "Foutcode: {code}", "Excel workbook (.xlsx), at most 10 MB, with the sheet \"Onbeh Applicaties CMDB\" or \"Beheerde Applicaties CMDB\".": "Excel-werkmap (.xlsx), hoogstens 10 MB, met het tabblad \"Onbeh Applicaties CMDB\" of \"Beheerde Applicaties CMDB\".", "Existing municipalities could not be loaded. You can still type the name of a municipality.": "Bestaande gemeenten konden niet worden geladen. U kunt nog steeds de naam van een gemeente typen.", - "Expected a sheet named \"{first}\" or \"{second}\". Sheet names must match exactly.": "Verwacht werd een tabblad met de naam \"{first}\" of \"{second}\". De naam van het tabblad moet precies overeenkomen.", "Failed": "Mislukt", "Import": "Importeren", "Import a TOPdesk CMDB export (.xlsx) as the applications one municipality uses": "Importeer een TOPdesk CMDB-export (.xlsx) als de applicaties die één gemeente gebruikt", @@ -1193,7 +1192,6 @@ "Split the rows over two copies of the export and import them one after the other.": "Verdeel de rijen over twee kopieën van de export en importeer die na elkaar.", "The sheet \"{sheet}\" is too large to import: unpacked it is {size}, and the import reads at most {limit} of one sheet.": "Het tabblad \"{sheet}\" is te groot om te importeren: uitgepakt is het {size}, en de import leest van één tabblad hooguit {limit}.", "The part {part} of the workbook is too large to import: unpacked it is {size}, and the import reads at most {limit} of one part.": "Het onderdeel {part} van de werkmap is te groot om te importeren: uitgepakt is het {size}, en de import leest van één onderdeel hooguit {limit}.", - "Delete the sheets the import does not read from a copy of the export (it reads only \"{first}\" and \"{second}\"), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen \"{first}\" en \"{second}\"), en importeer die kopie.", "The workbook holds more than {count} different texts, the most the import reads.": "De werkmap bevat meer dan {count} verschillende teksten, het maximum dat de import leest.", "Together, the cells of the workbook reference more than {size} of shared text, the most the import reads.": "Samen verwijzen de cellen van de werkmap naar meer dan {size} gedeelde tekst, het maximum dat de import leest.", "In development since": "In ontwikkeling sinds", @@ -1242,6 +1240,9 @@ "Excel workbook (.xlsx) with the sheet {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met het tabblad {sheets}. Standaard mag het bestand hoogstens {size} zijn.", "Excel workbook (.xlsx) with one of the sheets {sheets}. By default the file may be at most {size}.": "Excel-werkmap (.xlsx) met een van de tabbladen {sheets}. Standaard mag het bestand hoogstens {size} zijn.", "The sheet {sheets} is read; other sheets, including the \"Invoer\" sheets, are ignored.": "Het tabblad {sheets} wordt gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", - "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd." + "The sheets {sheets} are read; other sheets, including the \"Invoer\" sheets, are ignored.": "De tabbladen {sheets} worden gelezen; andere tabbladen, ook de \"Invoer\"-tabbladen, worden genegeerd.", + "Expected the sheet {sheets}. Sheet names must match exactly.": "Verwacht werd het tabblad {sheets}. De naam van het tabblad moet precies overeenkomen.", + "Expected one of the sheets {sheets}. Sheet names must match exactly.": "Verwacht werd een van de tabbladen {sheets}. De namen van de tabbladen moeten precies overeenkomen.", + "Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy.": "Verwijder in een kopie van de export de tabbladen die de import niet leest (hij leest alleen {sheets}), en importeer die kopie." } } diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 883d15e56..379eae833 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -490,10 +490,11 @@ export function isKnownError(code) { * to which sheet or part, and what to do about it. * * @param {object} details The error details: `maxPartBytes`, `part`, `size` and, for a sheet, `sheet`; `maxSharedStrings`; `maxReferencedStringBytes`; or `maxUncompressedBytes` + * @param {Array} sheetNames The sheets the import reads, for the hint * @return {{title: string, hint: string}} The words * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-the-workbook-shall-be-read-as-stored-data-without-evaluating-formulas-or-following-links-req-cmdb-002 */ -function workbookTooLarge(details) { +function workbookTooLarge(details, sheetNames) { const compressed = t( 'stackiq', 'An .xlsx file is compressed, so a file of a few MB can be much larger once unpacked. Nothing was imported.', @@ -532,11 +533,8 @@ function workbookTooLarge(details) { + ' ' + t( 'stackiq', - 'Delete the sheets the import does not read from a copy of the export (it reads only "{first}" and "{second}"), and import that copy.', - { - first: PROFILE_DEFAULTS.sheets[0], - second: PROFILE_DEFAULTS.sheets[1], - }, + 'Delete the sheets the import does not read from a copy of the export (it reads only {sheets}), and import that copy.', + { sheets: quotedSheetList(sheetNames) }, AS_TEXT, ) if (Number(details.maxSharedStrings) > 0) { @@ -584,10 +582,11 @@ function workbookTooLarge(details) { * when the server's message is not. * * @param {{error: string, message?: string, details?: object}} error The error in the server's shape + * @param {Array} [sheetNames] The sheets the import reads, from mappingSheetNames(); the shipped names when left out. Hints name all of them, whatever their number * @return {{title: string, hint: string}} The words * @spec openspec/changes/cmdb-export-import/specs/cmdb-export-import/spec.md#requirement-columns-shall-be-resolved-by-header-name-and-a-missing-required-column-shall-stop-the-import-with-422-req-cmdb-003 */ -export function errorText(error) { +export function errorText(error, sheetNames = PROFILE_DEFAULTS.sheets) { const details = error?.details || {} switch (error?.error) { case 'NO_FILE_UPLOADED': @@ -682,22 +681,28 @@ export function errorText(error) { case 'NO_SOURCE_SHEET': { const expected = Array.isArray(details.expected) && details.expected.length > 0 - ? details.expected - : PROFILE_DEFAULTS.sheets + ? details.expected.map((name) => String(name)) + : sheetNames + const values = { sheets: quotedSheetList(expected) } return { title: t( 'stackiq', 'The workbook has none of the sheets the import reads.', ), - hint: t( - 'stackiq', - 'Expected a sheet named "{first}" or "{second}". Sheet names must match exactly.', - { - first: String(expected[0] ?? PROFILE_DEFAULTS.sheets[0]), - second: String(expected[1] ?? PROFILE_DEFAULTS.sheets[1]), - }, - AS_TEXT, - ), + hint: + expected.length === 1 + ? t( + 'stackiq', + 'Expected the sheet {sheets}. Sheet names must match exactly.', + values, + AS_TEXT, + ) + : t( + 'stackiq', + 'Expected one of the sheets {sheets}. Sheet names must match exactly.', + values, + AS_TEXT, + ), } } case 'MISSING_COLUMN': @@ -767,7 +772,7 @@ export function errorText(error) { ), } case 'WORKBOOK_TOO_LARGE': - return workbookTooLarge(details) + return workbookTooLarge(details, sheetNames) case 'SCHEMA_OUTDATED': return { title: details.schema diff --git a/src/utils/cmdbImport.spec.js b/src/utils/cmdbImport.spec.js index 8f566d6cb..2e4ac7942 100644 --- a/src/utils/cmdbImport.spec.js +++ b/src/utils/cmdbImport.spec.js @@ -363,12 +363,37 @@ describe('errorText', () => { error: 'NO_SOURCE_SHEET', details: { expected: ['Blad A', 'Blad B'] }, }).hint, - ).toContain('"Blad A" or "Blad B"') + ).toContain('one of the sheets "Blad A", "Blad B"') expect(errorText({ error: 'NO_SOURCE_SHEET', details: {} }).hint).toContain( - PROFILE_DEFAULTS.sheets[1], + '"' + PROFILE_DEFAULTS.sheets.join('", "') + '"', ) }) + it.each([ + [1, ['Blad A'], 'Expected the sheet "Blad A".'], + [3, ['Blad A', 'Blad B', 'Blad C'], 'one of the sheets "Blad A", "Blad B", "Blad C".'], + ])( + 'names all %i sheets of the profile in the missing-sheet and too-large hints', + (count, names, expected) => { + const quoted = names.map((name) => '"' + name + '"').join(', ') + + const missing = errorText({ error: 'NO_SOURCE_SHEET', details: {} }, names) + expect(missing.hint).toContain(expected) + for (const shipped of PROFILE_DEFAULTS.sheets) { + expect(missing.hint).not.toContain(shipped) + } + + const tooLarge = errorText( + { error: 'WORKBOOK_TOO_LARGE', details: { maxSharedStrings: 200000 } }, + names, + ) + expect(tooLarge.hint).toContain('(it reads only ' + quoted + ')') + for (const shipped of PROFILE_DEFAULTS.sheets) { + expect(tooLarge.hint).not.toContain(shipped) + } + }, + ) + it('names the unpacked size limit the server applied', () => { expect( errorText({ @@ -408,7 +433,7 @@ describe('errorText', () => { expect( errorText({ error: 'WORKBOOK_TOO_LARGE', details }).hint, ).toContain( - '(it reads only "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB")', + '(it reads only "Onbeh Applicaties CMDB", "Beheerde Applicaties CMDB")', ) } }) diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index b20691f00..dbe41aa47 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -253,4 +253,23 @@ describe('CmdbImport sheet names in the help texts', () => { expect(help).not.toContain(shipped) } }) + + it('names the profile\'s sheets in the section\'s missing-sheet and too-large errors', async () => { + const wrapper = await mountWithSlots() + wrapper + .findComponent(CmdbImportMapping) + .vm.$emit('loaded', mappingOf(['Alpha', 'Beta', 'Gamma'])) + await flushPromises() + + await wrapper.setData({ error: { error: 'NO_SOURCE_SHEET', details: {} } }) + expect(wrapper.vm.errorView.hint).toContain('"Alpha", "Beta", "Gamma"') + + await wrapper.setData({ + error: { + error: 'WORKBOOK_TOO_LARGE', + details: { maxUncompressedBytes: 50 * 1024 * 1024 }, + }, + }) + expect(wrapper.vm.errorView.hint).toContain('(it reads only "Alpha", "Beta", "Gamma")') + }) }) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 8fb27fdc5..d2dc4b0a4 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -582,7 +582,7 @@ export default { if (!this.error) { return null } - const words = errorText(this.error) + const words = errorText(this.error, this.sheetNames) return { ...words, code: this.error.error, From d805c5bcaa5a50fa800f2c11e1ae55bf4b20209e Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 17:55:01 +0200 Subject: [PATCH 40/87] fix(cmdb-import): bring every usage of a leaving key to the same state onePerKey() reconciled only the best entry per import key and left the usages of the dropped entries untouched. A module trashed by hand while its usage stayed working, next to a working pair with the same key, kept that stray usage visible after the key left the export; so did a second usage of one module, and a working duplicate module. The dropped entries now travel as extras of the application: their usages, and their modules when those are not in the trash, are archived or soft-deleted with it, once, in the same error boundary and report row. Objects in the trash are still never restored for an extra, so a stale duplicate does not come back into the archive. REQ-CMDB-015, design D5 and task 4 say so. Review pre-push p4 Co-Authored-By: Claude Opus 5.5 --- lib/Service/CmdbExportImportService.php | 91 +++++++++++++++++-- .../design.md | 2 +- .../specs/cmdb-export-import/spec.md | 5 +- .../tasks.md | 2 +- .../Service/CmdbExportImportServiceTest.php | 60 +++++++++++- 5 files changed, 145 insertions(+), 15 deletions(-) diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index fc3b15e34..54c9e4b86 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -87,7 +87,9 @@ * * @phpstan-type ArchiveSheet array{sheet: string, present: bool, keyColumnMissing: bool, uncachedRows: array, keys: array} * - * @phpstan-type ScopeEntry array{key: string, usage: object, module: object, states: array} + * @phpstan-type ScopeExtra array{schemaKey: string, uuid: string, state: array{archived: bool, deleted: bool}} + * @phpstan-type ScopeStates array + * @phpstan-type ScopeEntry array{key: string, usage: object, module: object, states: ScopeStates, extras?: array} * * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) One import run resolves four linked * object kinds per row (supplier, module, owners, usage), each with its own match rule, @@ -2303,14 +2305,18 @@ private function reconciliationScope(string $municipalityUuid): array { }//end reconciliationScope() /** - * Keep one application per import key: the one the import itself would match. + * One application per import key: the entry the import would match, with the other usages of the key as extras. * * The import takes a working module before an archived one, and an archived * one before one in the trash (design D4), so a module deleted by hand next * to the one a later import created is never revived. The reconciliation - * follows the same order: the entry with the best module state wins, the - * best usage state breaks a tie, and the first one listed breaks the rest. - * A stale duplicate is left as it is, never restored into the archive. + * follows the same order: the entry with the best module state is the + * application, the best usage state breaks a tie, and the first one listed + * breaks the rest. The other entries of the key are not dropped: their + * usages, and their modules when those are not in the trash, are carried + * as `extras` and brought to the same state, so a stray usage of a stale + * duplicate does not stay visible. Nothing in the trash is restored for an + * extra, and nothing the application itself holds is listed twice. * * @param array $scope The entries. * @@ -2321,18 +2327,54 @@ private function reconciliationScope(string $municipalityUuid): array { private static function onePerKey(array $scope): array { $preferred = []; $ranks = []; + $entries = []; foreach ($scope as $application) { $rank = ((self::stateRank(state: $application['states']['module']) * 3) + self::stateRank(state: $application['states']['usage'])); $key = $application['key']; + $entries[$key][] = $application; if (isset($ranks[$key]) === false || $rank < $ranks[$key]) { $ranks[$key] = $rank; $preferred[$key] = $application; } } + foreach ($preferred as $key => $application) { + $preferred[$key]['extras'] = self::extrasOf(application: $application, entries: $entries[$key]); + } + return array_values($preferred); }//end onePerKey() + /** + * The usages and modules of the other entries of a key that the reconciliation brings to the same state. + * + * A usage or module appears once, and not when the application holds it + * itself. One in the trash is left out: it is not visible, and restoring + * a stale duplicate is what the preference rule exists to prevent. + * + * @param ScopeEntry $application The preferred entry of the key. + * @param array $entries Every entry of the key. + * + * @return array Usages first, then modules. + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private static function extrasOf(array $application, array $entries): array { + $seen = [(string)$application['usage']->getUuid() => true, (string)$application['module']->getUuid() => true]; + $found = ['usage' => [], 'module' => []]; + foreach ($entries as $entry) { + foreach (['usage', 'module'] as $schemaKey) { + $uuid = (string)$entry[$schemaKey]->getUuid(); + if (isset($seen[$uuid]) === false && $entry['states'][$schemaKey]['deleted'] === false) { + $seen[$uuid] = true; + $found[$schemaKey][] = ['schemaKey' => $schemaKey, 'uuid' => $uuid, 'state' => $entry['states'][$schemaKey]]; + } + } + } + + return array_merge($found['usage'], $found['module']); + }//end extrasOf() + /** * How far down the preference order a state is: working 0, archived 1, deleted 2. * @@ -2479,7 +2521,7 @@ private function modulesInChunk(array $chunk, array $coordinates): array { * leaves the municipality's view right and the module at worst visible. * An application already in the state the export asks for is not reported. * - * @param array{key: string, usage: object, module: object, states: array} $application Application. + * @param ScopeEntry $application The application. * @param ArchiveSheet $archive The archive sheet. * @param string $operationId The operation id, for the archive reason. * @param CmdbImportReport $report The report. @@ -2512,9 +2554,9 @@ private function reconcileApplication(array $application, array $archive, string $step = 'archive'; $changed = false; try { - foreach (['usage', 'module'] as $schemaKey) { - foreach (self::transitions(state: $application['states'][$schemaKey], target: $target, restoreDeleted: $onArchiveSheet) as $step) { - $this->transition(step: $step, schemaKey: $schemaKey, uuid: $uuids[$schemaKey], reason: $reason); + foreach (self::itemsOf(application: $application, restoreDeleted: $onArchiveSheet) as $item) { + foreach (self::transitions(state: $item['state'], target: $target, restoreDeleted: $item['restore']) as $step) { + $this->transition(step: $step, schemaKey: $item['schemaKey'], uuid: $item['uuid'], reason: $reason); $changed = true; } } @@ -2530,6 +2572,37 @@ private function reconcileApplication(array $application, array $archive, string } }//end reconcileApplication() + /** + * What one application brings to the state the export asks for: its usage and module, then the other usages and modules of its key. + * + * Only the application's own objects are restored from the trash, and only + * when `restoreDeleted` says so; the extras never are. + * + * @param ScopeEntry $application The application. + * @param bool $restoreDeleted Whether the application's own objects in the trash are restored to be archived. + * + * @return array + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + private static function itemsOf(array $application, bool $restoreDeleted): array { + $items = []; + foreach (['usage', 'module'] as $schemaKey) { + $items[] = [ + 'schemaKey' => $schemaKey, + 'uuid' => (string)$application[$schemaKey]->getUuid(), + 'state' => $application['states'][$schemaKey], + 'restore' => $restoreDeleted, + ]; + } + + foreach (($application['extras'] ?? []) as $extra) { + $items[] = $extra + ['restore' => false]; + } + + return $items; + }//end itemsOf() + /** * The steps that bring one module or usage to the state the export asks for (design D5). * diff --git a/openspec/changes/cmdb-import-archive-reconciliation/design.md b/openspec/changes/cmdb-import-archive-reconciliation/design.md index 79c9444dc..770c074e6 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/design.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/design.md @@ -97,7 +97,7 @@ Per application, with `processed` = every APPID match key on the CMDB sheets (wh | nowhere, sheet absent or incomplete (no key column, or `uncachedRows` not empty) | working | archive usage, then module | `archived` | | nowhere, sheet absent or incomplete | archived or deleted | nothing | — | -The table decides on one entry per key. When a municipality has several usages and modules under the same import key (D4 says earlier imports created exactly such duplicates, a module deleted by hand next to the one a later import created), the scope keeps the entry the import would match: the best module state wins (working, then archived, then deleted), the best usage state breaks a tie, the first listed breaks the rest. The other entries are left as they are: a stale trashed duplicate is not restored into the archive when the key is on the archive sheet, and is not touched when it is on no sheet. +The table decides on one entry per key. When a municipality has several usages and modules under the same import key (D4 says earlier imports created exactly such duplicates, a module deleted by hand next to the one a later import created), the scope keeps the entry the import would match: the best module state wins (working, then archived, then deleted), the best usage state breaks a tie, the first listed breaks the rest. The other usages of the key follow the application, so a stray usage stays visible nowhere: a working or archived usage of a stale duplicate, a second usage of the same module, and a working or archived duplicate module are archived or soft-deleted as the key asks, once, in the same error boundary and the same report row (usages first, then modules). What is in the trash is left as it is: a stale trashed duplicate is not restored into the archive when the key is on the archive sheet, and is not touched when it is on no sheet. The usage is written before the module, so a failure between the two leaves the municipality's usage in the new state and the module visible, never the reverse. A module another municipality still uses is archived or deleted all the same when this municipality's usage says so: the module carries this municipality's import key, so it is this municipality's record (REQ-CMDB-006 allows a shared module only when the other organisation's usage was added by hand). The archive reason is `cmdb-import: on sheet "Gearchiveerde Applicaties"` or `cmdb-import: not in the export`, plus the operation id, so the audit trail says why. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md index 48f43be64..d03c04ee7 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md @@ -16,7 +16,7 @@ OpenRegister: `OCA\OpenRegister\Contract\ObjectServiceInterface` (`searchObjects ### Requirement: An application missing from the CMDB sheets SHALL be archived when the archive sheet lists it and soft-deleted when no sheet does (REQ-CMDB-015) -With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, or a row of the sheet holds an APPID formula without a cached value (read as empty, as REQ-CMDB-002 has it for the CMDB sheets, and listed by the reader in `archive.uncachedRows`), the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report; for an uncached row the warning SHALL name the sheet and the row numbers. When a municipality has more than one module or usage under the same import key (a module deleted by hand next to the one a later import created), the import SHALL reconcile only the one it would match, a working one before an archived one and an archived one before one in the trash, and SHALL leave the others as they are, so a stale duplicate is never restored into the archive. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. +With `missingRecords` `archive`, after every row of an import that was not cancelled and did not fail, the import SHALL reconcile, under the register lock it already holds, every `usage` whose `consumer` is the imported municipality together with the `module` it points at, including archived and soft-deleted ones, and only those whose module carries this municipality's import key (`topdesk::`). It SHALL read the APPID column of the sheet "Gearchiveerde Applicaties" when the workbook has it, under the same size and row bounds as the CMDB sheets and without reading any other column of it. For an application whose APPID is on neither CMDB sheet: when the archive sheet lists the APPID, the import SHALL archive the usage and then the module through OpenRegister's archive state with a reason that names the import, restoring a soft-deleted one from the trash first; when no sheet lists it, the import SHALL soft-delete the usage and then the module through OpenRegister's trash. When the workbook has no archive sheet, or the sheet has no APPID column, or a row of the sheet holds an APPID formula without a cached value (read as empty, as REQ-CMDB-002 has it for the CMDB sheets, and listed by the reader in `archive.uncachedRows`), the import SHALL archive and SHALL NOT soft-delete, and SHALL add a warning to the report; for an uncached row the warning SHALL name the sheet and the row numbers. When a municipality has more than one module or usage under the same import key (a module deleted by hand next to the one a later import created), the import SHALL reconcile as the application the one it would match, a working one before an archived one and an archived one before one in the trash, and SHALL bring every other usage of that key, and every other module of it that is not in the trash, to the same state, so no stray usage stays visible; it SHALL leave what is in the trash as it is, so a stale duplicate is never restored into the archive. An application that is already in the state the export asks for SHALL be left alone and not reported. The import SHALL NOT touch contact persons, organisations, modules that carry another municipality's import key or no import key, or usages of another organisation. Each application SHALL be reconciled in its own error boundary: a failure SHALL be a `failed` report row naming the step and the exception class, and the other applications SHALL still be reconciled. The reconciliation SHALL report its progress as the phase `reconciling` of the import's operation. A cancel SHALL also be honoured after the last row and before each application of the reconciliation: the import SHALL stop there, SHALL keep what it archived or deleted so far, and SHALL report the run as cancelled. The report SHALL count `archived` and `deleted` in its summary and SHALL list each as a row with the APPID, the application name and both uuids, never person data. When OpenRegister's archive handler or object mapper is not available, an import with `missingRecords` `archive` SHALL be refused with 503 `ARCHIVE_UNAVAILABLE` before the file is read. #### Scenario: An application that moved to the archive sheet is archived @e2e tests/e2e/spec-coverage/cmdb-import.spec.ts @@ -37,12 +37,13 @@ With `missingRecords` `archive`, after every row of an import that was not cance - **AND** the report SHALL carry a warning that the sheet "Gearchiveerde Applicaties" was not found #### Scenario: A stale duplicate under the same import key is left in the trash -@e2e exclude Data condition from earlier imports; tests/Unit/Service/CmdbExportImportServiceTest.php testAStaleDuplicateIsNeverRestoredByTheReconciliation stores a working and a trashed module and usage for APPID `7`, lists `7` on the archive sheet, and asserts the working pair is archived, no `restore` happens and the trashed pair stays deleted. +@e2e exclude Data condition from earlier imports; tests/Unit/Service/CmdbExportImportServiceTest.php testAStaleDuplicateIsNeverRestoredByTheReconciliation stores a working and a trashed module and usage for APPID `7`, lists `7` on the archive sheet, and asserts the working pair is archived, no `restore` happens and the trashed pair stays deleted; testEveryOtherUsageOfAKeyFollowsTheApplication asserts the stray and second usages follow. - **GIVEN** a working module with APPID `7` and its usage, and a module with the same import key in the trash with its own usage - **WHEN** an export without `7` on the CMDB sheets, with `7` on "Gearchiveerde Applicaties", is imported with `archive` - **THEN** the working module and usage SHALL be archived - **AND** the module and the usage in the trash SHALL stay there, and nothing SHALL be restored +- **AND** a working usage that points at the module in the trash, or a second usage of the working module, SHALL be archived as well #### Scenario: An archive row whose APPID has no cached value never turns an archive into a deletion @e2e exclude Covered by the service test; tests/Unit/Service/CmdbExportImportServiceTest.php testAnUncachedArchiveRowArchivesInsteadOfDeleting asserts the archive state, no delete, and the warning naming the sheet and the row; tests/Unit/Service/Cmdb/CmdbWorkbookReaderTest.php testAnUncachedArchiveKeyIsListedNotDropped asserts the reader lists the row. diff --git a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md index 59914b112..ca4c335d5 100644 --- a/openspec/changes/cmdb-import-archive-reconciliation/tasks.md +++ b/openspec/changes/cmdb-import-archive-reconciliation/tasks.md @@ -45,7 +45,7 @@ Spec: `openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-imp - GIVEN APPIDs 1 and 7 imported WHEN an export with row 1 and 7 on the archive sheet is imported THEN module and usage of 7 are archived with a reason, the report row is `archived`, and the owner contact person is untouched - GIVEN APPID 7 on no sheet THEN module and usage are soft-deleted and the row is `deleted`; GIVEN the archive sheet absent, without an APPID column, or with an APPID that has no cached value THEN they are archived instead and the report warns - GIVEN APPID 7 soft-deleted and listed on the archive sheet THEN module and usage are restored and archived; GIVEN it already archived or, when on no sheet, already deleted THEN nothing is written - - GIVEN a working pair and a trashed pair under the same import key and the key on the archive sheet THEN the working pair is archived, nothing is restored and the trashed pair stays deleted + - GIVEN a working pair and a trashed pair under the same import key and the key on the archive sheet THEN the working pair is archived, nothing is restored and the trashed pair stays deleted; GIVEN a further working usage of the key (of the trashed module, or a second one of the working module) THEN it is archived or deleted with the application - GIVEN `missingRecords` `keep`, a cancelled import, or a usage of another municipality THEN nothing is archived or deleted - GIVEN a cancel during the last row THEN nothing is archived or deleted; GIVEN a cancel during the reconciliation THEN it stops before the next application and the report is cancelled - GIVEN the archive of one application throws THEN that application is a `failed` row and the others are reconciled; GIVEN no ArchiveHandler THEN 503 `ARCHIVE_UNAVAILABLE` before reading diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index da576de5d..6fb99267f 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -2153,7 +2153,8 @@ public function testAStaleDuplicateIsNeverRestoredByTheReconciliation(): void { * Of two usages under one import key, the one of the module the import would match is reconciled, whatever * state the usage is in: the module's state ranks first, the usage's second. * - * The working usage is listed before the one in the trash but belongs to the duplicate module in the trash. + * The working usage is listed before the one in the trash but belongs to the duplicate module in the trash; + * it is archived as well (a stray usage stays visible otherwise), and the module in the trash is not restored. * * @return void * @@ -2174,7 +2175,7 @@ public function testTheModulesStateDecidesWhichDuplicateIsReconciled(): void { $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven), 'the working module is archived'); $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $usage), 'its usage is restored from the trash and archived'); $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'module-stale')); - $this->assertSame([false, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-stale'), 'the working usage of the stale module is not touched'); + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: 'usage-stale'), 'the working usage of the stale module is archived with the key'); }//end testTheModulesStateDecidesWhichDuplicateIsReconciled() /** @@ -2367,6 +2368,61 @@ public function testKeepWithoutTheArchiveServicesFailsARowWhoseMatchIsArchived() $this->assertSame(0, $report['summary']['archived'] + $report['summary']['deleted']); }//end testKeepWithoutTheArchiveServicesFailsARowWhoseMatchIsArchived() + /** + * Every other usage of a key leaving the export follows the application: a working usage of a module deleted + * by hand, and a second usage of the same module. A stale module in the trash is neither restored nor touched. + * + * @return void + * + * @spec openspec/changes/cmdb-import-archive-reconciliation/specs/cmdb-export-import/spec.md#requirement-an-application-missing-from-the-cmdb-sheets-shall-be-archived-when-the-archive-sheet-lists-it-and-soft-deleted-when-no-sheet-does-req-cmdb-015 + */ + public function testEveryOtherUsageOfAKeyFollowsTheApplication(): void { + $this->importOneSevenAndEight(); + [$seven, $eight] = [$this->moduleOf(appId: '7'), $this->moduleOf(appId: '8')]; + $sevenUsage = $this->usageOf(moduleUuid: $seven); + // APPID 7: a module deleted by hand whose usage stayed working, and a second usage of the working module. + $this->store[self::MODULE]['module-stale'] = array_merge($this->store[self::MODULE][$seven], ['id' => 'module-stale']); + $this->states[self::MODULE]['module-stale'] = ['deleted' => ['deletedBy' => 'admin']]; + $this->store[self::USAGE]['usage-stray'] = array_merge($this->store[self::USAGE][$sevenUsage], ['id' => 'usage-stray', 'module' => 'module-stale']); + $this->store[self::USAGE]['usage-second'] = array_merge($this->store[self::USAGE][$sevenUsage], ['id' => 'usage-second']); + // APPID 8 also has a stale pair in the trash: nothing to follow. + $this->store[self::MODULE]['module-stale-8'] = array_merge($this->store[self::MODULE][$eight], ['id' => 'module-stale-8']); + $this->store[self::USAGE]['usage-stale-8'] = array_merge($this->store[self::USAGE][$this->usageOf(moduleUuid: $eight)], ['id' => 'usage-stale-8', 'module' => 'module-stale-8']); + $this->states[self::MODULE]['module-stale-8'] = ['deleted' => ['deletedBy' => 'admin']]; + $this->states[self::USAGE]['usage-stale-8'] = ['deleted' => ['deletedBy' => 'admin']]; + + // APPID 8 also has a second working module with its own usage, which a first-hit match never reaches. + $this->store[self::MODULE]['module-dup-8'] = array_merge($this->store[self::MODULE][$eight], ['id' => 'module-dup-8']); + $this->store[self::USAGE]['usage-dup-8'] = array_merge($this->store[self::USAGE][$this->usageOf(moduleUuid: $eight)], ['id' => 'usage-dup-8', 'module' => 'module-dup-8']); + + // 7 is on the archive sheet, 8 on no sheet. + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: ['7'])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + $this->assertSame([[false, true], [false, true]], [$this->stateOf(schema: self::MODULE, uuid: 'module-dup-8'), $this->stateOf(schema: self::USAGE, uuid: 'usage-dup-8')], 'a working duplicate of 8 is deleted with it'); + + foreach ([$sevenUsage, 'usage-stray', 'usage-second'] as $usage) { + $this->assertSame([true, false], $this->stateOf(schema: self::USAGE, uuid: $usage), $usage . ' is archived'); + } + + $this->assertSame([true, false], $this->stateOf(schema: self::MODULE, uuid: $seven)); + $this->assertSame([false, true], $this->stateOf(schema: self::MODULE, uuid: 'module-stale'), 'the stale module stays in the trash'); + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: 'usage-stale-8'), 'a usage in the trash is never restored'); + $this->assertNotContains('restore', array_column($this->transitions, 0)); + $this->assertSame([1, 1], [$report['summary']['archived'], $report['summary']['deleted']], 'one report row per application'); + + // The same key on no sheet: every usage is soft-deleted, once. + $this->transitions = []; + $this->states = array_replace_recursive($this->states, [self::USAGE => [$sevenUsage => ['archived' => null], 'usage-stray' => ['archived' => null], 'usage-second' => ['archived' => null]], self::MODULE => [$seven => ['archived' => null]]]); + $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')], archived: [])) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + foreach ([$sevenUsage, 'usage-stray', 'usage-second'] as $usage) { + $this->assertSame([false, true], $this->stateOf(schema: self::USAGE, uuid: $usage), $usage . ' is soft-deleted'); + } + + $deletes = array_column(array_filter($this->transitions, static fn (array $t): bool => $t[0] === 'delete'), 2); + $this->assertSame(count($deletes), count(array_unique($deletes)), 'nothing is deleted twice'); + }//end testEveryOtherUsageOfAKeyFollowsTheApplication() + /** * One application that cannot be archived is a failed row; the others are still reconciled. * From 6ecb274a9fda05940578fad316e20d7a99b53774 Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 18:05:19 +0200 Subject: [PATCH 41/87] fix(cmdb-import): keep a mapped field with a create default in createOnlyFields The overview subtracted every key of createOnlyDefaults from createOnlyFields. merge() guards a mapped create-only field on update, so a mapped field that also had a create default would have shown as written on every update. Only defaults that are not mapped fields leave the list now (the shipped module type); REQ-CMDB-020, design and the overlap assertions say the same. Mutation-checked: testAMappedCreateOnlyDefaultStaysACreateOnlyField fails on the previous subtraction and passes now. Review pre-push q1 Co-Authored-By: Claude Opus 5.5 --- lib/Service/Cmdb/CmdbImportProfile.php | 17 +++---- .../cmdb-import-mapping-view/design.md | 2 +- .../specs/cmdb-export-import/spec.md | 2 +- ...ettingsControllerCmdbImportMappingTest.php | 4 +- .../Service/Cmdb/CmdbImportProfileTest.php | 48 ++++++++++++++++++- 5 files changed, 58 insertions(+), 15 deletions(-) diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index 73dc75e5c..fb455ac7a 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -636,7 +636,7 @@ public function referencedColumns(): array { * `defaults` (pack values set on every write), `createOnlyDefaults` * (values set on create only, not mapped columns), `createOnlyFields` (mapped * fields written on create or, on update, only when the stored value is - * empty; never a key of `createOnlyDefaults`) and `neverWritten` (fields the + * empty; a key of `createOnlyDefaults` only when it is also mapped) and `neverWritten` (fields the * import never writes on an existing object). * * @return array{profile: array, packs: array>} @@ -671,16 +671,13 @@ public function mappingOverview(): array { ]; } - // A create-only default (the module type) is a value the import sets on - // create, not a mapped field: merge() never reads it on update, so it is - // listed under createOnlyDefaults only. + // A create-only default that is not a mapped field (the module type) is a + // value the import sets on create; merge() never reads it on update, so it + // is listed under createOnlyDefaults only. A mapped field keeps its place + // in createOnlyFields, since merge() guards it on update. $createOnlyDefaults = $this->createOnlyDefaults(target: $target); - $createOnlyFields = array_values( - array_diff( - $this->createOnlyFields(target: $target), - array_map('strval', array_keys($createOnlyDefaults)) - ) - ); + $unmappedDefaults = array_diff(array_map('strval', array_keys($createOnlyDefaults)), array_column($mappings, 'target')); + $createOnlyFields = array_values(array_diff($this->createOnlyFields(target: $target), $unmappedDefaults)); $defaults = $pack['defaults'] ?? []; if (is_array($defaults) === false) { diff --git a/openspec/changes/cmdb-import-mapping-view/design.md b/openspec/changes/cmdb-import-mapping-view/design.md index 0bd69a911..e0c8f5871 100644 --- a/openspec/changes/cmdb-import-mapping-view/design.md +++ b/openspec/changes/cmdb-import-mapping-view/design.md @@ -46,7 +46,7 @@ The method carries no auth attribute (Nextcloud's default: admins with a CSRF to `fieldMappings[].transform` is passed through as the pack stores it (`type` plus `map`/`default`, `fields`/`separator`, `value`, `sourceFormat`/`targetFormat`). Re-shaping per type would hide a key the engine reads. The page renders the known keys and lists any other scalar key as it is. `required` is normalised to a boolean. -The write rules the import applies besides the mapping are answered per pack, built from the accessors the import uses (`createOnlyDefaults()`, `createOnlyFields()`, `neverWrittenOnUpdate()`) and the pack's own `defaults`, so the view does not overclaim for a field the import writes only on create or never. A create-only default such as the module `type` is answered in `createOnlyDefaults` only: it is not a mapped field and `merge()` never reads it on update, so `createOnlyFields` leaves it out. +The write rules the import applies besides the mapping are answered per pack, built from the accessors the import uses (`createOnlyDefaults()`, `createOnlyFields()`, `neverWrittenOnUpdate()`) and the pack's own `defaults`, so the view does not overclaim for a field the import writes only on create or never. A create-only default that is not a mapped field, such as the module `type`, is answered in `createOnlyDefaults` only: `merge()` never reads it on update, so `createOnlyFields` leaves it out. A mapped field with a create default stays in `createOnlyFields`, since `merge()` guards it on update. ### D4. The section takes the sheet names from the endpoint, with the constant as fallback diff --git a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md index 9675bb693..e1e7f2430 100644 --- a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md @@ -16,7 +16,7 @@ Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\Mi ### Requirement: The admin settings SHALL show the mapping the import uses (REQ-CMDB-020) -`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack it names through the same loader and validator the import uses, and SHALL answer 200 with a flat envelope of two keys: `profile` (the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns`, `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`) and `packs`, one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings` and the write rules the import applies besides the mapping: `defaults` (the pack's values, field => value), `createOnlyDefaults` (values the import sets on create only, field => value; they are not mapped columns), `createOnlyFields` (mapped fields written on create or, on an existing object, only when the stored value is empty; never a field that is a key of `createOnlyDefaults`) and `neverWritten` (fields the import never writes on an existing object); the last three come from the profile's `createOnly` and `neverWritten` through the accessors the import itself uses. Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it: the `type`, and for `lookup` and `bool-map` the `map` and `default`, for `concat` the `fields` and `separator`, for `const` the `value`, for `date` the `sourceFormat` and `targetFormat`. The endpoint SHALL read nothing but those files and SHALL write nothing. When the validator is missing, or the profile or a pack is unreadable or invalid, it SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason`, so the administrator who edited a file learns what is wrong without the Nextcloud log. The "CMDB import" section SHALL show the answer in a collapsible block "Mapping (read-only)": the sheets, the key, name, required and date columns, and one table per pack with the source column, the target field, whether it is required, the transformation and its details. The block SHALL show a loading state until the endpoint answers and the error with its code when it fails, and SHALL offer no editing: the mapping is changed in the files on the server, as the documentation describes. +`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack it names through the same loader and validator the import uses, and SHALL answer 200 with a flat envelope of two keys: `profile` (the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns`, `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`) and `packs`, one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings` and the write rules the import applies besides the mapping: `defaults` (the pack's values, field => value), `createOnlyDefaults` (values the import sets on create only, field => value; they are not mapped columns), `createOnlyFields` (mapped fields written on create or, on an existing object, only when the stored value is empty; a key of `createOnlyDefaults` only when it is also a mapped field) and `neverWritten` (fields the import never writes on an existing object); the last three come from the profile's `createOnly` and `neverWritten` through the accessors the import itself uses. Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it: the `type`, and for `lookup` and `bool-map` the `map` and `default`, for `concat` the `fields` and `separator`, for `const` the `value`, for `date` the `sourceFormat` and `targetFormat`. The endpoint SHALL read nothing but those files and SHALL write nothing. When the validator is missing, or the profile or a pack is unreadable or invalid, it SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason`, so the administrator who edited a file learns what is wrong without the Nextcloud log. The "CMDB import" section SHALL show the answer in a collapsible block "Mapping (read-only)": the sheets, the key, name, required and date columns, and one table per pack with the source column, the target field, whether it is required, the transformation and its details. The block SHALL show a loading state until the endpoint answers and the error with its code when it fails, and SHALL offer no editing: the mapping is changed in the files on the server, as the documentation describes. #### Scenario: An admin reads the mapping the import uses @e2e tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts diff --git a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php index 46d6badd4..69bdc844c 100644 --- a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php +++ b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php @@ -243,11 +243,11 @@ public function testTheShippedMappingIsAnsweredFlat(): void { $this->assertSame([], $byTarget['module']['defaults']); foreach ($data['packs'] as $pack) { $this->assertSame( - array_values(array_diff($profile->createOnlyFields(target: $pack['target']), array_keys($pack['createOnlyDefaults']))), + array_values(array_diff($profile->createOnlyFields(target: $pack['target']), array_diff(array_keys($pack['createOnlyDefaults']), array_column($pack['fieldMappings'], 'target')))), $pack['createOnlyFields'], $pack['target'] ); - $this->assertSame([], array_intersect($pack['createOnlyFields'], array_keys($pack['createOnlyDefaults'])), $pack['target']); + $this->assertSame([], array_intersect($pack['createOnlyFields'], array_diff(array_keys($pack['createOnlyDefaults']), array_column($pack['fieldMappings'], 'target'))), $pack['target']); $this->assertSame($profile->neverWrittenOnUpdate(target: $pack['target']), $pack['neverWritten'], $pack['target']); } diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index d76e4203f..74fdbe784 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -297,6 +297,52 @@ public function testAMissingPackOrValidatorIsMappingUnavailable(): void { } }//end testAMissingPackOrValidatorIsMappingUnavailable() + /** + * A create-only default keeps its mapped field in createOnlyFields; only an unmapped default leaves it. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 + */ + public function testAMappedCreateOnlyDefaultStaysACreateOnlyField(): void { + CmdbTestSupport::loadMigrationPack(); + $profile = new class(container: $this->emptyContainer()) extends CmdbImportProfile { + /** + * The usage pack's create-only fields with a create default on one mapped field and one unmapped one. + * + * @param string $target The pack target. + * + * @return array + */ + public function createOnlyDefaults(string $target): array { + if ($target !== 'usage') { + return parent::createOnlyDefaults(target: $target); + } + + return ['timeClassification' => 'Tolerate', 'unmappedFlag' => true]; + } + + /** + * The usage pack's create-only fields plus the unmapped default's key. + * + * @param string $target The pack target. + * + * @return array + */ + public function createOnlyFields(string $target): array { + if ($target !== 'usage') { + return parent::createOnlyFields(target: $target); + } + + return [...parent::createOnlyFields(target: $target), 'unmappedFlag']; + } + }; + + $usage = array_values(array_filter($profile->mappingOverview()['packs'], static fn (array $pack): bool => $pack['target'] === 'usage'))[0]; + + $this->assertSame(['interneAnnotation', 'timeClassification'], $usage['createOnlyFields'], 'the mapped field stays, the unmapped default leaves'); + }//end testAMappedCreateOnlyDefaultStaysACreateOnlyField() + /** * The overview the admin settings show is built from the validated packs, and a broken pack breaks it too. * @@ -330,7 +376,7 @@ public function testTheOverviewShowsWhatTheImportRuns(): void { } $this->assertSame($profile->createOnlyDefaults(target: $pack['target']), $pack['createOnlyDefaults'], $pack['target']); - $this->assertSame([], array_intersect($pack['createOnlyFields'], array_keys($pack['createOnlyDefaults'])), $pack['target']); + $this->assertSame([], array_intersect($pack['createOnlyFields'], array_diff(array_keys($pack['createOnlyDefaults']), array_column($pack['fieldMappings'], 'target'))), $pack['target']); $this->assertSame($profile->neverWrittenOnUpdate(target: $pack['target']), $pack['neverWritten'], $pack['target']); } From 323d7615d0480448f351907a6e191cd873e6d72a Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 18:09:42 +0200 Subject: [PATCH 42/87] docs(cmdb-import): quote both forms of the uncached-APPID warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warning has a one-row form and an "in N rows (…)" form that names the first ten rows; the docs quoted only the first. Review pre-push q4 Co-Authored-By: Claude Opus 5.5 --- docs/features/cmdb-import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index 364bcf5f6..1f06a572f 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -277,7 +277,7 @@ APPID is on neither CMDB sheet: |---|---| | on the sheet `Gearchiveerde Applicaties` | are **archived**: OpenRegister's archive state, with the reason `cmdb-import: on sheet "Gearchiveerde Applicaties"`. Reported as *archived*. | | on no sheet at all | are **deleted**: moved to OpenRegister's trash, where they stay restorable for 30 days. Reported as *deleted*. | -| on no sheet, but the archive sheet cannot be relied on: the workbook has no sheet `Gearchiveerde Applicaties`, the sheet has no `APPID` column, or an `APPID` in it has no cached value (a formula whose result the file does not store) | are **archived**, never deleted. Each case has its own warning in the result: *sheet not found*, *no column APPID*, or *an APPID without a cached value in row …* (the first rows are named). Such a workbook therefore never moves anything to the trash. | +| on no sheet, but the archive sheet cannot be relied on: the workbook has no sheet `Gearchiveerde Applicaties`, the sheet has no `APPID` column, or an `APPID` in it has no cached value (a formula whose result the file does not store) | are **archived**, never deleted. Each case has its own warning in the result: *sheet not found*, *no column APPID*, or *an APPID without a cached value in row …* (for more rows: *in N rows (…)*, naming the first ten). Such a workbook therefore never moves anything to the trash. | An archived or deleted application disappears from the stackiq lists, from OpenCatalogi and from Portaliq's **Software we use**. In stackiq, the quick From 024122ba1292e83eade4feaff9990cbf12551deb Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 18:52:04 +0200 Subject: [PATCH 43/87] feat(cmdb-import): run the mapping packs stored in OpenRegister The module, manufacturer, usage and business-owner packs now live in OpenRegister's migration-pack store, where an administrator edits them, and the import runs what is stored there. A repair step seeds missing packs as built-in packs and replaces a pack nobody edited when a release ships a newer one; an x-stackiq marker (shipped version and the hash of the shipped pack) tells an edit from a release, and an edited pack is never overwritten. The shipped files stay the seed, and the fallback while nothing was stored. Every pack must follow the import's rules: the required name and APPID mappings, and only fields of its target schema, never object metadata, match keys or the relations the import sets. An invalid or rule-breaking pack, a renamed pack, or a store that is gone after a seed answers 503 MAPPING_UNAVAILABLE naming the pack and its id. A column a pack maps with a date transform is converted from an Excel date, and the note cleanup uses the pack's own separator. The municipality pack stays a shipped file. GET /api/settings/cmdb-import/mapping answers per pack its source, OpenRegister id, stored and shipped versions, edited and newerShipped, editable and an editUrl to OpenRegister's pack editor, plus openRegisterPacks. The OpenSpec change and the documentation describe editing in OpenRegister. Co-Authored-By: Claude Opus 5.5 (1M context) --- appinfo/info.xml | 10 + docs/features/cmdb-import.md | 108 ++-- l10n/en.js | 5 +- l10n/en.json | 5 +- l10n/nl.js | 5 +- l10n/nl.json | 5 +- lib/Controller/CmdbImportController.php | 29 +- lib/Controller/SettingsController.php | 79 ++- lib/Repair/SeedCmdbImportPacks.php | 146 ++++++ lib/Service/Cmdb/CmdbImportPackMarker.php | 175 +++++++ lib/Service/Cmdb/CmdbImportPackRules.php | 377 ++++++++++++++ lib/Service/Cmdb/CmdbImportPackStore.php | 462 ++++++++++++++++++ lib/Service/Cmdb/CmdbImportProfile.php | 364 ++++++++++++-- lib/Service/CmdbExportImportService.php | 25 +- .../cmdb-import-mapping-view/design.md | 151 ++++-- .../cmdb-import-mapping-view/proposal.md | 60 ++- .../specs/cmdb-export-import/spec.md | 197 ++++++-- .../changes/cmdb-import-mapping-view/tasks.md | 67 ++- openspec/specs/cmdb-export-import/spec.md | 2 +- .../Controller/CmdbImportControllerTest.php | 43 ++ ...ettingsControllerCmdbImportMappingTest.php | 131 ++++- tests/Unit/Repair/SeedCmdbImportPacksTest.php | 212 ++++++++ .../Service/Cmdb/CmdbImportPackRulesTest.php | 103 ++++ .../Service/Cmdb/CmdbImportPackStoreTest.php | 296 +++++++++++ .../Service/Cmdb/CmdbImportProfileTest.php | 337 ++++++++++++- .../Service/CmdbExportImportServiceTest.php | 85 +++- .../Unit/Support/FakeMigrationPackService.php | 286 +++++++++++ 27 files changed, 3537 insertions(+), 228 deletions(-) create mode 100644 lib/Repair/SeedCmdbImportPacks.php create mode 100644 lib/Service/Cmdb/CmdbImportPackMarker.php create mode 100644 lib/Service/Cmdb/CmdbImportPackRules.php create mode 100644 lib/Service/Cmdb/CmdbImportPackStore.php create mode 100644 tests/Unit/Repair/SeedCmdbImportPacksTest.php create mode 100644 tests/Unit/Service/Cmdb/CmdbImportPackRulesTest.php create mode 100644 tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php create mode 100644 tests/Unit/Support/FakeMigrationPackService.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 895cb48bc..4efc599c9 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -250,6 +250,14 @@ Vrij en open source onder de EUPL-licentie. instead of failing. --> OCA\Stackiq\Repair\RenameDutchCatalogValues + + OCA\Stackiq\Repair\SeedCmdbImportPacks OCA\Stackiq\Repair\RepointEolSyncConfig OCA\Stackiq\Repair\InitializeSettings + + OCA\Stackiq\Repair\SeedCmdbImportPacks diff --git a/docs/features/cmdb-import.md b/docs/features/cmdb-import.md index b9d505abc..0fe0f258a 100644 --- a/docs/features/cmdb-import.md +++ b/docs/features/cmdb-import.md @@ -150,6 +150,9 @@ date stored there, including one set by hand. ### Columns and where they go +This is the mapping as it ships; an administrator can change it in +OpenRegister (see [Viewing and editing the mapping](#viewing-and-editing-the-mapping)). + | Column | Goes to | Rule | |---|---|---| | APPID | module external number, and the match key | required, see [Repeat imports](#repeat-imports) | @@ -294,7 +297,7 @@ and the section shows the reason and the error code. | `FIELD_INVALID` | A form field of the request has a value the import does not accept, for example an `updateExisting` or `publish` that is neither `true` nor `false`. The message names the field. | Not reachable from the section; reported for API callers. | | `UPLOAD_FAILED` | The file reached the server but could not be stored there. | Try again; the Nextcloud log has the details. | | `MISSING_RECORDS_UNSUPPORTED` | The request asked to mark or remove records missing from the export. Only keeping them is supported. | Not reachable from the section; reported for API callers. | -| `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine is missing, or one of the mapping files is invalid. | Update OpenRegister. If you changed a mapping file, check it against the Nextcloud log. | +| `MAPPING_UNAVAILABLE` | OpenRegister's mapping engine or pack store is missing, or a mapping pack is invalid or breaks a rule of the import. The message names the pack when one is at fault. | Update or enable OpenRegister. If you changed a pack, correct it in OpenRegister; see [Viewing and editing the mapping](#viewing-and-editing-the-mapping). | | `READER_UNAVAILABLE` | The Excel reader that ships with OpenRegister cannot be loaded. | Make sure OpenRegister is installed and enabled. | | `NOT_CONFIGURED` | The stackiq register or its schemas cannot be found. | Run **Auto Configure** at the top of the stackiq admin settings. | | `SCHEMA_OUTDATED` | A stackiq schema lacks a property the import recognises records by, for example `externalKey` on the module schema, or the module schema is older than version 0.3.8: it has no `applicationType`, or `externalKey` lacks the rule that only an administrator may change it. Importing anyway would create every application again, or leave the import key open to every editor. The message names the schema. | Press **Force Update** at the top of the stackiq admin settings to import the register configuration again. | @@ -331,35 +334,82 @@ every import: The section's help text shows the defaults; when the server refuses a file, the message shows the limit the server applied. A larger file also has to pass PHP's `upload_max_filesize` and `post_max_size` and the web server's -request size limit. Like the mapping files, the profile is part of the app: -a change made on the server is overwritten by the next app update. +request size limit. The profile is part of the app: a change made on the +server is overwritten by the next app update. -## Adjusting the mapping +## Viewing and editing the mapping -The mapping from columns to fields is not in code. It is a set of JSON files -in `lib/Settings/cmdb-import/`, executed by OpenRegister's mapping engine: +The mapping from export columns to stackiq fields is not in code. It is a +set of mapping packs that OpenRegister's mapping engine runs, one per kind of +record the import writes. Four of them are kept in OpenRegister, where you +can view and change them: -| File | What it maps | +| Pack (in OpenRegister) | What it maps | |---|---| -| `topdesk-profile.json` | the sheets, the constant each sheet adds to its rows (`Beheer`) and the columns it is known to lack, the match column, the required, date and id columns, the placeholder values that mean empty, the limits, and which pack is used for which target | -| `topdesk-module.json` | a row to the module (hosting model and BBN lookups) | -| `topdesk-manufacturer.json` | "Vendor" to the Supplier organisation | -| `topdesk-municipality.json` | a typed municipality name to a new organisation | -| `topdesk-usage.json` | a row to the usage (status and TIME lookups, dates, annotation) | -| `topdesk-business-owner.json` | the owner columns | - -Each pack has a list of `fieldMappings`, one per column: `source` (the column -name in the export), `target` (the field), optionally `required`, and a -`transform` such as `trim`, `date` or a `lookup` with a `map` of export values -to stored values. For example, to accept a new "Applicatiesoort" value, add -it to the `map` of the hosting-model lookup in `topdesk-module.json`: - -```json -"Cloud": ["SaaS"] -``` - -To accept a new "Applicatie Status" value, add it to the `map` of the status -lookup in `topdesk-usage.json`. The packs are checked by OpenRegister when an import -starts; an invalid pack stops the import with `MAPPING_UNAVAILABLE` before -any row is read. A mapping file changed on the server is overwritten by the -next app update, so propose lasting changes to the app itself. +| `stackiq-topdesk-module` | a row to the application (hosting model and BBN lookups, dates) | +| `stackiq-topdesk-manufacturer` | "Vendor" to the Supplier organisation | +| `stackiq-topdesk-usage` | a row to the usage (status and TIME lookups, phase-out date, internal note) | +| `stackiq-topdesk-business-owner` | the owner columns | + +Stackiq puts these four packs into OpenRegister when it is installed or +updated. The **CMDB import** section shows, per pack, where the import takes +it from (OpenRegister, or the copy that ships with stackiq), its version, and +an **Edit in OpenRegister** link that opens the pack. The import always runs +what is saved in OpenRegister. + +Each pack has a list of field mappings, one per column: the column name in +the export, the field it goes to, whether it is required, and a +transformation such as trim, a date, or a lookup table of export values to +stored values. For example, to accept a new "Applicatie Status" value, add it +to the lookup table of the status mapping in `stackiq-topdesk-usage`. To +accept a new "Applicatiesoort" value as a hosting model, add it to the lookup +table of the hosting-model mapping in `stackiq-topdesk-module`, for example +`Cloud` → `SaaS`. When a pack maps a column with a date transformation, the +import reads that column as an Excel date. + +**What the import refuses.** The import writes past OpenRegister's access +rules, so it checks every pack before it reads a row. It refuses a pack, with +`MAPPING_UNAVAILABLE` and the name and id of the pack, when: + +- OpenRegister finds the pack invalid; +- the application name, the APPID (to the source number), the vendor name or + the owner name is no longer mapped, or no longer marked required; +- a mapping writes a field the import sets itself or that the record does not + have: the import key, the provider, the consumer, the application of a + usage, the business owner, the publication dates of an application, or + the type of an organisation (a Supplier stays a Supplier). + +The section shows the same reason, so you can correct the pack in +OpenRegister and try again. + +**What a pack does not change.** Which sheets are read, the APPID and +"Applicatie Naam" columns, the values that mean empty (`NB`), the constant +each sheet adds (`Beheer geregeld: ja` or `nee`), the limits, and which fields +are written only on create or never on an existing record are part of the app +(`lib/Settings/cmdb-import/topdesk-profile.json`), and the section shows them +read-only. The municipality is mapped from the name you type in the section, +not from the export; that pack is not edited in OpenRegister either. + +**Reading more columns.** The import reads only the columns the mapping +names. When you map a new column in a pack, the import reads that column from +then on. + +**One mapping per Nextcloud.** The packs in OpenRegister apply to every +import on this Nextcloud, whatever the municipality. + +**App updates.** A pack you have not changed follows the app: when an update +ships a newer version, the stored pack is replaced by it. A pack you changed +is never overwritten. When an update ships a newer version of a pack you +changed, the section says so; compare the two and copy what you need into +your pack in OpenRegister. + +**Back to the shipped pack.** Delete the pack in OpenRegister. The next +import, or the next app update, puts the shipped pack back. Do not rename a +pack: the import finds it by its id (such as `stackiq-topdesk-usage`), and +refuses to run while a pack it stored has another id. + +**Without OpenRegister's pack store.** On an OpenRegister version without +migration packs, the import uses the packs that ship with stackiq and the +result says so. Once the packs have been stored in OpenRegister, the import +does not fall back to those copies: when OpenRegister's pack store is gone, +it refuses with `MAPPING_UNAVAILABLE` until OpenRegister is back. diff --git a/l10n/en.js b/l10n/en.js index 8dd84a2f4..afd93e55c 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1046,7 +1046,10 @@ OC.L10N.register( "The file is larger than the maximum of %s MB.": "The file is larger than the maximum of %s MB.", "The file is not an Excel workbook (.xlsx).": "The file is not an Excel workbook (.xlsx).", "The import failed. The details are in the Nextcloud log.": "The import failed. The details are in the Nextcloud log.", - "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.", + "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.", + "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.": "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.", + "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.", + "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.": "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.", "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", diff --git a/l10n/en.json b/l10n/en.json index 313f968ea..349882f8f 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1045,7 +1045,10 @@ "The file is larger than the maximum of %s MB.": "The file is larger than the maximum of %s MB.", "The file is not an Excel workbook (.xlsx).": "The file is not an Excel workbook (.xlsx).", "The import failed. The details are in the Nextcloud log.": "The import failed. The details are in the Nextcloud log.", - "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.", + "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.", + "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.": "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.", + "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.", + "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.": "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.", "The workbook has neither of the sheets %s.": "The workbook has neither of the sheets %s.", "Column \"%1$s\": %2$s": "Column \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optional column \"%s\" not found", diff --git a/l10n/nl.js b/l10n/nl.js index 3151813d6..2cf474396 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1114,7 +1114,10 @@ OC.L10N.register( "The file is larger than the maximum of %s MB.": "Het bestand is groter dan het maximum van %s MB.", "The file is not an Excel workbook (.xlsx).": "Het bestand is geen Excel-werkmap (.xlsx).", "The import failed. The details are in the Nextcloud log.": "De import is mislukt. De details staan in het Nextcloud-logboek.", - "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt of een mappingbestand is ongeldig.", + "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt, of een mappingpakket is ongeldig of niet beschikbaar.", + "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.": "De mapping van de import kan niet worden uitgevoerd: het migratiepakket %s in OpenRegister is ongeldig of kan niet worden gelezen. De details staan in het Nextcloud-logboek.", + "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "De mapping van de import kan niet worden getoond: OpenRegister ontbreekt, of een mappingpakket is ongeldig of niet beschikbaar.", + "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.": "De opslag voor migratiepakketten van OpenRegister is niet beschikbaar, dus deze import gebruikte de mapping die met stackiq wordt meegeleverd.", "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", diff --git a/l10n/nl.json b/l10n/nl.json index 680f6f853..ff4bd963e 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1113,7 +1113,10 @@ "The file is larger than the maximum of %s MB.": "Het bestand is groter dan het maximum van %s MB.", "The file is not an Excel workbook (.xlsx).": "Het bestand is geen Excel-werkmap (.xlsx).", "The import failed. The details are in the Nextcloud log.": "De import is mislukt. De details staan in het Nextcloud-logboek.", - "The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt of een mappingbestand is ongeldig.", + "The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "De mapping van de import kan niet worden uitgevoerd: OpenRegister ontbreekt, of een mappingpakket is ongeldig of niet beschikbaar.", + "The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.": "De mapping van de import kan niet worden uitgevoerd: het migratiepakket %s in OpenRegister is ongeldig of kan niet worden gelezen. De details staan in het Nextcloud-logboek.", + "The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.": "De mapping van de import kan niet worden getoond: OpenRegister ontbreekt, of een mappingpakket is ongeldig of niet beschikbaar.", + "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq.": "De opslag voor migratiepakketten van OpenRegister is niet beschikbaar, dus deze import gebruikte de mapping die met stackiq wordt meegeleverd.", "The workbook has neither of the sheets %s.": "De werkmap bevat geen van de tabbladen %s.", "Column \"%1$s\": %2$s": "Kolom \"%1$s\": %2$s", "Optional column \"%s\" not found": "Optionele kolom \"%s\" niet gevonden", diff --git a/lib/Controller/CmdbImportController.php b/lib/Controller/CmdbImportController.php index 7d1c00a8f..cc81d612e 100644 --- a/lib/Controller/CmdbImportController.php +++ b/lib/Controller/CmdbImportController.php @@ -384,6 +384,8 @@ private function error(string $code, int $status, array $details = []): JSONResp * @return string * * @SuppressWarnings(PHPMD.CyclomaticComplexity) One branch per contract error code. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ private function message(string $code, array $details): string { $megabytes = (string)round((int)($details['maxBytes'] ?? $this->importService->maxFileBytes()) / 1048576, 1); @@ -401,7 +403,7 @@ private function message(string $code, array $details): string { 'NO_SOURCE_SHEET' => $this->l10n->t('The workbook has neither of the sheets %s.', [$expected]), 'MISSING_COLUMN' => $this->l10n->t('Sheet "%1$s" has no column "%2$s".', [(string)($details['sheet'] ?? ''), (string)($details['column'] ?? '')]), 'TOO_MANY_ROWS' => $this->l10n->t('Sheet "%1$s" has more than %2$s rows.', [(string)($details['sheet'] ?? ''), (string)($details['limit'] ?? '')]), - 'MAPPING_UNAVAILABLE' => $this->l10n->t('The import mapping cannot run: OpenRegister is missing or a mapping file is invalid.'), + 'MAPPING_UNAVAILABLE' => $this->mappingMessage(details: $details), 'READER_UNAVAILABLE' => $this->l10n->t('The Excel reader is not available: OpenRegister is missing or incomplete.'), 'NOT_CONFIGURED' => $this->l10n->t('Stackiq is not configured: the register or its schemas cannot be found.'), 'OPERATION_NOT_FOUND' => $this->l10n->t('No running CMDB import has this id.'), @@ -414,6 +416,31 @@ private function message(string $code, array $details): string { }; }//end message() + /** + * The message of MAPPING_UNAVAILABLE, naming the OpenRegister migration pack when one is at fault. + * + * @param array $details The details: `pack` and `packId` for a pack stored in OpenRegister. + * + * @return string + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function mappingMessage(array $details): string { + $pack = ($details['pack'] ?? null); + if (is_string($pack) === false || $pack === '') { + return $this->l10n->t('The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.'); + } + + if (is_int($details['packId'] ?? null) === true) { + $pack .= ' (id ' . $details['packId'] . ')'; + } + + return $this->l10n->t( + 'The import mapping cannot run: the OpenRegister migration pack %s is invalid or cannot be read. The details are in the Nextcloud log.', + [$pack] + ); + }//end mappingMessage() + /** * The message of FIELD_INVALID, naming the accepted values when the field has a fixed set. * diff --git a/lib/Controller/SettingsController.php b/lib/Controller/SettingsController.php index 13c9fc0ac..3f4c7c39b 100644 --- a/lib/Controller/SettingsController.php +++ b/lib/Controller/SettingsController.php @@ -46,6 +46,7 @@ use OCP\IGroupManager; use OCP\IL10N; use OCP\IRequest; +use OCP\IURLGenerator; use OCP\IUserSession; use Psr\Container\ContainerInterface; use Psr\Log\LoggerInterface; @@ -74,6 +75,13 @@ class SettingsController extends Controller { */ private const REASON_MAX_LENGTH = 500; + /** + * OpenRegister's single-page catch-all route, which serves its migration-pack editor. + * + * @var string + */ + private const OPENREGISTER_PAGE_ROUTE = 'openregister.dashboard.catchAll'; + /** * The OpenRegister object service. * @@ -100,6 +108,7 @@ class SettingsController extends Controller { * @param ConnectionReportService|null $connectionReports Asks integriq to look again after an email settings save. * @param CmdbImportProfile|null $cmdbImportProfile The CMDB import profile and packs; null builds the shipped one. * @param IL10N|null $l10n Translations of the CMDB mapping messages. + * @param IURLGenerator|null $urlGenerator Builds the link to a CMDB mapping pack's editor in OpenRegister. * * @SuppressWarnings(PHPMD.ExcessiveParameterList) * @@ -123,6 +132,7 @@ public function __construct( private readonly ?ConnectionReportService $connectionReports = null, private readonly ?CmdbImportProfile $cmdbImportProfile = null, private readonly ?IL10N $l10n = null, + private readonly ?IURLGenerator $urlGenerator = null, ) { parent::__construct(appName: $appName, request: $request); @@ -3900,23 +3910,29 @@ public function getEolSyncStatus(): JSONResponse { }//end getEolSyncStatus() /** - * The column mapping the CMDB import uses, read-only. + * The column mapping the CMDB import uses, read-only, with where each pack is edited. * * Loads the import profile and its packs through the loader the import - * uses (CmdbImportProfile), so the answer is what the next import runs. - * Success is the flat `{profile, packs}`; a failure uses the import's - * error envelope, so the section shows it like an import error, with the - * loader's reason in `details.reason` because the admin who edited a file - * needs to know which file and why. That reason is the loader's message cut - * to its first line and REASON_MAX_LENGTH characters: the file, the - * validator's rule and the configuration value it refused, never a - * workbook cell or a person. - * - * @return JSONResponse `{profile, packs}` (200), 503 MAPPING_UNAVAILABLE, or 500 IMPORT_FAILED. + * uses (CmdbImportProfile), so the answer is what the next import runs: + * each sheet pack from OpenRegister's migration-pack store when it holds + * it, otherwise the shipped file. Success is the flat + * `{profile, packs, openRegisterPacks}`; every pack carries its source and + * versions and an `editUrl`, the absolute link to its editor in + * OpenRegister, or null when it is not stored there. A failure uses the + * import's error envelope, so the section shows it like an import error, + * with the loader's reason in `details.reason` because the admin who + * edited a pack needs to know which pack and why; a pack stored in + * OpenRegister is also named in `details.pack` and `details.packId`. That reason is the + * loader's message cut to its first line and REASON_MAX_LENGTH characters: + * the file or the OpenRegister pack and its id, the broken rule and the + * configuration value it refused, never a workbook cell or a person. + * + * @return JSONResponse `{profile, packs, openRegisterPacks}` (200), 503 MAPPING_UNAVAILABLE, or 500 IMPORT_FAILED. * * @auth admin-only the import's configuration; the import writes with RBAC and multitenancy off, so its view keeps that posture. * * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 */ public function getCmdbImportMapping(): JSONResponse { $profile = ($this->cmdbImportProfile ?? new CmdbImportProfile(container: $this->container)); @@ -3932,8 +3948,8 @@ public function getCmdbImportMapping(): JSONResponse { [ 'success' => false, 'error' => $e->getErrorCode(), - 'message' => $this->translate(text: 'The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.'), - 'details' => (object)['reason' => self::boundedReason(message: $e->getMessage())], + 'message' => $this->translate(text: 'The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.'), + 'details' => (object)array_merge($e->getDetails(), ['reason' => self::boundedReason(message: $e->getMessage())]), ], $e->getHttpStatus() ); @@ -3950,9 +3966,46 @@ public function getCmdbImportMapping(): JSONResponse { ); }//end try + foreach ($overview['packs'] as $index => $pack) { + $overview['packs'][$index]['editUrl'] = $this->packEditUrl(packId: ($pack['packId'] ?? null)); + } + return new JSONResponse($overview, Http::STATUS_OK); }//end getCmdbImportMapping() + /** + * The absolute link to a migration pack's editor in OpenRegister. + * + * Built from OpenRegister's catch-all page route, so an instance that + * needs `/index.php` in its URLs gets it. + * + * @param int|null $packId The OpenRegister row id, or null for a pack that is not stored there. + * + * @return string|null Null when the pack is not stored in OpenRegister or the link cannot be built. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 + */ + private function packEditUrl(?int $packId): ?string { + if ($packId === null || $this->urlGenerator === null) { + return null; + } + + $path = 'migration-packs/' . $packId; + try { + $url = $this->urlGenerator->linkToRouteAbsolute(self::OPENREGISTER_PAGE_ROUTE, ['path' => $path]); + } catch (\Throwable $e) { + $this->logger->info('SettingsController: no link to the OpenRegister pack editor', ['exception' => get_class($e)]); + return null; + } + + // Nextcloud's router answers an unknown route with an empty path, which would link to the instance's front page. + if (str_ends_with($url, '/' . $path) === false) { + return null; + } + + return $url; + }//end packEditUrl() + /** * A loader message as the mapping endpoint answers it: its first line, at most REASON_MAX_LENGTH characters. * diff --git a/lib/Repair/SeedCmdbImportPacks.php b/lib/Repair/SeedCmdbImportPacks.php new file mode 100644 index 000000000..3d911e9b4 --- /dev/null +++ b/lib/Repair/SeedCmdbImportPacks.php @@ -0,0 +1,146 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Repair; + +use OCA\Stackiq\Service\Cmdb\CmdbImportPackStore; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use Psr\Log\LoggerInterface; +use Throwable; + +/** + * Seeds and updates the CMDB import packs in OpenRegister's migration-pack store. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ +class SeedCmdbImportPacks implements IRepairStep { + /** + * Constructor. + * + * @param CmdbImportProfile $profile Reads and validates the shipped packs. + * @param CmdbImportPackStore $store OpenRegister's migration-pack store. + * @param LoggerInterface $logger Logger for seed diagnostics. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ + public function __construct( + private readonly CmdbImportProfile $profile, + private readonly CmdbImportPackStore $store, + private readonly LoggerInterface $logger, + ) { + }//end __construct() + + /** + * The name of this repair step. + * + * @return string + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ + public function getName(): string { + return 'Seed the CMDB import mapping packs into OpenRegister'; + }//end getName() + + /** + * Seed the missing packs and update the unedited ones. + * + * @param IOutput $output Output interface for status messages. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ + public function run(IOutput $output): void { + if ($this->store->isAvailable() === false) { + $output->info('OpenRegister migration pack store not available; CMDB import packs not seeded'); + return; + } + + try { + $packs = $this->profile->shippedEditablePacks(); + } catch (Throwable $e) { + $this->logger->warning('[SeedCmdbImportPacks] shipped packs unreadable: ' . $e->getMessage()); + $output->warning('CMDB import packs not seeded: ' . $e->getMessage()); + return; + } + + foreach ($packs as $shipped) { + $this->seed(output: $output, shipped: $shipped); + } + }//end run() + + /** + * Seed or update one pack, and report what happened. + * + * @param IOutput $output Output interface for status messages. + * @param array $shipped The shipped pack. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ + private function seed(IOutput $output, array $shipped): void { + $slug = (string)($shipped['id'] ?? ''); + try { + $stored = $this->store->resolve(shipped: $shipped); + } catch (Throwable $e) { + $this->logger->warning('[SeedCmdbImportPacks] ' . $slug . ': ' . $e->getMessage()); + $output->warning('CMDB import pack ' . $slug . ' not seeded: ' . $e->getMessage()); + return; + } + + $where = $slug . ' (id ' . $stored['id'] . ')'; + if ($stored['outcome'] === CmdbImportPackStore::SEEDED) { + $output->info('Seeded CMDB import pack ' . $where); + return; + } + + if ($stored['outcome'] === CmdbImportPackStore::UPDATED) { + $output->info('Updated unedited CMDB import pack ' . $where . ' to version ' . $stored['version']); + return; + } + + if ($stored['newerShipped'] === true) { + $output->warning( + 'CMDB import pack ' . $where . ' was edited in OpenRegister and is kept; this release ships version ' + . (string)($shipped['version'] ?? '') . ', which is not applied' + ); + return; + } + + $output->info('CMDB import pack ' . $where . ' already present'); + }//end seed() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportPackMarker.php b/lib/Service/Cmdb/CmdbImportPackMarker.php new file mode 100644 index 000000000..5d41765f9 --- /dev/null +++ b/lib/Service/Cmdb/CmdbImportPackMarker.php @@ -0,0 +1,175 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +/** + * Adds, reads and compares the seed marker of a stored pack. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ +class CmdbImportPackMarker { + /** + * The definition key that records what stackiq seeded. + */ + public const KEY = 'x-stackiq'; + + /** + * The hash stackiq records for a shipped pack: sha256 of its canonical JSON. + * + * @param array $pack The pack, without the marker. + * + * @return string + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function hash(array $pack): string { + $json = json_encode($this->canonical(value: $pack), JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE | JSON_PRESERVE_ZERO_FRACTION); + return hash('sha256', (string)$json); + }//end hash() + + /** + * The shipped pack as it is stored: with the marker. + * + * @param array $shipped The shipped pack. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function seed(array $shipped): array { + $definition = $this->strip(definition: $shipped); + $definition[self::KEY] = [ + 'shippedVersion' => (string)($shipped['version'] ?? ''), + 'shippedHash' => $this->hash(pack: $definition), + ]; + + return $definition; + }//end seed() + + /** + * A stored definition without the marker: the pack the engine runs. + * + * @param array $definition The stored definition. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function strip(array $definition): array { + unset($definition[self::KEY]); + return $definition; + }//end strip() + + /** + * Whether a stored definition differs from what stackiq seeded. + * + * @param array $definition The stored definition. + * + * @return bool|null Null when it carries no marker, so nobody knows what was seeded. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function isEdited(array $definition): ?bool { + $recorded = $this->recordedHash(definition: $definition); + if ($recorded === null) { + return null; + } + + return $this->hash(pack: $this->strip(definition: $definition)) !== $recorded; + }//end isEdited() + + /** + * Whether a stored pack nobody edited was seeded from another shipped pack than this one. + * + * @param array $definition The stored definition. + * @param array $shipped The shipped pack. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function isOutdated(array $definition, array $shipped): bool { + return $this->isEdited(definition: $definition) === false + && $this->recordedHash(definition: $definition) !== $this->hash(pack: $shipped); + }//end isOutdated() + + /** + * Whether an edited stored pack was seeded from a shipped pack that has changed since. + * + * @param array $definition The stored definition. + * @param array $shipped The shipped pack. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function isNewerShipped(array $definition, array $shipped): bool { + return $this->isEdited(definition: $definition) === true + && $this->recordedHash(definition: $definition) !== $this->hash(pack: $shipped); + }//end isNewerShipped() + + /** + * The hash the marker of a stored definition records. + * + * @param array $definition The stored definition. + * + * @return string|null + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function recordedHash(array $definition): ?string { + $hash = ($definition[self::KEY]['shippedHash'] ?? null); + if (is_string($hash) === false || $hash === '') { + return null; + } + + return $hash; + }//end recordedHash() + + /** + * A value with every object's keys sorted, recursively. + * + * @param mixed $value The value. + * + * @return mixed + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function canonical(mixed $value): mixed { + if (is_array($value) === false) { + return $value; + } + + if (array_is_list($value) === false) { + ksort($value, SORT_STRING); + } + + return array_map(fn ($item) => $this->canonical(value: $item), $value); + }//end canonical() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportPackRules.php b/lib/Service/Cmdb/CmdbImportPackRules.php new file mode 100644 index 000000000..6e9b0e4e7 --- /dev/null +++ b/lib/Service/Cmdb/CmdbImportPackRules.php @@ -0,0 +1,377 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use OCA\Stackiq\Exception\CmdbImportException; + +/** + * Lists the import rules a pack breaks. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ +class CmdbImportPackRules { + /** + * The fields a pack must map with `required: true`, per target. + * + * The import keys a module on its APPID and recognises a manufacturer, + * a municipality and an owner by name; without them it cannot match. + * + * @var array> + */ + private const REQUIRED_TARGETS = [ + 'module' => ['name', 'externalNumber'], + 'manufacturer' => ['name'], + 'municipality' => ['name'], + 'usage' => [], + 'businessOwner' => ['name'], + ]; + + /** + * The register schema a target writes into; its properties are the fields a pack may map. + * + * @var array + */ + private const TARGET_SCHEMAS = [ + 'module' => 'module', + 'manufacturer' => 'organization', + 'municipality' => 'organization', + 'usage' => 'usage', + ]; + + /** + * Targets that are no schema: the fields such a pack may map. + * + * The business owner is resolved in Nextcloud Contacts by name, with the role on the contact person. + * + * @var array> + */ + private const FIXED_FIELDS = ['businessOwner' => ['name', 'role']]; + + /** + * Fields no pack may write: object metadata, and the keys and relations the import sets itself. + * + * @var array + */ + private const MANAGED_FIELDS = ['@self', 'id', 'provider', 'consumer', 'module', 'externalKey', 'businessOwner']; + + /** + * Further match keys per target: the import finds an organisation by name and type. + * + * @var array> + */ + private const MATCH_FIELDS = ['manufacturer' => ['type'], 'municipality' => ['type']]; + + /** + * Defaults a pack must keep as shipped, per target. + * + * @var array> + */ + private const PINNED_DEFAULTS = [ + 'manufacturer' => ['type' => 'Supplier'], + 'municipality' => ['type' => 'Municipality'], + ]; + + /** + * Property names per schema of the shipped register, once read. + * + * @var array>|null + */ + private ?array $schemaFields = null; + + /** + * Constructor. + * + * @param string|null $settingsDirectory Directory of the register file and its fragments; null is the shipped one. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function __construct( + private ?string $settingsDirectory = null, + ) { + if ($this->settingsDirectory === null) { + $this->settingsDirectory = __DIR__ . '/../../Settings'; + } + }//end __construct() + + /** + * The rules a pack breaks, as readable sentences; empty when it follows them all. + * + * @param string $target The profile target. + * @param array $pack The pack, validated by OpenRegister. + * @param array $profile The decoded import profile. + * + * @return array + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the register cannot be read. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function violations(string $target, array $pack, array $profile): array { + $never = $profile['neverWritten'][$target] ?? []; + if (is_array($never) === false) { + $never = []; + } + + $forbidden = array_merge(self::MANAGED_FIELDS, (self::MATCH_FIELDS[$target] ?? []), array_map('strval', $never)); + $allowed = array_values(array_diff($this->targetFields(target: $target), $forbidden)); + + return array_values( + array_unique( + array_merge( + $this->requiredViolations(target: $target, pack: $pack, keyColumn: (string)($profile['keyColumn'] ?? 'APPID')), + $this->mappingViolations(target: $target, pack: $pack, allowed: $allowed, forbidden: $forbidden), + $this->defaultViolations(target: $target, pack: $pack, allowed: $allowed, forbidden: $forbidden) + ) + ) + ); + }//end violations() + + /** + * The required mappings a pack lacks. + * + * @param string $target The target. + * @param array $pack The pack. + * @param string $keyColumn The match key column. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function requiredViolations(string $target, array $pack, string $keyColumn): array { + $violations = []; + foreach ((self::REQUIRED_TARGETS[$target] ?? []) as $field) { + if ($this->hasRequiredMapping(pack: $pack, field: $field, keyColumn: $keyColumn) === true) { + continue; + } + + if ($field === 'externalNumber') { + $violations[] = 'the key column "' . $keyColumn . '" must be mapped to "externalNumber" as required'; + continue; + } + + $violations[] = '"' . $field . '" must be mapped as required'; + } + + return $violations; + }//end requiredViolations() + + /** + * Whether a pack maps a field as required; `externalNumber` only from the key column. + * + * @param array $pack The pack. + * @param string $field The field. + * @param string $keyColumn The match key column. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function hasRequiredMapping(array $pack, string $field, string $keyColumn): bool { + foreach (($pack['fieldMappings'] ?? []) as $mapping) { + if (is_array($mapping) === false || ($mapping['target'] ?? null) !== $field || ($mapping['required'] ?? false) !== true) { + continue; + } + + if ($field !== 'externalNumber' || ($mapping['source'] ?? null) === $keyColumn) { + return true; + } + } + + return false; + }//end hasRequiredMapping() + + /** + * The mapped targets a pack may not write. + * + * @param string $target The target. + * @param array $pack The pack. + * @param array $allowed The fields it may write. + * @param array $forbidden The fields the import manages. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function mappingViolations(string $target, array $pack, array $allowed, array $forbidden): array { + $violations = []; + foreach (($pack['fieldMappings'] ?? []) as $mapping) { + $field = (string)($mapping['target'] ?? ''); + if (in_array($field, $allowed, true) === false) { + $violations[] = self::fieldViolation(field: $field, forbidden: in_array($field, $forbidden, true), target: $target); + } + } + + return $violations; + }//end mappingViolations() + + /** + * The defaults a pack may not set, and the pinned defaults it changed. + * + * @param string $target The target. + * @param array $pack The pack. + * @param array $allowed The fields it may write. + * @param array $forbidden The fields the import manages. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function defaultViolations(string $target, array $pack, array $allowed, array $forbidden): array { + $defaults = $pack['defaults'] ?? []; + if (is_array($defaults) === false) { + $defaults = []; + } + + $pinned = (self::PINNED_DEFAULTS[$target] ?? []); + $violations = []; + foreach (array_keys($defaults) as $field) { + $field = (string)$field; + if (array_key_exists($field, $pinned) === false && in_array($field, $allowed, true) === false) { + $violations[] = 'default ' . self::fieldViolation(field: $field, forbidden: in_array($field, $forbidden, true), target: $target); + } + } + + foreach ($pinned as $field => $value) { + if (($defaults[$field] ?? null) !== $value) { + $violations[] = 'default "' . $field . '" must stay "' . $value . '"'; + } + } + + return $violations; + }//end defaultViolations() + + /** + * Why a field may not be written. + * + * @param string $field The field. + * @param bool $forbidden Whether the import manages it. + * @param string $target The target. + * + * @return string + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private static function fieldViolation(string $field, bool $forbidden, string $target): string { + if ($forbidden === true) { + return '"' . $field . '" is set by the import and cannot be mapped'; + } + + return '"' . $field . '" is not a field the ' . $target . ' pack may map'; + }//end fieldViolation() + + /** + * The fields a target's pack may map before the forbidden ones are taken out. + * + * @param string $target The target. + * + * @return array + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the register cannot be read. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function targetFields(string $target): array { + if (isset(self::FIXED_FIELDS[$target]) === true) { + return self::FIXED_FIELDS[$target]; + } + + $schema = (self::TARGET_SCHEMAS[$target] ?? null); + if ($schema === null) { + return []; + } + + return ($this->schemaFields()[$schema] ?? []); + }//end targetFields() + + /** + * The property names per schema of the register stackiq ships, fragments included. + * + * @return array> + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the register file cannot be read. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function schemaFields(): array { + if ($this->schemaFields !== null) { + return $this->schemaFields; + } + + $register = $this->decodeJson(file: $this->settingsDirectory . '/softwarecatalogus_register.json'); + if ($register === null) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'The stackiq register definition cannot be read, so the CMDB import packs cannot be checked' + ); + } + + $fragments = glob($this->settingsDirectory . '/register.d/*.json'); + if ($fragments === false) { + $fragments = []; + } + + $fields = []; + foreach (array_merge([$register], array_map(fn (string $file): ?array => $this->decodeJson(file: $file), $fragments)) as $decoded) { + foreach (($decoded['components']['schemas'] ?? []) as $schema => $definition) { + $properties = array_keys((array)($definition['properties'] ?? [])); + $fields[(string)$schema] = array_values(array_unique(array_merge(($fields[(string)$schema] ?? []), array_map('strval', $properties)))); + } + } + + $this->schemaFields = $fields; + return $fields; + }//end schemaFields() + + /** + * A JSON file decoded to an array, or null when it cannot be read. + * + * @param string $file The file. + * + * @return array|null + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function decodeJson(string $file): ?array { + $content = false; + if (is_readable($file) === true) { + $content = file_get_contents($file); + } + + if (is_string($content) === false) { + return null; + } + + $decoded = json_decode($content, true); + if (is_array($decoded) === false) { + return null; + } + + return $decoded; + }//end decodeJson() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportPackStore.php b/lib/Service/Cmdb/CmdbImportPackStore.php new file mode 100644 index 000000000..c256e9919 --- /dev/null +++ b/lib/Service/Cmdb/CmdbImportPackStore.php @@ -0,0 +1,462 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Service\Cmdb; + +use OCA\Stackiq\AppInfo\Application; +use OCA\Stackiq\Exception\CmdbImportException; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\IAppConfig; +use Psr\Container\ContainerInterface; +use Throwable; + +/** + * Seeds, finds and updates the CMDB import packs in OpenRegister. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ +class CmdbImportPackStore { + /** + * OpenRegister's migration-pack service (not a public contract). + */ + public const SERVICE_CLASS = 'OCA\OpenRegister\Service\MigrationPackService'; + + /** + * App config key: the OpenRegister row id per pack slug, as JSON. + */ + public const CONFIG_KEY = 'cmdb_import_pack_ids'; + + /** + * The store resolved the pack as it was. + */ + public const FOUND = 'found'; + + /** + * The store created the pack from the shipped file. + */ + public const SEEDED = 'seeded'; + + /** + * The store replaced an unedited pack with a newer shipped one. + */ + public const UPDATED = 'updated'; + + /** + * The methods this store calls on OpenRegister's service. + * + * @var array + */ + private const SERVICE_METHODS = ['findByPackSlug', 'find', 'create', 'update']; + + /** + * OpenRegister's service, once resolved; false when it is not available. + * + * @var object|false|null + */ + private object|false|null $service = null; + + /** + * Reads and writes the seed marker. + * + * @var CmdbImportPackMarker + */ + private readonly CmdbImportPackMarker $marker; + + /** + * Constructor. + * + * @param ContainerInterface $container Resolves OpenRegister's service, and the app config when none is given. + * @param IAppConfig|null $appConfig Keeps the row ids; null resolves it from the container. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function __construct( + private readonly ContainerInterface $container, + private ?IAppConfig $appConfig = null, + ) { + $this->marker = new CmdbImportPackMarker(); + }//end __construct() + + /** + * Whether OpenRegister's migration-pack store can be used. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function isAvailable(): bool { + return $this->service() !== null; + }//end isAvailable() + + /** + * The OpenRegister row ids this store resolved before, per pack slug. + * + * Not empty means the packs were stored in OpenRegister once, so the + * import may no longer fall back to the shipped files. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function recordedPackIds(): array { + $config = $this->appConfig(); + if ($config === null) { + return []; + } + + $decoded = json_decode($config->getValueString(Application::APP_ID, self::CONFIG_KEY, ''), true); + if (is_array($decoded) === false) { + return []; + } + + $ids = []; + foreach ($decoded as $slug => $id) { + if (is_string($slug) === true && is_int($id) === true && $id > 0) { + $ids[$slug] = $id; + } + } + + return $ids; + }//end recordedPackIds() + + /** + * Refuse the shipped files once the packs were stored in OpenRegister. + * + * Called when the store is not available: before any pack was stored + * there the import may use the shipped files; afterwards that would + * silently drop an administrator's edit. + * + * @return void + * + * @throws CmdbImportException MAPPING_UNAVAILABLE naming the packs the store held. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function assertShippedFallbackAllowed(): void { + $named = []; + foreach ($this->recordedPackIds() as $slug => $id) { + $named[] = $slug . ' (id ' . $id . ')'; + } + + if ($named === []) { + return; + } + + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister migration pack store is not available, but it holds the CMDB import packs ' + . implode(', ', $named) . '; the import does not fall back to the shipped files', + details: ['packs' => $this->recordedPackIds()] + ); + }//end assertShippedFallbackAllowed() + + /** + * The stored pack for a shipped pack: found, seeded when missing, or updated when unedited and outdated. + * + * The answer holds the stored definition, the pack the engine runs (the + * definition without the marker), whether it was edited (null when it + * carries no marker) and whether an edit kept a newer shipped pack out. + * + * @param array $shipped The shipped pack, already validated. + * + * @return array id (int), slug, version, definition (stored, with the marker), + * pack (without it), edited (bool|null), newerShipped (bool) and outcome. + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the store is missing or fails, + * or when the pack was renamed in OpenRegister. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function resolve(array $shipped): array { + $service = $this->service(); + if ($service === null) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister MigrationPackService is not available' + ); + } + + $slug = (string)($shipped['id'] ?? ''); + $outcome = self::FOUND; + $entity = $this->findBySlug(service: $service, slug: $slug); + if ($entity === null) { + $this->assertNotRenamed(service: $service, slug: $slug); + $entity = $this->write( + slug: $slug, + action: 'seeded', + call: fn (): object => $service->create(definition: $this->marker->seed(shipped: $shipped), ownerUid: null, builtin: true) + ); + $outcome = self::SEEDED; + } elseif ($this->marker->isOutdated(definition: self::definitionOf(entity: $entity), shipped: $shipped) === true) { + $id = (int)$entity->getId(); + $entity = $this->write( + slug: $slug, + action: 'updated', + call: fn (): object => $service->update(id: $id, definition: $this->marker->seed(shipped: $shipped)) + ); + $outcome = self::UPDATED; + } + + $id = (int)$entity->getId(); + $this->record(slug: $slug, id: $id); + + $definition = self::definitionOf(entity: $entity); + return [ + 'id' => $id, + 'slug' => $slug, + 'version' => (string)($definition['version'] ?? ''), + 'definition' => $definition, + 'pack' => $this->marker->strip(definition: $definition), + 'edited' => $this->marker->isEdited(definition: $definition), + 'newerShipped' => $this->marker->isNewerShipped(definition: $definition, shipped: $shipped), + 'outcome' => $outcome, + ]; + }//end resolve() + + /** + * The decoded definition of a stored pack. + * + * @param object $entity The MigrationPack entity. + * + * @return array + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private static function definitionOf(object $entity): array { + $definition = $entity->getDefinitionArray(); + if (is_array($definition) === false) { + return []; + } + + return $definition; + }//end definitionOf() + + /** + * Find a stored pack by slug. + * + * @param object $service OpenRegister's service. + * @param string $slug The pack slug. + * + * @return object|null The entity, or null when no pack has the slug. + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the lookup fails otherwise. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function findBySlug(object $service, string $slug): ?object { + try { + return $service->findByPackSlug(packSlug: $slug); + } catch (DoesNotExistException $e) { + return null; + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister migration pack ' . $slug . ' could not be read: ' . get_class($e), + details: ['pack' => $slug], + previous: $e + ); + } + }//end findBySlug() + + /** + * Refuse when the row recorded for a slug still exists under another slug. + * + * A renamed pack is one the import no longer reads, so seeding a second + * one would let the administrator edit a pack that never runs. + * + * @param object $service OpenRegister's service. + * @param string $slug The pack slug. + * + * @return void + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the pack was renamed or the lookup fails. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function assertNotRenamed(object $service, string $slug): void { + $id = ($this->recordedPackIds()[$slug] ?? null); + if ($id === null) { + return; + } + + try { + $entity = $service->find(id: $id); + } catch (DoesNotExistException $e) { + // Deleted: seeding it again restores the shipped pack. + return; + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister migration pack ' . $slug . ' (id ' . $id . ') could not be read: ' . get_class($e), + details: ['pack' => $slug, 'packId' => $id], + previous: $e + ); + } + + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister migration pack id ' . $id . ' was renamed from ' . $slug . ' to ' + . (string)$entity->getPackSlug() . '; the import reads it by the id ' . $slug . ' only, so rename it back', + details: ['pack' => $slug, 'packId' => $id] + ); + }//end assertNotRenamed() + + /** + * Create or update a stored pack, turning a failure into MAPPING_UNAVAILABLE. + * + * @param string $slug The pack slug. + * @param string $action What the call does, for the message. + * @param callable(): object $call The service call. + * + * @return object The entity. + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the call fails. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function write(string $slug, string $action, callable $call): object { + try { + return $call(); + } catch (Throwable $e) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'OpenRegister migration pack ' . $slug . ' could not be ' . $action . ': ' . get_class($e), + details: ['pack' => $slug], + previous: $e + ); + } + }//end write() + + /** + * Remember the row id of a slug in the app config, when it changed. + * + * @param string $slug The pack slug. + * @param int $id The row id. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function record(string $slug, int $id): void { + $config = $this->appConfig(); + if ($config === null) { + return; + } + + $ids = $this->recordedPackIds(); + if (($ids[$slug] ?? null) === $id) { + return; + } + + $ids[$slug] = $id; + ksort($ids); + $config->setValueString(Application::APP_ID, self::CONFIG_KEY, (string)json_encode($ids)); + }//end record() + + /** + * OpenRegister's service, resolved once. + * + * @return object|null + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function service(): ?object { + if ($this->service === null) { + $this->service = false; + try { + if ($this->container->has(self::SERVICE_CLASS) === true) { + $service = $this->container->get(self::SERVICE_CLASS); + if (is_object($service) === true && self::hasMethods(service: $service) === true) { + $this->service = $service; + } + } + } catch (Throwable $e) { + // OpenRegister is missing or its service cannot be built: the store is not available. + $this->service = false; + } + } + + if ($this->service === false) { + return null; + } + + return $this->service; + }//end service() + + /** + * Whether a resolved service has every method this store calls. + * + * @param object $service The service. + * + * @return bool + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private static function hasMethods(object $service): bool { + foreach (self::SERVICE_METHODS as $method) { + if (method_exists($service, $method) === false) { + return false; + } + } + + return true; + }//end hasMethods() + + /** + * The app config, resolved from the container when it was not given. + * + * @return IAppConfig|null + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function appConfig(): ?IAppConfig { + if ($this->appConfig !== null) { + return $this->appConfig; + } + + try { + if ($this->container->has(IAppConfig::class) === true) { + $config = $this->container->get(IAppConfig::class); + if ($config instanceof IAppConfig) { + $this->appConfig = $config; + } + } + } catch (Throwable $e) { + // No app config in this container: nothing is recorded. + $this->appConfig = null; + } + + return $this->appConfig; + }//end appConfig() +}//end class diff --git a/lib/Service/Cmdb/CmdbImportProfile.php b/lib/Service/Cmdb/CmdbImportProfile.php index fb455ac7a..ecd4426e4 100644 --- a/lib/Service/Cmdb/CmdbImportProfile.php +++ b/lib/Service/Cmdb/CmdbImportProfile.php @@ -11,10 +11,18 @@ * mean empty, which constants a sheet adds to its rows, and which columns * may be read at all (the allowlist, design D3). * + * The four sheet packs come from OpenRegister's migration-pack store when it + * is available (CmdbImportPackStore), so an administrator's edit there is + * what the import runs; the shipped files are the seed. The municipality + * pack maps the request options and always comes from its file. A pack from + * either source must pass the validator and the import's own rules + * (CmdbImportPackRules). + * * `PackDefinitionValidator` is not part of OpenRegister's `Contract` - * namespace, so it is resolved defensively. When it is missing, or a shipped - * pack is invalid, loading throws `MAPPING_UNAVAILABLE` (503) before any - * row is read (REQ-CMDB-005). + * namespace, so it is resolved defensively. When it is missing, a pack is + * invalid or breaks a rule, or the store that holds the packs is gone, + * loading throws `MAPPING_UNAVAILABLE` (503) before any row is read + * (REQ-CMDB-005, REQ-CMDB-021). * * @category Service * @package OCA\Stackiq\Service\Cmdb @@ -24,6 +32,7 @@ * @link https://github.com/ConductionNL/stackiq * * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 * * SPDX-FileCopyrightText: 2026 Conduction B.V. * SPDX-License-Identifier: EUPL-1.2 @@ -47,6 +56,8 @@ * @SuppressWarnings(PHPMD.TooManyMethods) The same accessors, plus the loader's small private helpers. * @SuppressWarnings(PHPMD.ExcessiveClassComplexity) The accessors each guard against a * malformed profile value; the sum passes the threshold, no single method is complex. + * @SuppressWarnings(PHPMD.ExcessiveClassLength) The same accessors with their docblocks, and the + * loader that picks each pack's source; the store, the marker and the import rules are classes of their own. */ class CmdbImportProfile { /** @@ -86,6 +97,16 @@ class CmdbImportProfile { */ public const DEFAULT_MAX_REFERENCED_STRING_BYTES = 67108864; + /** + * The targets whose pack an administrator edits in OpenRegister. + * + * The municipality pack maps the request options, not a sheet, so it + * stays a shipped file. + * + * @var array + */ + public const EDITABLE_TARGETS = ['module', 'manufacturer', 'usage', 'businessOwner']; + /** * Sources of the municipality pack that come from the request, not from a sheet. * @@ -93,6 +114,11 @@ class CmdbImportProfile { */ private const OPTION_SOURCES = ['municipalityName']; + /** + * The separator of a concat mapping when the pack names none (OpenRegister's default). + */ + private const DEFAULT_CONCAT_SEPARATOR = ' '; + /** * The decoded profile, once loaded. * @@ -107,65 +133,164 @@ class CmdbImportProfile { */ private array $packs = []; + /** + * Where each loaded pack came from, per target. + * + * @var array> Per target: source, packId, storedVersion, shippedVersion, edited, newerShipped. + */ + private array $sources = []; + + /** + * Whether OpenRegister's pack store was available when the packs were loaded. + * + * @var bool + */ + private bool $storeAvailable = false; + + /** + * The import rules every pack must follow. + * + * @var CmdbImportPackRules + */ + private readonly CmdbImportPackRules $rules; + /** * Constructor. * * @param ContainerInterface $container Resolves OpenRegister's validator. * @param string|null $directory Directory of the profile and packs; null is the shipped one. * @param string $profileFile File name of the profile inside the directory. + * @param CmdbImportPackStore|null $packStore OpenRegister's pack store; null builds one on the container. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ public function __construct( private readonly ContainerInterface $container, private ?string $directory = null, private readonly string $profileFile = 'topdesk-profile.json', + private ?CmdbImportPackStore $packStore = null, ) { if ($this->directory === null) { $this->directory = __DIR__ . '/../../Settings/cmdb-import'; } + + $this->rules = new CmdbImportPackRules(); }//end __construct() /** * Load the profile and validate every pack it names. * + * Each sheet pack is taken from OpenRegister's pack store when the store + * is available, seeded from its shipped file when it is missing there, and + * otherwise from the shipped file. Once the packs were stored in + * OpenRegister, a store that is gone fails closed: the import never falls + * back to the files and silently drops an administrator's edit. + * * @return void * - * @throws CmdbImportException MAPPING_UNAVAILABLE when the validator is missing, - * or the profile or a pack is unreadable or invalid. + * @throws CmdbImportException MAPPING_UNAVAILABLE when the validator is missing, the profile + * or a pack is unreadable, invalid or breaks an import rule, or + * the store that holds the packs is not available. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-3 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ public function load(): void { $validator = $this->resolveValidator(); $profile = $this->decodeFile(fileName: $this->profileFile); + $store = $this->packStore(); + $storeAvailable = $store->isAvailable(); + if ($storeAvailable === false) { + $store->assertShippedFallbackAllowed(); + } $packs = []; + $sources = []; foreach (self::TARGETS as $target) { - $fileName = $profile['packs'][$target] ?? null; - if (is_string($fileName) === false || $fileName === '' || basename($fileName) !== $fileName) { - throw new CmdbImportException( - errorCode: CmdbImportException::MAPPING_UNAVAILABLE, - message: 'CMDB import profile names no pack for target ' . $target - ); - } + $fileName = (string)($profile['packs'][$target] ?? ''); + $shipped = $this->readShippedPack(profile: $profile, target: $target, validator: $validator); + $pack = $shipped; + $label = 'CMDB mapping pack ' . $fileName; + $details = []; + $source = [ + 'source' => 'file', + 'packId' => null, + 'storedVersion' => null, + 'shippedVersion' => (string)($shipped['version'] ?? ''), + 'edited' => null, + 'newerShipped' => false, + ]; - $pack = $this->decodeFile(fileName: $fileName); - $errors = $validator->validate($pack); - if (is_array($errors) === true && $errors !== []) { - throw new CmdbImportException( - errorCode: CmdbImportException::MAPPING_UNAVAILABLE, - message: 'CMDB mapping pack ' . $fileName . ' is invalid: ' . implode('; ', array_map('strval', $errors)) + if ($storeAvailable === true && in_array($target, self::EDITABLE_TARGETS, true) === true) { + $stored = $store->resolve(shipped: $shipped); + $pack = $stored['pack']; + $label = 'OpenRegister migration pack ' . $stored['slug'] . ' (id ' . $stored['id'] . ')'; + $details = ['pack' => $stored['slug'], 'packId' => $stored['id']]; + $this->assertValidPack(validator: $validator, pack: $pack, label: $label, details: $details); + $source = array_merge( + $source, + [ + 'source' => 'openregister', + 'packId' => $stored['id'], + 'storedVersion' => $stored['version'], + 'edited' => $stored['edited'], + 'newerShipped' => $stored['newerShipped'], + ] ); } + $this->assertImportRules(target: $target, pack: $pack, profile: $profile, label: $label, details: $details); $packs[$target] = $pack; - } + $sources[$target] = $source; + }//end foreach $this->profile = $profile; $this->packs = $packs; + $this->sources = $sources; + $this->storeAvailable = $storeAvailable; }//end load() + /** + * The shipped packs an administrator edits in OpenRegister, validated, per target. + * + * The seed step stores these; it reads them without touching the store. + * + * @return array> + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the validator is missing or a file is unreadable or invalid. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + */ + public function shippedEditablePacks(): array { + $validator = $this->resolveValidator(); + $profile = $this->decodeFile(fileName: $this->profileFile); + + $packs = []; + foreach (self::EDITABLE_TARGETS as $target) { + $packs[$target] = $this->readShippedPack(profile: $profile, target: $target, validator: $validator); + } + + return $packs; + }//end shippedEditablePacks() + + /** + * Whether the import uses the shipped files because OpenRegister's pack store is not available. + * + * Only true while the packs were never stored there; afterwards a missing + * store fails the load instead. + * + * @return bool + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when loading fails. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function usesShippedFallback(): bool { + $this->profile(); + return $this->storeAvailable === false; + }//end usesShippedFallback() + /** * The upload limit in bytes, readable without validating the packs. * @@ -469,14 +594,40 @@ public function requiredColumns(): array { }//end requiredColumns() /** - * Columns that hold Excel serial dates. + * Columns that hold Excel serial dates: the profile's, plus every column a pack maps with a date transform. + * + * A pack edited in OpenRegister can map a new column to a date; that + * column is converted too, so the date transform gets `Y-m-d` and not an + * Excel serial number. * * @return array * * @spec openspec/changes/cmdb-export-import/tasks.md#task-4 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ public function dateColumns(): array { - return $this->stringList(key: 'dateColumns'); + $columns = $this->stringList(key: 'dateColumns'); + foreach (self::TARGETS as $target) { + if ($target === 'municipality') { + continue; + } + + foreach (($this->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { + if (is_array($mapping) === true && ($mapping['transform']['type'] ?? null) === 'date') { + $columns[] = (string)($mapping['source'] ?? ''); + } + } + } + + $constants = $this->constantColumns(); + return array_values( + array_unique( + array_filter( + $columns, + fn (string $column): bool => $column !== '' && in_array($column, $constants, true) === false + ) + ) + ); }//end dateColumns() /** @@ -573,6 +724,39 @@ public function neverWrittenOnUpdate(string $target): array { return array_values(array_map('strval', $value)); }//end neverWrittenOnUpdate() + /** + * The separator of the concat mapping that writes a field, or null when no concat mapping writes it. + * + * The import drops the empty parts of a concatenated note; it splits on + * the separator the pack uses, which an administrator can change. + * + * @param string $target The profile target. + * @param string $field The target field. + * + * @return string|null + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function concatSeparator(string $target, string $field): ?string { + $separator = null; + foreach (($this->pack(target: $target)['fieldMappings'] ?? []) as $mapping) { + if (is_array($mapping) === false || ($mapping['target'] ?? null) !== $field) { + continue; + } + + // The last mapping of a field is the one whose value the engine keeps. + $separator = null; + if (($mapping['transform']['type'] ?? null) === 'concat') { + $separator = ($mapping['transform']['separator'] ?? self::DEFAULT_CONCAT_SEPARATOR); + if (is_string($separator) === false || $separator === '') { + $separator = self::DEFAULT_CONCAT_SEPARATOR; + } + } + } + + return $separator; + }//end concatSeparator() + /** * The accepted values of the missingRecords option. * @@ -637,14 +821,18 @@ public function referencedColumns(): array { * (values set on create only, not mapped columns), `createOnlyFields` (mapped * fields written on create or, on update, only when the stored value is * empty; a key of `createOnlyDefaults` only when it is also mapped) and `neverWritten` (fields the - * import never writes on an existing object). + * import never writes on an existing object). It also says where it came + * from (`source`, `packId`, the stored and the shipped version, whether it + * was edited and whether a newer shipped pack was left out because of + * that) and whether it is edited in OpenRegister (`editable`). * - * @return array{profile: array, packs: array>} + * @return array{profile: array, packs: array>, openRegisterPacks: bool} * * @throws CmdbImportException MAPPING_UNAVAILABLE when the validator is missing, * or the profile or a pack is unreadable or invalid. * * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 */ public function mappingOverview(): array { $profile = $this->profile(); @@ -684,19 +872,23 @@ public function mappingOverview(): array { $defaults = []; } - $packs[] = [ - 'target' => $target, - 'file' => (string)($profile['packs'][$target] ?? ''), - 'id' => (string)($pack['id'] ?? ''), - 'name' => (string)($pack['name'] ?? ''), - 'version' => (string)($pack['version'] ?? ''), - 'description' => (string)($pack['description'] ?? ''), - 'fieldMappings' => $mappings, - 'defaults' => $defaults, - 'createOnlyDefaults' => $createOnlyDefaults, - 'createOnlyFields' => $createOnlyFields, - 'neverWritten' => $this->neverWrittenOnUpdate(target: $target), - ]; + $packs[] = array_merge( + [ + 'target' => $target, + 'file' => (string)($profile['packs'][$target] ?? ''), + 'id' => (string)($pack['id'] ?? ''), + 'name' => (string)($pack['name'] ?? ''), + 'version' => (string)($pack['version'] ?? ''), + 'description' => (string)($pack['description'] ?? ''), + 'fieldMappings' => $mappings, + 'defaults' => $defaults, + 'createOnlyDefaults' => $createOnlyDefaults, + 'createOnlyFields' => $createOnlyFields, + 'neverWritten' => $this->neverWrittenOnUpdate(target: $target), + ], + $this->sources[$target], + ['editable' => in_array($target, self::EDITABLE_TARGETS, true)] + ); }//end foreach return [ @@ -717,6 +909,7 @@ public function mappingOverview(): array { 'missingRecords' => $this->missingRecordsModes(), ], 'packs' => $packs, + 'openRegisterPacks' => $this->storeAvailable, ]; }//end mappingOverview() @@ -751,6 +944,105 @@ private function stringList(string $key): array { return array_values(array_map('strval', $value)); }//end stringList() + /** + * OpenRegister's pack store, built on the container when none was injected. + * + * @return CmdbImportPackStore + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function packStore(): CmdbImportPackStore { + if ($this->packStore === null) { + $this->packStore = new CmdbImportPackStore(container: $this->container); + } + + return $this->packStore; + }//end packStore() + + /** + * Read and validate the shipped pack of a target. + * + * @param array $profile The decoded profile. + * @param string $target The target. + * @param object $validator OpenRegister's validator. + * + * @return array + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the profile names no pack or the file is unreadable or invalid. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function readShippedPack(array $profile, string $target, object $validator): array { + $fileName = $profile['packs'][$target] ?? null; + if (is_string($fileName) === false || $fileName === '' || basename($fileName) !== $fileName) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: 'CMDB import profile names no pack for target ' . $target + ); + } + + $pack = $this->decodeFile(fileName: $fileName); + $this->assertValidPack(validator: $validator, pack: $pack, label: 'CMDB mapping pack ' . $fileName); + + return $pack; + }//end readShippedPack() + + /** + * Refuse a pack OpenRegister's validator refuses. + * + * @param object $validator OpenRegister's validator. + * @param array $pack The pack. + * @param string $label The pack as the message names it: its file, or its OpenRegister slug and id. + * @param array $details The exception details: the OpenRegister pack and its id, for a stored pack. + * + * @return void + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the pack is invalid. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function assertValidPack(object $validator, array $pack, string $label, array $details = []): void { + $errors = $validator->validate($pack); + if (is_array($errors) === true && $errors !== []) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: $label . ' is invalid: ' . implode('; ', array_map('strval', $errors)), + details: $details + ); + } + }//end assertValidPack() + + /** + * Refuse a pack that breaks a rule of the import, which the validator does not know. + * + * The import writes with OpenRegister's RBAC and multitenancy off, so a + * pack may only map the fields of its target schema, never the object's + * metadata or the keys and relations the import sets itself, and must keep + * the mappings the import matches records on (REQ-CMDB-021). + * + * @param string $target The target. + * @param array $pack The validated pack. + * @param array $profile The decoded profile. + * @param string $label The pack as the message names it. + * @param array $details The exception details: the OpenRegister pack and its id, for a stored pack. + * + * @return void + * + * @throws CmdbImportException MAPPING_UNAVAILABLE when the pack breaks a rule, or the register cannot be read. + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + private function assertImportRules(string $target, array $pack, array $profile, string $label, array $details): void { + $violations = $this->rules->violations(target: $target, pack: $pack, profile: $profile); + if ($violations !== []) { + throw new CmdbImportException( + errorCode: CmdbImportException::MAPPING_UNAVAILABLE, + message: $label . ' breaks a rule of the CMDB import: ' . implode('; ', $violations), + details: $details + ); + } + }//end assertImportRules() + /** * Resolve OpenRegister's pack validator. * diff --git a/lib/Service/CmdbExportImportService.php b/lib/Service/CmdbExportImportService.php index 1efc049d7..18ef6a09e 100644 --- a/lib/Service/CmdbExportImportService.php +++ b/lib/Service/CmdbExportImportService.php @@ -182,11 +182,6 @@ class CmdbExportImportService { */ private const PAGE_SIZE = 500; - /** - * Separator of the concat mapping for the internal note. - */ - private const NOTE_SEPARATOR = ' / '; - /** * The mapping engine of the current run. * @@ -442,6 +437,7 @@ private function acquireImportLock(int $register): string { * @return array The report. * * @spec openspec/changes/cmdb-export-import/tasks.md#task-7 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ private function runImport(string $path, array $options, string $startedAt): array { $workbook = $this->reader->read(path: $path, profile: $this->profile); @@ -452,6 +448,17 @@ private function runImport(string $path, array $options, string $startedAt): arr $report = new CmdbImportReport(operationId: $operationId, rowsRead: count($rows)); $report->setMunicipality(uuid: $municipality['uuid'], name: $municipality['name'], created: $municipality['created']); $report->addImportWarnings(warnings: $this->translateImportWarnings(warnings: $workbook['importWarnings'])); + if ($this->profile->usesShippedFallback() === true) { + $report->addImportWarnings( + warnings: [ + [ + 'sheet' => '', + 'message' => $this->l10n->t('OpenRegister\'s migration pack store is not available, so this import used the mapping that ships with stackiq.'), + ], + ] + ); + } + if ($municipality['created'] === true) { $report->addImportWarnings( warnings: [ @@ -1176,15 +1183,17 @@ private function upsertModule( * @return array{uuid: string, outcome: string} * * @spec openspec/changes/cmdb-export-import/tasks.md#task-5 + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 */ private function upsertUsage(array $data, string $municipalityUuid, string $moduleUuid, ?string $providerUuid, array $owners): array { - if (isset($data['interneAnnotation']) === true && is_string($data['interneAnnotation']) === true) { + $separator = $this->profile->concatSeparator(target: 'usage', field: 'interneAnnotation'); + if ($separator !== null && isset($data['interneAnnotation']) === true && is_string($data['interneAnnotation']) === true) { // The concat mapping leaves an empty part for every empty column; drop those. $parts = array_filter( - array_map('trim', explode(self::NOTE_SEPARATOR, $data['interneAnnotation'])), + array_map('trim', explode($separator, $data['interneAnnotation'])), fn (string $part): bool => $part !== '' ); - $data['interneAnnotation'] = implode(self::NOTE_SEPARATOR, $parts); + $data['interneAnnotation'] = implode($separator, $parts); } $data['consumer'] = $municipalityUuid; diff --git a/openspec/changes/cmdb-import-mapping-view/design.md b/openspec/changes/cmdb-import-mapping-view/design.md index e0c8f5871..b632419d9 100644 --- a/openspec/changes/cmdb-import-mapping-view/design.md +++ b/openspec/changes/cmdb-import-mapping-view/design.md @@ -2,106 +2,159 @@ ## Context -The CMDB import (archived change `2026-10-05-cmdb-export-import`) maps a TOPdesk export through `lib/Settings/cmdb-import/topdesk-profile.json` and five migration packs, loaded and validated by `CmdbImportProfile` and executed by OpenRegister's `MappingEngine`. The section `CmdbImport.vue` shows the upload form, the progress and the report; its help text names the two sheets from `PROFILE_DEFAULTS` in `src/utils/cmdbImport.js`. Nothing in stackiq or OpenRegister shows the profile or the packs (verified on OpenRegister `beta` ff4dad5c, 2026-10-09: APIs for mappings and migration packs exist, a screen does not). +The CMDB import (archived change `2026-10-05-cmdb-export-import`) maps a TOPdesk export through `lib/Settings/cmdb-import/topdesk-profile.json` and five migration packs, loaded and validated by `CmdbImportProfile` and executed by OpenRegister's `MappingEngine`. The packs are OpenRegister migration-pack documents (`fieldMappings` with `trim`, `date`, `lookup`, `concat` transforms, `defaults`, `idStrategy`). OpenRegister already stores such documents in `openregister_migration_packs` through `MigrationPackService`, with CRUD at `/api/migration-packs` for admins. On 2026-10-12 it was decided that the mapping lives in OpenRegister: the administrator edits the packs there, and the import runs what is stored. ## Goals / Non-Goals **Goals** -- An administrator sees, in the section, exactly the mapping the next import runs. -- The section's sheet names cannot drift from the profile. -- Nothing is written; nothing new is parsed. +- The import runs the pack the administrator saved in OpenRegister, unchanged. +- An edit is never lost: not overwritten by an update, never silently replaced by the shipped file. +- An edit cannot make the import write what it must not write. +- The settings section tells the administrator where each pack comes from and links to its editor. **Non-Goals** -- Editing the mapping in a UI (see proposal, Out of Scope). -- Showing a pack from OpenRegister's migration-pack store; the import does not read that store. +- The editor screens (OpenRegister's change). +- Editing the municipality pack or the import profile. +- A mapping per municipality. ## Architecture Overview ``` -Admin settings, "CMDB import" section (CmdbImport.vue) - └─ CmdbImportMapping.vue collapsible "Mapping (read-only)", one CnDataTable per pack - │ GET /api/settings/cmdb-import/mapping (+ requesttoken, via @nextcloud/axios) - ▼ -SettingsController::getCmdbImportMapping() Nextcloud admins only, CSRF - └─ CmdbImportProfile::mappingOverview() load() → PackDefinitionValidator → accessors - lib/Settings/cmdb-import/topdesk-profile.json + topdesk-{module,manufacturer,municipality,usage,business-owner}.json +Install / upgrade Import or GET /api/settings/cmdb-import/mapping + SeedCmdbImportPacks (repair) CmdbImportProfile::load() + │ │ for module, manufacturer, usage, businessOwner + ▼ ▼ + CmdbImportPackStore::resolve(shipped) ◄────────┘ municipality: shipped file only + │ findByPackSlug → absent: create(builtin, + x-stackiq) + │ unedited + outdated: update(shipped + x-stackiq) + │ edited: keep (newerShipped) + ▼ + OpenRegister MigrationPackService ── openregister_migration_packs + │ + CmdbImportPackRules (required mappings, field allowlist) + PackDefinitionValidator + │ + MappingEngine::mapRow() (import) SettingsController: + editUrl (IURLGenerator) ``` ## Decisions ### D1. The endpoint lives in `SettingsController` and reuses `CmdbImportProfile` -The route sits with the other `/api/settings/*` reads, and `SettingsController` gets `CmdbImportProfile` and `IL10N` as optional, trailing constructor arguments, so the eight existing tests that build the controller positionally keep working and Nextcloud's container injects both. The method loads through `CmdbImportProfile::load()`, the loader the import uses, so the view can never show a profile the import would refuse. - -- **Alternative: `CmdbImportController::mapping()`.** Rejected: the brief and the plan put the read with the settings reads; the profile is injected directly, so the import service is not needed. -- **Alternative: read the JSON files in the controller.** Rejected: that would bypass the validator and could show a pack the import refuses. +The route sits with the other `/api/settings/*` reads, and `SettingsController` gets `CmdbImportProfile`, `IL10N` and `IURLGenerator` as optional, trailing constructor arguments, so the existing tests that build the controller positionally keep working and Nextcloud's container injects them. The method loads through `CmdbImportProfile::load()`, the loader the import uses, so the view can never show a mapping the import would refuse. ### D2. Admin-only, CSRF, the import's error envelope -The method carries no auth attribute (Nextcloud's default: admins with a CSRF token), with `@auth admin-only` and its reason, as `CmdbImportController` does. The settings page is admin-only anyway, so a delegated group never sees the section; the endpoint follows the import's posture so an API caller gets the same answer from both. Success is the flat `{profile, packs}` (ADR-050). Failure is the import's envelope `{success: false, error, message, details}`, so `cmdbImport.js`'s `normaliseError()` and `errorText()` handle it unchanged; `details.reason` carries the loader's message, because an administrator who edited a file needs to know which file and why. The message is cut to its first line and 500 characters; it names the file, the validator's rule and the configuration value it refused, never a workbook cell or a person. - -### D3. The transform is shown as stored +The method carries no auth attribute (Nextcloud's default: admins with a CSRF token), with `@auth admin-only` and its reason, as `CmdbImportController` does. Success is the flat `{profile, packs, openRegisterPacks}` (ADR-050). Failure is the import's envelope `{success: false, error, message, details}`; `details.reason` carries the loader's message cut to its first line and 500 characters, and a pack stored in OpenRegister is also named in `details.pack` and `details.packId`. The import's own 503 names the pack in its `message` from the same details; the full reason goes to the Nextcloud log. The reason names the file or the pack, the broken rule and the configuration value it refused, never a workbook cell or a person. -`fieldMappings[].transform` is passed through as the pack stores it (`type` plus `map`/`default`, `fields`/`separator`, `value`, `sourceFormat`/`targetFormat`). Re-shaping per type would hide a key the engine reads. The page renders the known keys and lists any other scalar key as it is. `required` is normalised to a boolean. +### D3. The transform and the write rules are shown as stored -The write rules the import applies besides the mapping are answered per pack, built from the accessors the import uses (`createOnlyDefaults()`, `createOnlyFields()`, `neverWrittenOnUpdate()`) and the pack's own `defaults`, so the view does not overclaim for a field the import writes only on create or never. A create-only default that is not a mapped field, such as the module `type`, is answered in `createOnlyDefaults` only: `merge()` never reads it on update, so `createOnlyFields` leaves it out. A mapped field with a create default stays in `createOnlyFields`, since `merge()` guards it on update. +`fieldMappings[].transform` is passed through as the pack stores it. The write rules the import applies besides the mapping are answered per pack (`defaults`, `createOnlyDefaults`, `createOnlyFields`, `neverWritten`), built from the accessors the import uses. A create-only default that is not a mapped field, such as the module `type`, is answered in `createOnlyDefaults` only: `merge()` never reads it on update, so `createOnlyFields` leaves it out. A mapped field with a create default stays in `createOnlyFields`. ### D4. The section takes the sheet names from the endpoint, with the constant as fallback -`CmdbImportMapping.vue` fetches once on creation and emits `loaded` with the answer; `CmdbImport.vue` computes `sheetNames` from it, or from `PROFILE_DEFAULTS.sheets` until it arrives or when it fails. The constant stays, as the design of cmdb-export-import intended: a fallback, never the source. +Unchanged from the first version: the help text computes the sheet names from the endpoint's answer, or from `PROFILE_DEFAULTS.sheets` until it arrives or when it fails. + +### D5. The packs live in OpenRegister's migration-pack store, found by slug + +The four sheet packs are stored as OpenRegister migration packs and found by their slug (`stackiq-topdesk-module`, …). That is the format the packs already have and the engine the import already calls, so what the administrator edits is exactly what runs, with OpenRegister's validator on save (422) and on import (503). + +- **Alternative: OpenRegister's Twig mapping entity (`/api/mappings`).** Rejected: lookups, `required` and dates would become Twig templates, validation would no longer be structural, and errors would lose their row and column; the import engine would have to be rewritten. +- **Alternative: packs as register objects, like Integriq's mappings.** Rejected: a third mapping store next to the one OpenRegister already has, with the validator duplicated in a listener. + +The municipality pack maps the import options (`municipalityName`), not a sheet, so it stays a shipped file and is answered with `editable: false`. The import profile stays a shipped file: its limits guard memory and zip bombs, and its sheets, key column and write rules are stackiq semantics. + +OpenRegister's pack store has no organisation column: an instance has one TOPdesk mapping, used for every municipality it imports. + +### D6. Seeding, and following the release only while unedited + +A stored pack carries `x-stackiq: {shippedVersion, shippedHash}`: the version of the shipped pack it was seeded from and the sha256 of that pack's canonical JSON (keys of every object sorted recursively, list order kept, because the order of `fieldMappings` decides which mapping wins a target). OpenRegister's validator ignores keys it does not know, so the marker travels with the definition. A stored definition that, without the marker, still hashes to `shippedHash` is unedited. + +`CmdbImportPackStore::resolve()` is the one place that writes: it creates a missing pack as built-in with the marker (never through OpenRegister's import, which upserts and would overwrite an edit), replaces an unedited pack whose marker hash differs from the current shipped pack, and leaves an edited pack alone, reporting `newerShipped`. A pack without a marker (created by hand under the same slug) counts as edited with `edited: null`, and is never replaced. The repair step `SeedCmdbImportPacks` runs it on install and upgrade, after `InitializeSettings`; the loader runs it too, so a pack deleted in OpenRegister is seeded again on the next import, and an instance whose OpenRegister gained the store after stackiq's upgrade is seeded lazily. + +Version numbers alone cannot tell an edit from a release: OpenRegister's editor bumps the version on save, independently of stackiq's releases. The hash can. + +- **Alternative: seed only when missing, never update.** Rejected: an unedited pack would never get a fix that a release ships. +- **Alternative: always overwrite on upgrade.** Rejected: it destroys the administrator's edits. + +### D7. Fail closed once the packs were stored + +The store keeps the row id it resolved per slug in stackiq's app config (`cmdb_import_pack_ids`). Without OpenRegister's store and with no recorded ids, the import uses the shipped files and adds one import warning saying so. With recorded ids, a missing store is 503 `MAPPING_UNAVAILABLE` naming the packs: falling back would silently drop an edit. A slug lookup that misses while the recorded row still exists under another slug is a rename, and is 503 naming the row and both slugs, not a second seeded pack the import would read while the administrator edits the renamed one. A recorded row that was deleted is seeded again. Any other failure of the store is 503 naming the pack and the exception class, never its message. + +### D8. The import's rules on every pack + +OpenRegister's validator checks structure only, and the import writes with OpenRegister's RBAC and multitenancy off. `CmdbImportPackRules` therefore checks every pack, stored or shipped: + +- required mappings with `required: true`: module `name` and `externalNumber` (from the profile's key column), manufacturer, municipality and business-owner `name`; +- an allowlist per target: the properties of the target schema in the register stackiq ships (fragments included, ADR-037), minus `@self`, `id`, the match keys and the relations the import sets (`externalKey`, `provider`, `consumer`, `module`, `businessOwner`, the organisation `type`) and the profile's `neverWritten`; the business owner may map `name` and `role` only, because it is resolved in Nextcloud Contacts; +- the same allowlist for `defaults`, and the organisation `type` default pinned (`Supplier`, `Municipality`), because the import finds organisations by name and type. + +A broken rule is 503 `MAPPING_UNAVAILABLE` naming the pack, its id and every rule it breaks. + +### D9. What a pack controls besides its mappings + +Only the profile's date columns used to be converted from Excel serial numbers, so a date transform on a new column would have received `45111`. The date columns are now the profile's plus every column a pack maps with a `date` transform. The cleanup of empty parts in the usage note splits on the separator of the concat mapping that writes `interneAnnotation`, not on a constant. `idColumns`, `emptyValues` and the sheet constants (`Beheer`) stay profile semantics. -### D5. One component, fetched once, collapsed by default +### D10. The editor link is built by Nextcloud -The block is collapsed by default so the import form stays the section's first thing; the fetch happens on creation anyway, because the help text needs the sheet names. The toggle is an `NcButton` with a visible label and `aria-expanded`/`aria-controls`, the pattern of `CollapsibleSection.vue`. +`editUrl` is `IURLGenerator::linkToRouteAbsolute('openregister.dashboard.catchAll', ['path' => 'migration-packs/'])`, so an instance that needs `/index.php` gets it. Nextcloud's router answers an unknown route with an empty path; a link that does not end in the pack path is answered as null rather than as the instance's front page. ## Declarative-vs-imperative decision (ADR-031) -- **No new rule.** The change reads the declarative mapping (JSON packs in OpenRegister's format, executed by `MappingEngine`) and shows it; it adds no `x-openregister-*` block and no imperative rule. -- **Imperative, because it is an endpoint:** one controller method and one accessor that shape the loaded JSON into the answer. That is presentation of configuration, not object lifecycle, aggregation, notification or relation logic. -- **The mapping itself stays declarative:** a change to a pack changes the view and the import together, without PHP. +- **The mapping stays declarative:** OpenRegister migration packs, executed by `MappingEngine`; an edit changes the import without PHP. +- **Imperative, because they guard the import:** the pack resolution, the seed and update policy and the import's rules. They are configuration management and security checks on writes with RBAC off, not object lifecycle, aggregation, notification or relation logic, and no `x-openregister-*` vocabulary covers them. ## Seed Data -No schema changes. No register fragment, seed object or migration is added or changed; the endpoint reads the six files that ship under `lib/Settings/cmdb-import/`. +No schema changes and no register fragment. The repair step stores four migration packs in OpenRegister (`stackiq-topdesk-module`, `stackiq-topdesk-manufacturer`, `stackiq-topdesk-usage`, `stackiq-topdesk-business-owner`) from the shipped files, each with the `x-stackiq` marker. ## Nextcloud Integration - Controller: `SettingsController::getCmdbImportMapping()`; route `settings#getCmdbImportMapping`, `GET /api/settings/cmdb-import/mapping`. -- Service: `CmdbImportProfile::mappingOverview()`. -- OCP: `OCP\IL10N`. -- OpenRegister: `Service\MigrationPack\PackDefinitionValidator`, through the existing guard; absent, the endpoint answers 503 `MAPPING_UNAVAILABLE`. +- Services: `CmdbImportProfile`, `CmdbImportPackStore`, `CmdbImportPackMarker`, `CmdbImportPackRules`. +- Repair step: `SeedCmdbImportPacks`, in `` and ``. +- OCP: `OCP\IL10N`, `OCP\IURLGenerator`, `OCP\IAppConfig`, `OCP\Migration\IRepairStep`, `OCP\AppFramework\Db\DoesNotExistException`. +- OpenRegister: `Service\MigrationPackService`, `Service\MigrationPack\PackDefinitionValidator`, `Service\MigrationPack\MappingEngine`, resolved defensively; the page route `dashboard#catchAll`. ## Security Considerations -- Admin-only with CSRF (D2). The answer is configuration that ships with the app: column names, field names, lookup tables, file names. No object, no person. -- `details.reason` is stackiq's own composed message or the validator's structural messages; `message` is static and translated. An unexpected exception is logged and answered as 500 `IMPORT_FAILED` with a static message, never with `getMessage()`. -- Every value is rendered as text; nothing goes into `v-html`. +- Admin-only with CSRF (D2). OpenRegister's pack writes are admin-only too. +- The import's rules (D8) keep an edited pack from writing object metadata, match keys or relations, or fields outside its schema. +- An edit can add a column to what the import reads: the read allowlist is every column the profile or a pack names. Only Nextcloud admins can edit packs and run the import. +- Fail closed (D7): no silent fallback that would drop an administrator's edit. +- Messages name packs, rules and configuration values, never a workbook cell or a person; a store failure is named by its exception class. ## NL Design System -Nextcloud and `@conduction/nextcloud-vue` components only (ADR-012): `NcButton` for the toggle, `NcLoadingIcon`, `NcNoteCard` for the error and `CnDataTable` for the tables, the same table the report uses. +Nextcloud and `@conduction/nextcloud-vue` components only (ADR-012), as in the first version of the block. ## File Structure ``` -appinfo/routes.php (+ settings#getCmdbImportMapping) -lib/Controller/SettingsController.php (+ getCmdbImportMapping) -lib/Service/Cmdb/CmdbImportProfile.php (+ mappingOverview) -src/utils/cmdbImport.js (+ mappingUrl, loadCmdbMapping, mappingSheetNames, packTargetLabel, transformLabel, mappingRows) -src/views/settings/sections/CmdbImportMapping.vue (new) -src/views/settings/sections/CmdbImport.vue (mounts the block; sheetNames) -tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php -tests/Unit/Service/Cmdb/CmdbImportProfileTest.php (+ overview) -tests/vitest/cmdbImportMapping.spec.js -tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts +appinfo/info.xml (+ SeedCmdbImportPacks in install and post-migration) +lib/Repair/SeedCmdbImportPacks.php (new) +lib/Service/Cmdb/CmdbImportPackStore.php (new) +lib/Service/Cmdb/CmdbImportPackMarker.php (new) +lib/Service/Cmdb/CmdbImportPackRules.php (new) +lib/Service/Cmdb/CmdbImportProfile.php (resolution, sources, dateColumns, concatSeparator, overview) +lib/Service/CmdbExportImportService.php (fallback warning, separator) +lib/Controller/SettingsController.php (editUrl, details) +lib/Controller/CmdbImportController.php (MAPPING_UNAVAILABLE names the pack) +src/… (the section's mapping block) +tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php, CmdbImportPackRulesTest.php, CmdbImportProfileTest.php +tests/Unit/Repair/SeedCmdbImportPacksTest.php +tests/Unit/Support/FakeMigrationPackService.php +tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php, CmdbImportControllerTest.php +tests/Unit/Service/CmdbExportImportServiceTest.php docs/features/cmdb-import.md l10n/en.json, en.js, nl.json, nl.js ``` ## Testing -- PHPUnit: the route and its posture (no attribute, `@auth admin-only`, no exemption annotation); the shipped directory answers five packs that match the profile's accessors; a broken pack answers 503 with the reason; a missing validator answers 503; an unexpected error answers 500 and is logged. -- vitest: the component renders one table per pack from a mocked answer, emits `loaded`, and shows the error and code on a 503. -- Playwright (written, not run here: no Nextcloud reachable): an admin sees the five tables and the usage row; a non-admin gets 403 and no settings page. +- PHPUnit against an in-memory `MigrationPackService`: seeding with the marker, an unedited pack following a newer release, an edited pack kept with `newerShipped`, a pack without marker never replaced, a renamed pack and a failing store failing closed, a deleted pack seeded again, the canonical hash. +- PHPUnit on the loader: a stored pack is what runs, an invalid stored pack and thirteen broken rules are 503 naming the pack and id, no store before a seed uses the files, no store after a seed fails closed, a stored date column is converted, the concat separator is the pack's. +- PHPUnit on the import: an edit in OpenRegister changes the mapped usage, the separator and a new date column; without a store the report carries the fallback warning. +- PHPUnit on the endpoint and the repair step: the extended answer, `editUrl` from the page route and null when it cannot be built, 503 naming the pack; seeding four packs, updating and keeping, no store, a failing store, the registration order. +- Playwright: the section shows the source and the editor link per pack (UI). diff --git a/openspec/changes/cmdb-import-mapping-view/proposal.md b/openspec/changes/cmdb-import-mapping-view/proposal.md index e651febe6..7c7925b01 100644 --- a/openspec/changes/cmdb-import-mapping-view/proposal.md +++ b/openspec/changes/cmdb-import-mapping-view/proposal.md @@ -7,11 +7,11 @@ depends_on: [] ## Summary -The "CMDB import" section of stackiq's admin settings gets a collapsible, read-only block "Mapping (read-only)" that shows the column mapping the import really uses: the sheets, the key and required columns, and one table per migration pack with the source column, the target field, whether it is required, and the transformation with its lookup values. A new admin endpoint `GET /api/settings/cmdb-import/mapping` loads the import profile and the five packs through the same loader and validator the import uses, so what the administrator sees is what the next import runs. Editing stays in the JSON files on the server; the documentation says where they are and what an administrator can change. +A Nextcloud administrator can view and edit the column mapping of the CMDB import in OpenRegister. The four sheet packs of the import (module, manufacturer, usage and business owner) move into OpenRegister's existing migration-pack store, where OpenRegister's pack editor shows and changes them. The import reads each pack from that store first and runs exactly what the administrator saved. The packs that ship under `lib/Settings/cmdb-import/` become the seed and the fallback: a repair step stores them in OpenRegister as built-in packs, and replaces a stored pack nobody edited when a later release ships a newer one. An edited pack is never overwritten. A stored pack that is invalid, or that breaks a rule of the import, stops the import with 503 `MAPPING_UNAVAILABLE` naming the pack; it never falls back to the files silently. `GET /api/settings/cmdb-import/mapping` says per pack where it comes from, which versions are stored and shipped, whether it was edited, and links to its editor in OpenRegister. The "CMDB import" section of stackiq's admin settings shows that per pack, with the sheet names the import reads. ## Motivation -The municipality's application manager (Jira WOO-588, sub-task of WOO-586) wants to see which TOPdesk column lands in which stackiq field, and which export values the import recognises, without reading JSON on the server. The mapping is declarative (REQ-CMDB-005) and OpenRegister has APIs for migration packs, but no screen for them, and stackiq offered no view of its profile or packs either: the section's help text even carried the sheet names as a constant in the page, which could drift from the profile. Options weighed on 2026-10-09: (A) an existing OpenRegister screen, which does not exist; (B) a read-only view in the section; (C) documentation only; (B+) editing through OpenRegister's packs. B was chosen: the question was "make it visible", the change stays in stackiq, and editing a mapping stays a rare, file-based act. +The municipality's application manager (Jira WOO-588, sub-task of WOO-586) must be able to see and change which TOPdesk column lands in which stackiq field, and which export values the import recognises, without editing JSON on the server. The first version of this change (2026-10-09) only showed the mapping read-only and left editing in the files, which an app update overwrites. On 2026-10-12 it was decided that mapping lives in OpenRegister: the packs are already OpenRegister migration packs, executed by OpenRegister's `MappingEngine` and validated by its `PackDefinitionValidator`, and OpenRegister already stores and serves such packs (`/api/migration-packs`). Storing the import's packs there makes the edited object exactly the object the import runs, and gives the administrator one place for mappings. Integriq is not changed; its mapping screens are the pattern for OpenRegister's editor. ## Capabilities @@ -21,31 +21,38 @@ None. ### Modified Capabilities -- `cmdb-export-import`: ADDED REQ-CMDB-020 (the mapping overview endpoint and the read-only block); MODIFIED REQ-CMDB-014 (the section shows the mapping, and its help text takes the sheet names from the endpoint). +- `cmdb-export-import`: ADDED REQ-CMDB-020 (the mapping endpoint and section, now with source, versions and the editor link), REQ-CMDB-021 (the import takes the sheet packs from OpenRegister's store, with the import's rules and fail-closed answers) and REQ-CMDB-022 (seeding and updating the packs in OpenRegister); MODIFIED REQ-CMDB-005 (where the packs come from, the date columns a pack adds, the pack's own concat separator) and REQ-CMDB-014 (the section shows the mapping summary). ## Affected Projects -- [ ] Project: `stackiq`: one endpoint in `SettingsController`, one accessor on `CmdbImportProfile`, a `CmdbImportMapping.vue` block in the "CMDB import" section, helpers in `src/utils/cmdbImport.js`, documentation, translations, tests. +- [ ] Project: `stackiq`: the pack resolution and rules in `lib/Service/Cmdb/`, a repair step, the extended endpoint, the slimmed mapping block in the "CMDB import" section, documentation, translations, tests. +- [ ] Project: `openregister` (separate change and PR): the migration-pack list and editor pages that the `editUrl` opens, the envelope body on `PUT /api/migration-packs/{id}`, and a builtin pack keeping its slug. ## Scope ### In Scope -- `GET /api/settings/cmdb-import/mapping`, for Nextcloud admins only and CSRF-protected, answering the profile and the packs as loaded by `CmdbImportProfile`, or 503 `MAPPING_UNAVAILABLE` with the loader's reason when a pack is broken. -- The collapsible block in `CmdbImport.vue`, with loading and error states, one table per pack and a `data-testid` per table. -- The sheet names in the section's help text come from the endpoint; the shipped names stay as the fallback. -- `docs/features/cmdb-import.md`: "Viewing the mapping", and "Adjusting the mapping" updated. -- PHPUnit, vitest and a Playwright spec. +- Reading the module, manufacturer, usage and business-owner packs from OpenRegister's migration-pack store by slug, with the shipped file as the seed and, while nothing was ever stored, as the fallback with one import warning. +- The import's own rules on every pack, stored or shipped: the required mappings, and an allowlist of the fields a pack may write. +- Failing closed with 503 `MAPPING_UNAVAILABLE` when a stored pack is invalid or breaks a rule, when it was renamed, or when the store is gone after the packs were stored there. +- The repair step `SeedCmdbImportPacks` on install and upgrade: seeds missing packs as built-in, replaces unedited packs with the shipped pack of the release, keeps edited ones and reports them; the row ids are kept in stackiq's app config. +- The `x-stackiq` marker in a stored definition: the shipped version and the hash of the shipped pack it was seeded from. +- Excel date conversion for every column a pack maps with a date transform; the note cleanup on the pack's own concat separator. +- `GET /api/settings/cmdb-import/mapping` extended per pack with `source`, `packId`, `storedVersion`, `shippedVersion`, `edited`, `newerShipped`, `editable` and `editUrl`, and `openRegisterPacks` at the top level. +- The section's mapping block: per pack its source, version, a hint when a newer shipped pack was not applied, and an "Edit in OpenRegister" link; the sheet names in the help text come from the endpoint. +- `docs/features/cmdb-import.md`: viewing and editing the mapping in OpenRegister. ### Out of Scope -- Editing the mapping in the UI, or through OpenRegister's migration-pack store (B+, rejected for now). -- New transformations in OpenRegister's mapping engine, or promoting `MappingEngine` to an OpenRegister contract. -- A sixth pack (technical owner): the functional administrator is not imported (cmdb-export-import, design D2). +- The editor itself: OpenRegister's change for its migration-pack pages. +- The municipality pack: it maps the import options, not a sheet, and stays a shipped file that is not editable. +- The import profile (`topdesk-profile.json`): sheets, key and required columns, limits and write rules stay a shipped file, shown read-only. +- More than one mapping per instance: OpenRegister's pack store has no organisation scope, so an instance has one TOPdesk mapping for every municipality it imports. +- Promoting `MappingEngine`, `PackDefinitionValidator` or `MigrationPackService` to an OpenRegister contract. ## Approach -`CmdbImportProfile::mappingOverview()` builds the answer from the loaded profile and packs through the existing accessors, after `load()` validated every pack. `SettingsController::getCmdbImportMapping()` returns it as a flat 200, or translates `CmdbImportException` into the import's error envelope with the reason in `details.reason`. The section mounts a `CmdbImportMapping` component that fetches the endpoint once, renders the tables with `CnDataTable`, and emits the loaded mapping so the parent can take the sheet names from it. The transform is shown as the pack stores it: type, map or fields or value, and the formats of a date. +`CmdbImportProfile::load()` reads each shipped pack, validates it, and for the four sheet targets asks `CmdbImportPackStore` for the stored pack. The store resolves OpenRegister's `MigrationPackService` defensively from the container, finds the pack by slug, seeds it when it is missing, and replaces it when its `x-stackiq` marker shows it was not edited and the shipped pack has changed. `CmdbImportPackRules` checks every pack against the import's rules; `CmdbImportPackMarker` computes the canonical hash. The repair step calls the same store. `SettingsController` adds `editUrl` from OpenRegister's page route through `IURLGenerator`. ## New Dependencies @@ -53,26 +60,33 @@ None. ## Impact -- **Backend**: `lib/Controller/SettingsController.php` (one method, two optional constructor arguments), `lib/Service/Cmdb/CmdbImportProfile.php` (one method), one route in `appinfo/routes.php`. -- **Frontend**: `src/views/settings/sections/CmdbImportMapping.vue` (new), `src/views/settings/sections/CmdbImport.vue` (mounts it, takes the sheet names from it), `src/utils/cmdbImport.js` (URL, request, row and label helpers). -- **Data**: none. The endpoint reads six files that ship with the app and writes nothing. +- **Backend**: `lib/Service/Cmdb/CmdbImportProfile.php` (resolution, sources, date columns, concat separator), new `CmdbImportPackStore.php`, `CmdbImportPackMarker.php`, `CmdbImportPackRules.php`, new `lib/Repair/SeedCmdbImportPacks.php` and its registration in `appinfo/info.xml`, `lib/Controller/SettingsController.php` (`editUrl`, details), `lib/Controller/CmdbImportController.php` (the message names the pack), `lib/Service/CmdbExportImportService.php` (fallback warning, separator). +- **Frontend**: the mapping block in the "CMDB import" section. +- **Data**: four rows in OpenRegister's `openregister_migration_packs`, created by the repair step or the first import; one app config key, `cmdb_import_pack_ids`. ## Cross-Project Dependencies -- **openregister** (consumed, not changed): `MigrationPack\PackDefinitionValidator`, through the existing guard in `CmdbImportProfile`. +- **openregister** (consumed): `Service\MigrationPackService` (`findByPackSlug`, `find`, `create`, `update`), `Db\MigrationPack::getDefinitionArray()`, `Service\MigrationPack\PackDefinitionValidator`, `Service\MigrationPack\MappingEngine`, and the page route `dashboard#catchAll` for `/migration-packs/{id}`. +- **openregister** (needed for editing): the migration-pack pages and the envelope body of OpenRegister's change for WOO-588. Without them stackiq still seeds and reads the packs, and the link opens a page OpenRegister does not have yet. ## Risks -### Risk 1: The view shows a mapping the import does not use -**Severity:** Low — **Mitigation:** the endpoint and the import share one loader (`CmdbImportProfile`), one directory and one validator; the unit test reads the shipped directory through the controller and compares it with the profile's own accessors. +### Risk 1: An edit in OpenRegister writes fields the import must not write +**Severity:** High — **Mitigation:** the import writes with OpenRegister's RBAC and multitenancy off, so every pack must follow the import's rules (REQ-CMDB-021): only the fields of its target schema, never object metadata, match keys or the relations the import sets, and the required mappings kept. A pack that breaks a rule stops the import with 503 naming the pack. -### Risk 2: The loader's reason leaks something it should not -**Severity:** Low — **Mitigation:** the reason is composed by stackiq (file name plus the validator's messages about the pack's structure) or is the fixed text that the validator is missing; it never holds a cell value or a person. The `message` stays static and translated (ADR-050). +### Risk 2: An administrator's edit is lost or silently ignored +**Severity:** High — **Mitigation:** the repair step never overwrites an edited pack; a renamed pack, or a store that is gone after the packs were stored, fails closed instead of falling back to the files. + +### Risk 3: An edit widens what the import reads +**Severity:** Medium — **Mitigation:** the read allowlist is every column the profile or a pack names, so an administrator who maps a new column makes the import read it. Only Nextcloud admins can edit packs and run the import; the documentation says that a mapped column is read. + +### Risk 4: OpenRegister's internal classes change +**Severity:** Medium — **Mitigation:** resolved defensively and checked for the methods called; a missing service before any seed falls back to the files with a warning, afterwards fails closed. ## Rollback Strategy -Additive. Revert the PR to remove the route, the method, the component and the strings; the section then shows its constant sheet names again, as before. +Revert the PR. The import then reads the shipped files again; the four rows in OpenRegister's pack store stay and are ignored, and can be deleted there. The app config key `cmdb_import_pack_ids` can be deleted with `occ config:app:delete stackiq cmdb_import_pack_ids`. ## Open Questions -- Should a later change let an administrator override a pack through OpenRegister's migration-pack store (`POST /api/migration-packs/import`), as cmdb-export-import design D2 noted? Not for this change. +- Does any instance import for more than one municipality? The store holds one mapping per instance; per-municipality packs would need a slug per municipality or a pack chosen in the import options. diff --git a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md index e1e7f2430..51dcaa9f4 100644 --- a/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md +++ b/openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md @@ -8,43 +8,56 @@ ## Purpose -The CMDB import maps the columns of a TOPdesk export to stackiq fields through an import profile and five migration packs, JSON files under `lib/Settings/cmdb-import/` that OpenRegister's mapping engine executes (REQ-CMDB-005). Until now an administrator could only learn that mapping by reading the JSON on the server. This delta adds a read-only view of it to the "CMDB import" section of the admin settings, served by an endpoint that loads the files through the same loader the import uses, so what the administrator sees is what the next import runs (Jira WOO-588, sub-task of WOO-586). +The CMDB import maps the columns of a TOPdesk export to stackiq fields through an import profile and five migration packs that OpenRegister's mapping engine executes (REQ-CMDB-005). Until now the packs were JSON files under `lib/Settings/cmdb-import/`, which an administrator could only read on the server and an app update overwrote. This delta moves the four sheet packs into OpenRegister's migration-pack store, where an administrator views and edits them, makes the import run what is stored there, and shows in the "CMDB import" section where each pack comes from with a link to its editor (Jira WOO-588, sub-task of WOO-586; decision 2026-10-12: mapping lives in OpenRegister). -Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\MigrationPack\PackDefinitionValidator` (through the existing `CmdbImportProfile`, guarded). +Nextcloud OCP interfaces used: `OCP\IL10N` (messages), `OCP\IURLGenerator` (the editor link), `OCP\IAppConfig` (the stored pack ids), `OCP\Migration\IRepairStep` (seeding). OpenRegister: `Service\MigrationPackService`, `Service\MigrationPack\PackDefinitionValidator` and `Service\MigrationPack\MappingEngine`, resolved defensively. ## ADDED Requirements ### Requirement: The admin settings SHALL show the mapping the import uses (REQ-CMDB-020) -`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack it names through the same loader and validator the import uses, and SHALL answer 200 with a flat envelope of two keys: `profile` (the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns`, `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`) and `packs`, one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings` and the write rules the import applies besides the mapping: `defaults` (the pack's values, field => value), `createOnlyDefaults` (values the import sets on create only, field => value; they are not mapped columns), `createOnlyFields` (mapped fields written on create or, on an existing object, only when the stored value is empty; a key of `createOnlyDefaults` only when it is also a mapped field) and `neverWritten` (fields the import never writes on an existing object); the last three come from the profile's `createOnly` and `neverWritten` through the accessors the import itself uses. Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it: the `type`, and for `lookup` and `bool-map` the `map` and `default`, for `concat` the `fields` and `separator`, for `const` the `value`, for `date` the `sourceFormat` and `targetFormat`. The endpoint SHALL read nothing but those files and SHALL write nothing. When the validator is missing, or the profile or a pack is unreadable or invalid, it SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason`, so the administrator who edited a file learns what is wrong without the Nextcloud log. The "CMDB import" section SHALL show the answer in a collapsible block "Mapping (read-only)": the sheets, the key, name, required and date columns, and one table per pack with the source column, the target field, whether it is required, the transformation and its details. The block SHALL show a loading state until the endpoint answers and the error with its code when it fails, and SHALL offer no editing: the mapping is changed in the files on the server, as the documentation describes. +`GET /api/settings/cmdb-import/mapping` SHALL be reachable only by Nextcloud admins and SHALL require Nextcloud's CSRF token; like the import routes (REQ-CMDB-001) it SHALL NOT carry `#[AuthorizedAdminSetting]`, `#[NoAdminRequired]` or `#[NoCSRFRequired]`. It SHALL load the import profile and every pack through the same loader, store and validator the import uses (REQ-CMDB-021), and SHALL answer 200 with a flat envelope of three keys: + +- `profile`: the profile's `id`, `name`, `version`, `sheets` with each sheet's `name`, `constants` and `absentColumns`, `sheetPrecedence`, `keyColumn`, `nameColumn`, `requiredColumns`, `dateColumns` (the columns the import converts from Excel dates, REQ-CMDB-005), `idColumns`, `emptyValues`, `externalKeyPrefix`, `missingRecords` and `profileFile`; +- `packs`: one entry per target in the order module, manufacturer, municipality, usage, businessOwner, each with `target`, `file`, `id`, `name`, `version`, `description`, `fieldMappings`, the write rules `defaults`, `createOnlyDefaults`, `createOnlyFields` and `neverWritten`, and where the pack comes from: `source` (`openregister` or `file`), `packId` (the OpenRegister row id, or null), `storedVersion` (or null), `shippedVersion`, `edited` (whether the stored pack differs from what stackiq seeded; null when unknown or for a file), `newerShipped` (an edited pack kept a newer shipped pack out), `editable` (false for the municipality pack) and `editUrl` (the absolute link to the pack's editor in OpenRegister, built by Nextcloud's URL generator so an instance that needs `/index.php` gets it, or null when the pack is not stored there or the link cannot be built); +- `openRegisterPacks`: whether OpenRegister's migration-pack store is available. + +Each field mapping SHALL carry `source`, `target`, `required` (a boolean) and `transform` as the pack stores it. `createOnlyDefaults` holds values the import sets on create only, `createOnlyFields` the mapped fields written on create or, on an existing object, only when the stored value is empty (a key of `createOnlyDefaults` only when it is also a mapped field), and `neverWritten` the fields the import never writes on an existing object. When the validator is missing, a pack is unreadable, invalid or breaks a rule of the import, or the store that holds the packs is gone, the endpoint SHALL answer 503 `MAPPING_UNAVAILABLE` in the import's error envelope (`success`, `error`, `message`, `details`) with the loader's reason in `details.reason` (its first line, at most 500 characters), and for a pack stored in OpenRegister its slug in `details.pack` and its row id in `details.packId`. The "CMDB import" section SHALL show, per pack, its source, its version, a hint when a newer shipped pack was not applied, and a link "Edit in OpenRegister" when the pack has an `editUrl`; it SHALL offer no editing itself. #### Scenario: An admin reads the mapping the import uses @e2e tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts -- **GIVEN** a Nextcloud admin with the shipped profile and packs +- **GIVEN** a Nextcloud admin, with OpenRegister's migration-pack store holding the seeded packs - **WHEN** they request `GET /api/settings/cmdb-import/mapping` -- **THEN** the endpoint SHALL answer 200 with `profile.sheets` naming "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB", `profile.keyColumn` `APPID` and `profile.requiredColumns` `APPID` and `Applicatie Naam` -- **AND** `packs` SHALL hold five entries with targets module, manufacturer, municipality, usage and businessOwner, each with its pack id, name and version +- **THEN** the endpoint SHALL answer 200 with `profile.sheets` naming "Onbeh Applicaties CMDB" and "Beheerde Applicaties CMDB", `profile.keyColumn` `APPID` and `openRegisterPacks` `true` +- **AND** `packs` SHALL hold five entries with targets module, manufacturer, municipality, usage and businessOwner +- **AND** the module, manufacturer, usage and businessOwner packs SHALL have `source` `openregister`, a `packId` and an `editUrl` ending in `/apps/openregister/migration-packs/` +- **AND** the municipality pack SHALL have `source` `file`, `editable` `false` and `editUrl` `null` - **AND** the usage pack SHALL list a mapping from "Applicatie Status" to `status` with a `lookup` transform whose `map` sends "In productie" to "In production" -- **AND** the usage pack's `createOnlyFields` SHALL be `interneAnnotation` and `timeClassification`, the module pack's `createOnlyDefaults` SHALL be `type` = `Application` and its `neverWritten` SHALL be `publicationDate` and `depublicationDate` -#### Scenario: The section shows one table per pack +#### Scenario: Without OpenRegister's store the shipped packs are shown +@e2e exclude Removing OpenRegister's store on a shared instance is not done in e2e; tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php asserts the answer. + +- **GIVEN** OpenRegister has no migration-pack store and the packs were never stored there +- **WHEN** a Nextcloud admin requests the mapping +- **THEN** every pack SHALL have `source` `file` and `editUrl` `null` +- **AND** `openRegisterPacks` SHALL be `false` + +#### Scenario: The section links each pack to its editor @e2e tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts - **GIVEN** a Nextcloud admin on stackiq's admin settings page -- **WHEN** they open "Mapping (read-only)" in the "CMDB import" section -- **THEN** the section SHALL show five tables, one per pack, each named after its target and its file -- **AND** the usage table SHALL have a row for the column "Applicatie Status" with the field `status`, the transformation Lookup and the pair "In productie" to "In production" -- **AND** the block SHALL have no control that edits a mapping +- **WHEN** the mapping block of the "CMDB import" section has loaded +- **THEN** each of the four sheet packs SHALL show its source and version and a link "Edit in OpenRegister" to its `editUrl` +- **AND** the municipality pack SHALL show no such link -#### Scenario: A broken pack is reported with its reason -@e2e exclude Breaking a shipped file on a shared instance is not done in e2e; tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php asserts the 503 and the reason, and tests/vitest/cmdbImportMapping.spec.js asserts the section's error state. +#### Scenario: A broken stored pack is reported with its name +@e2e exclude Breaking a stored pack on a shared instance is not done in e2e; tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php asserts the 503, the reason and the pack. -- **GIVEN** a pack file on the server that OpenRegister's validator refuses +- **GIVEN** the usage pack in OpenRegister maps a column to `provider` - **WHEN** a Nextcloud admin requests the mapping - **THEN** the endpoint SHALL answer 503 with error `MAPPING_UNAVAILABLE` -- **AND** `details.reason` SHALL name the pack file and the validator's reason, on one line of at most 500 characters -- **AND** the section SHALL show the error and the code `MAPPING_UNAVAILABLE` instead of the tables +- **AND** `details.pack` SHALL be `stackiq-topdesk-usage`, `details.packId` its row id, and `details.reason` SHALL name the pack, its id and the broken rule on one line of at most 500 characters #### Scenario: A user who is not a Nextcloud admin cannot read the mapping @e2e tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts @@ -52,13 +65,135 @@ Nextcloud OCP interfaces used: `OCP\IL10N` (messages). OpenRegister: `Service\Mi - **GIVEN** a signed-in user who is not a Nextcloud admin - **WHEN** they request `GET /api/settings/cmdb-import/mapping` - **THEN** Nextcloud SHALL answer 403 -- **AND** the admin settings page that holds the section SHALL NOT be shown to them + +### Requirement: The import SHALL run the mapping packs stored in OpenRegister (REQ-CMDB-021) + +For the module, manufacturer, usage and business-owner targets the import SHALL use the pack stored in OpenRegister's migration-pack store under the shipped pack's slug (`stackiq-topdesk-module`, `stackiq-topdesk-manufacturer`, `stackiq-topdesk-usage`, `stackiq-topdesk-business-owner`), without the `x-stackiq` marker (REQ-CMDB-022). When the store is available and such a pack is missing, the import SHALL first seed it from the shipped file. The municipality pack maps the import options, not a sheet; it SHALL always come from its shipped file. An instance SHALL have one mapping, used for every municipality it imports. + +Every pack, stored or shipped, SHALL pass OpenRegister's `PackDefinitionValidator` and the import's own rules, or the import SHALL stop with 503 `MAPPING_UNAVAILABLE` before any row is read, naming the pack (its file, or its OpenRegister slug and row id) and every rule it breaks; it SHALL NOT fall back to the shipped file. The rules are: + +- the module pack SHALL map `name`, and SHALL map the profile's key column (`APPID`) to `externalNumber`, both with `required: true`; the manufacturer, municipality and business-owner packs SHALL map `name` with `required: true`; +- a pack SHALL map, and set as a default, only properties of its target schema in the register stackiq ships (module; organisation for manufacturer and municipality; usage), and the business-owner pack only `name` and `role`; +- no pack SHALL map or default `@self`, `id`, `externalKey`, `provider`, `consumer`, `module`, `businessOwner` or a field the profile lists as never written; the manufacturer and municipality packs SHALL NOT map `type` and SHALL keep the default `type` `Supplier` and `Municipality`. + +When OpenRegister's store is not available and no pack was ever stored there, the import SHALL use the shipped files and SHALL add one warning to the report saying so. Once the packs were stored there, a store that is not available SHALL stop the import with 503 `MAPPING_UNAVAILABLE` naming the stored packs. A pack whose stored row now carries another slug SHALL stop the import with 503 naming the row and both slugs; a pack whose row was deleted SHALL be seeded again. The import's 503 message SHALL name the OpenRegister pack and its id when one is at fault. + +#### Scenario: An edit in OpenRegister is what the import runs +@e2e exclude Editing the shared mapping changes every import on the instance; tests/Unit/Service/CmdbExportImportServiceTest.php imports a row against an edited stored usage pack. + +- **GIVEN** the usage pack in OpenRegister maps "In productie" to `Planned` +- **WHEN** a row with "Applicatie Status" `In productie` is imported +- **THEN** the usage SHALL have `status` `Planned` +- **AND** the report SHALL carry no warning about the mapping + +#### Scenario: A stored pack that writes a field the import manages is refused +@e2e exclude Breaking the shared mapping is not done in e2e; tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts thirteen broken rules. + +- **GIVEN** the usage pack in OpenRegister maps "Applicatie Status" to `provider` +- **WHEN** an import starts +- **THEN** the import SHALL answer 503 `MAPPING_UNAVAILABLE` before any row is read +- **AND** the reason SHALL name `stackiq-topdesk-usage`, its row id and `"provider" is set by the import and cannot be mapped` + +#### Scenario: A stored module pack without the required name mapping is refused +@e2e exclude Breaking the shared mapping is not done in e2e; tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts it. + +- **GIVEN** the module pack in OpenRegister maps "Applicatie Naam" to `name` without `required: true` +- **WHEN** an import starts +- **THEN** the import SHALL answer 503 `MAPPING_UNAVAILABLE` naming the pack and `"name" must be mapped as required` + +#### Scenario: Without OpenRegister's store, before any seed, the shipped mapping is used with a warning +@e2e exclude Requires removing OpenRegister's store; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the warning. + +- **GIVEN** OpenRegister has no migration-pack store and stackiq never stored a pack there +- **WHEN** an export is imported +- **THEN** the rows SHALL be mapped by the shipped packs +- **AND** the report SHALL carry the warning "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq." + +#### Scenario: Without OpenRegister's store, after a seed, the import fails closed +@e2e exclude Requires removing OpenRegister's store; tests/Unit/Service/Cmdb/CmdbImportProfileTest.php asserts it. + +- **GIVEN** stackiq stored the packs in OpenRegister, and OpenRegister's store is no longer available +- **WHEN** an import starts +- **THEN** the import SHALL answer 503 `MAPPING_UNAVAILABLE` naming each stored pack and its id +- **AND** it SHALL NOT use the shipped files + +#### Scenario: A renamed pack is refused, a deleted one is seeded again +@e2e exclude Renaming or deleting the shared pack is not done in e2e; tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php asserts both. + +- **GIVEN** the usage pack's row in OpenRegister was renamed to `my-usage` +- **WHEN** an import starts +- **THEN** the import SHALL answer 503 `MAPPING_UNAVAILABLE` naming the row, `stackiq-topdesk-usage` and `my-usage`, and SHALL NOT seed a second pack +- **AND** when the row was deleted instead, the import SHALL seed the shipped usage pack again and use it + +### Requirement: Stackiq SHALL seed its mapping packs into OpenRegister and keep unedited ones current (REQ-CMDB-022) + +On install and on every upgrade, after its register is initialised, stackiq SHALL create each of the four sheet packs that is missing from OpenRegister's migration-pack store as a built-in pack, from the shipped file, never through OpenRegister's import (which overwrites). Each stored definition SHALL carry `x-stackiq` with `shippedVersion` and `shippedHash`: the version of the shipped pack and the sha256 of its canonical JSON (every object's keys sorted, recursively; list order kept). A stored pack whose definition without `x-stackiq` still hashes to its `shippedHash` is unedited: when the shipped pack of the release differs, stackiq SHALL replace it with the shipped pack and a new marker. An edited pack, or one without marker, SHALL NOT be changed; for an edited pack whose marker differs from the shipped pack, the upgrade SHALL warn that the shipped version was not applied, and the mapping endpoint SHALL answer `newerShipped` `true`. Stackiq SHALL record the row id of every pack it stored or found in its app config. Seeding SHALL never throw: without OpenRegister's store, or when the store fails, it SHALL report and leave the instance as it was. + +#### Scenario: A fresh install seeds the four sheet packs +@e2e exclude Install-time behaviour; tests/Unit/Repair/SeedCmdbImportPacksTest.php asserts it. + +- **GIVEN** OpenRegister's migration-pack store holds no stackiq pack +- **WHEN** stackiq is installed +- **THEN** the store SHALL hold `stackiq-topdesk-module`, `stackiq-topdesk-manufacturer`, `stackiq-topdesk-usage` and `stackiq-topdesk-business-owner` as built-in packs with the `x-stackiq` marker +- **AND** no municipality pack SHALL be stored +- **AND** running the step again SHALL write nothing + +#### Scenario: An upgrade updates an unedited pack and keeps an edited one +@e2e exclude Upgrade-time behaviour; tests/Unit/Repair/SeedCmdbImportPacksTest.php and tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php assert it. + +- **GIVEN** the stored usage pack is unedited and was seeded from version 1.9.0, and the stored business-owner pack was edited by an administrator +- **WHEN** stackiq is upgraded to a release that ships both packs at version 2.0.0 +- **THEN** the usage pack SHALL be the shipped 2.0.0 pack with a new marker +- **AND** the business-owner pack SHALL keep the administrator's edit, and the upgrade SHALL warn that version 2.0.0 was not applied ## MODIFIED Requirements +### Requirement: Field mapping SHALL be declarative and executed by OpenRegister's mapping engine (REQ-CMDB-005) + +The service SHALL map each normalised row with OpenRegister's `MigrationPack\MappingEngine::mapRow()`, once per target pack: module, manufacturer, municipality, usage, business owner. The packs and the import profile SHALL ship as JSON under `lib/Settings/cmdb-import/`; the module, manufacturer, usage and business-owner packs SHALL be taken from OpenRegister's migration-pack store when it holds them (REQ-CMDB-021), the shipped files being their seed. Each pack SHALL pass OpenRegister's `PackDefinitionValidator` and the import's rules when the import starts; an invalid pack, or a missing `MappingEngine`, SHALL stop the import with 503 `MAPPING_UNAVAILABLE` before any row is read. Before mapping, the service SHALL convert the cells of the date columns from Excel serial numbers to `Y-m-d`: the profile's date columns and every column a pack maps with a `date` transform. It SHALL turn numeric id cells into strings without a decimal part, SHALL read a value the profile lists as empty for its column (`NB` in "BNN Classificatie"; dates, "End-of-Life Functioneel" included, are kept as the file has them) as empty, and SHALL add the constants of the row's sheet (`Beheer` = `Beheer geregeld: nee` or `ja`). A mapping error on a mapping marked `required` in the module pack SHALL skip the row. In the manufacturer and owner packs it SHALL mean the row has no manufacturer or no such owner, without a warning. A mapping error on any other mapping SHALL drop only that field and add a row warning naming the column and the value. The empty parts of the usage's internal note SHALL be dropped on the separator of the concat mapping that writes it. The reader SHALL keep only the columns that the profile or a pack references, and SHALL discard every other cell when it reads the row. + +(Previously: the packs were read from the shipped files only, only the profile's date columns were converted, and the note was split on a fixed ` / `.) + +#### Scenario: Excel serial dates are converted before mapping +@e2e exclude Pure transformation; tests/Unit/Service/Cmdb/CmdbRowNormaliserTest.php asserts the conversions below. + +- **GIVEN** the "Onbeh" row of the anonymised export with "Datum" = `45111.380322627316` and "Referentie datum wijziging" = `46232.552113113423`, and the "Beheerde" row with "End-of-Life Functioneel" = `53359` +- **WHEN** the rows are normalised +- **THEN** "Datum" SHALL be `2023-07-04`, "Referentie datum wijziging" SHALL be `2026-07-29`, and "End-of-Life Functioneel" SHALL be `2046-02-01` +- **AND** "APPID" `1234` SHALL be the string `"1234"` +- **AND** "BNN Classificatie" `NB` SHALL be empty +- **AND** "End-of-Life Functioneel" `49675` SHALL be `2036-01-01` + +#### Scenario: Changing a pack changes the mapping without code +@e2e exclude Configuration behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php imports against an alternate module pack and against an edited stored usage pack. + +- **GIVEN** the usage pack in OpenRegister is edited to add a mapping from "In productie sinds" to `startDateInProduction` with a `date` transform, and its note separator is changed to ` | ` +- **WHEN** an export is imported whose row has "In productie sinds" = `45111` and "Cluster" = `B10` +- **THEN** the usage SHALL have `startDateInProduction` = `2023-07-04` +- **AND** its `interneAnnotation` SHALL be `Beheer geregeld: ja | B10` +- **AND** no PHP code SHALL have changed + +#### Scenario: An unknown status value drops only that field +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php asserts the row outcome and warning. + +- **GIVEN** a row whose "Applicatie Status" is `Onbekende status`, which the usage pack's lookup does not contain +- **WHEN** the row is imported +- **THEN** the module and the usage SHALL be saved without a status from the export +- **AND** the row's report entry SHALL carry a warning naming column "Applicatie Status" and value `Onbekende status` + +#### Scenario: The classifications map to the stackiq fields +@e2e exclude Mapping behaviour; tests/Unit/Service/CmdbExportImportServiceTest.php imports the fixture and asserts the fields. + +- **GIVEN** the "Beheerde" row of the anonymised export with "Applicatiesoort" `Saas`, "BNN Classificatie" `BBN2`, "Classificatie" `Tolereren` and "End-of-Life Functioneel" `53359` +- **WHEN** it is imported +- **THEN** the module SHALL have `applicationType` = `Saas`, `cloudDienstverleningsmodel` = `["SaaS"]` and `bbnLevel` = `BBN2` +- **AND** the usage SHALL have `timeClassification` = `Tolerate` and `startDateOutPhased` = `2046-02-01` +- **AND** the "Onbeh" row's "Applicatiesoort" `Webapplicatie`, which is not a hosting model, SHALL be stored as `applicationType` and SHALL leave `cloudDienstverleningsmodel` empty without a warning +- **AND** "BNN Classificatie" `1`, `2`, `2+` SHALL be `BBN1`, `BBN2`, `BBN2+` + ### Requirement: The admin settings SHALL offer a CMDB import section (REQ-CMDB-014) -Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. The section SHALL also show the mapping the import uses in a collapsible block "Mapping (read-only)" (REQ-CMDB-020), and the sheet names in its help text SHALL come from that mapping, with the shipped names as the fallback until the mapping has loaded or when it cannot be loaded. Every control SHALL have a visible label, and every string SHALL be translatable. +Stackiq's admin settings page SHALL show a section "CMDB import", rendered by the settings page and not registered as an in-app route. The section SHALL let the admin choose an existing municipality or type the name of a new one, choose an `.xlsx` file, and start the import. While the import runs it SHALL show a progress bar and a Cancel button. Afterwards it SHALL show the summary and a report table that can be filtered by outcome. The section SHALL also show the mapping the import uses (REQ-CMDB-020): per pack its source, its version and a link to its editor in OpenRegister. The sheet names in its help text SHALL come from that mapping, with the shipped names as the fallback until the mapping has loaded or when it cannot be loaded. Every control SHALL have a visible label, and every string SHALL be translatable. (Previously: the section had no mapping block, and the sheet names in its help text were a constant in the page, which could drift from the profile on the server.) @@ -81,20 +216,24 @@ Stackiq's admin settings page SHALL show a section "CMDB import", rendered by th ## Non-Functional Requirements -- **Performance:** the endpoint reads six small JSON files and validates five packs; it SHALL answer within the time an admin settings page loads and SHALL be called once per page view. -- **Security:** admin-only with CSRF, as the import routes. The answer holds column names, field names and lookup tables from files that ship with the app; no object data and no person data. The loader's reason in `details.reason` is its message's first line, at most 500 characters: the file, the validator's rule and the configuration value it refused, never a cell value. -- **Accessibility:** Target WCAG 2.2 AA. The block's toggle is an `NcButton` with a visible label and `aria-expanded` (SC 4.1.2; gate `button-name`), the content it controls is referenced with `aria-controls`, and each table has header cells (SC 1.3.1; gate `table-headers`). New in 2.2: 2.4.11 Focus Not Obscured applies (the tables are in the page flow, nothing sticky covers them); 2.5.7 Dragging Movements does not apply; 2.5.8 Target Size applies to the toggle (Nextcloud default); 3.2.6 Consistent Help does not apply (no help mechanism added); 3.3.7 Redundant Entry does not apply (nothing is entered); 3.3.8 Accessible Authentication does not apply. -- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the block, the table headers, the target and transformation names and the error. +- **Performance:** loading the mapping reads seven small JSON files and the register definition, and makes at most one OpenRegister lookup per sheet pack; it SHALL answer within the time an admin settings page loads. Writes happen only to seed or update a pack. +- **Security:** admin-only with CSRF, as the import routes; OpenRegister's pack writes are admin-only. The import writes with OpenRegister's RBAC and multitenancy off, so every pack SHALL follow the import's rules (REQ-CMDB-021). An edit that maps a new column makes the import read that column. The answer and the reasons hold column names, field names, lookup tables, pack names and ids; no object data and no person data. A store failure is named by its exception class, never its message. +- **Reliability:** an edit is never overwritten by an upgrade and never silently replaced by the shipped files (REQ-CMDB-021, REQ-CMDB-022). +- **Accessibility:** Target WCAG 2.2 AA. The mapping block's link has a visible, descriptive label (SC 2.4.4, 4.1.2; gate `link-name`); a status hint is text, not colour alone (SC 1.4.1). New in 2.2: 2.4.11 Focus Not Obscured applies (the block is in the page flow); 2.5.7 Dragging Movements does not apply; 2.5.8 Target Size applies to the link (Nextcloud default); 3.2.6 Consistent Help does not apply; 3.3.7 Redundant Entry does not apply (nothing is entered); 3.3.8 Accessible Authentication does not apply. +- **Internationalization:** Dutch and English MUST be supported (ADR-005) for the block, the messages and the import warning. ## Acceptance Criteria -- [ ] A Nextcloud admin opens "Mapping (read-only)" in the "CMDB import" section and sees five tables, one per pack, with the columns of the shipped packs. -- [ ] The usage table shows "Applicatie Status" mapped to `status` with its lookup values. -- [ ] With a pack file broken on the server, the block shows `MAPPING_UNAVAILABLE` with the reason and the import refuses to run with the same code. +- [ ] After install, OpenRegister lists the four `stackiq-topdesk-*` packs as built-in packs. +- [ ] A Nextcloud admin edits a lookup value of the usage pack in OpenRegister, and the next CMDB import uses it. +- [ ] A pack edited to map a field the import manages makes the import answer 503 naming the pack; the mapping block shows the same. +- [ ] Deleting the stored pack in OpenRegister makes the next import seed the shipped pack again. +- [ ] The "CMDB import" section shows per pack its source and an "Edit in OpenRegister" link that opens the pack's editor. - [ ] A signed-in user who is not a Nextcloud admin gets 403 from the endpoint. -- [ ] The help text under the file control names the sheets from the endpoint. ## Notes -- Editing stays file-based (choice B, 2026-10-09): OpenRegister has APIs for migration packs but no screen, so a stackiq editor would have been a larger change for a mapping that changes rarely. A pack edited on the server is overwritten by the next app update; lasting changes go to the app itself. -- The transform is passed through as the pack stores it, not re-shaped per type, so a transform key the engine reads is never hidden from the administrator. +- Decision 2026-10-12: the mapping lives in OpenRegister. The packs use OpenRegister's existing migration-pack store, not its Twig mapping entity: the packs are already migration packs, executed by `MappingEngine` with structural validation and row and column errors. +- One mapping per instance: OpenRegister's pack store has no organisation scope. An instance that imports for more than one municipality shares the mapping between them. +- `MigrationPackService`, `PackDefinitionValidator` and `MappingEngine` are not part of OpenRegister's `Contract` namespace; stackiq resolves them defensively. +- Deleting a stored pack means "back to the shipped pack": the next import or upgrade seeds it again. diff --git a/openspec/changes/cmdb-import-mapping-view/tasks.md b/openspec/changes/cmdb-import-mapping-view/tasks.md index 36fe883c7..b36e6dc94 100644 --- a/openspec/changes/cmdb-import-mapping-view/tasks.md +++ b/openspec/changes/cmdb-import-mapping-view/tasks.md @@ -4,50 +4,65 @@ Spec: `openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.m ## Implementation Tasks -### Task 1: Mapping overview on the profile loader -- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) -- **files**: `lib/Service/Cmdb/CmdbImportProfile.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php` +### Task 1: The import reads the sheet packs from OpenRegister's migration-pack store +- **spec_ref**: `SPEC#requirement-the-import-shall-run-the-mapping-packs-stored-in-openregister-req-cmdb-021` (cmdb-export-import#REQ-CMDB-021) and `SPEC#requirement-field-mapping-shall-be-declarative-and-executed-by-openregisters-mapping-engine-req-cmdb-005` (cmdb-export-import#REQ-CMDB-005) +- **files**: `lib/Service/Cmdb/CmdbImportProfile.php`, `lib/Service/Cmdb/CmdbImportPackStore.php`, `lib/Service/Cmdb/CmdbImportPackMarker.php`, `lib/Service/Cmdb/CmdbImportPackRules.php`, `lib/Service/CmdbExportImportService.php`, `lib/Controller/CmdbImportController.php`, `tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php`, `tests/Unit/Service/Cmdb/CmdbImportPackRulesTest.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php`, `tests/Unit/Service/CmdbExportImportServiceTest.php`, `tests/Unit/Controller/CmdbImportControllerTest.php`, `tests/Unit/Support/FakeMigrationPackService.php` - **acceptance_criteria**: - - GIVEN the shipped directory WHEN `mappingOverview()` is called THEN it loads and validates every pack first and answers `profile` and `packs` with five entries in TARGETS order, each with target, file, id, name, version, description, fieldMappings and the write rules (defaults, createOnlyDefaults, createOnlyFields, neverWritten) - - GIVEN a field mapping WHEN it is shaped THEN `required` is a boolean and `transform` is the stored array - - GIVEN a broken pack WHEN `mappingOverview()` is called THEN it throws `CmdbImportException` MAPPING_UNAVAILABLE with the file in its message + - GIVEN OpenRegister's store holds an edited module pack WHEN the profile loads THEN the module pack is the stored one without its marker, and the missing sheet packs are seeded; the municipality pack is the shipped file + - GIVEN an unedited stored pack seeded from an older release WHEN the profile loads THEN it is replaced by the shipped pack; GIVEN an edited one THEN it is kept and reported `newerShipped` + - GIVEN a stored pack the validator refuses, or that breaks a rule of the import WHEN the profile loads THEN 503 `MAPPING_UNAVAILABLE` names the pack and its id in the message and the details + - GIVEN no store and no recorded pack ids THEN the shipped files are used and the import report carries one warning; GIVEN no store and recorded ids THEN 503 naming the packs + - GIVEN a renamed stored pack THEN 503 naming the row and both slugs; GIVEN a deleted one THEN it is seeded again + - GIVEN a stored pack that maps a new column with a date transform WHEN a row is imported THEN that column is converted from an Excel date; the note cleanup uses the pack's concat separator - [x] Implement - [x] Test -### Task 2: Endpoint `GET /api/settings/cmdb-import/mapping` -- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) -- **files**: `lib/Controller/SettingsController.php`, `appinfo/routes.php`, `tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php` +### Task 2: Seed and update the packs in OpenRegister on install and upgrade +- **spec_ref**: `SPEC#requirement-stackiq-shall-seed-its-mapping-packs-into-openregister-and-keep-unedited-ones-current-req-cmdb-022` (cmdb-export-import#REQ-CMDB-022) +- **files**: `lib/Repair/SeedCmdbImportPacks.php`, `appinfo/info.xml`, `tests/Unit/Repair/SeedCmdbImportPacksTest.php` - **acceptance_criteria**: - - GIVEN the route WHEN its method is reflected THEN it carries no attribute, declares `@auth admin-only` with a reason and no `AuthorizedAdminSetting`, `NoAdminRequired`, `NoCSRFRequired` or `PublicPage` annotation - - GIVEN the shipped directory WHEN an admin calls it THEN 200 with the flat `{profile, packs}` and five packs - - GIVEN a broken pack THEN 503 `MAPPING_UNAVAILABLE` with the reason in `details.reason`; GIVEN an unexpected error THEN 500 `IMPORT_FAILED`, logged, with a static message + - GIVEN an empty store WHEN the step runs THEN the four sheet packs are created as built-in packs with the `x-stackiq` marker and their ids are recorded; a second run writes nothing + - GIVEN an unedited pack from an older release THEN it is updated; GIVEN an edited one THEN it is kept and the step warns that the shipped version was not applied + - GIVEN no store, or a failing store THEN the step reports and never throws + - GIVEN `appinfo/info.xml` THEN the step runs after `InitializeSettings` in `` and `` - [x] Implement - [x] Test -### Task 3: The "Mapping (read-only)" block and the sheet names -- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) and `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014` (cmdb-export-import#REQ-CMDB-014) -- **files**: `src/views/settings/sections/CmdbImportMapping.vue`, `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `tests/vitest/cmdbImportMapping.spec.js` +### Task 3: The mapping endpoint says where each pack comes from and links to its editor +- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) +- **files**: `lib/Service/Cmdb/CmdbImportProfile.php`, `lib/Controller/SettingsController.php`, `appinfo/routes.php`, `tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php`, `tests/Unit/Service/Cmdb/CmdbImportProfileTest.php` - **acceptance_criteria**: - - GIVEN the endpoint's answer WHEN the block is expanded THEN one table per pack renders with `data-testid="cmdb-import-mapping-"`, the usage table holding "Applicatie Status" → `status` with the lookup pairs - - GIVEN a 503 WHEN the block loads THEN it shows the error and the code and no table - - GIVEN the answer WHEN the section renders its help text THEN the sheet names are the endpoint's; before and on failure they are `PROFILE_DEFAULTS.sheets` + - GIVEN the route WHEN its method is reflected THEN it carries no attribute, declares `@auth admin-only` with a reason and no exemption annotation + - GIVEN the shipped files and no store THEN 200 with `{profile, packs, openRegisterPacks: false}`, every pack `source: file`, `editUrl: null` + - GIVEN the store THEN the four sheet packs answer `source: openregister`, `packId`, `storedVersion`, `shippedVersion`, `edited`, `newerShipped`, `editable: true` and `editUrl` built from OpenRegister's page route; the municipality pack answers `editable: false` + - GIVEN a broken stored pack THEN 503 with `details.reason`, `details.pack` and `details.packId` - [x] Implement - [x] Test -### Task 4: Translations and documentation +### Task 4: The section's mapping block shows the source and the editor link per pack +- **spec_ref**: `SPEC#requirement-the-admin-settings-shall-offer-a-cmdb-import-section-req-cmdb-014` (cmdb-export-import#REQ-CMDB-014) and `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) +- **files**: `src/views/settings/sections/CmdbImportMapping.vue`, `src/views/settings/sections/CmdbImport.vue`, `src/utils/cmdbImport.js`, `tests/vitest/cmdbImportMapping.spec.js` +- **acceptance_criteria**: + - GIVEN the endpoint's answer WHEN the block renders THEN each pack shows its source, its version, a hint when `newerShipped`, and an "Edit in OpenRegister" link to `editUrl` when there is one + - GIVEN a 503 THEN the block shows the error, the code and the reason + - GIVEN the answer THEN the help text names the endpoint's sheets; before and on failure `PROFILE_DEFAULTS.sheets` +- [ ] Implement +- [ ] Test + +### Task 5: Translations and documentation - **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) - **files**: `l10n/en.json`, `l10n/nl.json`, `l10n/en.js`, `l10n/nl.js`, `docs/features/cmdb-import.md` - **acceptance_criteria**: - GIVEN the new strings WHEN `npm run test:l10n` and `npm run check:l10n-js` run THEN both pass, with Dutch for every string - - GIVEN the docs WHEN an administrator reads "Viewing the mapping" and "Adjusting the mapping" THEN they know where the block is, what it shows, which file to change for which column and that an app update overwrites a changed file -- [x] Implement + - GIVEN the docs WHEN an administrator reads "Viewing and editing the mapping" THEN they know where to edit a pack, what the import refuses, what happens on an app update and how to go back to the shipped pack +- [x] Implement (backend strings and documentation; the section's strings come with Task 4) - [x] Test -### Task 5: End-to-end coverage +### Task 6: End-to-end coverage - **spec_ref**: `SPEC#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020` (cmdb-export-import#REQ-CMDB-020) - **files**: `tests/e2e/spec-coverage/cmdb-import-mapping.spec.ts` - **acceptance_criteria**: - - GIVEN an admin on the settings page WHEN they expand the block THEN five tables are visible and the usage row for "Applicatie Status" is shown - - GIVEN a signed-in non-admin WHEN they call the endpoint THEN 403, and the admin settings page is not shown to them -- [x] Implement -- [ ] Test (written, not run: no Nextcloud was reachable in this build) + - GIVEN an admin on the settings page WHEN the mapping has loaded THEN every sheet pack shows its source and an "Edit in OpenRegister" link + - GIVEN a signed-in non-admin WHEN they call the endpoint THEN 403 +- [ ] Implement +- [ ] Test diff --git a/openspec/specs/cmdb-export-import/spec.md b/openspec/specs/cmdb-export-import/spec.md index 8818d842b..d285d7409 100644 --- a/openspec/specs/cmdb-export-import/spec.md +++ b/openspec/specs/cmdb-export-import/spec.md @@ -10,7 +10,7 @@ built_by: openspec/changes/archive/2026-10-05-cmdb-export-import **Scope**: stackiq **OpenSpec changes**: - [cmdb-export-import](../../changes/archive/2026-10-05-cmdb-export-import/) _(archived 2026-10-05)_ — admin uploads a TOPdesk CMDB export (xlsx); stackiq upserts modules, vendor organisations, usages and owner contact persons for one municipality from the two CMDB sheets, matched on APPID, mapped by OpenRegister migration packs (kind: code) -- [cmdb-import-mapping-view](../../changes/cmdb-import-mapping-view/) — the "CMDB import" section shows, read-only, the mapping the import uses, served by `GET /api/settings/cmdb-import/mapping` through the import's own loader; editing stays file-based (kind: code) +- [cmdb-import-mapping-view](../../changes/cmdb-import-mapping-view/) — the four sheet packs are seeded into OpenRegister's migration-pack store, where an administrator edits them, and the import runs what is stored there (shipped files as seed and fallback, fail-closed on an invalid pack); the "CMDB import" section shows per pack its source and a link to its editor, served by `GET /api/settings/cmdb-import/mapping` (kind: code) ## Purpose diff --git a/tests/Unit/Controller/CmdbImportControllerTest.php b/tests/Unit/Controller/CmdbImportControllerTest.php index a27b7cfbf..b18e7d8f5 100644 --- a/tests/Unit/Controller/CmdbImportControllerTest.php +++ b/tests/Unit/Controller/CmdbImportControllerTest.php @@ -396,6 +396,7 @@ public function testAMunicipalityIsRequired(): void { public static function serviceErrors(): array { return [ 'mapping' => [CmdbImportException::MAPPING_UNAVAILABLE, 503, []], + 'stored mapping pack' => [CmdbImportException::MAPPING_UNAVAILABLE, 503, ['pack' => 'stackiq-topdesk-usage', 'packId' => 43]], 'reader' => [CmdbImportException::READER_UNAVAILABLE, 503, []], 'config' => [CmdbImportException::NOT_CONFIGURED, 503, []], 'no sheet' => [CmdbImportException::NO_SOURCE_SHEET, 422, ['expected' => ['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB']]], @@ -442,6 +443,48 @@ public function testEngineCodesHaveTheirOwnMessage(string $code, string $message $this->assertSame($message, $response->getData()['message']); }//end testEngineCodesHaveTheirOwnMessage() + /** + * MAPPING_UNAVAILABLE names the OpenRegister migration pack at fault, with its id when known. + * + * @return array, string}> + */ + public static function mappingMessages(): array { + return [ + 'no pack' => [[], 'The import mapping cannot run: OpenRegister is missing, or a mapping pack is invalid or unavailable.'], + 'stored pack' => [ + ['pack' => 'stackiq-topdesk-usage', 'packId' => 43], + 'The import mapping cannot run: the OpenRegister migration pack stackiq-topdesk-usage (id 43) is invalid or cannot be read. The details are in the Nextcloud log.', + ], + 'pack without id' => [ + ['pack' => 'stackiq-topdesk-usage'], + 'The import mapping cannot run: the OpenRegister migration pack stackiq-topdesk-usage is invalid or cannot be read. The details are in the Nextcloud log.', + ], + ]; + }//end mappingMessages() + + /** + * The MAPPING_UNAVAILABLE message names the pack from the details. + * + * @param array $details The exception details. + * @param string $message The expected message. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + #[DataProvider('mappingMessages')] + public function testTheMappingMessageNamesThePack(array $details, string $message): void { + $service = $this->service(); + $service->method('import')->willThrowException( + new CmdbImportException(errorCode: CmdbImportException::MAPPING_UNAVAILABLE, message: 'internal', details: $details) + ); + + $response = $this->controller(file: $this->file(path: $this->upload()), params: ['municipalityName' => 'Gemeente Voorbeeldstad'], service: $service)->import(); + + $this->assertSame(503, $response->getStatus()); + $this->assertSame($message, $response->getData()['message']); + }//end testTheMappingMessageNamesThePack() + /** * A service exception becomes its contract response. * diff --git a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php index 69bdc844c..7f84ca17e 100644 --- a/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php +++ b/tests/Unit/Controller/SettingsControllerCmdbImportMappingTest.php @@ -23,6 +23,7 @@ namespace OCA\Stackiq\Tests\Unit\Controller; require_once __DIR__ . '/../Support/CmdbTestSupport.php'; +require_once __DIR__ . '/../Support/FakeMigrationPackService.php'; use OCA\Stackiq\Controller\SettingsController; use OCA\Stackiq\Exception\CmdbImportException; @@ -33,6 +34,7 @@ use OCA\Stackiq\Service\ProgressTracker; use OCA\Stackiq\Service\SettingsService; use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\FakeMigrationPackService; use OCP\App\IAppManager; use OCP\AppFramework\Http\Attribute\AuthorizedAdminSetting; use OCP\AppFramework\Http\Attribute\NoAdminRequired; @@ -42,6 +44,7 @@ use OCP\IGroupManager; use OCP\IL10N; use OCP\IRequest; +use OCP\IURLGenerator; use OCP\IUserSession; use PHPUnit\Framework\MockObject\MockObject; use PHPUnit\Framework\TestCase; @@ -113,10 +116,16 @@ private function copyOfShippedDirectory(): string { * @param CmdbImportProfile|null $profile The profile; null lets the controller build the shipped one. * @param LoggerInterface|MockObject|null $logger The logger. * @param IL10N|null $l10n The translator; null answers the English text. + * @param IURLGenerator|null $urlGenerator The URL generator; null builds no edit links. * * @return SettingsController */ - private function controller(?CmdbImportProfile $profile, LoggerInterface|MockObject|null $logger = null, ?IL10N $l10n = null): SettingsController { + private function controller( + ?CmdbImportProfile $profile, + LoggerInterface|MockObject|null $logger = null, + ?IL10N $l10n = null, + ?IURLGenerator $urlGenerator = null + ): SettingsController { $request = $this->createMock(IRequest::class); $request->method('getParams')->willReturn([]); @@ -136,7 +145,8 @@ private function controller(?CmdbImportProfile $profile, LoggerInterface|MockObj ($logger ?? $this->createMock(LoggerInterface::class)), null, $profile, - $l10n + $l10n, + $urlGenerator ); }//end controller() @@ -197,7 +207,8 @@ public function testTheShippedMappingIsAnsweredFlat(): void { $data = $response->getData(); $this->assertSame(200, $response->getStatus()); - $this->assertSame(['profile', 'packs'], array_keys($data), 'flat envelope, no success or message key'); + $this->assertSame(['profile', 'packs', 'openRegisterPacks'], array_keys($data), 'flat envelope, no success or message key'); + $this->assertFalse($data['openRegisterPacks']); $this->assertSame(['Onbeh Applicaties CMDB', 'Beheerde Applicaties CMDB'], array_column($data['profile']['sheets'], 'name')); $this->assertSame(['Beheer' => 'Beheer geregeld: nee'], $data['profile']['sheets'][0]['constants']); @@ -225,6 +236,8 @@ public function testTheShippedMappingIsAnsweredFlat(): void { // What is shown is what runs: every pack lists as many mappings as the loader hands the engine. foreach ($data['packs'] as $pack) { + $this->assertSame('file', $pack['source'], $pack['target']); + $this->assertNull($pack['editUrl'], $pack['target'] . ': a file is not edited in OpenRegister'); $this->assertNotSame('', $pack['name'], $pack['target']); $this->assertNotSame('', $pack['version'], $pack['target']); $this->assertCount(count($profile->pack(target: $pack['target'])['fieldMappings']), $pack['fieldMappings'], $pack['target']); @@ -293,7 +306,7 @@ public function testABrokenPackIs503WithTheReason(): void { $this->assertSame(503, $response->getStatus()); $this->assertFalse($data['success']); $this->assertSame('MAPPING_UNAVAILABLE', $data['error']); - $this->assertSame('NL The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.', $data['message']); + $this->assertSame('NL The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.', $data['message']); $this->assertStringContainsString('topdesk-usage.json', $data['details']->reason); $this->assertStringContainsString('uppercase', $data['details']->reason, 'the validator names the unknown transform'); }//end testABrokenPackIs503WithTheReason() @@ -313,7 +326,7 @@ public function testAMissingValidatorIs503(): void { $this->assertSame(503, $response->getStatus()); $this->assertSame('MAPPING_UNAVAILABLE', $data['error']); - $this->assertSame('The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.', $data['message']); + $this->assertSame('The import mapping cannot be shown: OpenRegister is missing, or a mapping pack is invalid or unavailable.', $data['message']); $this->assertSame('OpenRegister PackDefinitionValidator is not available', $data['details']->reason); }//end testAMissingValidatorIs503() @@ -348,6 +361,114 @@ public function mappingOverview(): array { $this->assertSame(500, mb_strlen($reason)); }//end testTheReasonIsOneBoundedLine() + /** + * A pack stored in OpenRegister answers its source, versions and the absolute link to its editor. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 + */ + public function testAStoredPackLinksToItsEditorInOpenRegister(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $config = []; + $profile = new CmdbImportProfile( + container: $service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $config)) + ); + $routes = []; + $urls = $this->createMock(IURLGenerator::class); + $urls->method('linkToRouteAbsolute')->willReturnCallback( + function (string $route, array $parameters) use (&$routes): string { + $routes[] = $route; + return 'https://cloud.example/index.php/apps/openregister/' . $parameters['path']; + } + ); + + $data = $this->controller(profile: $profile, urlGenerator: $urls)->getCmdbImportMapping()->getData(); + $byTarget = array_column($data['packs'], null, 'target'); + + $this->assertTrue($data['openRegisterPacks']); + foreach (CmdbImportProfile::EDITABLE_TARGETS as $target) { + $pack = $byTarget[$target]; + $this->assertSame('openregister', $pack['source'], $target); + $this->assertIsInt($pack['packId'], $target); + $this->assertSame( + 'https://cloud.example/index.php/apps/openregister/migration-packs/' . $pack['packId'], + $pack['editUrl'], + $target + ); + $this->assertTrue($pack['editable'], $target); + $this->assertFalse($pack['edited'], $target . ' is as seeded'); + $this->assertFalse($pack['newerShipped'], $target); + $this->assertSame($pack['shippedVersion'], $pack['storedVersion'], $target); + } + + $this->assertSame(['openregister.dashboard.catchAll'], array_values(array_unique($routes)), "OpenRegister's page route"); + $this->assertSame('file', $byTarget['municipality']['source']); + $this->assertNull($byTarget['municipality']['editUrl']); + $this->assertFalse($byTarget['municipality']['editable']); + $this->assertSame( + ['target', 'file', 'id', 'name', 'version', 'description', 'fieldMappings', 'defaults', 'createOnlyDefaults', 'createOnlyFields', 'neverWritten', 'source', 'packId', 'storedVersion', 'shippedVersion', 'edited', 'newerShipped', 'editable', 'editUrl'], + array_keys($byTarget['module']), + 'the keys the settings section reads' + ); + }//end testAStoredPackLinksToItsEditorInOpenRegister() + + /** + * When OpenRegister's route cannot be built the link is null, never the instance's front page. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 + */ + public function testAnUnknownRouteGivesNoLink(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $config = []; + $profile = new CmdbImportProfile( + container: $service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $config)) + ); + $urls = $this->createMock(IURLGenerator::class); + $urls->method('linkToRouteAbsolute')->willReturn('https://cloud.example/'); + + $data = $this->controller(profile: $profile, urlGenerator: $urls)->getCmdbImportMapping()->getData(); + + $this->assertSame([null], array_values(array_unique(array_column($data['packs'], 'editUrl')))); + + $throwing = $this->createMock(IURLGenerator::class); + $throwing->method('linkToRouteAbsolute')->willThrowException(new RuntimeException('no router')); + $data = $this->controller(profile: $profile, urlGenerator: $throwing)->getCmdbImportMapping()->getData(); + $this->assertSame([null], array_values(array_unique(array_column($data['packs'], 'editUrl')))); + }//end testAnUnknownRouteGivesNoLink() + + /** + * A stored pack that breaks a rule answers 503 naming the OpenRegister pack and its id. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-3 + */ + public function testABrokenStoredPackIs503NamingThePack(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $usage = json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/topdesk-usage.json'), true); + $usage['fieldMappings'][0]['target'] = 'provider'; + $id = $service->put(definition: $usage); + $config = []; + $profile = new CmdbImportProfile( + container: $service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $config)) + ); + + $response = $this->controller(profile: $profile)->getCmdbImportMapping(); + + $this->assertSame(503, $response->getStatus()); + $this->assertSame('MAPPING_UNAVAILABLE', $response->getData()['error']); + $this->assertStringStartsWith('OpenRegister migration pack stackiq-topdesk-usage (id ' . $id . ')', $response->getData()['details']->reason); + $this->assertStringContainsString('"provider" is set by the import', $response->getData()['details']->reason); + $this->assertSame('stackiq-topdesk-usage', $response->getData()['details']->pack); + $this->assertSame($id, $response->getData()['details']->packId); + }//end testABrokenStoredPackIs503NamingThePack() + /** * An unexpected error is 500 IMPORT_FAILED with a static message, and the exception is logged. * diff --git a/tests/Unit/Repair/SeedCmdbImportPacksTest.php b/tests/Unit/Repair/SeedCmdbImportPacksTest.php new file mode 100644 index 000000000..e792f054b --- /dev/null +++ b/tests/Unit/Repair/SeedCmdbImportPacksTest.php @@ -0,0 +1,212 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-2 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Repair; + +require_once __DIR__ . '/../Support/CmdbTestSupport.php'; +require_once __DIR__ . '/../Support/FakeMigrationPackService.php'; + +use OCA\Stackiq\Repair\SeedCmdbImportPacks; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackMarker; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackStore; +use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\FakeMigrationPackService; +use OCP\Migration\IOutput; +use OCP\Migration\IRepairStep; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use Psr\Log\LoggerInterface; +use RuntimeException; + +/** + * SeedCmdbImportPacks against an in-memory MigrationPackService. + */ +class SeedCmdbImportPacksTest extends TestCase { + /** + * What the step wrote to its output, as "info: …" and "warning: …". + * + * @var array + */ + private array $lines = []; + + /** + * An output that records its lines. + * + * @return IOutput + */ + private function recordingOutput(): IOutput { + $this->lines = []; + $output = $this->createMock(IOutput::class); + $output->method('info')->willReturnCallback(function (string $message): void { + $this->lines[] = 'info: ' . $message; + }); + $output->method('warning')->willReturnCallback(function (string $message): void { + $this->lines[] = 'warning: ' . $message; + }); + + return $output; + }//end output() + + /** + * The step on the fake service. + * + * @param FakeMigrationPackService $service The fake service. + * @param array $config The app config values. + * @param LoggerInterface|null $logger The logger. + * + * @return SeedCmdbImportPacks + */ + private function step(FakeMigrationPackService $service, array &$config, ?LoggerInterface $logger = null): SeedCmdbImportPacks { + $container = $service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $config)); + return new SeedCmdbImportPacks( + profile: new CmdbImportProfile(container: $container), + store: new CmdbImportPackStore(container: $container), + logger: ($logger ?? $this->createMock(LoggerInterface::class)) + ); + }//end step() + + /** + * The step is a repair step and names itself. + * + * @return void + */ + public function testItIsARepairStepWithAName(): void { + $config = []; + $step = $this->step(service: new FakeMigrationPackService(), config: $config); + + $this->assertInstanceOf(IRepairStep::class, $step); + $this->assertSame('Seed the CMDB import mapping packs into OpenRegister', $step->getName()); + }//end testItIsARepairStepWithAName() + + /** + * A fresh store gets the four sheet packs as built-in packs, and their ids are recorded; the municipality pack never. + * + * @return void + */ + public function testTheSheetPacksAreSeeded(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $config = []; + + $this->step(service: $service, config: $config)->run($this->recordingOutput()); + + $this->assertSame( + ['create stackiq-topdesk-module', 'create stackiq-topdesk-manufacturer', 'create stackiq-topdesk-usage', 'create stackiq-topdesk-business-owner'], + $service->writes + ); + $this->assertSame([true], array_values(array_unique(array_column($service->rows, 'builtin')))); + $this->assertCount(4, json_decode($config[CmdbImportPackStore::CONFIG_KEY], true)); + $this->assertCount(4, preg_grep('/^info: Seeded CMDB import pack stackiq-topdesk-/', $this->lines)); + + $this->step(service: $service, config: $config)->run($this->recordingOutput()); + $this->assertCount(4, $service->writes, 'a second run writes nothing'); + $this->assertCount(4, preg_grep('/already present$/', $this->lines)); + }//end testTheSheetPacksAreSeeded() + + /** + * An unedited pack from an older release is updated; an edited one is kept with a warning. + * + * @return void + */ + public function testUneditedPacksAreUpdatedAndEditedOnesKept(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $older = json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/topdesk-usage.json'), true); + $older['version'] = '1.9.0'; + unset($older['fieldMappings'][0]['transform']['map']['Besteld']); + $usageId = $service->put(definition: (new CmdbImportPackMarker())->seed(shipped: $older)); + + $olderOwner = json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/topdesk-business-owner.json'), true); + $olderOwner['version'] = '1.9.0'; + $edited = (new CmdbImportPackMarker())->seed(shipped: $olderOwner); + $edited['fieldMappings'][1]['source'] = 'Functie eigenaar'; + $ownerId = $service->put(definition: $edited); + $config = []; + + $this->step(service: $service, config: $config)->run($this->recordingOutput()); + + $this->assertContains('update ' . $usageId, $service->writes); + $this->assertNotContains('update ' . $ownerId, $service->writes); + $this->assertSame('Functie eigenaar', $service->definition(id: $ownerId)['fieldMappings'][1]['source']); + $this->assertContains( + 'info: Updated unedited CMDB import pack stackiq-topdesk-usage (id ' . $usageId . ') to version 2.0.0', + $this->lines + ); + $this->assertContains( + 'warning: CMDB import pack stackiq-topdesk-business-owner (id ' . $ownerId . ') was edited in OpenRegister and is kept; ' + . 'this release ships version 2.0.0, which is not applied', + $this->lines + ); + }//end testUneditedPacksAreUpdatedAndEditedOnesKept() + + /** + * Without OpenRegister's store the step does nothing and does not throw. + * + * @return void + */ + public function testWithoutTheStoreNothingIsSeeded(): void { + $container = $this->createMock(ContainerInterface::class); + $container->method('has')->willReturn(false); + $step = new SeedCmdbImportPacks( + profile: new CmdbImportProfile(container: $container), + store: new CmdbImportPackStore(container: $container), + logger: $this->createMock(LoggerInterface::class) + ); + + $step->run($this->recordingOutput()); + + $this->assertSame(['info: OpenRegister migration pack store not available; CMDB import packs not seeded'], $this->lines); + }//end testWithoutTheStoreNothingIsSeeded() + + /** + * A failing store is reported per pack and logged, never thrown, so an install is not aborted. + * + * @return void + */ + public function testAFailingStoreIsReportedNotThrown(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $service->failure = new RuntimeException('database gone'); + $config = []; + $logger = $this->createMock(LoggerInterface::class); + $logger->expects($this->exactly(4))->method('warning'); + + $this->step(service: $service, config: $config, logger: $logger)->run($this->recordingOutput()); + + $this->assertCount(4, preg_grep('/^warning: CMDB import pack stackiq-topdesk-[a-z-]+ not seeded: /', $this->lines)); + $this->assertSame([], $service->writes); + }//end testAFailingStoreIsReportedNotThrown() + + /** + * The step is registered after InitializeSettings, on install and on upgrade. + * + * @return void + */ + public function testItIsRegisteredOnInstallAndUpgrade(): void { + $info = simplexml_load_file(__DIR__ . '/../../../appinfo/info.xml'); + foreach (['install', 'post-migration'] as $block) { + $steps = array_map('strval', iterator_to_array($info->{'repair-steps'}->{$block}->step, false)); + $seed = array_search(SeedCmdbImportPacks::class, $steps, true); + $this->assertIsInt($seed, $block); + $this->assertGreaterThan(array_search('OCA\Stackiq\Repair\InitializeSettings', $steps, true), $seed, $block); + } + }//end testItIsRegisteredOnInstallAndUpgrade() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbImportPackRulesTest.php b/tests/Unit/Service/Cmdb/CmdbImportPackRulesTest.php new file mode 100644 index 000000000..5c039f807 --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbImportPackRulesTest.php @@ -0,0 +1,103 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +require_once __DIR__ . '/../../Support/CmdbTestSupport.php'; + +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackRules; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use PHPUnit\Framework\TestCase; + +/** + * CmdbImportPackRules. + */ +class CmdbImportPackRulesTest extends TestCase { + /** + * A shipped file, decoded. + * + * @param string $file The file name. + * + * @return array + */ + private function shipped(string $file): array { + return json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/' . $file), true); + }//end shipped() + + /** + * Every shipped pack follows the rules, including the module fields only a register fragment declares. + * + * @return void + */ + public function testTheShippedPacksFollowTheRules(): void { + $rules = new CmdbImportPackRules(); + $profile = $this->shipped(file: 'topdesk-profile.json'); + + foreach ($profile['packs'] as $target => $file) { + $this->assertSame([], $rules->violations(target: $target, pack: $this->shipped(file: $file), profile: $profile), $target); + } + }//end testTheShippedPacksFollowTheRules() + + /** + * Several broken rules are all named, once each. + * + * @return void + */ + public function testEveryBrokenRuleIsNamedOnce(): void { + $profile = $this->shipped(file: 'topdesk-profile.json'); + $pack = $this->shipped(file: 'topdesk-manufacturer.json'); + $pack['fieldMappings'][0]['required'] = false; + $pack['fieldMappings'][] = ['source' => 'Leverancier', 'target' => 'provider']; + $pack['fieldMappings'][] = ['source' => 'Hostingpartij', 'target' => 'provider']; + unset($pack['defaults']['type']); + + $this->assertSame( + [ + '"name" must be mapped as required', + '"provider" is set by the import and cannot be mapped', + 'default "type" must stay "Supplier"', + ], + (new CmdbImportPackRules())->violations(target: 'manufacturer', pack: $pack, profile: $profile) + ); + }//end testEveryBrokenRuleIsNamedOnce() + + /** + * Without the register definition no pack can be checked, so the import fails closed. + * + * @return void + */ + public function testAnUnreadableRegisterIsMappingUnavailable(): void { + $rules = new CmdbImportPackRules(settingsDirectory: '/nonexistent'); + + try { + $rules->violations(target: 'usage', pack: $this->shipped(file: 'topdesk-usage.json'), profile: []); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame('The stackiq register definition cannot be read, so the CMDB import packs cannot be checked', $e->getMessage()); + } + }//end testAnUnreadableRegisterIsMappingUnavailable() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php b/tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php new file mode 100644 index 000000000..ac32ce3f1 --- /dev/null +++ b/tests/Unit/Service/Cmdb/CmdbImportPackStoreTest.php @@ -0,0 +1,296 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; + +require_once __DIR__ . '/../../Support/CmdbTestSupport.php'; +require_once __DIR__ . '/../../Support/FakeMigrationPackService.php'; + +use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackMarker; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackStore; +use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\FakeMigrationPackService; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use RuntimeException; + +/** + * CmdbImportPackStore against an in-memory MigrationPackService. + */ +class CmdbImportPackStoreTest extends TestCase { + /** + * The fake OpenRegister service. + * + * @var FakeMigrationPackService + */ + private FakeMigrationPackService $service; + + /** + * The app config values the store wrote. + * + * @var array + */ + private array $config = []; + + /** + * A fresh service and config per test. + * + * @return void + */ + protected function setUp(): void { + parent::setUp(); + $this->service = new FakeMigrationPackService(); + $this->config = []; + }//end setUp() + + /** + * The store on the fake service, with an app config. + * + * @return CmdbImportPackStore + */ + private function store(): CmdbImportPackStore { + return new CmdbImportPackStore( + container: $this->service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $this->config)) + ); + }//end store() + + /** + * The shipped usage pack. + * + * @return array + */ + private function shippedUsage(): array { + return json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/topdesk-usage.json'), true); + }//end shippedUsage() + + /** + * The shipped usage pack as a later release would ship it: one more status value, a newer version. + * + * @return array + */ + private function newerUsage(): array { + $pack = $this->shippedUsage(); + $pack['version'] = '2.1.0'; + $pack['fieldMappings'][0]['transform']['map']['Gepland'] = 'Planned'; + return $pack; + }//end newerUsage() + + /** + * Without OpenRegister's service the store is not available and refuses to resolve. + * + * @return void + */ + public function testWithoutTheServiceTheStoreIsNotAvailable(): void { + $container = $this->createMock(ContainerInterface::class); + $container->method('has')->willReturn(false); + $store = new CmdbImportPackStore(container: $container); + + $this->assertFalse($store->isAvailable()); + $this->assertSame([], $store->recordedPackIds(), 'no app config: nothing recorded'); + + $this->expectException(CmdbImportException::class); + $this->expectExceptionMessage('OpenRegister MigrationPackService is not available'); + $store->resolve(shipped: $this->shippedUsage()); + }//end testWithoutTheServiceTheStoreIsNotAvailable() + + /** + * A missing pack is created as a built-in pack with the marker, and its row id is recorded. + * + * @return void + */ + public function testAMissingPackIsSeededWithTheMarker(): void { + $shipped = $this->shippedUsage(); + + $stored = $this->store()->resolve(shipped: $shipped); + + $this->assertSame(CmdbImportPackStore::SEEDED, $stored['outcome']); + $this->assertSame(['create stackiq-topdesk-usage'], $this->service->writes); + $this->assertTrue($this->service->rows[$stored['id']]['builtin']); + $definition = $this->service->definition(id: $stored['id']); + $this->assertSame( + ['shippedVersion' => '2.0.0', 'shippedHash' => (new CmdbImportPackMarker())->hash(pack: $shipped)], + $definition[CmdbImportPackMarker::KEY] + ); + $this->assertSame($shipped, (new CmdbImportPackMarker())->strip(definition: $definition), 'the pack itself is stored as shipped'); + $this->assertFalse((new CmdbImportPackMarker())->isEdited(definition: $definition)); + $this->assertSame(['stackiq-topdesk-usage' => $stored['id']], json_decode($this->config[CmdbImportPackStore::CONFIG_KEY], true)); + $this->assertSame(['stackiq-topdesk-usage' => $stored['id']], $this->store()->recordedPackIds()); + }//end testAMissingPackIsSeededWithTheMarker() + + /** + * An unedited pack that matches the shipped one is found and not written. + * + * @return void + */ + public function testAnUneditedCurrentPackIsLeftAlone(): void { + $shipped = $this->shippedUsage(); + $id = $this->service->put(definition: (new CmdbImportPackMarker())->seed(shipped: $shipped)); + + $stored = $this->store()->resolve(shipped: $shipped); + + $this->assertSame(CmdbImportPackStore::FOUND, $stored['outcome']); + $this->assertSame($id, $stored['id']); + $this->assertSame([], $this->service->writes); + }//end testAnUneditedCurrentPackIsLeftAlone() + + /** + * An unedited pack seeded from an older release is replaced by the newer shipped pack. + * + * @return void + */ + public function testAnUneditedPackFollowsANewerShippedPack(): void { + $id = $this->service->put(definition: (new CmdbImportPackMarker())->seed(shipped: $this->shippedUsage())); + $newer = $this->newerUsage(); + + $stored = $this->store()->resolve(shipped: $newer); + + $this->assertSame(CmdbImportPackStore::UPDATED, $stored['outcome']); + $this->assertSame(['update ' . $id], $this->service->writes); + $this->assertSame('2.1.0', $stored['version']); + $definition = $this->service->definition(id: $id); + $this->assertSame('Planned', $definition['fieldMappings'][0]['transform']['map']['Gepland']); + $this->assertSame('2.1.0', $definition[CmdbImportPackMarker::KEY]['shippedVersion']); + $this->assertFalse((new CmdbImportPackMarker())->isEdited(definition: $definition)); + $this->assertFalse((new CmdbImportPackMarker())->isNewerShipped(definition: $definition, shipped: $newer)); + }//end testAnUneditedPackFollowsANewerShippedPack() + + /** + * An edited pack is kept when a newer pack ships, and reported as such. + * + * @return void + */ + public function testAnEditedPackIsKeptAndReportsTheNewerShippedPack(): void { + $edited = (new CmdbImportPackMarker())->seed(shipped: $this->shippedUsage()); + $edited['fieldMappings'][0]['transform']['map']['In beheer'] = 'In production'; + $edited['version'] = '2.0.1'; + $id = $this->service->put(definition: $edited); + $newer = $this->newerUsage(); + + $stored = $this->store()->resolve(shipped: $newer); + + $this->assertSame(CmdbImportPackStore::FOUND, $stored['outcome']); + $this->assertSame([], $this->service->writes, 'an edit is never overwritten'); + $this->assertSame('2.0.1', $stored['version']); + $this->assertTrue((new CmdbImportPackMarker())->isEdited(definition: $stored['definition'])); + $this->assertTrue((new CmdbImportPackMarker())->isNewerShipped(definition: $stored['definition'], shipped: $newer)); + $this->assertFalse( + (new CmdbImportPackMarker())->isNewerShipped(definition: $stored['definition'], shipped: $this->shippedUsage()), + 'edited, but seeded from the pack that still ships' + ); + $this->assertSame('In production', $this->service->definition(id: $id)['fieldMappings'][0]['transform']['map']['In beheer']); + }//end testAnEditedPackIsKeptAndReportsTheNewerShippedPack() + + /** + * A stored pack without the marker is never updated: nobody knows what was seeded. + * + * @return void + */ + public function testAPackWithoutTheMarkerIsNeverUpdated(): void { + $this->service->put(definition: $this->shippedUsage()); + $newer = $this->newerUsage(); + + $stored = $this->store()->resolve(shipped: $newer); + + $this->assertSame(CmdbImportPackStore::FOUND, $stored['outcome']); + $this->assertSame([], $this->service->writes); + $this->assertNull((new CmdbImportPackMarker())->isEdited(definition: $stored['definition'])); + $this->assertFalse((new CmdbImportPackMarker())->isNewerShipped(definition: $stored['definition'], shipped: $newer)); + }//end testAPackWithoutTheMarkerIsNeverUpdated() + + /** + * A pack whose recorded row now carries another slug is refused, naming the row and both slugs. + * + * @return void + */ + public function testARenamedPackFailsClosed(): void { + $renamed = $this->shippedUsage(); + $renamed['id'] = 'my-usage'; + $id = $this->service->put(definition: $renamed); + $this->config[CmdbImportPackStore::CONFIG_KEY] = (string)json_encode(['stackiq-topdesk-usage' => $id]); + + try { + $this->store()->resolve(shipped: $this->shippedUsage()); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + $this->assertStringContainsString('id ' . $id, $e->getMessage()); + $this->assertStringContainsString('stackiq-topdesk-usage', $e->getMessage()); + $this->assertStringContainsString('my-usage', $e->getMessage()); + $this->assertSame(['pack' => 'stackiq-topdesk-usage', 'packId' => $id], $e->getDetails()); + } + + $this->assertSame([], $this->service->writes, 'no second pack under the old slug'); + }//end testARenamedPackFailsClosed() + + /** + * A pack whose recorded row was deleted is seeded again, and the new row id is recorded. + * + * @return void + */ + public function testADeletedPackIsSeededAgain(): void { + $this->config[CmdbImportPackStore::CONFIG_KEY] = (string)json_encode(['stackiq-topdesk-usage' => 7]); + + $stored = $this->store()->resolve(shipped: $this->shippedUsage()); + + $this->assertSame(CmdbImportPackStore::SEEDED, $stored['outcome']); + $this->assertNotSame(7, $stored['id']); + $this->assertSame(['stackiq-topdesk-usage' => $stored['id']], json_decode($this->config[CmdbImportPackStore::CONFIG_KEY], true)); + }//end testADeletedPackIsSeededAgain() + + /** + * A store that fails on a lookup is MAPPING_UNAVAILABLE naming the pack, never a fallback. + * + * @return void + */ + public function testAFailingStoreIsMappingUnavailable(): void { + $this->service->failure = new RuntimeException('database gone'); + + try { + $this->store()->resolve(shipped: $this->shippedUsage()); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertStringContainsString('stackiq-topdesk-usage', $e->getMessage()); + $this->assertStringContainsString('RuntimeException', $e->getMessage()); + $this->assertStringNotContainsString('database gone', $e->getMessage(), 'the class, not the message'); + } + }//end testAFailingStoreIsMappingUnavailable() + + /** + * The hash ignores the order of an object's keys but not the order of the field mappings. + * + * @return void + */ + public function testTheHashIsCanonical(): void { + $pack = $this->shippedUsage(); + $reordered = array_reverse($pack, true); + $reordered['fieldMappings'][0]['transform'] = array_reverse($reordered['fieldMappings'][0]['transform'], true); + $swapped = $pack; + $swapped['fieldMappings'] = array_reverse($pack['fieldMappings']); + + $this->assertSame((new CmdbImportPackMarker())->hash(pack: $pack), (new CmdbImportPackMarker())->hash(pack: $reordered)); + $this->assertNotSame((new CmdbImportPackMarker())->hash(pack: $pack), (new CmdbImportPackMarker())->hash(pack: $swapped)); + $this->assertMatchesRegularExpression('/^[0-9a-f]{64}$/', (new CmdbImportPackMarker())->hash(pack: $pack)); + }//end testTheHashIsCanonical() +}//end class diff --git a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php index 74fdbe784..f144495bf 100644 --- a/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php +++ b/tests/Unit/Service/Cmdb/CmdbImportProfileTest.php @@ -21,12 +21,16 @@ namespace OCA\Stackiq\Tests\Unit\Service\Cmdb; require_once __DIR__ . '/../../Support/CmdbTestSupport.php'; +require_once __DIR__ . '/../../Support/FakeMigrationPackService.php'; use OCA\OpenRegister\Service\MigrationPack\MappingEngine; use OCA\OpenRegister\Service\MigrationPack\PackDefinitionValidator; use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackMarker; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackStore; use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\FakeMigrationPackService; use PHPUnit\Framework\TestCase; use Psr\Container\ContainerInterface; @@ -356,7 +360,8 @@ public function testTheOverviewShowsWhatTheImportRuns(): void { $overview = $profile->mappingOverview(); - $this->assertSame(['profile', 'packs'], array_keys($overview)); + $this->assertSame(['profile', 'packs', 'openRegisterPacks'], array_keys($overview)); + $this->assertFalse($overview['openRegisterPacks'], 'no OpenRegister pack store in this container'); $this->assertSame('topdesk-cmdb', $overview['profile']['id']); $this->assertSame($profile->sheets(), $overview['profile']['sheets']); $this->assertSame(['Beheerde Applicaties CMDB', 'Onbeh Applicaties CMDB'], $overview['profile']['sheetPrecedence']); @@ -378,6 +383,14 @@ public function testTheOverviewShowsWhatTheImportRuns(): void { $this->assertSame($profile->createOnlyDefaults(target: $pack['target']), $pack['createOnlyDefaults'], $pack['target']); $this->assertSame([], array_intersect($pack['createOnlyFields'], array_diff(array_keys($pack['createOnlyDefaults']), array_column($pack['fieldMappings'], 'target'))), $pack['target']); $this->assertSame($profile->neverWrittenOnUpdate(target: $pack['target']), $pack['neverWritten'], $pack['target']); + + $this->assertSame('file', $pack['source'], $pack['target']); + $this->assertNull($pack['packId'], $pack['target']); + $this->assertNull($pack['storedVersion'], $pack['target']); + $this->assertSame($shipped['version'], $pack['shippedVersion'], $pack['target']); + $this->assertNull($pack['edited'], $pack['target']); + $this->assertFalse($pack['newerShipped'], $pack['target']); + $this->assertSame($pack['target'] !== 'municipality', $pack['editable'], $pack['target']); } $directory = $this->copyOfShippedDirectory(); @@ -393,6 +406,328 @@ public function testTheOverviewShowsWhatTheImportRuns(): void { } }//end testTheOverviewShowsWhatTheImportRuns() + /** + * A shipped pack, decoded. + * + * @param string $file The file name. + * + * @return array + */ + private function shipped(string $file): array { + return json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/' . $file), true); + }//end shipped() + + /** + * A profile whose container holds the fake OpenRegister service and an app config. + * + * @param FakeMigrationPackService $service The fake service. + * @param array $config The app config values. + * + * @return CmdbImportProfile + */ + private function storedProfile(FakeMigrationPackService $service, array &$config): CmdbImportProfile { + return new CmdbImportProfile( + container: $service->container(test: $this, appConfig: FakeMigrationPackService::appConfig(test: $this, values: $config)) + ); + }//end storedProfile() + + /** + * The import runs the pack an administrator edited in OpenRegister; missing packs are seeded, the municipality stays a file. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testAStoredPackIsWhatTheImportRuns(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $module = (new CmdbImportPackMarker())->seed(shipped: $this->shipped(file: 'topdesk-module.json')); + $module['fieldMappings'][7]['transform']['map']['Cloud'] = ['SaaS']; + $module['version'] = '2.0.1'; + $moduleId = $service->put(definition: $module); + $config = []; + + $profile = $this->storedProfile(service: $service, config: $config); + $overview = $profile->mappingOverview(); + + $this->assertSame(['SaaS'], $profile->pack(target: 'module')['fieldMappings'][7]['transform']['map']['Cloud']); + $this->assertArrayNotHasKey(CmdbImportPackMarker::KEY, $profile->pack(target: 'module'), 'the engine gets the pack without the marker'); + $this->assertSame( + ['create stackiq-topdesk-manufacturer', 'create stackiq-topdesk-usage', 'create stackiq-topdesk-business-owner'], + $service->writes, + 'the missing sheet packs are seeded; the municipality pack never is' + ); + $this->assertTrue($overview['openRegisterPacks']); + $this->assertFalse($profile->usesShippedFallback()); + + $byTarget = array_column($overview['packs'], null, 'target'); + $this->assertSame('openregister', $byTarget['module']['source']); + $this->assertSame($moduleId, $byTarget['module']['packId']); + $this->assertSame('2.0.1', $byTarget['module']['storedVersion']); + $this->assertSame('2.0.1', $byTarget['module']['version']); + $this->assertSame('2.0.0', $byTarget['module']['shippedVersion']); + $this->assertTrue($byTarget['module']['edited']); + $this->assertFalse($byTarget['module']['newerShipped'], 'seeded from the pack that still ships'); + $this->assertTrue($byTarget['module']['editable']); + $this->assertSame('openregister', $byTarget['usage']['source']); + $this->assertFalse($byTarget['usage']['edited']); + $this->assertSame('file', $byTarget['municipality']['source']); + $this->assertNull($byTarget['municipality']['packId']); + $this->assertFalse($byTarget['municipality']['editable']); + + $recorded = json_decode($config[CmdbImportPackStore::CONFIG_KEY], true); + $this->assertSame(['stackiq-topdesk-business-owner', 'stackiq-topdesk-manufacturer', 'stackiq-topdesk-module', 'stackiq-topdesk-usage'], array_keys($recorded)); + $this->assertSame($moduleId, $recorded['stackiq-topdesk-module']); + }//end testAStoredPackIsWhatTheImportRuns() + + /** + * An unedited stored pack from an older release follows the shipped pack; an edited one is kept and reported. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testUneditedPacksFollowTheReleaseAndEditedOnesAreReported(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $olderUsage = $this->shipped(file: 'topdesk-usage.json'); + $olderUsage['version'] = '1.9.0'; + unset($olderUsage['fieldMappings'][0]['transform']['map']['Besteld']); + $usageId = $service->put(definition: (new CmdbImportPackMarker())->seed(shipped: $olderUsage)); + + $olderModule = $this->shipped(file: 'topdesk-module.json'); + $olderModule['version'] = '1.9.0'; + $editedModule = (new CmdbImportPackMarker())->seed(shipped: $olderModule); + $editedModule['fieldMappings'][0]['transform'] = ['type' => 'trim']; + $editedModule['description'] = 'Edited by the administrator'; + $moduleId = $service->put(definition: $editedModule); + $config = []; + + $profile = $this->storedProfile(service: $service, config: $config); + $byTarget = array_column($profile->mappingOverview()['packs'], null, 'target'); + + $this->assertContains('update ' . $usageId, $service->writes); + $this->assertNotContains('update ' . $moduleId, $service->writes); + $this->assertSame('Acquisition', $profile->pack(target: 'usage')['fieldMappings'][0]['transform']['map']['Besteld']); + $this->assertFalse($byTarget['usage']['edited']); + $this->assertFalse($byTarget['usage']['newerShipped']); + $this->assertSame('2.0.0', $byTarget['usage']['storedVersion']); + + $this->assertSame('Edited by the administrator', $profile->pack(target: 'module')['description']); + $this->assertTrue($byTarget['module']['edited']); + $this->assertTrue($byTarget['module']['newerShipped']); + $this->assertSame('1.9.0', $byTarget['module']['storedVersion']); + $this->assertSame('2.0.0', $byTarget['module']['shippedVersion']); + }//end testUneditedPacksFollowTheReleaseAndEditedOnesAreReported() + + /** + * A stored pack OpenRegister's validator refuses stops the import, naming the pack and its id. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testAnInvalidStoredPackIsMappingUnavailable(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $usage = (new CmdbImportPackMarker())->seed(shipped: $this->shipped(file: 'topdesk-usage.json')); + $usage['fieldMappings'][0]['transform'] = ['type' => 'uppercase']; + $id = $service->put(definition: $usage); + $config = []; + + try { + $this->storedProfile(service: $service, config: $config)->load(); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertSame(503, $e->getHttpStatus()); + $this->assertStringStartsWith('OpenRegister migration pack stackiq-topdesk-usage (id ' . $id . ') is invalid: ', $e->getMessage()); + $this->assertStringContainsString('uppercase', $e->getMessage()); + $this->assertSame(['pack' => 'stackiq-topdesk-usage', 'packId' => $id], $e->getDetails()); + } + }//end testAnInvalidStoredPackIsMappingUnavailable() + + /** + * Edits a stored pack may not make, and the rule each one breaks. + * + * @return array + */ + public static function brokenRules(): array { + $target = static fn (int $index, string $field): callable => static function (array $pack) use ($index, $field): array { + $pack['fieldMappings'][$index]['target'] = $field; + return $pack; + }; + + return [ + 'module name no longer required' => ['topdesk-module.json', static function (array $pack): array { + $pack['fieldMappings'][0]['required'] = false; + return $pack; + }, '"name" must be mapped as required'], + 'externalNumber from another column' => ['topdesk-module.json', static function (array $pack): array { + $pack['fieldMappings'][1]['source'] = 'Applicatie Code'; + return $pack; + }, 'the key column "APPID" must be mapped to "externalNumber" as required'], + 'the object metadata' => ['topdesk-module.json', $target(2, '@self'), '"@self" is set by the import and cannot be mapped'], + 'the match key' => ['topdesk-module.json', $target(2, 'externalKey'), '"externalKey" is set by the import'], + 'a field never written' => ['topdesk-module.json', $target(2, 'publicationDate'), '"publicationDate" is set by the import'], + 'a field the schema lacks' => ['topdesk-module.json', $target(2, 'secretNotes'), '"secretNotes" is not a field the module pack may map'], + 'the usage provider' => ['topdesk-usage.json', $target(0, 'provider'), '"provider" is set by the import'], + 'the usage consumer' => ['topdesk-usage.json', $target(1, 'consumer'), '"consumer" is set by the import'], + 'the usage business owner' => ['topdesk-usage.json', $target(1, 'businessOwner'), '"businessOwner" is set by the import'], + 'a default the import sets' => ['topdesk-usage.json', static function (array $pack): array { + $pack['defaults'] = ['module' => 'x']; + return $pack; + }, 'default "module" is set by the import'], + 'the manufacturer type mapped' => ['topdesk-manufacturer.json', $target(0, 'type'), '"type" is set by the import'], + 'the manufacturer type default changed' => ['topdesk-manufacturer.json', static function (array $pack): array { + $pack['defaults']['type'] = 'Municipality'; + return $pack; + }, 'default "type" must stay "Supplier"'], + 'an owner field that is no contact field' => ['topdesk-business-owner.json', $target(1, 'email'), '"email" is not a field the businessOwner pack may map'], + ]; + }//end brokenRules() + + /** + * A stored pack that breaks a rule of the import stops it, naming the pack, its id and the rule. + * + * @param string $file The shipped file the stored pack starts from. + * @param callable $edit The edit. + * @param string $rule The text of the broken rule. + * + * @return void + * + * @dataProvider brokenRules + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testAStoredPackThatBreaksAnImportRuleIsRefused(string $file, callable $edit, string $rule): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $pack = $edit((new CmdbImportPackMarker())->seed(shipped: $this->shipped(file: $file))); + $id = $service->put(definition: $pack); + $config = []; + + try { + $this->storedProfile(service: $service, config: $config)->load(); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertStringStartsWith('OpenRegister migration pack ' . $pack['id'] . ' (id ' . $id . ') breaks a rule of the CMDB import: ', $e->getMessage()); + $this->assertStringContainsString($rule, $e->getMessage()); + $this->assertSame(['pack' => $pack['id'], 'packId' => $id], $e->getDetails()); + } + }//end testAStoredPackThatBreaksAnImportRuleIsRefused() + + /** + * Without OpenRegister's store, and no pack ever stored there, the import uses the shipped files and says so. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testWithoutTheStoreBeforeASeedTheShippedFilesAreUsed(): void { + CmdbTestSupport::loadMigrationPack(); + $config = []; + $container = $this->createMock(ContainerInterface::class); + $appConfig = FakeMigrationPackService::appConfig(test: $this, values: $config); + $container->method('has')->willReturnCallback(fn (string $id): bool => $id === \OCP\IAppConfig::class); + $container->method('get')->willReturn($appConfig); + + $profile = new CmdbImportProfile(container: $container); + $overview = $profile->mappingOverview(); + + $this->assertTrue($profile->usesShippedFallback()); + $this->assertFalse($overview['openRegisterPacks']); + $this->assertSame(['file'], array_values(array_unique(array_column($overview['packs'], 'source')))); + $this->assertSame($this->shipped(file: 'topdesk-module.json'), $profile->pack(target: 'module')); + }//end testWithoutTheStoreBeforeASeedTheShippedFilesAreUsed() + + /** + * Once the packs were stored in OpenRegister, a store that is gone fails closed, naming the packs. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testWithoutTheStoreAfterASeedTheImportFailsClosed(): void { + CmdbTestSupport::loadMigrationPack(); + $config = [CmdbImportPackStore::CONFIG_KEY => (string)json_encode(['stackiq-topdesk-module' => 41, 'stackiq-topdesk-usage' => 43])]; + $container = $this->createMock(ContainerInterface::class); + $appConfig = FakeMigrationPackService::appConfig(test: $this, values: $config); + $container->method('has')->willReturnCallback(fn (string $id): bool => $id === \OCP\IAppConfig::class); + $container->method('get')->willReturn($appConfig); + + try { + (new CmdbImportProfile(container: $container))->load(); + $this->fail('MAPPING_UNAVAILABLE expected'); + } catch (CmdbImportException $e) { + $this->assertSame('MAPPING_UNAVAILABLE', $e->getErrorCode()); + $this->assertStringContainsString('OpenRegister migration pack store is not available', $e->getMessage()); + $this->assertStringContainsString('stackiq-topdesk-module (id 41)', $e->getMessage()); + $this->assertStringContainsString('stackiq-topdesk-usage (id 43)', $e->getMessage()); + $this->assertSame(['packs' => ['stackiq-topdesk-module' => 41, 'stackiq-topdesk-usage' => 43]], $e->getDetails()); + } + }//end testWithoutTheStoreAfterASeedTheImportFailsClosed() + + /** + * A column a stored pack maps with a date transform is converted from an Excel date, and is read. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testADateColumnOfAStoredPackIsConverted(): void { + CmdbTestSupport::loadMigrationPack(); + $service = new FakeMigrationPackService(); + $usage = (new CmdbImportPackMarker())->seed(shipped: $this->shipped(file: 'topdesk-usage.json')); + $usage['fieldMappings'][] = [ + 'source' => 'In productie sinds', + 'target' => 'startDateInProduction', + 'transform' => ['type' => 'date', 'sourceFormat' => '!Y-m-d', 'targetFormat' => 'Y-m-d'], + ]; + $service->put(definition: $usage); + $config = []; + + $profile = $this->storedProfile(service: $service, config: $config); + + $this->assertSame( + ['Datum', 'Referentie datum wijziging', 'End-of-Life Functioneel', 'In productie sinds'], + $profile->dateColumns() + ); + $this->assertContains('In productie sinds', $profile->referencedColumns(), 'a mapped column is read'); + $this->assertSame( + ['Datum', 'Referentie datum wijziging', 'End-of-Life Functioneel'], + (new CmdbImportProfile(container: $this->emptyContainer()))->dateColumns(), + 'the shipped packs add no column the profile does not list' + ); + }//end testADateColumnOfAStoredPackIsConverted() + + /** + * The separator the import drops empty note parts on is the pack's own. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testTheConcatSeparatorIsThePacks(): void { + CmdbTestSupport::loadMigrationPack(); + $shipped = new CmdbImportProfile(container: $this->emptyContainer()); + $this->assertSame(' / ', $shipped->concatSeparator(target: 'usage', field: 'interneAnnotation')); + $this->assertNull($shipped->concatSeparator(target: 'usage', field: 'status'), 'no concat mapping writes it'); + + $service = new FakeMigrationPackService(); + $usage = (new CmdbImportPackMarker())->seed(shipped: $this->shipped(file: 'topdesk-usage.json')); + $usage['fieldMappings'][3]['transform']['separator'] = ' | '; + $service->put(definition: $usage); + $config = []; + $this->assertSame(' | ', $this->storedProfile(service: $service, config: $config)->concatSeparator(target: 'usage', field: 'interneAnnotation')); + + unset($usage['fieldMappings'][3]['transform']['separator']); + $service = new FakeMigrationPackService(); + $service->put(definition: $usage); + $this->assertSame(' ', $this->storedProfile(service: $service, config: $config)->concatSeparator(target: 'usage', field: 'interneAnnotation'), "OpenRegister's default"); + }//end testTheConcatSeparatorIsThePacks() + /** * The upload limit is readable without OpenRegister. * diff --git a/tests/Unit/Service/CmdbExportImportServiceTest.php b/tests/Unit/Service/CmdbExportImportServiceTest.php index d81417ea7..31c23aa10 100644 --- a/tests/Unit/Service/CmdbExportImportServiceTest.php +++ b/tests/Unit/Service/CmdbExportImportServiceTest.php @@ -32,10 +32,13 @@ namespace OCA\Stackiq\Tests\Unit\Service; require_once __DIR__ . '/../Support/CmdbTestSupport.php'; +require_once __DIR__ . '/../Support/FakeMigrationPackService.php'; use OCA\OpenRegister\Contract\ObjectEntityInterface; use OCA\OpenRegister\Contract\ObjectServiceInterface; use OCA\Stackiq\Exception\CmdbImportException; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackMarker; +use OCA\Stackiq\Service\Cmdb\CmdbImportPackStore; use OCA\Stackiq\Service\Cmdb\CmdbImportProfile; use OCA\Stackiq\Service\Cmdb\CmdbRowNormaliser; use OCA\Stackiq\Service\Cmdb\CmdbWorkbookReader; @@ -44,6 +47,7 @@ use OCA\Stackiq\Service\SettingsService; use OCA\Stackiq\Service\StackiqContactSyncService; use OCA\Stackiq\Tests\Unit\Support\CmdbTestSupport; +use OCA\Stackiq\Tests\Unit\Support\FakeMigrationPackService; use OCP\IAppConfig; use OCP\ICache; use OCP\ICacheFactory; @@ -710,16 +714,26 @@ public function read(string $path, CmdbImportProfile $profile): array { * @param CmdbWorkbookReader|null $reader The reader; null is the real one. * @param string|null $profileDir A profile directory other than the shipped one. * @param array $config The voorzieningen config. + * @param FakeMigrationPackService|false|null $packs OpenRegister's pack store: null is an empty one, false none at all. * * @return CmdbExportImportService */ - private function service(?CmdbWorkbookReader $reader = null, ?string $profileDir = null, array $config = ['register' => '20']): CmdbExportImportService { + private function service( + ?CmdbWorkbookReader $reader = null, + ?string $profileDir = null, + array $config = ['register' => '20'], + FakeMigrationPackService|false|null $packs = null + ): CmdbExportImportService { $objectService = $this->objectService(); $schemaMapper = $this->schemaMapper(); + if ($packs === null) { + $packs = new FakeMigrationPackService(); + } + $container = $this->createMock(ContainerInterface::class); - $container->method('has')->willReturn(false); + $container->method('has')->willReturnCallback(fn (string $id): bool => $packs !== false && $id === CmdbImportPackStore::SERVICE_CLASS); $container->method('get')->willReturnCallback( - function (string $id) use ($objectService, $schemaMapper) { + function (string $id) use ($objectService, $schemaMapper, $packs) { if ($id === ObjectServiceInterface::class) { return $objectService; } @@ -728,6 +742,10 @@ function (string $id) use ($objectService, $schemaMapper) { return $schemaMapper; } + if ($packs !== false && $id === CmdbImportPackStore::SERVICE_CLASS) { + return $packs; + } + throw new RuntimeException('not in this container: ' . $id); } ); @@ -970,6 +988,67 @@ public function testAnAmbiguousMunicipalityNameIsRefused(): void { $this->assertSame([], $this->saves); }//end testAnAmbiguousMunicipalityNameIsRefused() + /** + * Without OpenRegister's pack store, and nothing stored there before, the import runs the shipped mapping and says so once. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testWithoutThePackStoreTheShippedMappingIsUsedWithAWarning(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Rotterdam', type: 'Municipality'); + + $report = $this->service(reader: $this->rowsReader(rows: [$this->row(appId: '1')]), packs: false) + ->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame(1, $report['summary']['created']); + $this->assertSame( + [['sheet' => '', 'message' => "OpenRegister's migration pack store is not available, so this import used the mapping that ships with stackiq."]], + $report['importWarnings'] + ); + $this->assertSame('In production', $this->objects(self::USAGE)[0]['status']); + }//end testWithoutThePackStoreTheShippedMappingIsUsedWithAWarning() + + /** + * A pack edited in OpenRegister is what the import runs: its lookup, its concat separator and a new date column. + * + * @return void + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + */ + public function testAPackEditedInOpenRegisterIsWhatTheImportRuns(): void { + $this->seedOrganisation(uuid: 'muni-1', name: 'Gemeente Rotterdam', type: 'Municipality'); + $packs = new FakeMigrationPackService(); + $usage = (new CmdbImportPackMarker())->seed( + shipped: json_decode((string)file_get_contents(CmdbTestSupport::appRoot() . '/lib/Settings/cmdb-import/topdesk-usage.json'), true) + ); + $usage['fieldMappings'][0]['transform']['map']['In productie'] = 'Planned'; + $usage['fieldMappings'][3]['transform']['separator'] = ' | '; + $usage['fieldMappings'][] = [ + 'source' => 'In productie sinds', + 'target' => 'startDateInProduction', + 'transform' => ['type' => 'date', 'sourceFormat' => '!Y-m-d', 'targetFormat' => 'Y-m-d'], + ]; + $packs->put(definition: $usage); + + $report = $this->service( + reader: $this->rowsReader(rows: [$this->row(appId: '1', cells: ['Cluster' => 'B10', 'In productie sinds' => 45111])]), + packs: $packs + )->import(path: '', options: ['municipalityUuid' => 'muni-1']); + + $this->assertSame([], $report['importWarnings'], 'the packs came from OpenRegister'); + $this->assertSame([], $report['rows'][0]['warnings']); + $stored = $this->objects(self::USAGE)[0]; + $this->assertSame('Planned', $stored['status'], 'the edited lookup'); + $this->assertSame('Beheer geregeld: ja | B10', $stored['interneAnnotation'], 'empty parts dropped on the edited separator'); + $this->assertSame('2023-07-04', $stored['startDateInProduction'], 'the new column was converted from an Excel date'); + $this->assertSame( + ['create stackiq-topdesk-module', 'create stackiq-topdesk-manufacturer', 'create stackiq-topdesk-business-owner'], + $packs->writes, + 'the other sheet packs were seeded on the first import' + ); + }//end testAPackEditedInOpenRegisterIsWhatTheImportRuns() + /** * A name that matches no municipality creates it, and the report warns about that. * diff --git a/tests/Unit/Support/FakeMigrationPackService.php b/tests/Unit/Support/FakeMigrationPackService.php new file mode 100644 index 000000000..4dea695be --- /dev/null +++ b/tests/Unit/Support/FakeMigrationPackService.php @@ -0,0 +1,286 @@ + + * @copyright 2026 Conduction B.V. + * @license EUPL-1.2 https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * @link https://github.com/ConductionNL/stackiq + * + * @spec openspec/changes/cmdb-import-mapping-view/tasks.md#task-1 + * + * SPDX-FileCopyrightText: 2026 Conduction B.V. + * SPDX-License-Identifier: EUPL-1.2 + */ + +declare(strict_types=1); + +namespace OCA\Stackiq\Tests\Unit\Support; + +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\IAppConfig; +use PHPUnit\Framework\TestCase; +use Psr\Container\ContainerInterface; +use RuntimeException; +use Throwable; + +/** + * Migration packs in memory. + */ +final class FakeMigrationPackService { + /** + * Stored rows by id: slug, definition JSON and builtin flag. + * + * @var array + */ + public array $rows = []; + + /** + * Every write, as "create " or "update ". + * + * @var array + */ + public array $writes = []; + + /** + * When set, every lookup throws it. + * + * @var Throwable|null + */ + public ?Throwable $failure = null; + + /** + * The next row id. + * + * @var int + */ + private int $nextId = 40; + + /** + * Store a row directly, as an administrator's edit or an older seed would have. + * + * @param array $definition The definition. + * @param int|null $id The row id; null takes the next one. + * + * @return int The row id. + */ + public function put(array $definition, ?int $id = null): int { + $id = ($id ?? $this->nextId++); + $this->rows[$id] = ['slug' => (string)$definition['id'], 'definition' => (string)json_encode($definition), 'builtin' => false]; + return $id; + }//end put() + + /** + * The stored definition of a row. + * + * @param int $id The row id. + * + * @return array + */ + public function definition(int $id): array { + return json_decode($this->rows[$id]['definition'], true); + }//end definition() + + /** + * Find a pack by slug. + * + * @param string $packSlug The slug. + * + * @return object + * + * @throws DoesNotExistException When no row has the slug. + */ + public function findByPackSlug(string $packSlug): object { + $this->failIfAsked(); + foreach ($this->rows as $id => $row) { + if ($row['slug'] === $packSlug) { + return $this->entity(id: $id); + } + } + + throw new DoesNotExistException('No migration pack ' . $packSlug); + }//end findByPackSlug() + + /** + * Find a pack by id. + * + * @param int $id The row id. + * + * @return object + * + * @throws DoesNotExistException When no row has the id. + */ + public function find(int $id): object { + $this->failIfAsked(); + if (isset($this->rows[$id]) === false) { + throw new DoesNotExistException('No migration pack ' . $id); + } + + return $this->entity(id: $id); + }//end find() + + /** + * Create a pack. + * + * @param array $definition The definition. + * @param string|null $ownerUid The owner. + * @param bool $builtin Whether it is built in. + * + * @return object + * + * @throws RuntimeException When the slug exists. + */ + public function create(array $definition, ?string $ownerUid = null, bool $builtin = false): object { + foreach ($this->rows as $row) { + if ($row['slug'] === $definition['id']) { + throw new RuntimeException('A migration pack with id "' . $definition['id'] . '" already exists'); + } + } + + $id = $this->put(definition: $definition); + $this->rows[$id]['builtin'] = $builtin; + $this->writes[] = 'create ' . $definition['id']; + return $this->entity(id: $id); + }//end create() + + /** + * Replace a pack's definition. + * + * @param int $id The row id. + * @param array $definition The definition. + * + * @return object + */ + public function update(int $id, array $definition): object { + $this->rows[$id]['slug'] = (string)$definition['id']; + $this->rows[$id]['definition'] = (string)json_encode($definition); + $this->writes[] = 'update ' . $id; + return $this->entity(id: $id); + }//end update() + + /** + * A container that hands out this service and, when given, an app config. + * + * @param TestCase $test The test, to build the mock with. + * @param IAppConfig|null $appConfig The app config, or null for none. + * + * @return ContainerInterface + */ + public function container(TestCase $test, ?IAppConfig $appConfig = null): ContainerInterface { + $builder = new \ReflectionMethod(TestCase::class, 'createMock'); + $container = $builder->invoke($test, ContainerInterface::class); + $known = ['OCA\OpenRegister\Service\MigrationPackService' => $this]; + if ($appConfig !== null) { + $known[IAppConfig::class] = $appConfig; + } + + $container->method('has')->willReturnCallback(fn (string $id): bool => isset($known[$id])); + $container->method('get')->willReturnCallback( + function (string $id) use ($known) { + if (isset($known[$id]) === true) { + return $known[$id]; + } + + throw new RuntimeException('not in this container: ' . $id); + } + ); + + return $container; + }//end container() + + /** + * An app config backed by an array, for the store's row ids. + * + * @param TestCase $test The test, to build the mock with. + * @param array $values Initial values by key. + * + * @return IAppConfig + */ + public static function appConfig(TestCase $test, array &$values): IAppConfig { + $builder = new \ReflectionMethod(TestCase::class, 'createMock'); + $config = $builder->invoke($test, IAppConfig::class); + $config->method('getValueString')->willReturnCallback( + function (string $app, string $key, string $default = '') use (&$values): string { + return ($values[$key] ?? $default); + } + ); + $config->method('setValueString')->willReturnCallback( + function (string $app, string $key, string $value) use (&$values): bool { + $values[$key] = $value; + return true; + } + ); + + return $config; + }//end appConfig() + + /** + * Throw the configured failure. + * + * @return void + * + * @throws Throwable The configured failure. + */ + private function failIfAsked(): void { + if ($this->failure !== null) { + throw $this->failure; + } + }//end failIfAsked() + + /** + * The entity of a row, with the getters the store calls. + * + * @param int $id The row id. + * + * @return object + */ + private function entity(int $id): object { + $row = $this->rows[$id]; + return new class($id, $row['slug'], $row['definition']) { + /** + * Constructor. + * + * @param int $id The row id. + * @param string $slug The slug. + * @param string $definition The definition JSON. + */ + public function __construct(private int $id, private string $slug, private string $definition) { + } + + /** + * The row id. + * + * @return int + */ + public function getId(): int { + return $this->id; + } + + /** + * The slug. + * + * @return string + */ + public function getPackSlug(): string { + return $this->slug; + } + + /** + * The decoded definition. + * + * @return array + */ + public function getDefinitionArray(): array { + return json_decode($this->definition, true); + } + }; + }//end entity() +}//end class From 9450c8497177ea285758d1c60fada2a1aa9b756c Mon Sep 17 00:00:00 2001 From: WilcoLouwerse Date: Mon, 12 Oct 2026 18:52:16 +0200 Subject: [PATCH 44/87] feat(cmdb-import): show where each import pack comes from and link to its editor in OpenRegister The Mapping block of the CMDB import section no longer lists each pack's columns. Per pack it now shows one line: what the pack's rows become, whether the import takes it from OpenRegister's pack store or from the shipped file, the stored and shipped versions, an Edited marker, a hint when a newer shipped version was not applied because the pack was edited, and an Edit in OpenRegister link (http(s) or server-relative addresses only). The municipality pack says it maps the import options and is not edited there. When OpenRegister's pack store is not available, the block says the shipped mapping is used. The transform helpers the tables used are removed, the unused strings are dropped from l10n, and the vitest and jest specs cover the new view. Co-Authored-By: Claude Opus 5.5 (1M context) --- l10n/en.js | 35 +- l10n/en.json | 35 +- l10n/nl.js | 35 +- l10n/nl.json | 35 +- src/utils/cmdbImport.js | 169 ++------ .../settings/sections/CmdbImport.spec.js | 3 + src/views/settings/sections/CmdbImport.vue | 2 +- .../settings/sections/CmdbImportMapping.vue | 229 ++++++----- tests/vitest/cmdbImportMapping.spec.js | 366 +++++++++++------- 9 files changed, 469 insertions(+), 440 deletions(-) diff --git a/l10n/en.js b/l10n/en.js index 60a730e51..571939b25 100644 --- a/l10n/en.js +++ b/l10n/en.js @@ -1137,35 +1137,28 @@ OC.L10N.register( "Supplier organisation (Vendor)": "Supplier organisation (Vendor)", "Municipality (from the import options)": "Municipality (from the import options)", "Business owner (contact person)": "Business owner (contact person)", - "As is": "As is", - "Trim": "Trim", - "Date": "Date", - "Lookup": "Lookup", - "Yes/no lookup": "Yes/no lookup", - "Join": "Join", - "Constant": "Constant", - "Any other value: {value}": "Any other value: {value}", - "Joined with the columns {columns}, separated by \"{separator}\"": "Joined with the columns {columns}, separated by \"{separator}\"", - "Value: {value}": "Value: {value}", - "Read as {source}, stored as {target}": "Read as {source}, stored as {target}", - "Mapping (read-only)": "Mapping (read-only)", - "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.": "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.", "Loading the mapping…": "Loading the mapping…", "The mapping could not be loaded.": "The mapping could not be loaded.", - "{file}: {name}, version {version}": "{file}: {name}, version {version}", - "This pack maps no columns": "This pack maps no columns", - "Yes": "Yes", - "No": "No", "Sheets read": "Sheets read", "Match column": "Match column", "Name column": "Name column", "Required columns": "Required columns", "Date columns": "Date columns", - "Column in the export": "Column in the export", - "Field": "Field", - "Transformation": "Transformation", "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.", - "The import mapping could not be read. The details are in the Nextcloud log.": "The import mapping could not be read. The details are in the Nextcloud log." + "The import mapping could not be read. The details are in the Nextcloud log.": "The import mapping could not be read. The details are in the Nextcloud log.", + "Mapping": "Mapping", + "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.": "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.", + "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.": "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.", + "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.": "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.", + "OpenRegister": "OpenRegister", + "Shipped file": "Shipped file", + "Version {version}": "Version {version}", + "Version {version} (shipped: {shipped})": "Version {version} (shipped: {shipped})", + "Edited": "Edited", + "Edit in OpenRegister": "Edit in OpenRegister", + "Edit in OpenRegister: {target} (opens in a new tab)": "Edit in OpenRegister: {target} (opens in a new tab)", + "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.": "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.", + "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost.": "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/en.json b/l10n/en.json index 5e5a32434..9e3614a50 100644 --- a/l10n/en.json +++ b/l10n/en.json @@ -1136,34 +1136,27 @@ "Supplier organisation (Vendor)": "Supplier organisation (Vendor)", "Municipality (from the import options)": "Municipality (from the import options)", "Business owner (contact person)": "Business owner (contact person)", - "As is": "As is", - "Trim": "Trim", - "Date": "Date", - "Lookup": "Lookup", - "Yes/no lookup": "Yes/no lookup", - "Join": "Join", - "Constant": "Constant", - "Any other value: {value}": "Any other value: {value}", - "Joined with the columns {columns}, separated by \"{separator}\"": "Joined with the columns {columns}, separated by \"{separator}\"", - "Value: {value}": "Value: {value}", - "Read as {source}, stored as {target}": "Read as {source}, stored as {target}", - "Mapping (read-only)": "Mapping (read-only)", - "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.": "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.", "Loading the mapping…": "Loading the mapping…", "The mapping could not be loaded.": "The mapping could not be loaded.", - "{file}: {name}, version {version}": "{file}: {name}, version {version}", - "This pack maps no columns": "This pack maps no columns", - "Yes": "Yes", - "No": "No", "Sheets read": "Sheets read", "Match column": "Match column", "Name column": "Name column", "Required columns": "Required columns", "Date columns": "Date columns", - "Column in the export": "Column in the export", - "Field": "Field", - "Transformation": "Transformation", "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.": "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.", - "The import mapping could not be read. The details are in the Nextcloud log.": "The import mapping could not be read. The details are in the Nextcloud log." + "The import mapping could not be read. The details are in the Nextcloud log.": "The import mapping could not be read. The details are in the Nextcloud log.", + "Mapping": "Mapping", + "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.": "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.", + "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.": "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.", + "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.": "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.", + "OpenRegister": "OpenRegister", + "Shipped file": "Shipped file", + "Version {version}": "Version {version}", + "Version {version} (shipped: {shipped})": "Version {version} (shipped: {shipped})", + "Edited": "Edited", + "Edit in OpenRegister": "Edit in OpenRegister", + "Edit in OpenRegister: {target} (opens in a new tab)": "Edit in OpenRegister: {target} (opens in a new tab)", + "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.": "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.", + "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost.": "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost." } } diff --git a/l10n/nl.js b/l10n/nl.js index e51e4af38..e3cb1682f 100644 --- a/l10n/nl.js +++ b/l10n/nl.js @@ -1205,35 +1205,28 @@ OC.L10N.register( "Supplier organisation (Vendor)": "Leveranciersorganisatie (Vendor)", "Municipality (from the import options)": "Gemeente (uit de importopties)", "Business owner (contact person)": "Business owner (contactpersoon)", - "As is": "Ongewijzigd", - "Trim": "Spaties weghalen", - "Date": "Datum", - "Lookup": "Opzoektabel", - "Yes/no lookup": "Ja/nee-opzoektabel", - "Join": "Samenvoegen", - "Constant": "Vaste waarde", - "Any other value: {value}": "Elke andere waarde: {value}", - "Joined with the columns {columns}, separated by \"{separator}\"": "Samengevoegd met de kolommen {columns}, gescheiden door \"{separator}\"", - "Value: {value}": "Waarde: {value}", - "Read as {source}, stored as {target}": "Gelezen als {source}, opgeslagen als {target}", - "Mapping (read-only)": "Mapping (alleen-lezen)", - "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.": "De mapping die de volgende import uitvoert: de tabbladen die hij leest en, per pack, welke kolom van de export naar welk veld gaat. Hij wordt gelezen uit de bestanden onder lib/Settings/cmdb-import van de app op de server; een bestand dat daar is aangepast wordt bij de volgende app-update overschreven.", "Loading the mapping…": "Mapping laden…", "The mapping could not be loaded.": "De mapping kon niet worden geladen.", - "{file}: {name}, version {version}": "{file}: {name}, versie {version}", - "This pack maps no columns": "Dit pack koppelt geen kolommen", - "Yes": "Ja", - "No": "Nee", "Sheets read": "Gelezen tabbladen", "Match column": "Koppelkolom", "Name column": "Naamkolom", "Required columns": "Verplichte kolommen", "Date columns": "Datumkolommen", - "Column in the export": "Kolom in de export", - "Field": "Veld", - "Transformation": "Transformatie", "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden getoond: OpenRegister ontbreekt of een mappingbestand is ongeldig.", - "The import mapping could not be read. The details are in the Nextcloud log.": "De mapping van de import kon niet worden gelezen. De details staan in het Nextcloud-logboek." + "The import mapping could not be read. The details are in the Nextcloud log.": "De mapping van de import kon niet worden gelezen. De details staan in het Nextcloud-logboek.", + "Mapping": "Mapping", + "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.": "De mapping die de volgende import uitvoert: de tabbladen die hij leest en, per soort gegeven, het pack dat bepaalt welke kolom van de export naar welk veld gaat.", + "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.": "De packs staan in OpenRegister, waar ze kunnen worden bekeken en bewerkt. Een pack dat daar is verwijderd, wordt bij de volgende import hersteld uit het meegeleverde bestand.", + "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.": "De packopslag van OpenRegister is niet beschikbaar, dus de import gebruikt de mapping die met stackiq wordt meegeleverd. Werk OpenRegister bij of schakel het in om de mapping daar te bekijken en te bewerken.", + "OpenRegister": "OpenRegister", + "Shipped file": "Meegeleverd bestand", + "Version {version}": "Versie {version}", + "Version {version} (shipped: {shipped})": "Versie {version} (meegeleverd: {shipped})", + "Edited": "Bewerkt", + "Edit in OpenRegister": "Bewerken in OpenRegister", + "Edit in OpenRegister: {target} (opens in a new tab)": "Bewerken in OpenRegister: {target} (opent in een nieuw tabblad)", + "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.": "Dit pack koppelt de importopties, geen kolom van de export, en wordt daarom niet in OpenRegister bewerkt.", + "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost.": "Stackiq levert een nieuwere versie van dit pack mee ({version}). Die is niet toegepast, omdat het pack in OpenRegister is bewerkt. Verwijder het pack daar om vanaf de volgende import de meegeleverde versie te gebruiken; de wijzigingen gaan dan verloren." }, "nplurals=2; plural=(n != 1);" ) diff --git a/l10n/nl.json b/l10n/nl.json index 6ab4bbf98..58b39d6d4 100644 --- a/l10n/nl.json +++ b/l10n/nl.json @@ -1204,34 +1204,27 @@ "Supplier organisation (Vendor)": "Leveranciersorganisatie (Vendor)", "Municipality (from the import options)": "Gemeente (uit de importopties)", "Business owner (contact person)": "Business owner (contactpersoon)", - "As is": "Ongewijzigd", - "Trim": "Spaties weghalen", - "Date": "Datum", - "Lookup": "Opzoektabel", - "Yes/no lookup": "Ja/nee-opzoektabel", - "Join": "Samenvoegen", - "Constant": "Vaste waarde", - "Any other value: {value}": "Elke andere waarde: {value}", - "Joined with the columns {columns}, separated by \"{separator}\"": "Samengevoegd met de kolommen {columns}, gescheiden door \"{separator}\"", - "Value: {value}": "Waarde: {value}", - "Read as {source}, stored as {target}": "Gelezen als {source}, opgeslagen als {target}", - "Mapping (read-only)": "Mapping (alleen-lezen)", - "The mapping the next import runs: the sheets it reads and, per pack, which column of the export goes to which field. It is read from the files under lib/Settings/cmdb-import of the app on the server; a file changed there is overwritten by the next app update.": "De mapping die de volgende import uitvoert: de tabbladen die hij leest en, per pack, welke kolom van de export naar welk veld gaat. Hij wordt gelezen uit de bestanden onder lib/Settings/cmdb-import van de app op de server; een bestand dat daar is aangepast wordt bij de volgende app-update overschreven.", "Loading the mapping…": "Mapping laden…", "The mapping could not be loaded.": "De mapping kon niet worden geladen.", - "{file}: {name}, version {version}": "{file}: {name}, versie {version}", - "This pack maps no columns": "Dit pack koppelt geen kolommen", - "Yes": "Ja", - "No": "Nee", "Sheets read": "Gelezen tabbladen", "Match column": "Koppelkolom", "Name column": "Naamkolom", "Required columns": "Verplichte kolommen", "Date columns": "Datumkolommen", - "Column in the export": "Kolom in de export", - "Field": "Veld", - "Transformation": "Transformatie", "The import mapping cannot be shown: OpenRegister is missing or a mapping file is invalid.": "De mapping van de import kan niet worden getoond: OpenRegister ontbreekt of een mappingbestand is ongeldig.", - "The import mapping could not be read. The details are in the Nextcloud log.": "De mapping van de import kon niet worden gelezen. De details staan in het Nextcloud-logboek." + "The import mapping could not be read. The details are in the Nextcloud log.": "De mapping van de import kon niet worden gelezen. De details staan in het Nextcloud-logboek.", + "Mapping": "Mapping", + "The mapping the next import runs: the sheets it reads and, for each kind of record, the pack that says which column of the export goes to which field.": "De mapping die de volgende import uitvoert: de tabbladen die hij leest en, per soort gegeven, het pack dat bepaalt welke kolom van de export naar welk veld gaat.", + "The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.": "De packs staan in OpenRegister, waar ze kunnen worden bekeken en bewerkt. Een pack dat daar is verwijderd, wordt bij de volgende import hersteld uit het meegeleverde bestand.", + "OpenRegister's pack store is not available, so the import uses the mapping shipped with stackiq. Update or enable OpenRegister to view and edit the mapping there.": "De packopslag van OpenRegister is niet beschikbaar, dus de import gebruikt de mapping die met stackiq wordt meegeleverd. Werk OpenRegister bij of schakel het in om de mapping daar te bekijken en te bewerken.", + "OpenRegister": "OpenRegister", + "Shipped file": "Meegeleverd bestand", + "Version {version}": "Versie {version}", + "Version {version} (shipped: {shipped})": "Versie {version} (meegeleverd: {shipped})", + "Edited": "Bewerkt", + "Edit in OpenRegister": "Bewerken in OpenRegister", + "Edit in OpenRegister: {target} (opens in a new tab)": "Bewerken in OpenRegister: {target} (opent in een nieuw tabblad)", + "This pack maps the import options, not a column of the export, so it is not edited in OpenRegister.": "Dit pack koppelt de importopties, geen kolom van de export, en wordt daarom niet in OpenRegister bewerkt.", + "Stackiq ships a newer version of this pack ({version}). It was not applied, because the pack was edited in OpenRegister. Delete the pack there to use the shipped version from the next import on; your edits are then lost.": "Stackiq levert een nieuwere versie van dit pack mee ({version}). Die is niet toegepast, omdat het pack in OpenRegister is bewerkt. Verwijder het pack daar om vanaf de volgende import de meegeleverde versie te gebruiken; de wijzigingen gaan dan verloren." } } diff --git a/src/utils/cmdbImport.js b/src/utils/cmdbImport.js index 4646c5a53..228d5a1fe 100644 --- a/src/utils/cmdbImport.js +++ b/src/utils/cmdbImport.js @@ -1052,157 +1052,66 @@ export function packTargetLabel(target) { } /** - * The words for a transformation type of OpenRegister's mapping engine. + * The words for where the import takes a pack from: OpenRegister's pack + * store, or the file shipped with stackiq. * - * @param {string|null|undefined} type The transform type, or nothing when the value is copied as is - * @return {string} The translated name, or the type itself when it is unknown + * @param {string} source The pack's `source` from the endpoint (`openregister` or `file`) + * @return {string} The translated name, or the source itself when it is unknown * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 */ -export function transformLabel(type) { - switch (type) { - case undefined: - case null: - case '': - return t('stackiq', 'As is') - case 'trim': - return t('stackiq', 'Trim') - case 'date': - return t('stackiq', 'Date') - case 'lookup': - return t('stackiq', 'Lookup') - case 'bool-map': - return t('stackiq', 'Yes/no lookup') - case 'concat': - return t('stackiq', 'Join') - case 'const': - return t('stackiq', 'Constant') +export function packSourceLabel(source) { + switch (source) { + case 'openregister': + return t('stackiq', 'OpenRegister') + case 'file': + return t('stackiq', 'Shipped file') default: - return String(type) + return String(source ?? '') } } /** - * One text per value, for the details column: a JSON array is shown as its - * members, a null as a dash, anything else as a string. + * The version of a pack as the import runs it: the stored version for a + * pack from OpenRegister, with the shipped version next to it when the two + * differ, and the shipped version for a pack read from the file. * - * @param {unknown} value The stored value - * @return {string} The text - */ -function valueText(value) { - if (value === null || value === undefined) { - return '—' - } - if (Array.isArray(value)) { - return value.map((member) => valueText(member)).join(', ') - } - if (typeof value === 'object') { - return JSON.stringify(value) - } - return String(value) -} - -/** - * The details of a transformation, as lines: the pairs of a lookup, the - * extra columns of a join, the value of a constant, the formats of a date. - * A key the page does not know is listed as it is, so nothing the engine - * reads is hidden. - * - * @param {object|null} transform The transform as the pack stores it - * @return {Array} The lines, empty for a plain trim + * @param {object} pack One entry of the endpoint's `packs` + * @return {string} The text, empty when the pack names no version * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 */ -export function transformDetails(transform) { - if (!transform || typeof transform !== 'object') { - return [] - } - const lines = [] - const known = new Set(['type']) - if (transform.map && typeof transform.map === 'object') { - known.add('map') - for (const [from, to] of Object.entries(transform.map)) { - lines.push(`${from} → ${valueText(to)}`) - } - } - if (Object.hasOwn(transform, 'default')) { - known.add('default') - lines.push( - t( - 'stackiq', - 'Any other value: {value}', - { value: valueText(transform.default) }, - AS_TEXT, - ), - ) - } - if (Array.isArray(transform.fields)) { - known.add('fields') - known.add('separator') - lines.push( - t( +export function packVersionText(pack) { + const shipped = String(pack?.shippedVersion ?? pack?.version ?? '') + const stored = String(pack?.storedVersion ?? '') + if (pack?.source === 'openregister' && stored !== '') { + if (shipped !== '' && shipped !== stored) { + return t( 'stackiq', - 'Joined with the columns {columns}, separated by "{separator}"', - { - columns: transform.fields - .map((field) => String(field)) - .join(', '), - separator: String(transform.separator ?? ''), - }, + 'Version {version} (shipped: {shipped})', + { version: stored, shipped }, AS_TEXT, - ), - ) - } - if (Object.hasOwn(transform, 'value')) { - known.add('value') - lines.push( - t( - 'stackiq', - 'Value: {value}', - { value: valueText(transform.value) }, - AS_TEXT, - ), - ) - } - if (transform.sourceFormat || transform.targetFormat) { - known.add('sourceFormat') - known.add('targetFormat') - lines.push( - t( - 'stackiq', - 'Read as {source}, stored as {target}', - { - source: String(transform.sourceFormat ?? '—'), - target: String(transform.targetFormat ?? 'Y-m-d'), - }, - AS_TEXT, - ), - ) - } - for (const [key, value] of Object.entries(transform)) { - if (!known.has(key)) { - lines.push(`${key}: ${valueText(value)}`) + ) } + return t('stackiq', 'Version {version}', { version: stored }, AS_TEXT) } - return lines + if (shipped === '') { + return '' + } + return t('stackiq', 'Version {version}', { version: shipped }, AS_TEXT) } /** - * The rows of one pack's table: one per field mapping, in the pack's order. + * The address of OpenRegister's editor for a pack, when the page may link + * to it: the pack is editable and the server gave an http(s) or + * server-relative address. Anything else (a `javascript:` address, a + * protocol-relative one, no address at all) gives no link. * * @param {object} pack One entry of the endpoint's `packs` - * @return {Array<{key: string, source: string, target: string, required: boolean, transform: string, details: Array}>} The rows + * @return {string|null} The address, or null for no link * @spec openspec/changes/cmdb-import-mapping-view/specs/cmdb-export-import/spec.md#requirement-the-admin-settings-shall-show-the-mapping-the-import-uses-req-cmdb-020 */ -export function mappingRows(pack) { - const mappings = pack?.fieldMappings - if (!Array.isArray(mappings)) { - return [] +export function packEditUrl(pack) { + if (pack?.editable !== true || typeof pack.editUrl !== 'string') { + return null } - return mappings.map((mapping, index) => ({ - key: `${mapping?.source ?? ''}:${mapping?.target ?? ''}:${index}`, - source: String(mapping?.source ?? ''), - target: String(mapping?.target ?? ''), - required: Boolean(mapping?.required), - transform: transformLabel(mapping?.transform?.type), - details: transformDetails(mapping?.transform), - })) + return /^(https?:\/\/|\/(?!\/))/i.test(pack.editUrl) ? pack.editUrl : null } diff --git a/src/views/settings/sections/CmdbImport.spec.js b/src/views/settings/sections/CmdbImport.spec.js index d0598b407..b1b0b884a 100644 --- a/src/views/settings/sections/CmdbImport.spec.js +++ b/src/views/settings/sections/CmdbImport.spec.js @@ -61,6 +61,9 @@ jest.mock('vue-material-design-icons/ChevronDown.vue', () => ({ jest.mock('vue-material-design-icons/ChevronUp.vue', () => ({ render: () => null, })) +jest.mock('vue-material-design-icons/OpenInNew.vue', () => ({ + render: () => null, +})) jest.mock('../../../components/AlwaysVisibleSection.vue', () => ({ render: () => null, })) diff --git a/src/views/settings/sections/CmdbImport.vue b/src/views/settings/sections/CmdbImport.vue index 8b5a6ff62..294062392 100644 --- a/src/views/settings/sections/CmdbImport.vue +++ b/src/views/settings/sections/CmdbImport.vue @@ -362,7 +362,7 @@ - + diff --git a/src/views/settings/sections/CmdbImportMapping.vue b/src/views/settings/sections/CmdbImportMapping.vue index 127226a9a..77861847f 100644 --- a/src/views/settings/sections/CmdbImportMapping.vue +++ b/src/views/settings/sections/CmdbImportMapping.vue @@ -15,7 +15,7 @@ - {{ t('stackiq', 'Mapping (read-only)') }} + {{ t('stackiq', 'Mapping') }}
    +

    + {{ + t( + 'stackiq', + 'The packs are kept in OpenRegister, where you can view and edit them. A pack deleted there is restored from the shipped file at the next import.', ) }}

    @@ -39,7 +49,7 @@ {{ t('stackiq', 'Loading the mapping…') }}
    - +