From d46729d8dd5847b7cd52a847a5c19d6c5ddaa31c Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Mon, 7 Sep 2026 08:14:12 +0300 Subject: [PATCH 01/18] ILLDEV-484 Add directory import endpoint --- directory/api.yaml | 437 +++++++++++++++++ directory/api/entries.go | 24 +- directory/api/impl.go | 17 +- directory/api/import.go | 65 +++ directory/api/import_contract_test.go | 39 ++ directory/api/import_test.go | 139 ++++++ directory/app/app.go | 9 +- directory/app/import_limit.go | 29 ++ directory/app/import_limit_test.go | 92 ++++ .../ModuleDescriptor-template.json | 8 + directory/domain/hierarchy.go | 18 + directory/domain/hierarchy_test.go | 25 + .../enhancedcontext/enhancedcontext_test.go | 8 +- directory/import/db/entry.go | 285 +++++++++++ directory/import/db/repo.go | 59 +++ directory/import/db/repo_test.go | 445 ++++++++++++++++++ directory/import/db/tier_network.go | 164 +++++++ directory/import/model/config.go | 141 ++++++ directory/import/model/models.go | 275 +++++++++++ directory/import/model/models_test.go | 88 ++++ directory/import/service/decode.go | 344 ++++++++++++++ directory/import/service/importer.go | 130 +++++ directory/import/service/importer_test.go | 184 ++++++++ .../006_import_business_keys.down.sql | 5 + .../006_import_business_keys.up.sql | 29 ++ directory/oapi-codegen.yaml | 1 + directory/query.sql | 37 ++ directory/sqlc.yaml | 10 +- directory/test/import_test.go | 204 ++++++++ 29 files changed, 3280 insertions(+), 31 deletions(-) create mode 100644 directory/api/import.go create mode 100644 directory/api/import_contract_test.go create mode 100644 directory/api/import_test.go create mode 100644 directory/app/import_limit.go create mode 100644 directory/app/import_limit_test.go create mode 100644 directory/domain/hierarchy.go create mode 100644 directory/domain/hierarchy_test.go create mode 100644 directory/import/db/entry.go create mode 100644 directory/import/db/repo.go create mode 100644 directory/import/db/repo_test.go create mode 100644 directory/import/db/tier_network.go create mode 100644 directory/import/model/config.go create mode 100644 directory/import/model/models.go create mode 100644 directory/import/model/models_test.go create mode 100644 directory/import/service/decode.go create mode 100644 directory/import/service/importer.go create mode 100644 directory/import/service/importer_test.go create mode 100644 directory/migrations/006_import_business_keys.down.sql create mode 100644 directory/migrations/006_import_business_keys.up.sql create mode 100644 directory/test/import_test.go diff --git a/directory/api.yaml b/directory/api.yaml index e72599ac9..60b66a8c5 100644 --- a/directory/api.yaml +++ b/directory/api.yaml @@ -5,6 +5,53 @@ info: servers: - url: /directory paths: + /import: + post: + summary: Import complete directory aggregates + description: Imports ordered NDJSON entry, tier, and network aggregates. Parent entries must appear in earlier committed records. + operationId: postImport + parameters: + - name: conflictPolicy + in: query + required: false + schema: + $ref: '#/components/schemas/ConflictPolicy' + requestBody: + required: true + content: + application/x-ndjson: + schema: + type: string + format: binary + x-ndjson-item-schema: + $ref: '#/components/schemas/ImportRecord' + responses: + '200': + description: Import result + content: + application/json: + schema: + $ref: '#/components/schemas/ImportResult' + '400': + description: Invalid import request + content: + text/plain: + schema: { type: string } + '401': + description: Access denied + content: + text/plain: + schema: { type: string } + '413': + description: Import request or record is too large + content: + text/plain: + schema: { type: string } + '500': + description: Failed to read the import stream + content: + text/plain: + schema: { type: string } /entries: get: summary: Returns all entries @@ -1320,6 +1367,396 @@ components: type: string schemas: + ConflictPolicy: + type: string + enum: [fail, skip, update] + default: fail + + ImportItemType: + type: string + enum: [entry, tier, network] + + ImportSectionResult: + type: object + additionalProperties: false + required: [imported, failed, skipped] + properties: + imported: { type: integer, format: int32 } + failed: { type: integer, format: int32 } + skipped: { type: integer, format: int32 } + + ImportItemError: + type: object + additionalProperties: false + required: [line, error] + properties: + line: { type: integer, format: int32 } + type: + $ref: '#/components/schemas/ImportItemType' + key: { type: string } + error: { type: string } + + ImportResult: + type: object + additionalProperties: false + required: [entries, tiers, networks, errors] + properties: + entries: { $ref: '#/components/schemas/ImportSectionResult' } + tiers: { $ref: '#/components/schemas/ImportSectionResult' } + networks: { $ref: '#/components/schemas/ImportSectionResult' } + errors: + type: array + items: { $ref: '#/components/schemas/ImportItemError' } + + ImportSymbolRef: + type: object + additionalProperties: false + required: [authority, symbol] + properties: + authority: { type: string, minLength: 1 } + symbol: { type: string, minLength: 1 } + + ImportEntryKey: + $ref: '#/components/schemas/ImportSymbolRef' + + ImportTierKey: + type: object + additionalProperties: false + required: [consortium, name] + properties: + consortium: { $ref: '#/components/schemas/ImportSymbolRef' } + name: { type: string, minLength: 1 } + + ImportNetworkKey: + type: object + additionalProperties: false + required: [consortium, name] + properties: + consortium: { $ref: '#/components/schemas/ImportSymbolRef' } + name: { type: string, minLength: 1 } + + ImportServiceEndpoint: + type: object + additionalProperties: false + required: [name, type, address] + properties: + name: { type: string } + type: { type: string } + address: { type: string } + + ImportAddressComponent: + type: object + additionalProperties: false + required: [seq, type, value] + properties: + seq: { type: integer, format: int32 } + type: + type: string + pattern: '^(Thoroughfare|Locality|AdministrativeArea|PostalCode|CountryCode|Other)$' + value: { type: string } + + ImportAddress: + type: object + additionalProperties: false + required: [type, addressComponents] + properties: + type: + type: string + pattern: '^(Default|Shipping|Billing|Other)$' + addressComponents: + type: array + items: { $ref: '#/components/schemas/ImportAddressComponent' } + + ImportClosure: + type: object + additionalProperties: false + required: [startDate, endDate, reason] + properties: + startDate: { type: string, format: date } + endDate: { type: string, format: date } + reason: { type: string } + + ImportLmsConfig: + type: object + additionalProperties: false + required: [address, fromAgency, fromAgencyAuthentication, toAgency, lookupUserEnabled, acceptItemEnabled, checkInItemEnabled, checkOutItemEnabled, itemLocation, requestItemRequestType, requestItemRequestScopeType, requestItemBibIdCode, requestItemEnabled, requestItemPickupLocationEnabled, requesterPickupLocation, supplierPickupLocation, requesterPatronPattern] + properties: + address: { type: string } + fromAgency: { type: string } + fromAgencyAuthentication: { type: string, nullable: true } + toAgency: { type: string, nullable: true } + lookupUserEnabled: { type: boolean, nullable: true } + acceptItemEnabled: { type: boolean, nullable: true } + checkInItemEnabled: { type: boolean, nullable: true } + checkOutItemEnabled: { type: boolean, nullable: true } + itemLocation: { type: string, nullable: true } + requestItemRequestType: { type: string, nullable: true } + requestItemRequestScopeType: { type: string, nullable: true } + requestItemBibIdCode: { type: string, nullable: true } + requestItemEnabled: { type: boolean, nullable: true } + requestItemPickupLocationEnabled: { type: boolean, nullable: true } + requesterPickupLocation: { type: string, nullable: true } + supplierPickupLocation: { type: string, nullable: true } + requesterPatronPattern: { type: string, nullable: true } + + ImportIllConfig: + type: object + additionalProperties: false + required: [iso18626Url, iso18626Vendor, lendersOfLastResort, includeRequestingAgencyInfo, includeSupplierInfo, includeReturnInfo, includeVendorNote, useOfferedCosts, noteFieldSeparator, supplierPatronPattern, duplicateCheckWindowHours] + properties: + iso18626Url: { type: string, nullable: true } + iso18626Vendor: + allOf: [{ $ref: '#/components/schemas/EntryVendor' }] + nullable: true + lendersOfLastResort: + type: array + items: { $ref: '#/components/schemas/ImportSymbolRef' } + includeRequestingAgencyInfo: { type: boolean, nullable: true } + includeSupplierInfo: { type: boolean, nullable: true } + includeReturnInfo: { type: boolean, nullable: true } + includeVendorNote: { type: boolean, nullable: true } + useOfferedCosts: { type: boolean, nullable: true } + noteFieldSeparator: { type: string, nullable: true } + supplierPatronPattern: { type: string, nullable: true } + duplicateCheckWindowHours: { type: integer, format: int32, minimum: 0, nullable: true } + + ImportCatalogConfig: + type: object + additionalProperties: false + required: [metadataUpdateMode, sru, zoom, queryConfig, holdingsFormat, metadataFormat] + properties: + metadataUpdateMode: + allOf: [{ $ref: '#/components/schemas/MetadataUpdateMode' }] + nullable: true + sru: + allOf: [{ $ref: '#/components/schemas/ImportSruConfig' }] + nullable: true + zoom: + allOf: [{ $ref: '#/components/schemas/ImportZoomConfig' }] + nullable: true + queryConfig: + allOf: [{ $ref: '#/components/schemas/ImportQueryConfig' }] + nullable: true + holdingsFormat: + allOf: [{ $ref: '#/components/schemas/ImportHoldingsParserConfig' }] + nullable: true + metadataFormat: + allOf: [{ $ref: '#/components/schemas/ImportMetadataParserConfig' }] + nullable: true + + ImportSruConfig: + type: object + additionalProperties: false + required: [address, recordSchema] + properties: + address: { type: string } + recordSchema: { type: string, nullable: true } + + ImportZoomConfig: + type: object + additionalProperties: false + required: [address, options] + properties: + address: { type: string } + options: + type: object + nullable: true + additionalProperties: { type: string } + + ImportQueryConfig: + type: object + additionalProperties: false + required: [type, identifier, isbn, issn, title] + properties: + type: { type: string, pattern: '^(cql|pqf)$', nullable: true } + identifier: { type: string, nullable: true } + isbn: { type: string, nullable: true } + issn: { type: string, nullable: true } + title: { type: string, nullable: true } + + ImportHoldingsParserConfig: + type: object + additionalProperties: false + required: [marc, marc21plus1, opac, reservoir] + properties: + marc: + allOf: [{ $ref: '#/components/schemas/ImportMarcHoldingsParserConfig' }] + nullable: true + marc21plus1: { type: object, nullable: true } + opac: { type: object, nullable: true } + reservoir: { type: object, nullable: true } + + ImportMarcHoldingsParserConfig: + type: object + additionalProperties: false + required: [callNumberSubField, itemIdSubField, locationSubField, mainField, restrictedSubField, shelvingLocationSubField] + properties: + callNumberSubField: { type: string, nullable: true } + itemIdSubField: { type: string, nullable: true } + locationSubField: { type: string, nullable: true } + mainField: { type: string, nullable: true } + restrictedSubField: { type: string, nullable: true } + shelvingLocationSubField: { type: string, nullable: true } + + ImportMetadataParserConfig: + type: object + additionalProperties: false + required: [marc21] + properties: + marc21: + allOf: [{ $ref: '#/components/schemas/ImportMarcMetadataParserConfig' }] + nullable: true + + ImportMarcMetadataParserConfig: + type: object + additionalProperties: false + required: [author, edition, identifier, isbn, issn, subtitle, title] + properties: + author: { type: string, nullable: true } + edition: { type: string, nullable: true } + identifier: { type: string, nullable: true } + isbn: { type: string, nullable: true } + issn: { type: string, nullable: true } + subtitle: { type: string, nullable: true } + title: { type: string, nullable: true } + + ImportHoldingsPolicy: + type: object + additionalProperties: false + required: [locations, shelvingLocations, locationPolicies, itemLoanPolicies] + properties: + locations: + type: array + items: { $ref: '#/components/schemas/HoldingsLocation' } + shelvingLocations: + type: array + items: { $ref: '#/components/schemas/HoldingsShelvingLocation' } + locationPolicies: + type: array + items: { $ref: '#/components/schemas/ImportHoldingsLocationPolicy' } + itemLoanPolicies: + type: array + items: { $ref: '#/components/schemas/HoldingsItemLoanPolicy' } + + ImportHoldingsLocationPolicy: + type: object + additionalProperties: false + required: [locationCode, shelvingLocationCode, supplyPreference] + properties: + locationCode: { type: string, nullable: true } + shelvingLocationCode: { type: string } + supplyPreference: { $ref: '#/components/schemas/HoldingsSupplyPreference' } + + ImportEntryData: + type: object + additionalProperties: false + required: [name, type, parent, description, organizationId, contactName, email, fromEmail, tenant, vendor, phoneNumber, lmsLocationCode, hrid, timeZone, symbols, endpoints, addresses, closures, lmsConfig, catalogConfig, illConfig, holdingsPolicy] + properties: + name: { type: string, minLength: 1 } + type: { type: string, pattern: '^(Institution|Consortium|Branch)$' } + parent: + allOf: [{ $ref: '#/components/schemas/ImportSymbolRef' }] + nullable: true + description: { type: string, nullable: true } + organizationId: { type: string, nullable: true } + contactName: { type: string, nullable: true } + email: { type: string, nullable: true } + fromEmail: { type: string, nullable: true } + tenant: { type: string, nullable: true } + vendor: + allOf: [{ $ref: '#/components/schemas/EntryVendor' }] + nullable: true + phoneNumber: { type: string, nullable: true } + lmsLocationCode: { type: string, nullable: true } + hrid: { type: string, nullable: true } + timeZone: { type: string, nullable: true } + symbols: + type: array + items: { $ref: '#/components/schemas/ImportSymbolRef' } + endpoints: + type: array + items: { $ref: '#/components/schemas/ImportServiceEndpoint' } + addresses: + type: array + items: { $ref: '#/components/schemas/ImportAddress' } + closures: + type: array + items: { $ref: '#/components/schemas/ImportClosure' } + lmsConfig: + allOf: [{ $ref: '#/components/schemas/ImportLmsConfig' }] + nullable: true + catalogConfig: + allOf: [{ $ref: '#/components/schemas/ImportCatalogConfig' }] + nullable: true + illConfig: + allOf: [{ $ref: '#/components/schemas/ImportIllConfig' }] + nullable: true + holdingsPolicy: + allOf: [{ $ref: '#/components/schemas/ImportHoldingsPolicy' }] + nullable: true + + ImportTierData: + type: object + additionalProperties: false + required: [level, type, cost, entries] + properties: + level: { type: string, pattern: '^(express|normal|rush|secondarymail|standard|urgent)$' } + type: { type: string, pattern: '^(loan|copy)$' } + cost: { type: number, format: double } + entries: + type: array + items: { $ref: '#/components/schemas/ImportSymbolRef' } + + ImportNetworkData: + type: object + additionalProperties: false + required: [priority, reciprocal, entries] + properties: + priority: { type: integer, format: int32 } + reciprocal: { type: boolean, nullable: true } + entries: + type: array + items: { $ref: '#/components/schemas/ImportSymbolRef' } + + ImportEntryRecord: + type: object + additionalProperties: false + required: [type, key, data] + properties: + type: { type: string, enum: [entry] } + key: { $ref: '#/components/schemas/ImportEntryKey' } + data: { $ref: '#/components/schemas/ImportEntryData' } + + ImportTierRecord: + type: object + additionalProperties: false + required: [type, key, data] + properties: + type: { type: string, enum: [tier] } + key: { $ref: '#/components/schemas/ImportTierKey' } + data: { $ref: '#/components/schemas/ImportTierData' } + + ImportNetworkRecord: + type: object + additionalProperties: false + required: [type, key, data] + properties: + type: { type: string, enum: [network] } + key: { $ref: '#/components/schemas/ImportNetworkKey' } + data: { $ref: '#/components/schemas/ImportNetworkData' } + + ImportRecord: + oneOf: + - $ref: '#/components/schemas/ImportEntryRecord' + - $ref: '#/components/schemas/ImportTierRecord' + - $ref: '#/components/schemas/ImportNetworkRecord' + discriminator: + propertyName: type + mapping: + entry: '#/components/schemas/ImportEntryRecord' + tier: '#/components/schemas/ImportTierRecord' + network: '#/components/schemas/ImportNetworkRecord' + NetworksResponse: type: object required: diff --git a/directory/api/entries.go b/directory/api/entries.go index 638081916..3b197d675 100644 --- a/directory/api/entries.go +++ b/directory/api/entries.go @@ -17,6 +17,7 @@ import ( "github.com/indexdata/crosslink/directory/auth" "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/domain" ) const defaultSymbolAuthority string = "TEST" @@ -41,23 +42,6 @@ func maybeUpdateEntryVendor(cur *string, patch nullable.Nullable[EntryVendor]) * return &value } -func isValidParentForType(entryType EntryType, parentEntry *db.Entry) (bool, string) { - switch entryType { - case "Institution": - if parentEntry.Type == "Consortium" { - return true, "" - } - return false, "Institution parent must be of type Consortium" - case "Branch": - if parentEntry.Type == "Institution" { - return true, "" - } - return false, "Branch parent must be of type Institution" - default: - return false, "Invalid type to have parent" - } -} - func scanEntryRow(rows pgx.Rows) (Entry, int, error) { var ( id uuid.UUID @@ -677,7 +661,7 @@ func (a ApiImpl) AddEntry(ctx context.Context, request AddEntryRequestObject) (A slog.ErrorContext(ctx, "failed to fetch parent entry", "error", err) return AddEntry500TextResponse("Internal server error"), nil } - validParent, reason := isValidParentForType(entryType, &parentEntry) + validParent, reason := domain.ValidParentForType(string(entryType), parentEntry.Type) if !validParent { return AddEntry400TextResponse("Invalid entry for parent: " + reason), nil } @@ -911,7 +895,7 @@ func (a ApiImpl) UpdateEntry(ctx context.Context, request UpdateEntryRequestObje } if parent != nil { - validParent, reason := isValidParentForType(EntryType(resultingType), &parentEntry) + validParent, reason := domain.ValidParentForType(resultingType, parentEntry.Type) if !validParent { return UpdateEntry400TextResponse("Invalid entry for parent: " + reason), nil } @@ -926,7 +910,7 @@ func (a ApiImpl) UpdateEntry(ctx context.Context, request UpdateEntryRequestObje resultingParent := orig resultingParent.Type = resultingType for _, child := range children { - valid, reason := isValidParentForType(EntryType(child.Type), &resultingParent) + valid, reason := domain.ValidParentForType(child.Type, resultingParent.Type) if !valid { return UpdateEntry400TextResponse("Entry type is invalid for existing child: " + reason), nil } diff --git a/directory/api/impl.go b/directory/api/impl.go index d3fa023c1..c58630bb8 100644 --- a/directory/api/impl.go +++ b/directory/api/impl.go @@ -1,19 +1,28 @@ package api import ( + "context" + "io" + "github.com/jackc/pgx/v5/pgxpool" "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/import/model" ) +type AggregateImporter interface { + Import(context.Context, model.ConflictPolicy, io.Reader) (model.ImportResult, error) +} + type ApiImpl struct { - pool *pgxpool.Pool - queries *db.Queries + pool *pgxpool.Pool + queries *db.Queries + importer AggregateImporter } // Make sure we conform to StrictServerInterface var _ StrictServerInterface = (*ApiImpl)(nil) -func NewApiImpl(pool *pgxpool.Pool, queries *db.Queries) ApiImpl { - return ApiImpl{pool: pool, queries: queries} +func NewApiImpl(pool *pgxpool.Pool, queries *db.Queries, importer AggregateImporter) ApiImpl { + return ApiImpl{pool: pool, queries: queries, importer: importer} } diff --git a/directory/api/import.go b/directory/api/import.go new file mode 100644 index 000000000..dce9b6fc7 --- /dev/null +++ b/directory/api/import.go @@ -0,0 +1,65 @@ +package api + +import ( + "context" + "errors" + "net/http" + + "github.com/indexdata/crosslink/directory/auth" + "github.com/indexdata/crosslink/directory/import/model" + importservice "github.com/indexdata/crosslink/directory/import/service" +) + +func (a ApiImpl) PostImport(ctx context.Context, request PostImportRequestObject) (PostImportResponseObject, error) { + authData := auth.GetAuthData(ctx) + if authData == nil || !authData.HasRole(auth.ConsortialAdminRole) { + return PostImport401TextResponse("Access denied"), nil + } + + policyValue := "" + if request.Params.ConflictPolicy != nil { + policyValue = string(*request.Params.ConflictPolicy) + } + policy, err := model.ParseConflictPolicy(policyValue) + if err != nil { + return PostImport400TextResponse(err.Error()), nil + } + if request.Body == nil || request.Body == http.NoBody { + return PostImport400TextResponse("body is required"), nil + } + if a.importer == nil { + return PostImport500TextResponse("import service is unavailable"), nil + } + + result, err := a.importer.Import(ctx, policy, request.Body) + if err != nil { + var maxBytesError *http.MaxBytesError + if errors.Is(err, importservice.ErrRecordTooLarge) || errors.As(err, &maxBytesError) { + return PostImport413TextResponse("import request too large"), nil + } + return PostImport500TextResponse("failed to read import request"), nil + } + return PostImport200JSONResponse(mapImportResult(result)), nil +} + +func mapImportResult(result model.ImportResult) ImportResult { + errors := make([]ImportItemError, 0, len(result.Errors)) + for _, source := range result.Errors { + item := ImportItemError{Line: source.Line, Key: source.Key, Error: source.Error} + if source.Type != nil { + value := ImportItemType(*source.Type) + item.Type = &value + } + errors = append(errors, item) + } + return ImportResult{ + Entries: mapImportSection(result.Entries), + Tiers: mapImportSection(result.Tiers), + Networks: mapImportSection(result.Networks), + Errors: errors, + } +} + +func mapImportSection(section model.ImportSectionResult) ImportSectionResult { + return ImportSectionResult{Imported: section.Imported, Failed: section.Failed, Skipped: section.Skipped} +} diff --git a/directory/api/import_contract_test.go b/directory/api/import_contract_test.go new file mode 100644 index 000000000..bf4e19794 --- /dev/null +++ b/directory/api/import_contract_test.go @@ -0,0 +1,39 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImportOpenAPIContract(t *testing.T) { + spec, err := GetSpec() + require.NoError(t, err) + operation := spec.Paths.Find("/import") + require.NotNil(t, operation) + require.NotNil(t, operation.Post) + + post := operation.Post + require.Len(t, post.Parameters, 1) + require.Equal(t, "conflictPolicy", post.Parameters[0].Value.Name) + require.False(t, post.Parameters[0].Value.Required) + require.Equal(t, "fail", post.Parameters[0].Value.Schema.Value.Default) + require.NotNil(t, post.RequestBody) + media := post.RequestBody.Value.Content.Get("application/x-ndjson") + require.NotNil(t, media) + require.NotNil(t, media.Schema) + require.Contains(t, media.Extensions, "x-ndjson-item-schema") + for _, status := range []string{"200", "400", "401", "413", "500"} { + require.NotNil(t, post.Responses.Value(status), status) + } + + for _, name := range []string{"ImportEntryRecord", "ImportTierRecord", "ImportNetworkRecord"} { + schema := spec.Components.Schemas[name].Value + require.False(t, schema.AdditionalProperties.Has != nil && *schema.AdditionalProperties.Has, name) + require.ElementsMatch(t, []string{"type", "key", "data"}, schema.Required, name) + } + entryData := spec.Components.Schemas["ImportEntryData"].Value + require.Contains(t, entryData.Required, "parent") + require.Contains(t, entryData.Required, "lmsConfig") + require.Contains(t, entryData.Required, "holdingsPolicy") +} diff --git a/directory/api/import_test.go b/directory/api/import_test.go new file mode 100644 index 000000000..5c3c4c918 --- /dev/null +++ b/directory/api/import_test.go @@ -0,0 +1,139 @@ +package api + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/indexdata/crosslink/directory/auth" + "github.com/indexdata/crosslink/directory/import/model" + importservice "github.com/indexdata/crosslink/directory/import/service" + apiValidator "github.com/oapi-codegen/nethttp-middleware" + "github.com/stretchr/testify/require" +) + +type recordingAggregateImporter struct { + policy model.ConflictPolicy + result model.ImportResult + err error + calls int +} + +func (i *recordingAggregateImporter) Import(_ context.Context, policy model.ConflictPolicy, input io.Reader) (model.ImportResult, error) { + i.calls++ + i.policy = policy + _, _ = io.Copy(io.Discard, input) + return i.result, i.err +} + +func TestPostImportDefaultsPolicyAndMapsResult(t *testing.T) { + importer := &recordingAggregateImporter{result: model.ImportResult{ + Entries: model.ImportSectionResult{Imported: 1}, + Errors: []model.ImportItemError{}, + }} + impl := NewApiImpl(nil, nil, importer) + + response, err := impl.PostImport(consortialAdminContext(t), PostImportRequestObject{Body: strings.NewReader("record")}) + + require.NoError(t, err) + require.IsType(t, PostImport200JSONResponse{}, response) + require.Equal(t, model.ConflictPolicyFail, importer.policy) + mapped := ImportResult(response.(PostImport200JSONResponse)) + require.Equal(t, int32(1), mapped.Entries.Imported) + require.Empty(t, mapped.Errors) +} + +func TestPostImportRejectsUnauthorizedCallerBeforeReadingBody(t *testing.T) { + importer := &recordingAggregateImporter{} + impl := NewApiImpl(nil, nil, importer) + + response, err := impl.PostImport(context.Background(), PostImportRequestObject{Body: strings.NewReader("record")}) + + require.NoError(t, err) + require.IsType(t, PostImport401TextResponse(""), response) + require.Zero(t, importer.calls) +} + +func TestPostImportValidatesPolicyAndBody(t *testing.T) { + importer := &recordingAggregateImporter{} + impl := NewApiImpl(nil, nil, importer) + unknown := ConflictPolicy("unknown") + + response, err := impl.PostImport(consortialAdminContext(t), PostImportRequestObject{Params: PostImportParams{ConflictPolicy: &unknown}, Body: strings.NewReader("record")}) + require.NoError(t, err) + require.IsType(t, PostImport400TextResponse(""), response) + require.Zero(t, importer.calls) + + response, err = impl.PostImport(consortialAdminContext(t), PostImportRequestObject{Body: nil}) + require.NoError(t, err) + require.IsType(t, PostImport400TextResponse(""), response) + require.Zero(t, importer.calls) +} + +func TestPostImportMapsRecordAndBodyLimitsTo413(t *testing.T) { + for name, importErr := range map[string]error{ + "record": importservice.ErrRecordTooLarge, + "body": &http.MaxBytesError{Limit: 128}, + } { + t.Run(name, func(t *testing.T) { + importer := &recordingAggregateImporter{err: importErr} + impl := NewApiImpl(nil, nil, importer) + response, err := impl.PostImport(consortialAdminContext(t), PostImportRequestObject{Body: strings.NewReader("record")}) + require.NoError(t, err) + require.IsType(t, PostImport413TextResponse(""), response) + }) + } +} + +func TestPostImportMapsFatalReaderErrorTo500(t *testing.T) { + importer := &recordingAggregateImporter{err: errors.New("read failed")} + impl := NewApiImpl(nil, nil, importer) + response, err := impl.PostImport(consortialAdminContext(t), PostImportRequestObject{Body: strings.NewReader("record")}) + require.NoError(t, err) + require.IsType(t, PostImport500TextResponse(""), response) +} + +func TestPostImportHTTPValidatesBodyAndContentType(t *testing.T) { + importer := &recordingAggregateImporter{result: model.ImportResult{Errors: []model.ImportItemError{}}} + handler := importHTTPHandler(t, importer) + + missingBody := httptest.NewRequest(http.MethodPost, "/directory/import", http.NoBody) + missingBody.Header.Set("Content-Type", "application/x-ndjson") + missingBody.Header.Set(auth.FolioPermissionsHeader, `["directory.consortium.all"]`) + missingResponse := httptest.NewRecorder() + handler.ServeHTTP(missingResponse, missingBody) + require.Equal(t, http.StatusBadRequest, missingResponse.Code) + + wrongType := httptest.NewRequest(http.MethodPost, "/directory/import", strings.NewReader("record")) + wrongType.Header.Set("Content-Type", "application/json") + wrongType.Header.Set(auth.FolioPermissionsHeader, `["directory.consortium.all"]`) + wrongResponse := httptest.NewRecorder() + handler.ServeHTTP(wrongResponse, wrongType) + require.Equal(t, http.StatusBadRequest, wrongResponse.Code) + require.Zero(t, importer.calls) +} + +func importHTTPHandler(t *testing.T, importer AggregateImporter) http.Handler { + t.Helper() + spec, err := GetSpec() + require.NoError(t, err) + strict := NewStrictHandler(&ApiImpl{importer: importer}, nil) + routes := HandlerWithOptions(strict, StdHTTPServerOptions{BaseURL: "/directory", BaseRouter: http.NewServeMux()}) + return auth.FolioTokenAwareMiddleware(apiValidator.OapiRequestValidator(spec)(routes)) +} + +func consortialAdminContext(t *testing.T) context.Context { + t.Helper() + request := httptest.NewRequest(http.MethodGet, "/", nil) + request.Header.Set(auth.FolioPermissionsHeader, `["directory.consortium.all"]`) + var result context.Context + auth.FolioTokenAwareMiddleware(http.HandlerFunc(func(_ http.ResponseWriter, request *http.Request) { + result = request.Context() + })).ServeHTTP(httptest.NewRecorder(), request) + require.NotNil(t, result) + return result +} diff --git a/directory/app/app.go b/directory/app/app.go index 818020fbb..3b154dc04 100644 --- a/directory/app/app.go +++ b/directory/app/app.go @@ -23,6 +23,8 @@ import ( "github.com/indexdata/crosslink/directory/auth" "github.com/indexdata/crosslink/directory/db" "github.com/indexdata/crosslink/directory/enhancedcontext" + importdb "github.com/indexdata/crosslink/directory/import/db" + importservice "github.com/indexdata/crosslink/directory/import/service" ) var Host = cmp.Or(os.Getenv("HOST"), "localhost") @@ -62,7 +64,9 @@ func InitHandler(ctx context.Context, dbpool *pgxpool.Pool) http.Handler { } queries := db.New(dbpool) - impl := api.NewApiImpl(dbpool, queries) + importRepo := importdb.New(dbpool) + importer := importservice.New(importRepo) + impl := api.NewApiImpl(dbpool, queries, importer) si := api.NewStrictHandler(impl, nil) m := http.NewServeMux() h := api.HandlerWithOptions(si, api.StdHTTPServerOptions{ @@ -72,7 +76,8 @@ func InitHandler(ctx context.Context, dbpool *pgxpool.Pool) http.Handler { handlerWithValidation := apiValidator.OapiRequestValidator(swagger) handlerWithLogging := httpLoggingMiddleware(handlerWithValidation(h)) handlerWithHelper := enhancedcontext.EnhancedContextMiddleware(handlerWithLogging) - handlerWithAuth := auth.FolioTokenAwareMiddleware(handlerWithHelper) + handlerWithLimit := ImportBodyLimitMiddleware(MaxImportBodyBytes, handlerWithHelper) + handlerWithAuth := auth.FolioTokenAwareMiddleware(handlerWithLimit) return handlerWithAuth } diff --git a/directory/app/import_limit.go b/directory/app/import_limit.go new file mode 100644 index 000000000..c7ad99041 --- /dev/null +++ b/directory/app/import_limit.go @@ -0,0 +1,29 @@ +package app + +import ( + "net/http" + + "github.com/indexdata/crosslink/directory/auth" +) + +const MaxImportBodyBytes int64 = 2 << 30 + +func ImportBodyLimitMiddleware(maxBytes int64, next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodPost || request.URL.Path != BasePath+"/import" { + next.ServeHTTP(writer, request) + return + } + authData := auth.GetAuthData(request.Context()) + if authData == nil || !authData.HasRole(auth.ConsortialAdminRole) { + http.Error(writer, "Access denied", http.StatusUnauthorized) + return + } + if request.ContentLength > maxBytes { + http.Error(writer, "import request too large", http.StatusRequestEntityTooLarge) + return + } + request.Body = http.MaxBytesReader(writer, request.Body, maxBytes) + next.ServeHTTP(writer, request) + }) +} diff --git a/directory/app/import_limit_test.go b/directory/app/import_limit_test.go new file mode 100644 index 000000000..38a0a6a5e --- /dev/null +++ b/directory/app/import_limit_test.go @@ -0,0 +1,92 @@ +package app + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/indexdata/crosslink/directory/auth" + "github.com/stretchr/testify/require" +) + +func TestImportBodyLimitRejectsKnownAndChunkedOverflow(t *testing.T) { + for name, contentLength := range map[string]int64{"known": 129, "chunked": -1} { + t.Run(name, func(t *testing.T) { + called := false + next := http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + called = true + _, err := io.ReadAll(request.Body) + if err != nil { + var maxErr *http.MaxBytesError + require.ErrorAs(t, err, &maxErr) + http.Error(w, "too large", http.StatusRequestEntityTooLarge) + } + }) + handler := auth.FolioTokenAwareMiddleware(ImportBodyLimitMiddleware(128, next)) + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", strings.NewReader(strings.Repeat("x", 129))) + request.ContentLength = contentLength + request.Header.Set(auth.FolioPermissionsHeader, `["directory.consortium.all"]`) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusRequestEntityTooLarge, response.Code) + if contentLength > 128 { + require.False(t, called) + } else { + require.True(t, called) + } + }) + } +} + +func TestImportBodyLimitDoesNotReadUnauthorizedOrAffectOtherRoutes(t *testing.T) { + for name, values := range map[string][2]string{ + "unauthorized": {`["directory.public.all"]`, BasePath + "/import"}, + "other route": {`["directory.consortium.all"]`, BasePath + "/entries"}, + } { + t.Run(name, func(t *testing.T) { + permissions, path := values[0], values[1] + body := &trackingReadCloser{Reader: strings.NewReader(strings.Repeat("x", 129))} + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusUnauthorized) }) + handler := auth.FolioTokenAwareMiddleware(ImportBodyLimitMiddleware(128, next)) + request := httptest.NewRequest(http.MethodPost, path, body) + request.ContentLength = 129 + request.Header.Set(auth.FolioPermissionsHeader, permissions) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusUnauthorized, response.Code) + require.False(t, body.read) + }) + } +} + +func TestImportHandlerRejectsUnauthorizedBeforeValidationReadsBody(t *testing.T) { + body := &trackingReadCloser{Reader: strings.NewReader("record")} + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", body) + request.Header.Set("Content-Type", "application/x-ndjson") + request.Header.Set(auth.FolioPermissionsHeader, `["directory.public.all"]`) + response := httptest.NewRecorder() + + InitHandler(context.Background(), nil).ServeHTTP(response, request) + + require.Equal(t, http.StatusUnauthorized, response.Code) + require.False(t, body.read) +} + +type trackingReadCloser struct { + io.Reader + read bool +} + +func (r *trackingReadCloser) Read(data []byte) (int, error) { + r.read = true + return r.Reader.Read(data) +} + +func (r *trackingReadCloser) Close() error { return nil } diff --git a/directory/descriptors/ModuleDescriptor-template.json b/directory/descriptors/ModuleDescriptor-template.json index 049a630f1..ef8a9a9a6 100644 --- a/directory/descriptors/ModuleDescriptor-template.json +++ b/directory/descriptors/ModuleDescriptor-template.json @@ -6,6 +6,14 @@ "id": "entries", "version": "1.0", "handlers": [ + { + "methods": ["POST"], + "pathPattern": "/directory/import", + "permissionsRequired": [], + "permissionsDesired": [ + "directory.consortium.all" + ] + }, { "methods": ["GET"], "pathPattern": "/directory/entries", diff --git a/directory/domain/hierarchy.go b/directory/domain/hierarchy.go new file mode 100644 index 000000000..cf98d9fa1 --- /dev/null +++ b/directory/domain/hierarchy.go @@ -0,0 +1,18 @@ +package domain + +func ValidParentForType(entryType, parentType string) (bool, string) { + switch entryType { + case "Institution": + if parentType == "Consortium" { + return true, "" + } + return false, "Institution parent must be of type Consortium" + case "Branch": + if parentType == "Institution" { + return true, "" + } + return false, "Branch parent must be of type Institution" + default: + return false, "Invalid type to have parent" + } +} diff --git a/directory/domain/hierarchy_test.go b/directory/domain/hierarchy_test.go new file mode 100644 index 000000000..1288940c8 --- /dev/null +++ b/directory/domain/hierarchy_test.go @@ -0,0 +1,25 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidParentForType(t *testing.T) { + tests := []struct { + entryType string + parentType string + valid bool + }{ + {"Institution", "Consortium", true}, + {"Branch", "Institution", true}, + {"Institution", "Institution", false}, + {"Branch", "Consortium", false}, + {"Consortium", "Consortium", false}, + } + for _, test := range tests { + valid, _ := ValidParentForType(test.entryType, test.parentType) + require.Equal(t, test.valid, valid, "%s -> %s", test.entryType, test.parentType) + } +} diff --git a/directory/enhancedcontext/enhancedcontext_test.go b/directory/enhancedcontext/enhancedcontext_test.go index bfc4e09dc..717d3ce94 100644 --- a/directory/enhancedcontext/enhancedcontext_test.go +++ b/directory/enhancedcontext/enhancedcontext_test.go @@ -8,11 +8,11 @@ import ( func TestEnhancedContext(t *testing.T) { dummyHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - originalRequestPtr := GetRequest(r.Context()) + originalRequestPtr := GetRequest(r.Context()) - if originalRequestPtr == nil { - t.Fatal("Tried to retrieve stored request, but got nil") - } + if originalRequestPtr == nil { + t.Fatal("Tried to retrieve stored request, but got nil") + } if originalRequestPtr.URL != r.URL { t.Errorf("Expected URL %s, but got %s", r.URL, originalRequestPtr.URL) diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go new file mode 100644 index 000000000..6a0996e53 --- /dev/null +++ b/directory/import/db/entry.go @@ -0,0 +1,285 @@ +package importdb + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/domain" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" +) + +func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { + if err := aggregate.NormalizeAndValidate(); err != nil { + return model.RepoResult{}, err + } + key := aggregate.Key.String() + tx, queries, err := r.begin(ctx) + if err != nil { + return model.RepoResult{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + + existing, lookupErr := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{ + Authority: aggregate.Key.Authority, + Symbol: aggregate.Key.Symbol, + }) + exists := lookupErr == nil + if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { + return model.RepoResult{}, fmt.Errorf("resolve entry %s", key) + } + if exists && policy != model.ConflictPolicyUpdate { + return conflictResult("entry", key, policy) + } + if !exists && policy != model.ConflictPolicyFail && policy != model.ConflictPolicySkip && policy != model.ConflictPolicyUpdate { + return model.RepoResult{}, fmt.Errorf("invalid conflict policy") + } + + if aggregate.Data.Type == "Consortium" || (exists && existing.Type == "Consortium") { + if err := queries.LockConsortiumEntryChanges(ctx); err != nil { + return model.RepoResult{}, fmt.Errorf("lock consortium entry changes") + } + } + if aggregate.Data.Type == "Consortium" && (!exists || existing.Type != "Consortium") { + consortium, err := queries.GetConsortialEntry(ctx) + if err == nil && (!exists || consortium.ID != existing.ID) { + return model.RepoResult{}, fmt.Errorf("consortium already exists") + } + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return model.RepoResult{}, fmt.Errorf("check existing consortium") + } + } + + parentID, err := resolveParent(ctx, queries, aggregate.Data.Parent, aggregate.Data.Type) + if err != nil { + return model.RepoResult{}, err + } + if exists { + if err := validateEntryUpdateHierarchy(ctx, queries, existing, aggregate.Data.Type, parentID); err != nil { + return model.RepoResult{}, err + } + } + + entryID, err := writeEntry(ctx, queries, existing, exists, parentID, aggregate.Data) + if err != nil { + return model.RepoResult{}, persistenceError("entry", key, err) + } + if err := replaceEntryChildren(ctx, queries, entryID, aggregate.Data); err != nil { + return model.RepoResult{}, persistenceError("entry", key, err) + } + if err := tx.Commit(ctx); err != nil { + return model.RepoResult{}, fmt.Errorf("commit entry %s import", key) + } + return model.RepoResult{Outcome: model.OutcomeImported}, nil +} + +func resolveParent(ctx context.Context, queries *db.Queries, parent *model.SymbolRef, entryType string) (*uuid.UUID, error) { + if parent == nil { + return nil, nil + } + entry, err := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{Authority: parent.Authority, Symbol: parent.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("parent %s does not exist", parent.String()) + } + if err != nil { + return nil, fmt.Errorf("resolve parent %s", parent.String()) + } + if valid, reason := domain.ValidParentForType(entryType, entry.Type); !valid { + return nil, fmt.Errorf("invalid parent %s: %s", parent.String(), reason) + } + return &entry.ID, nil +} + +func validateEntryUpdateHierarchy(ctx context.Context, queries *db.Queries, existing db.Entry, resultingType string, parentID *uuid.UUID) error { + if parentID != nil { + cycle, err := queries.WouldCreateEntryCycle(ctx, db.WouldCreateEntryCycleParams{Child: existing.ID, Parent: *parentID}) + if err != nil { + return fmt.Errorf("validate entry hierarchy") + } + if cycle != nil && *cycle { + return fmt.Errorf("entry parent would create a cycle") + } + } + if resultingType != existing.Type { + children, err := queries.EntriesByParent(ctx, &existing.ID) + if err != nil { + return fmt.Errorf("validate entry children") + } + for _, child := range children { + if valid, reason := domain.ValidParentForType(child.Type, resultingType); !valid { + return fmt.Errorf("entry type is invalid for existing child: %s", reason) + } + } + } + return nil +} + +func writeEntry(ctx context.Context, queries *db.Queries, existing db.Entry, exists bool, parentID *uuid.UUID, data model.EntryData) (uuid.UUID, error) { + if exists { + err := queries.UpdateEntry(ctx, db.UpdateEntryParams{ + Name: data.Name, Description: data.Description, ContactName: data.ContactName, Email: data.Email, FromEmail: data.FromEmail, + Tenant: data.Tenant, Vendor: data.Vendor, PhoneNumber: data.PhoneNumber, TimeZone: data.TimeZone, + OrganizationID: data.OrganizationID, Type: data.Type, Parent: parentID, LmsLocationCode: data.LMSLocationCode, + Hrid: data.HRID, ID: existing.ID, + }) + return existing.ID, err + } + created, err := queries.CreateEntry(ctx, db.CreateEntryParams{ + Name: data.Name, Description: data.Description, ContactName: data.ContactName, Email: data.Email, FromEmail: data.FromEmail, + Tenant: data.Tenant, Vendor: data.Vendor, PhoneNumber: data.PhoneNumber, TimeZone: data.TimeZone, + OrganizationID: data.OrganizationID, Type: data.Type, Parent: parentID, LmsLocationCode: data.LMSLocationCode, + Hrid: data.HRID, + }) + return created.ID, err +} + +func replaceEntryChildren(ctx context.Context, queries *db.Queries, entryID uuid.UUID, data model.EntryData) error { + if err := queries.DeleteAllOwnedSymbols(ctx, entryID); err != nil { + return err + } + for _, symbol := range data.Symbols { + if _, err := queries.UpsertSymbol(ctx, db.UpsertSymbolParams{Owner: entryID, Authority: symbol.Authority, Symbol: symbol.Symbol}); err != nil { + return err + } + } + if err := queries.DeleteAllOwnedServiceEndpoints(ctx, entryID); err != nil { + return err + } + for _, endpoint := range data.Endpoints { + if _, err := queries.UpsertServiceEndpoint(ctx, db.UpsertServiceEndpointParams{Entry: entryID, Name: endpoint.Name, Type: endpoint.Type, Address: endpoint.Address}); err != nil { + return err + } + } + if err := queries.DeleteAllOwnedAddresses(ctx, entryID); err != nil { + return err + } + for _, address := range data.Addresses { + created, err := queries.UpsertAddress(ctx, db.UpsertAddressParams{Entry: entryID, Type: address.Type}) + if err != nil { + return err + } + for _, component := range address.Components { + if _, err := queries.CreateAddressComponent(ctx, db.CreateAddressComponentParams{Address: created.ID, Seq: component.Seq, Type: component.Type, Value: component.Value}); err != nil { + return err + } + } + } + if err := queries.DeleteClosuresByEntry(ctx, entryID); err != nil { + return err + } + for _, closure := range data.Closures { + start, _ := time.Parse(time.DateOnly, closure.StartDate) + end, _ := time.Parse(time.DateOnly, closure.EndDate) + if _, err := queries.CreateClosure(ctx, db.CreateClosureParams{ + Entry: entryID, StartDate: pgtype.Timestamp{Time: start, Valid: true}, EndDate: pgtype.Timestamp{Time: end, Valid: true}, Reason: closure.Reason, + }); err != nil { + return err + } + } + return replaceEntryConfigs(ctx, queries, entryID, data) +} + +func replaceEntryConfigs(ctx context.Context, queries *db.Queries, entryID uuid.UUID, data model.EntryData) error { + if err := queries.DeleteLMSConfigByEntry(ctx, entryID); err != nil { + return err + } + if data.LMSConfig != nil { + cfg := data.LMSConfig + if _, err := queries.UpsertLMSConfig(ctx, db.UpsertLMSConfigParams{ + Entry: &entryID, Address: cfg.Address, FromAgency: cfg.FromAgency, FromAgencyAuthentication: cfg.FromAgencyAuthentication, + ToAgency: cfg.ToAgency, LookupUserEnabled: cfg.LookupUserEnabled, AcceptItemEnabled: cfg.AcceptItemEnabled, + CheckinItemEnabled: cfg.CheckInItemEnabled, CheckoutItemEnabled: cfg.CheckOutItemEnabled, ItemLocation: cfg.ItemLocation, + RequestItemRequestType: cfg.RequestItemRequestType, RequestItemScopeType: cfg.RequestItemRequestScopeType, + RequestItemBibCode: cfg.RequestItemBibIDCode, RequestItemEnabled: cfg.RequestItemEnabled, + RequestItemPickupLocationEnabled: cfg.RequestItemPickupLocationEnabled, RequesterPickupLocation: cfg.RequesterPickupLocation, + SupplierPickupLocation: cfg.SupplierPickupLocation, RequesterPatronPattern: cfg.RequesterPatronPattern, + }); err != nil { + return err + } + } + if err := replaceCatalogConfig(ctx, queries, entryID, data.CatalogConfig); err != nil { + return err + } + if err := replaceILLConfig(ctx, queries, entryID, data.ILLConfig); err != nil { + return err + } + if err := queries.DeleteHoldingsPolicyByEntry(ctx, entryID); err != nil { + return err + } + if data.HoldingsPolicy != nil { + policy, err := json.Marshal(data.HoldingsPolicy) + if err != nil { + return err + } + if _, err := queries.UpsertHoldingsPolicy(ctx, db.UpsertHoldingsPolicyParams{Entry: entryID, Policy: policy}); err != nil { + return err + } + } + return nil +} + +func replaceCatalogConfig(ctx context.Context, queries *db.Queries, entryID uuid.UUID, config *model.CatalogConfig) error { + if err := queries.DeleteCatalogConfigByEntry(ctx, entryID); err != nil || config == nil { + return err + } + params := db.UpsertCatalogConfigParams{Entry: &entryID, MetadataUpdateMode: config.MetadataUpdateMode} + if config.SRU != nil { + params.SruAddress, params.SruRecordSchema = &config.SRU.Address, config.SRU.RecordSchema + } + if config.Zoom != nil { + params.ZoomAddress = &config.Zoom.Address + if config.Zoom.Options != nil { + params.ZoomOptions, _ = json.Marshal(config.Zoom.Options) + } + } + if config.Query != nil { + params.QueryType, params.QueryIdentifier, params.QueryIsbn, params.QueryIssn, params.QueryTitle = config.Query.Type, config.Query.Identifier, config.Query.ISBN, config.Query.ISSN, config.Query.Title + } + if config.HoldingsFormat != nil { + if config.HoldingsFormat.Marc != nil { + marc := config.HoldingsFormat.Marc + params.HoldingsMarcCallNumberSubfield, params.HoldingsMarcItemIDSubfield = marc.CallNumberSubField, marc.ItemIDSubField + params.HoldingsMarcLocationSubfield, params.HoldingsMarcMainField = marc.LocationSubField, marc.MainField + params.HoldingsMarcRestrictedSubfield, params.HoldingsMarcShelvingLocationSubfield = marc.RestrictedSubField, marc.ShelvingLocationSubField + } + params.HoldingsMarc21plus1Enabled = boolPointer(config.HoldingsFormat.Marc21Plus1 != nil) + params.HoldingsOpacEnabled = boolPointer(config.HoldingsFormat.OPAC != nil) + params.HoldingsReservoirEnabled = boolPointer(config.HoldingsFormat.Reservoir != nil) + } + if config.MetadataFormat != nil && config.MetadataFormat.Marc21 != nil { + marc := config.MetadataFormat.Marc21 + params.MetadataMarc21Author, params.MetadataMarc21Edition, params.MetadataMarc21Identifier = marc.Author, marc.Edition, marc.Identifier + params.MetadataMarc21Isbn, params.MetadataMarc21Issn, params.MetadataMarc21Subtitle, params.MetadataMarc21Title = marc.ISBN, marc.ISSN, marc.Subtitle, marc.Title + } + _, err := queries.UpsertCatalogConfig(ctx, params) + return err +} + +func replaceILLConfig(ctx context.Context, queries *db.Queries, entryID uuid.UUID, config *model.ILLConfig) error { + if err := queries.DeleteIllConfigByEntry(ctx, entryID); err != nil || config == nil { + return err + } + lenders := make([]string, 0, len(config.LendersOfLastResort)) + for _, lender := range config.LendersOfLastResort { + if _, err := resolveEntry(ctx, queries, lender); err != nil { + return fmt.Errorf("lender of last resort %s does not exist", lender.String()) + } + lenders = append(lenders, lender.String()) + } + _, err := queries.UpsertIllConfig(ctx, db.UpsertIllConfigParams{ + Entry: entryID, Iso18626Url: config.ISO18626URL, Iso18626Vendor: config.ISO18626Vendor, LendersOfLastResort: lenders, + IncludeRequestingAgencyInfo: config.IncludeRequestingAgencyInfo, IncludeSupplierInfo: config.IncludeSupplierInfo, + IncludeReturnInfo: config.IncludeReturnInfo, IncludeVendorNote: config.IncludeVendorNote, UseOfferedCosts: config.UseOfferedCosts, + NoteFieldSeparator: config.NoteFieldSeparator, SupplierPatronPattern: config.SupplierPatronPattern, + DuplicateCheckWindowHours: config.DuplicateCheckWindowHours, + }) + return err +} + +func boolPointer(value bool) *bool { return &value } diff --git a/directory/import/db/repo.go b/directory/import/db/repo.go new file mode 100644 index 000000000..f1d058b8d --- /dev/null +++ b/directory/import/db/repo.go @@ -0,0 +1,59 @@ +package importdb + +import ( + "context" + "errors" + "fmt" + + "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type PgImportRepo struct { + pool *pgxpool.Pool +} + +func New(pool *pgxpool.Pool) *PgImportRepo { + return &PgImportRepo{pool: pool} +} + +func (r *PgImportRepo) begin(ctx context.Context) (pgx.Tx, *db.Queries, error) { + tx, err := r.pool.Begin(ctx) + if err != nil { + return nil, nil, fmt.Errorf("begin directory import transaction: %w", err) + } + return tx, db.New(tx), nil +} + +func conflictResult(resource, key string, policy model.ConflictPolicy) (model.RepoResult, error) { + switch policy { + case model.ConflictPolicyFail: + return model.RepoResult{}, fmt.Errorf("%s %s already exists", resource, key) + case model.ConflictPolicySkip: + return model.RepoResult{Outcome: model.OutcomeSkipped, Diagnostic: fmt.Sprintf("%s %s already exists", resource, key)}, nil + case model.ConflictPolicyUpdate: + return model.RepoResult{}, nil + default: + return model.RepoResult{}, fmt.Errorf("invalid conflict policy") + } +} + +func resolveEntry(ctx context.Context, queries *db.Queries, key model.SymbolRef) (db.Entry, error) { + entry, err := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{Authority: key.Authority, Symbol: key.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + return db.Entry{}, fmt.Errorf("entry %s does not exist", key.String()) + } + if err != nil { + return db.Entry{}, fmt.Errorf("resolve entry %s", key.String()) + } + return entry, nil +} + +func persistenceError(resource, key string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("persist %s %s aggregate", resource, key) +} diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go new file mode 100644 index 000000000..72e0b1b16 --- /dev/null +++ b/directory/import/db/repo_test.go @@ -0,0 +1,445 @@ +package importdb_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/indexdata/crosslink/directory/app" + importdb "github.com/indexdata/crosslink/directory/import/db" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/postgres" + "github.com/testcontainers/testcontainers-go/wait" +) + +var testPool *pgxpool.Pool + +func TestMain(m *testing.M) { + ctx := context.Background() + container, err := postgres.Run(ctx, "postgres", + postgres.WithDatabase("directory_import_test"), + postgres.WithUsername("directory"), + postgres.WithPassword("directory"), + testcontainers.WithWaitStrategy(wait.ForLog("database system is ready to accept connections").WithOccurrence(2).WithStartupTimeout(10*time.Second)), + ) + if err != nil { + panic(fmt.Sprintf("start postgres: %v", err)) + } + connectionString, err := container.ConnectionString(ctx, "sslmode=disable") + if err != nil { + panic(fmt.Sprintf("get postgres connection string: %v", err)) + } + app.ConnectionString = connectionString + app.MigrationsFolder = "file://../../migrations" + app.RunMigrateScripts() + testPool = app.InitDbPool() + + code := m.Run() + testPool.Close() + if err := container.Terminate(ctx); err != nil { + panic(fmt.Sprintf("terminate postgres: %v", err)) + } + os.Exit(code) +} + +func TestImportBusinessKeyConstraints(t *testing.T) { + ctx := context.Background() + consortiumID := uuid.New() + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, 'Consortium', 'Consortium')`, consortiumID) + require.NoError(t, err) + + _, err = testPool.Exec(ctx, ` + INSERT INTO tiers (consortium, name, level, type, cost) + VALUES ($1, 'Loan', 'standard', 'loan', 0), ($1, 'Loan', 'standard', 'loan', 0)`, consortiumID) + requirePgCode(t, err, "23505") + + _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name, priority) VALUES ($1, NULL, 0)`, consortiumID) + requirePgCode(t, err, "23502") +} + +func TestImportEntryCreatesCompleteAggregateWithGeneratedIDs(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + aggregate := completeEntryAggregate("CON") + + result, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + + require.NoError(t, err) + require.Equal(t, model.OutcomeImported, result.Outcome) + entryID := entryIDBySymbol(t, aggregate.Key) + require.NotEqual(t, uuid.Nil, entryID) + for table, expected := range map[string]int{ + "symbols": 1, "service_endpoints": 1, "addresses": 1, "address_components": 1, "closures": 1, + "lms_configs": 1, "catalog_configs": 1, "ill_configs": 1, "holdings_policies": 1, + } { + assertOwnedCount(t, table, entryID, expected) + } + var authentication string + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT from_agency_authentication FROM lms_configs WHERE entry=$1`, entryID).Scan(&authentication)) + require.Equal(t, "credential-value", authentication) + var zoomOptions map[string]string + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT zoom_options FROM catalog_configs WHERE entry=$1`, entryID).Scan(&zoomOptions)) + require.Equal(t, map[string]string{"user": "private"}, zoomOptions) +} + +func TestImportEntryConflictPoliciesAndUpdateFullSynchronization(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + aggregate := completeEntryAggregate("CON") + _, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + entryID := entryIDBySymbol(t, aggregate.Key) + var originalEndpointID uuid.UUID + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT id FROM service_endpoints WHERE entry=$1`, entryID).Scan(&originalEndpointID)) + + _, err = repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + require.ErrorContains(t, err, "already exists") + skipped, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicySkip) + require.NoError(t, err) + require.Equal(t, model.OutcomeSkipped, skipped.Outcome) + + aggregate.Data.Name = "Updated Consortium" + aggregate.Data.Endpoints = []model.ServiceEndpoint{} + aggregate.Data.LMSConfig = nil + aggregate.Data.CatalogConfig = nil + aggregate.Data.ILLConfig = nil + aggregate.Data.HoldingsPolicy = nil + updated, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyUpdate) + require.NoError(t, err) + require.Equal(t, model.OutcomeImported, updated.Outcome) + require.Equal(t, entryID, entryIDBySymbol(t, aggregate.Key)) + assertOwnedCount(t, "service_endpoints", entryID, 0) + assertOwnedCount(t, "lms_configs", entryID, 0) + assertOwnedCount(t, "catalog_configs", entryID, 0) + assertOwnedCount(t, "ill_configs", entryID, 0) + assertOwnedCount(t, "holdings_policies", entryID, 0) + var name string + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT name FROM entries WHERE id=$1`, entryID).Scan(&name)) + require.Equal(t, "Updated Consortium", name) + + aggregate.Data.Endpoints = []model.ServiceEndpoint{{Name: "Replacement", Type: "ISO18626", Address: "https://replacement.test"}} + _, err = repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyUpdate) + require.NoError(t, err) + var replacementEndpointID uuid.UUID + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT id FROM service_endpoints WHERE entry=$1`, entryID).Scan(&replacementEndpointID)) + require.NotEqual(t, originalEndpointID, replacementEndpointID) +} + +func TestImportEntryRejectsInvalidHierarchy(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + consortium := minimalEntryAggregate("CON", "Consortium") + _, err := repo.ImportEntry(context.Background(), consortium, model.ConflictPolicyFail) + require.NoError(t, err) + + branch := minimalEntryAggregate("BRANCH", "Branch") + branch.Data.Parent = &consortium.Key + _, err = repo.ImportEntry(context.Background(), branch, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "Branch parent must be of type Institution") + assertEntryDoesNotExist(t, branch.Key) +} + +func TestImportEntryRejectsCycle(t *testing.T) { + resetImportDatabase(t) + ctx := context.Background() + repo := importdb.New(testPool) + branchID, institutionID := uuid.New(), uuid.New() + _, err := testPool.Exec(ctx, `INSERT INTO entries (id,name,type,parent) VALUES ($1,'Branch','Branch',NULL),($2,'Institution','Institution',$1)`, branchID, institutionID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner,authority,symbol) VALUES ($1,'ISIL','BRANCH'),($2,'ISIL','INST')`, branchID, institutionID) + require.NoError(t, err) + + branch := minimalEntryAggregate("BRANCH", "Branch") + branch.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "INST"} + _, err = repo.ImportEntry(ctx, branch, model.ConflictPolicyUpdate) + + require.ErrorContains(t, err, "would create a cycle") +} + +func TestImportEntryRejectsMissingParentWithoutWriting(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + aggregate := minimalEntryAggregate("LIB", "Institution") + aggregate.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "MISSING"} + + _, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "parent ISIL:MISSING does not exist") + assertEntryDoesNotExist(t, aggregate.Key) +} + +func TestImportEntryRollsBackAfterLateSymbolConflict(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + consortium := completeEntryAggregate("CON") + _, err := repo.ImportEntry(context.Background(), consortium, model.ConflictPolicyFail) + require.NoError(t, err) + + aggregate := minimalEntryAggregate("LIB", "Institution") + aggregate.Data.Parent = &consortium.Key + aggregate.Data.Symbols = append(aggregate.Data.Symbols, consortium.Key) + _, err = repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + + require.Error(t, err) + assertEntryDoesNotExist(t, aggregate.Key) +} + +func TestImportEntryAllowsOnlyOneConsortium(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + _, err := repo.ImportEntry(context.Background(), minimalEntryAggregate("CON1", "Consortium"), model.ConflictPolicyFail) + require.NoError(t, err) + + second := minimalEntryAggregate("CON2", "Consortium") + _, err = repo.ImportEntry(context.Background(), second, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "consortium already exists") + assertEntryDoesNotExist(t, second.Key) +} + +func TestImportTierConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { + repo, consortium, first, second := importRepoFixture(t) + aggregate := model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Loan"}, + Data: model.TierData{Level: "standard", Type: "loan", Cost: 1.5, Entries: []model.SymbolRef{first}}, + } + + result, err := repo.ImportTier(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + require.Equal(t, model.OutcomeImported, result.Outcome) + id := tierIDByKey(t, consortium, "Loan") + require.NotEqual(t, uuid.Nil, id) + require.Equal(t, []model.SymbolRef{first}, tierAssignments(t, id)) + + _, err = repo.ImportTier(context.Background(), aggregate, model.ConflictPolicyFail) + require.ErrorContains(t, err, "already exists") + skipped, err := repo.ImportTier(context.Background(), aggregate, model.ConflictPolicySkip) + require.NoError(t, err) + require.Equal(t, model.OutcomeSkipped, skipped.Outcome) + + aggregate.Data.Level = "rush" + aggregate.Data.Cost = 2.5 + aggregate.Data.Entries = []model.SymbolRef{second} + _, err = repo.ImportTier(context.Background(), aggregate, model.ConflictPolicyUpdate) + require.NoError(t, err) + require.Equal(t, id, tierIDByKey(t, consortium, "Loan")) + require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) +} + +func TestImportTierRollsBackWhenMemberIsMissing(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + missing := model.SymbolRef{Authority: "ISIL", Symbol: "MISSING"} + aggregate := model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Loan"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{first, missing}}, + } + + _, err := repo.ImportTier(context.Background(), aggregate, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "entry ISIL:MISSING does not exist") + assertTierDoesNotExist(t, consortium, "Loan") +} + +func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { + repo, consortium, first, second := importRepoFixture(t) + reciprocal := true + aggregate := model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Main"}, + Data: model.NetworkData{Priority: 1, Reciprocal: &reciprocal, Entries: []model.SymbolRef{first}}, + } + + result, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + require.Equal(t, model.OutcomeImported, result.Outcome) + id := networkIDByKey(t, consortium, "Main") + require.NotEqual(t, uuid.Nil, id) + require.Equal(t, []model.SymbolRef{first}, networkAssignments(t, id)) + + _, err = repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) + require.ErrorContains(t, err, "already exists") + skipped, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicySkip) + require.NoError(t, err) + require.Equal(t, model.OutcomeSkipped, skipped.Outcome) + + aggregate.Data.Priority = 2 + aggregate.Data.Reciprocal = nil + aggregate.Data.Entries = []model.SymbolRef{second} + _, err = repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyUpdate) + require.NoError(t, err) + require.Equal(t, id, networkIDByKey(t, consortium, "Main")) + require.Equal(t, []model.SymbolRef{second}, networkAssignments(t, id)) +} + +func TestImportNetworkRejectsNonConsortiumOwner(t *testing.T) { + repo, _, first, _ := importRepoFixture(t) + aggregate := model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: first, Name: "Main"}, + Data: model.NetworkData{Entries: []model.SymbolRef{}}, + } + + _, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "is not a consortium") +} + +func importRepoFixture(t *testing.T) (*importdb.PgImportRepo, model.SymbolRef, model.SymbolRef, model.SymbolRef) { + t.Helper() + resetImportDatabase(t) + repo := importdb.New(testPool) + consortium := minimalEntryAggregate("CON", "Consortium") + _, err := repo.ImportEntry(context.Background(), consortium, model.ConflictPolicyFail) + require.NoError(t, err) + first := minimalEntryAggregate("FIRST", "Institution") + first.Data.Parent = &consortium.Key + _, err = repo.ImportEntry(context.Background(), first, model.ConflictPolicyFail) + require.NoError(t, err) + second := minimalEntryAggregate("SECOND", "Institution") + second.Data.Parent = &consortium.Key + _, err = repo.ImportEntry(context.Background(), second, model.ConflictPolicyFail) + require.NoError(t, err) + return repo, consortium.Key, first.Key, second.Key +} + +func tierIDByKey(t *testing.T, consortium model.SymbolRef, name string) uuid.UUID { + t.Helper() + return aggregateIDByKey(t, "tiers", consortium, name) +} + +func networkIDByKey(t *testing.T, consortium model.SymbolRef, name string) uuid.UUID { + t.Helper() + return aggregateIDByKey(t, "networks", consortium, name) +} + +func aggregateIDByKey(t *testing.T, table string, consortium model.SymbolRef, name string) uuid.UUID { + t.Helper() + query := fmt.Sprintf(`SELECT a.id FROM %s a JOIN symbols s ON s.owner=a.consortium WHERE s.authority=$1 AND s.symbol=$2 AND a.name=$3`, table) //nolint:gosec // table names are fixed test constants + var id uuid.UUID + require.NoError(t, testPool.QueryRow(context.Background(), query, consortium.Authority, consortium.Symbol, name).Scan(&id)) + return id +} + +func tierAssignments(t *testing.T, id uuid.UUID) []model.SymbolRef { + t.Helper() + return aggregateAssignments(t, "entry_tiers", "tier", id) +} + +func networkAssignments(t *testing.T, id uuid.UUID) []model.SymbolRef { + t.Helper() + return aggregateAssignments(t, "entry_networks", "network", id) +} + +func aggregateAssignments(t *testing.T, table, aggregateColumn string, id uuid.UUID) []model.SymbolRef { + t.Helper() + query := fmt.Sprintf(`SELECT s.authority,s.symbol FROM %s a JOIN symbols s ON s.owner=a.entry WHERE a.%s=$1 ORDER BY s.authority,s.symbol`, table, aggregateColumn) //nolint:gosec // table and column names are fixed test constants + rows, err := testPool.Query(context.Background(), query, id) + require.NoError(t, err) + defer rows.Close() + var result []model.SymbolRef + for rows.Next() { + var ref model.SymbolRef + require.NoError(t, rows.Scan(&ref.Authority, &ref.Symbol)) + result = append(result, ref) + } + require.NoError(t, rows.Err()) + return result +} + +func assertTierDoesNotExist(t *testing.T, consortium model.SymbolRef, name string) { + t.Helper() + var count int + err := testPool.QueryRow(context.Background(), `SELECT count(*) FROM tiers t JOIN symbols s ON s.owner=t.consortium WHERE s.authority=$1 AND s.symbol=$2 AND t.name=$3`, consortium.Authority, consortium.Symbol, name).Scan(&count) + require.NoError(t, err) + require.Zero(t, count) +} + +func completeEntryAggregate(symbol string) model.EntryAggregate { + aggregate := minimalEntryAggregate(symbol, "Consortium") + text := "value" + metadataMode := "replace" + truth := true + aggregate.Data.Endpoints = []model.ServiceEndpoint{{Name: "ISO", Type: "ISO18626", Address: "https://example.test/ill"}} + aggregate.Data.Addresses = []model.Address{{Type: "Default", Components: []model.AddressComponent{{Seq: 1, Type: "Locality", Value: "Riga"}}}} + aggregate.Data.Closures = []model.Closure{{StartDate: "2026-12-24", EndDate: "2026-12-26", Reason: "Holiday"}} + aggregate.Data.LMSConfig = &model.LMSConfig{Address: "https://example.test/ncip", FromAgency: "FROM", FromAgencyAuthentication: stringPointer("credential-value")} + aggregate.Data.CatalogConfig = &model.CatalogConfig{ + MetadataUpdateMode: &metadataMode, + SRU: &model.SRUConfig{Address: "https://example.test/sru"}, + Zoom: &model.ZoomConfig{Address: "example.test:210", Options: &map[string]string{"user": "private"}}, + Query: &model.QueryConfig{Identifier: &text}, + HoldingsFormat: &model.HoldingsParserConfig{Marc: &model.MarcHoldingsParserConfig{MainField: &text}}, + MetadataFormat: &model.MetadataParserConfig{Marc21: &model.MarcMetadataParserConfig{Title: &text}}, + } + aggregate.Data.ILLConfig = &model.ILLConfig{ISO18626URL: &text, LendersOfLastResort: []model.SymbolRef{}, IncludeSupplierInfo: &truth} + aggregate.Data.HoldingsPolicy = &model.HoldingsPolicy{ + Locations: []model.HoldingsLocation{{Code: "MAIN", Name: "Main", SupplyPreference: 1}}, + ShelvingLocations: []model.HoldingsShelvingLocation{}, + LocationPolicies: []model.HoldingsLocationPolicy{}, + ItemLoanPolicies: []model.HoldingsItemLoanPolicy{{Code: "BOOK", Name: "Book", Lendable: true}}, + } + return aggregate +} + +func stringPointer(value string) *string { return &value } + +func minimalEntryAggregate(symbol, entryType string) model.EntryAggregate { + key := model.SymbolRef{Authority: "ISIL", Symbol: symbol} + return model.EntryAggregate{ + Key: key, + Data: model.EntryData{ + Name: "Entry " + symbol, Type: entryType, Symbols: []model.SymbolRef{key}, Endpoints: []model.ServiceEndpoint{}, + Addresses: []model.Address{}, Closures: []model.Closure{}, + }, + } +} + +func resetImportDatabase(t *testing.T) { + t.Helper() + _, err := testPool.Exec(context.Background(), `TRUNCATE entries CASCADE`) + require.NoError(t, err) +} + +func entryIDBySymbol(t *testing.T, key model.SymbolRef) uuid.UUID { + t.Helper() + var id uuid.UUID + err := testPool.QueryRow(context.Background(), `SELECT owner FROM symbols WHERE authority=$1 AND symbol=$2`, key.Authority, key.Symbol).Scan(&id) + require.NoError(t, err) + return id +} + +func assertEntryDoesNotExist(t *testing.T, key model.SymbolRef) { + t.Helper() + var count int + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT count(*) FROM symbols WHERE authority=$1 AND symbol=$2`, key.Authority, key.Symbol).Scan(&count)) + require.Zero(t, count) +} + +func assertOwnedCount(t *testing.T, table string, entryID uuid.UUID, expected int) { + t.Helper() + ownerColumn := "entry" + if table == "symbols" { + ownerColumn = "owner" + } + query := fmt.Sprintf(`SELECT count(*) FROM %s WHERE %s=$1`, table, ownerColumn) //nolint:gosec // table names are fixed test constants + if table == "address_components" { + query = `SELECT count(*) FROM address_components ac JOIN addresses a ON a.id=ac.address WHERE a.entry=$1` + } + var count int + require.NoError(t, testPool.QueryRow(context.Background(), query, entryID).Scan(&count)) + require.Equal(t, expected, count, table) +} + +func requirePgCode(t *testing.T, err error, code string) { + t.Helper() + require.Error(t, err) + pgErr, ok := err.(*pgconn.PgError) + require.True(t, ok, "expected PostgreSQL error, got %T: %v", err, err) + require.Equal(t, code, pgErr.Code) +} diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go new file mode 100644 index 000000000..fc30d4b02 --- /dev/null +++ b/directory/import/db/tier_network.go @@ -0,0 +1,164 @@ +package importdb + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/google/uuid" + "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" +) + +func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { + if err := aggregate.NormalizeAndValidate(); err != nil { + return model.RepoResult{}, err + } + key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + tx, queries, err := r.begin(ctx) + if err != nil { + return model.RepoResult{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + + consortium, err := resolveConsortium(ctx, queries, aggregate.Key.Consortium) + if err != nil { + return model.RepoResult{}, err + } + name := aggregate.Key.Name + existing, lookupErr := queries.LockTierByBusinessKey(ctx, db.LockTierByBusinessKeyParams{Consortium: consortium.ID, Name: &name}) + exists := lookupErr == nil + if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { + return model.RepoResult{}, fmt.Errorf("resolve tier %s", key) + } + if exists && policy != model.ConflictPolicyUpdate { + return conflictResult("tier", key, policy) + } + if !exists && !validPolicy(policy) { + return model.RepoResult{}, fmt.Errorf("invalid conflict policy") + } + + var tierID uuid.UUID + if exists { + tierID = existing.ID + err = queries.UpdateImportedTier(ctx, db.UpdateImportedTierParams{ID: tierID, Level: aggregate.Data.Level, Type: aggregate.Data.Type, Cost: aggregate.Data.Cost}) + } else { + var created db.Tier + created, err = queries.CreateTier(ctx, db.CreateTierParams{Name: &name, Consortium: consortium.ID, Level: aggregate.Data.Level, Type: aggregate.Data.Type, Cost: aggregate.Data.Cost}) + tierID = created.ID + } + if err != nil { + return model.RepoResult{}, persistenceError("tier", key, err) + } + if err := replaceTierAssignments(ctx, queries, tierID, aggregate.Data.Entries); err != nil { + return model.RepoResult{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.RepoResult{}, fmt.Errorf("commit tier %s import", key) + } + return model.RepoResult{Outcome: model.OutcomeImported}, nil +} + +func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.NetworkAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { + if err := aggregate.NormalizeAndValidate(); err != nil { + return model.RepoResult{}, err + } + key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + tx, queries, err := r.begin(ctx) + if err != nil { + return model.RepoResult{}, err + } + defer func() { _ = tx.Rollback(ctx) }() + + consortium, err := resolveConsortium(ctx, queries, aggregate.Key.Consortium) + if err != nil { + return model.RepoResult{}, err + } + name := aggregate.Key.Name + existing, lookupErr := queries.LockNetworkByBusinessKey(ctx, db.LockNetworkByBusinessKeyParams{Consortium: consortium.ID, Name: &name}) + exists := lookupErr == nil + if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { + return model.RepoResult{}, fmt.Errorf("resolve network %s", key) + } + if exists && policy != model.ConflictPolicyUpdate { + return conflictResult("network", key, policy) + } + if !exists && !validPolicy(policy) { + return model.RepoResult{}, fmt.Errorf("invalid conflict policy") + } + + var networkID uuid.UUID + if exists { + networkID = existing.ID + err = queries.UpdateImportedNetwork(ctx, db.UpdateImportedNetworkParams{ID: networkID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) + } else { + var created db.Network + created, err = queries.CreateNetwork(ctx, db.CreateNetworkParams{Name: &name, Consortium: consortium.ID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) + networkID = created.ID + } + if err != nil { + return model.RepoResult{}, persistenceError("network", key, err) + } + if err := replaceNetworkAssignments(ctx, queries, networkID, aggregate.Data.Entries); err != nil { + return model.RepoResult{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.RepoResult{}, fmt.Errorf("commit network %s import", key) + } + return model.RepoResult{Outcome: model.OutcomeImported}, nil +} + +func resolveConsortium(ctx context.Context, queries *db.Queries, key model.SymbolRef) (db.Entry, error) { + entry, err := resolveEntry(ctx, queries, key) + if err != nil { + return db.Entry{}, fmt.Errorf("consortium %s does not exist", key.String()) + } + if entry.Type != "Consortium" { + return db.Entry{}, fmt.Errorf("entry %s is not a consortium", key.String()) + } + return entry, nil +} + +func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uuid.UUID, refs []model.SymbolRef) error { + if err := queries.DeleteEntryTiersByTier(ctx, tierID); err != nil { + return fmt.Errorf("replace tier assignments") + } + for _, ref := range sortedRefs(refs) { + entry, err := resolveEntry(ctx, queries, ref) + if err != nil { + return err + } + if _, err := queries.CreateEntryTier(ctx, db.CreateEntryTierParams{Entry: entry.ID, Tier: tierID}); err != nil { + return fmt.Errorf("replace tier assignments") + } + } + return nil +} + +func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, networkID uuid.UUID, refs []model.SymbolRef) error { + if err := queries.DeleteEntryNetworksByNetwork(ctx, networkID); err != nil { + return fmt.Errorf("replace network assignments") + } + for _, ref := range sortedRefs(refs) { + entry, err := resolveEntry(ctx, queries, ref) + if err != nil { + return err + } + if _, err := queries.CreateEntryNetwork(ctx, db.CreateEntryNetworkParams{Entry: entry.ID, Network: networkID}); err != nil { + return fmt.Errorf("replace network assignments") + } + } + return nil +} + +func sortedRefs(refs []model.SymbolRef) []model.SymbolRef { + result := append([]model.SymbolRef(nil), refs...) + sort.Slice(result, func(i, j int) bool { return result[i].String() < result[j].String() }) + return result +} + +func validPolicy(policy model.ConflictPolicy) bool { + return policy == model.ConflictPolicyFail || policy == model.ConflictPolicySkip || policy == model.ConflictPolicyUpdate +} diff --git a/directory/import/model/config.go b/directory/import/model/config.go new file mode 100644 index 000000000..8da1d3cca --- /dev/null +++ b/directory/import/model/config.go @@ -0,0 +1,141 @@ +package model + +import "fmt" + +type LMSConfig struct { + Address string `json:"address"` + FromAgency string `json:"fromAgency"` + FromAgencyAuthentication *string `json:"fromAgencyAuthentication"` + ToAgency *string `json:"toAgency"` + LookupUserEnabled *bool `json:"lookupUserEnabled"` + AcceptItemEnabled *bool `json:"acceptItemEnabled"` + CheckInItemEnabled *bool `json:"checkInItemEnabled"` + CheckOutItemEnabled *bool `json:"checkOutItemEnabled"` + ItemLocation *string `json:"itemLocation"` + RequestItemRequestType *string `json:"requestItemRequestType"` + RequestItemRequestScopeType *string `json:"requestItemRequestScopeType"` + RequestItemBibIDCode *string `json:"requestItemBibIdCode"` + RequestItemEnabled *bool `json:"requestItemEnabled"` + RequestItemPickupLocationEnabled *bool `json:"requestItemPickupLocationEnabled"` + RequesterPickupLocation *string `json:"requesterPickupLocation"` + SupplierPickupLocation *string `json:"supplierPickupLocation"` + RequesterPatronPattern *string `json:"requesterPatronPattern"` +} + +type ILLConfig struct { + ISO18626URL *string `json:"iso18626Url"` + ISO18626Vendor *string `json:"iso18626Vendor"` + LendersOfLastResort []SymbolRef `json:"lendersOfLastResort"` + IncludeRequestingAgencyInfo *bool `json:"includeRequestingAgencyInfo"` + IncludeSupplierInfo *bool `json:"includeSupplierInfo"` + IncludeReturnInfo *bool `json:"includeReturnInfo"` + IncludeVendorNote *bool `json:"includeVendorNote"` + UseOfferedCosts *bool `json:"useOfferedCosts"` + NoteFieldSeparator *string `json:"noteFieldSeparator"` + SupplierPatronPattern *string `json:"supplierPatronPattern"` + DuplicateCheckWindowHours *int32 `json:"duplicateCheckWindowHours"` +} + +type CatalogConfig struct { + MetadataUpdateMode *string `json:"metadataUpdateMode"` + SRU *SRUConfig `json:"sru"` + Zoom *ZoomConfig `json:"zoom"` + Query *QueryConfig `json:"queryConfig"` + HoldingsFormat *HoldingsParserConfig `json:"holdingsFormat"` + MetadataFormat *MetadataParserConfig `json:"metadataFormat"` +} + +type SRUConfig struct { + Address string `json:"address"` + RecordSchema *string `json:"recordSchema"` +} + +type ZoomConfig struct { + Address string `json:"address"` + Options *map[string]string `json:"options"` +} + +type QueryConfig struct { + Type *string `json:"type"` + Identifier *string `json:"identifier"` + ISBN *string `json:"isbn"` + ISSN *string `json:"issn"` + Title *string `json:"title"` +} + +type HoldingsParserConfig struct { + Marc *MarcHoldingsParserConfig `json:"marc"` + Marc21Plus1 *map[string]any `json:"marc21plus1"` + OPAC *map[string]any `json:"opac"` + Reservoir *map[string]any `json:"reservoir"` +} + +type MarcHoldingsParserConfig struct { + CallNumberSubField *string `json:"callNumberSubField"` + ItemIDSubField *string `json:"itemIdSubField"` + LocationSubField *string `json:"locationSubField"` + MainField *string `json:"mainField"` + RestrictedSubField *string `json:"restrictedSubField"` + ShelvingLocationSubField *string `json:"shelvingLocationSubField"` +} + +type MetadataParserConfig struct { + Marc21 *MarcMetadataParserConfig `json:"marc21"` +} + +type MarcMetadataParserConfig struct { + Author *string `json:"author"` + Edition *string `json:"edition"` + Identifier *string `json:"identifier"` + ISBN *string `json:"isbn"` + ISSN *string `json:"issn"` + Subtitle *string `json:"subtitle"` + Title *string `json:"title"` +} + +type HoldingsPolicy struct { + Locations []HoldingsLocation `json:"locations"` + ShelvingLocations []HoldingsShelvingLocation `json:"shelvingLocations"` + LocationPolicies []HoldingsLocationPolicy `json:"locationPolicies"` + ItemLoanPolicies []HoldingsItemLoanPolicy `json:"itemLoanPolicies"` +} + +type HoldingsLocation struct { + Code string `json:"code"` + Name string `json:"name"` + SupplyPreference int `json:"supplyPreference"` +} + +type HoldingsShelvingLocation = HoldingsLocation + +type HoldingsLocationPolicy struct { + LocationCode *string `json:"locationCode"` + ShelvingLocationCode string `json:"shelvingLocationCode"` + SupplyPreference int `json:"supplyPreference"` +} + +type HoldingsItemLoanPolicy struct { + Code string `json:"code"` + Name string `json:"name"` + Lendable bool `json:"lendable"` +} + +func validateConfigEnums(catalog *CatalogConfig, ill *ILLConfig) error { + if ill != nil && ill.ISO18626Vendor != nil && !oneOf(*ill.ISO18626Vendor, "Alma", "ReShare", "CrossLink", "ILLiad", "Unknown") { + return fmt.Errorf("invalid ILL vendor: %s", *ill.ISO18626Vendor) + } + if ill != nil { + for index := range ill.LendersOfLastResort { + if err := ill.LendersOfLastResort[index].NormalizeAndValidate(); err != nil { + return fmt.Errorf("lender of last resort %d: %w", index+1, err) + } + } + } + if catalog != nil && catalog.MetadataUpdateMode != nil && !oneOf(*catalog.MetadataUpdateMode, "replace", "merge", "none", "auto") { + return fmt.Errorf("invalid metadata update mode: %s", *catalog.MetadataUpdateMode) + } + if catalog != nil && catalog.Query != nil && catalog.Query.Type != nil && !oneOf(*catalog.Query.Type, "cql", "pqf") { + return fmt.Errorf("invalid query type: %s", *catalog.Query.Type) + } + return nil +} diff --git a/directory/import/model/models.go b/directory/import/model/models.go new file mode 100644 index 000000000..f49ee07ca --- /dev/null +++ b/directory/import/model/models.go @@ -0,0 +1,275 @@ +package model + +import ( + "fmt" + "strings" + "time" +) + +type ConflictPolicy string + +const ( + ConflictPolicyFail ConflictPolicy = "fail" + ConflictPolicySkip ConflictPolicy = "skip" + ConflictPolicyUpdate ConflictPolicy = "update" +) + +func ParseConflictPolicy(value string) (ConflictPolicy, error) { + switch ConflictPolicy(value) { + case "", ConflictPolicyFail: + return ConflictPolicyFail, nil + case ConflictPolicySkip: + return ConflictPolicySkip, nil + case ConflictPolicyUpdate: + return ConflictPolicyUpdate, nil + default: + return "", fmt.Errorf("unknown conflict policy: %s", value) + } +} + +type Outcome string + +const ( + OutcomeImported Outcome = "imported" + OutcomeSkipped Outcome = "skipped" +) + +type RepoResult struct { + Outcome Outcome + Diagnostic string +} + +type ImportSectionResult struct { + Imported int32 `json:"imported"` + Failed int32 `json:"failed"` + Skipped int32 `json:"skipped"` +} + +type ImportItemError struct { + Line int32 `json:"line"` + Type *string `json:"type,omitempty"` + Key *string `json:"key,omitempty"` + Error string `json:"error"` +} + +type ImportResult struct { + Entries ImportSectionResult `json:"entries"` + Tiers ImportSectionResult `json:"tiers"` + Networks ImportSectionResult `json:"networks"` + Errors []ImportItemError `json:"errors"` +} + +type SymbolRef struct { + Authority string `json:"authority"` + Symbol string `json:"symbol"` +} + +func (s *SymbolRef) NormalizeAndValidate() error { + s.Authority = strings.ToUpper(strings.TrimSpace(s.Authority)) + s.Symbol = strings.ToUpper(strings.TrimSpace(s.Symbol)) + if s.Authority == "" || s.Symbol == "" { + return fmt.Errorf("symbol authority and symbol are required") + } + return nil +} + +func (s SymbolRef) String() string { return s.Authority + ":" + s.Symbol } + +type EntryAggregate struct { + Key SymbolRef + Data EntryData +} + +type EntryData struct { + Name string `json:"name"` + Type string `json:"type"` + Parent *SymbolRef `json:"parent"` + Description *string `json:"description"` + OrganizationID *string `json:"organizationId"` + ContactName *string `json:"contactName"` + Email *string `json:"email"` + FromEmail *string `json:"fromEmail"` + Tenant *string `json:"tenant"` + Vendor *string `json:"vendor"` + PhoneNumber *string `json:"phoneNumber"` + LMSLocationCode *string `json:"lmsLocationCode"` + HRID *string `json:"hrid"` + TimeZone *string `json:"timeZone"` + Symbols []SymbolRef `json:"symbols"` + Endpoints []ServiceEndpoint `json:"endpoints"` + Addresses []Address `json:"addresses"` + Closures []Closure `json:"closures"` + LMSConfig *LMSConfig `json:"lmsConfig"` + CatalogConfig *CatalogConfig `json:"catalogConfig"` + ILLConfig *ILLConfig `json:"illConfig"` + HoldingsPolicy *HoldingsPolicy `json:"holdingsPolicy"` +} + +type ServiceEndpoint struct { + Name string `json:"name"` + Type string `json:"type"` + Address string `json:"address"` +} + +type Address struct { + Type string `json:"type"` + Components []AddressComponent `json:"addressComponents"` +} + +type AddressComponent struct { + Seq int32 `json:"seq"` + Type string `json:"type"` + Value string `json:"value"` +} + +type Closure struct { + StartDate string `json:"startDate"` + EndDate string `json:"endDate"` + Reason string `json:"reason"` +} + +func (a *EntryAggregate) NormalizeAndValidate() error { + if err := a.Key.NormalizeAndValidate(); err != nil { + return fmt.Errorf("entry key: %w", err) + } + if strings.TrimSpace(a.Data.Name) == "" { + return fmt.Errorf("entry name is required") + } + if !oneOf(a.Data.Type, "Institution", "Consortium", "Branch") { + return fmt.Errorf("invalid entry type: %s", a.Data.Type) + } + if a.Data.Vendor != nil && !oneOf(*a.Data.Vendor, "Alma", "ReShare", "CrossLink", "ILLiad", "Unknown") { + return fmt.Errorf("invalid entry vendor: %s", *a.Data.Vendor) + } + if a.Data.Parent != nil { + if err := a.Data.Parent.NormalizeAndValidate(); err != nil { + return fmt.Errorf("parent: %w", err) + } + } + seen := make(map[string]struct{}, len(a.Data.Symbols)) + keyCount := 0 + for index := range a.Data.Symbols { + if err := a.Data.Symbols[index].NormalizeAndValidate(); err != nil { + return fmt.Errorf("symbol %d: %w", index+1, err) + } + value := a.Data.Symbols[index].String() + if _, exists := seen[value]; exists { + return fmt.Errorf("duplicate entry symbol %s", value) + } + seen[value] = struct{}{} + if value == a.Key.String() { + keyCount++ + } + } + if keyCount != 1 { + return fmt.Errorf("entry key %s must appear exactly once in symbols", a.Key.String()) + } + for _, address := range a.Data.Addresses { + if !oneOf(address.Type, "Default", "Shipping", "Billing", "Other") { + return fmt.Errorf("invalid address type: %s", address.Type) + } + for _, component := range address.Components { + if !oneOf(component.Type, "Thoroughfare", "Locality", "AdministrativeArea", "PostalCode", "CountryCode", "Other") { + return fmt.Errorf("invalid address component type: %s", component.Type) + } + } + } + for index, closure := range a.Data.Closures { + start, err := time.Parse(time.DateOnly, closure.StartDate) + if err != nil { + return fmt.Errorf("closure %d has invalid startDate", index+1) + } + end, err := time.Parse(time.DateOnly, closure.EndDate) + if err != nil { + return fmt.Errorf("closure %d has invalid endDate", index+1) + } + if end.Before(start) { + return fmt.Errorf("closure %d endDate must not precede startDate", index+1) + } + } + return validateConfigEnums(a.Data.CatalogConfig, a.Data.ILLConfig) +} + +type TierKey struct { + Consortium SymbolRef `json:"consortium"` + Name string `json:"name"` +} + +type TierData struct { + Level string `json:"level"` + Type string `json:"type"` + Cost float64 `json:"cost"` + Entries []SymbolRef `json:"entries"` +} + +type TierAggregate struct { + Key TierKey + Data TierData +} + +func (a *TierAggregate) NormalizeAndValidate() error { + if err := a.Key.Consortium.NormalizeAndValidate(); err != nil { + return fmt.Errorf("tier consortium: %w", err) + } + if strings.TrimSpace(a.Key.Name) == "" { + return fmt.Errorf("tier name is required") + } + if !oneOf(a.Data.Level, "express", "normal", "rush", "secondarymail", "standard", "urgent") { + return fmt.Errorf("invalid tier level: %s", a.Data.Level) + } + if !oneOf(a.Data.Type, "loan", "copy") { + return fmt.Errorf("invalid tier type: %s", a.Data.Type) + } + return normalizeUniqueRefs(a.Data.Entries, "tier", func(refs []SymbolRef) { a.Data.Entries = refs }) +} + +type NetworkKey struct { + Consortium SymbolRef `json:"consortium"` + Name string `json:"name"` +} + +type NetworkData struct { + Priority int32 `json:"priority"` + Reciprocal *bool `json:"reciprocal"` + Entries []SymbolRef `json:"entries"` +} + +type NetworkAggregate struct { + Key NetworkKey + Data NetworkData +} + +func (a *NetworkAggregate) NormalizeAndValidate() error { + if err := a.Key.Consortium.NormalizeAndValidate(); err != nil { + return fmt.Errorf("network consortium: %w", err) + } + if strings.TrimSpace(a.Key.Name) == "" { + return fmt.Errorf("network name is required") + } + return normalizeUniqueRefs(a.Data.Entries, "network", func(refs []SymbolRef) { a.Data.Entries = refs }) +} + +func normalizeUniqueRefs(refs []SymbolRef, resource string, assign func([]SymbolRef)) error { + seen := make(map[string]struct{}, len(refs)) + for index := range refs { + if err := refs[index].NormalizeAndValidate(); err != nil { + return fmt.Errorf("%s entry %d: %w", resource, index+1, err) + } + key := refs[index].String() + if _, exists := seen[key]; exists { + return fmt.Errorf("duplicate %s entry %s", resource, key) + } + seen[key] = struct{}{} + } + assign(refs) + return nil +} + +func oneOf(value string, valid ...string) bool { + for _, candidate := range valid { + if value == candidate { + return true + } + } + return false +} diff --git a/directory/import/model/models_test.go b/directory/import/model/models_test.go new file mode 100644 index 000000000..7573036ec --- /dev/null +++ b/directory/import/model/models_test.go @@ -0,0 +1,88 @@ +package model + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseConflictPolicy(t *testing.T) { + tests := map[string]ConflictPolicy{ + "": ConflictPolicyFail, + "fail": ConflictPolicyFail, + "skip": ConflictPolicySkip, + "update": ConflictPolicyUpdate, + } + for input, expected := range tests { + actual, err := ParseConflictPolicy(input) + require.NoError(t, err) + assert.Equal(t, expected, actual) + } + _, err := ParseConflictPolicy("replace") + require.EqualError(t, err, "unknown conflict policy: replace") +} + +func TestEntryAggregateNormalizesAndRequiresKeySymbol(t *testing.T) { + aggregate := validEntryAggregate() + aggregate.Key = SymbolRef{Authority: "isil", Symbol: "missing"} + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "entry key ISIL:MISSING must appear exactly once in symbols") + assert.Equal(t, SymbolRef{Authority: "ISIL", Symbol: "MISSING"}, aggregate.Key) +} + +func TestEntryAggregateRejectsDuplicateNormalizedSymbols(t *testing.T) { + aggregate := validEntryAggregate() + aggregate.Data.Symbols = append(aggregate.Data.Symbols, SymbolRef{Authority: "isil", Symbol: "lib"}) + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "duplicate entry symbol ISIL:LIB") +} + +func TestTierAggregateRejectsInvalidEnum(t *testing.T) { + aggregate := TierAggregate{ + Key: TierKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Loan"}, + Data: TierData{Level: "instant", Type: "loan", Entries: []SymbolRef{}}, + } + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "invalid tier level: instant") +} + +func TestNetworkAggregateRejectsDuplicateEntries(t *testing.T) { + aggregate := NetworkAggregate{ + Key: NetworkKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Main"}, + Data: NetworkData{Entries: []SymbolRef{ + {Authority: "isil", Symbol: "lib"}, + {Authority: "ISIL", Symbol: "LIB"}, + }}, + } + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "duplicate network entry ISIL:LIB") +} + +func TestEntryAggregateRejectsInvalidClosureRange(t *testing.T) { + aggregate := validEntryAggregate() + aggregate.Data.Closures = []Closure{{StartDate: "2026-09-03", EndDate: "2026-09-02", Reason: "maintenance"}} + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "closure 1 endDate must not precede startDate") +} + +func validEntryAggregate() EntryAggregate { + return EntryAggregate{ + Key: SymbolRef{Authority: "isil", Symbol: "lib"}, + Data: EntryData{ + Name: "Library", + Type: "Institution", + Symbols: []SymbolRef{{Authority: "isil", Symbol: "lib"}}, + }, + } +} diff --git a/directory/import/service/decode.go b/directory/import/service/decode.go new file mode 100644 index 000000000..93b083027 --- /dev/null +++ b/directory/import/service/decode.go @@ -0,0 +1,344 @@ +package service + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/indexdata/crosslink/directory/import/model" +) + +type envelope struct { + Type string `json:"type"` + Key json.RawMessage `json:"key"` + Data json.RawMessage `json:"data"` +} + +type decodedRecord struct { + recordType string + key string + entry *model.EntryAggregate + tier *model.TierAggregate + network *model.NetworkAggregate +} + +var requiredProperties = map[string][]string{ + "envelope": {"type", "key", "data"}, + "entryKey": {"authority", "symbol"}, + "entryData": { + "name", "type", "parent", "description", "organizationId", "contactName", "email", "fromEmail", + "tenant", "vendor", "phoneNumber", "lmsLocationCode", "hrid", "timeZone", "symbols", "endpoints", + "addresses", "closures", "lmsConfig", "catalogConfig", "illConfig", "holdingsPolicy", + }, + "tierKey": {"consortium", "name"}, + "tierData": {"level", "type", "cost", "entries"}, + "networkKey": {"consortium", "name"}, + "networkData": {"priority", "reciprocal", "entries"}, +} + +func decodeRecord(data []byte) (decodedRecord, error) { + var record decodedRecord + + properties, err := objectProperties(data) + if err != nil { + return record, fmt.Errorf("invalid JSON record: %w", err) + } + if rawType, exists := properties["type"]; exists { + _ = json.Unmarshal(rawType, &record.recordType) + } + if err := requireExactProperties(properties, requiredProperties["envelope"]); err != nil { + return record, fmt.Errorf("invalid import envelope: %w", err) + } + + var env envelope + if err := decodeStrict(data, &env); err != nil { + return record, fmt.Errorf("invalid import envelope: %w", err) + } + record.recordType = env.Type + + switch env.Type { + case "entry": + var aggregate model.EntryAggregate + if err := validateEntryShape(env.Key, env.Data); err != nil { + return record, fmt.Errorf("invalid entry aggregate: %w", err) + } + if err := decodeAggregate(env.Key, env.Data, requiredProperties["entryKey"], requiredProperties["entryData"], &aggregate.Key, &aggregate.Data); err != nil { + return record, fmt.Errorf("invalid entry aggregate: %w", err) + } + record.entry = &aggregate + validationErr := aggregate.NormalizeAndValidate() + if aggregate.Key.Authority != "" && aggregate.Key.Symbol != "" { + record.key = aggregate.Key.String() + } + if validationErr != nil { + return record, validationErr + } + case "tier": + var aggregate model.TierAggregate + if err := validateMembershipShape(env.Key, env.Data, "tier"); err != nil { + return record, fmt.Errorf("invalid tier aggregate: %w", err) + } + if err := decodeAggregate(env.Key, env.Data, requiredProperties["tierKey"], requiredProperties["tierData"], &aggregate.Key, &aggregate.Data); err != nil { + return record, fmt.Errorf("invalid tier aggregate: %w", err) + } + record.tier = &aggregate + validationErr := aggregate.NormalizeAndValidate() + if aggregate.Key.Consortium.Authority != "" && aggregate.Key.Consortium.Symbol != "" && aggregate.Key.Name != "" { + record.key = aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + } + if validationErr != nil { + return record, validationErr + } + case "network": + var aggregate model.NetworkAggregate + if err := validateMembershipShape(env.Key, env.Data, "network"); err != nil { + return record, fmt.Errorf("invalid network aggregate: %w", err) + } + if err := decodeAggregate(env.Key, env.Data, requiredProperties["networkKey"], requiredProperties["networkData"], &aggregate.Key, &aggregate.Data); err != nil { + return record, fmt.Errorf("invalid network aggregate: %w", err) + } + record.network = &aggregate + validationErr := aggregate.NormalizeAndValidate() + if aggregate.Key.Consortium.Authority != "" && aggregate.Key.Consortium.Symbol != "" && aggregate.Key.Name != "" { + record.key = aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + } + if validationErr != nil { + return record, validationErr + } + default: + return record, fmt.Errorf("unknown import record type: %s", env.Type) + } + return record, nil +} + +func validateEntryShape(keyJSON, dataJSON []byte) error { + if err := validateObject(keyJSON, requiredProperties["entryKey"], nil); err != nil { + return fmt.Errorf("key: %w", err) + } + return validateObject(dataJSON, requiredProperties["entryData"], func(properties map[string]json.RawMessage) error { + if err := validateNullableObject(properties["parent"], []string{"authority", "symbol"}, nil); err != nil { + return fmt.Errorf("parent: %w", err) + } + if err := validateObjectArray(properties["symbols"], []string{"authority", "symbol"}, nil); err != nil { + return fmt.Errorf("symbols: %w", err) + } + if err := validateObjectArray(properties["endpoints"], []string{"name", "type", "address"}, nil); err != nil { + return fmt.Errorf("endpoints: %w", err) + } + if err := validateObjectArray(properties["addresses"], []string{"type", "addressComponents"}, func(address map[string]json.RawMessage) error { + return validateObjectArray(address["addressComponents"], []string{"seq", "type", "value"}, nil) + }); err != nil { + return fmt.Errorf("addresses: %w", err) + } + if err := validateObjectArray(properties["closures"], []string{"startDate", "endDate", "reason"}, nil); err != nil { + return fmt.Errorf("closures: %w", err) + } + if err := validateNullableObject(properties["lmsConfig"], []string{ + "address", "fromAgency", "fromAgencyAuthentication", "toAgency", "lookupUserEnabled", "acceptItemEnabled", + "checkInItemEnabled", "checkOutItemEnabled", "itemLocation", "requestItemRequestType", "requestItemRequestScopeType", + "requestItemBibIdCode", "requestItemEnabled", "requestItemPickupLocationEnabled", "requesterPickupLocation", + "supplierPickupLocation", "requesterPatronPattern", + }, nil); err != nil { + return fmt.Errorf("lmsConfig: %w", err) + } + if err := validateCatalogShape(properties["catalogConfig"]); err != nil { + return fmt.Errorf("catalogConfig: %w", err) + } + if err := validateNullableObject(properties["illConfig"], []string{ + "iso18626Url", "iso18626Vendor", "lendersOfLastResort", "includeRequestingAgencyInfo", "includeSupplierInfo", + "includeReturnInfo", "includeVendorNote", "useOfferedCosts", "noteFieldSeparator", "supplierPatronPattern", + "duplicateCheckWindowHours", + }, func(config map[string]json.RawMessage) error { + return validateObjectArray(config["lendersOfLastResort"], []string{"authority", "symbol"}, nil) + }); err != nil { + return fmt.Errorf("illConfig: %w", err) + } + if err := validateHoldingsPolicyShape(properties["holdingsPolicy"]); err != nil { + return fmt.Errorf("holdingsPolicy: %w", err) + } + return nil + }) +} + +func validateMembershipShape(keyJSON, dataJSON []byte, aggregateType string) error { + keyRequired := requiredProperties[aggregateType+"Key"] + dataRequired := requiredProperties[aggregateType+"Data"] + if err := validateObject(keyJSON, keyRequired, func(key map[string]json.RawMessage) error { + return validateObject(key["consortium"], []string{"authority", "symbol"}, nil) + }); err != nil { + return fmt.Errorf("key: %w", err) + } + if err := validateObject(dataJSON, dataRequired, func(data map[string]json.RawMessage) error { + return validateObjectArray(data["entries"], []string{"authority", "symbol"}, nil) + }); err != nil { + return fmt.Errorf("data: %w", err) + } + return nil +} + +func validateCatalogShape(raw json.RawMessage) error { + return validateNullableObject(raw, []string{"metadataUpdateMode", "sru", "zoom", "queryConfig", "holdingsFormat", "metadataFormat"}, func(config map[string]json.RawMessage) error { + checks := []struct { + name string + properties []string + nested func(map[string]json.RawMessage) error + }{ + {"sru", []string{"address", "recordSchema"}, nil}, + {"zoom", []string{"address", "options"}, nil}, + {"queryConfig", []string{"type", "identifier", "isbn", "issn", "title"}, nil}, + {"holdingsFormat", []string{"marc", "marc21plus1", "opac", "reservoir"}, func(format map[string]json.RawMessage) error { + return validateNullableObject(format["marc"], []string{"callNumberSubField", "itemIdSubField", "locationSubField", "mainField", "restrictedSubField", "shelvingLocationSubField"}, nil) + }}, + {"metadataFormat", []string{"marc21"}, func(format map[string]json.RawMessage) error { + return validateNullableObject(format["marc21"], []string{"author", "edition", "identifier", "isbn", "issn", "subtitle", "title"}, nil) + }}, + } + for _, check := range checks { + if err := validateNullableObject(config[check.name], check.properties, check.nested); err != nil { + return fmt.Errorf("%s: %w", check.name, err) + } + } + return nil + }) +} + +func validateHoldingsPolicyShape(raw json.RawMessage) error { + return validateNullableObject(raw, []string{"locations", "shelvingLocations", "locationPolicies", "itemLoanPolicies"}, func(policy map[string]json.RawMessage) error { + if err := validateObjectArray(policy["locations"], []string{"code", "name", "supplyPreference"}, nil); err != nil { + return fmt.Errorf("locations: %w", err) + } + if err := validateObjectArray(policy["shelvingLocations"], []string{"code", "name", "supplyPreference"}, nil); err != nil { + return fmt.Errorf("shelvingLocations: %w", err) + } + if err := validateObjectArray(policy["locationPolicies"], []string{"locationCode", "shelvingLocationCode", "supplyPreference"}, nil); err != nil { + return fmt.Errorf("locationPolicies: %w", err) + } + if err := validateObjectArray(policy["itemLoanPolicies"], []string{"code", "name", "lendable"}, nil); err != nil { + return fmt.Errorf("itemLoanPolicies: %w", err) + } + return nil + }) +} + +func validateNullableObject(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return nil + } + return validateObject(raw, required, nested) +} + +func validateObject(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { + properties, err := objectProperties(raw) + if err != nil { + return err + } + if err := requireExactProperties(properties, required); err != nil { + return err + } + if nested != nil { + return nested(properties) + } + return nil +} + +func validateObjectArray(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { + if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return fmt.Errorf("must be an array, not null") + } + var values []json.RawMessage + if err := decodeStrict(raw, &values); err != nil { + return err + } + if values == nil { + return fmt.Errorf("must be an array") + } + for index, value := range values { + if err := validateObject(value, required, nested); err != nil { + return fmt.Errorf("item %d: %w", index+1, err) + } + } + return nil +} + +func decodeAggregate(keyJSON, dataJSON []byte, keyProperties, dataProperties []string, key, data any) error { + keyObject, err := objectProperties(keyJSON) + if err != nil { + return fmt.Errorf("key must be an object: %w", err) + } + if err := requireExactProperties(keyObject, keyProperties); err != nil { + return fmt.Errorf("key: %w", err) + } + dataObject, err := objectProperties(dataJSON) + if err != nil { + return fmt.Errorf("data must be an object: %w", err) + } + if err := requireExactProperties(dataObject, dataProperties); err != nil { + return fmt.Errorf("data: %w", err) + } + if err := decodeStrict(keyJSON, key); err != nil { + return fmt.Errorf("key: %w", err) + } + if err := decodeStrict(dataJSON, data); err != nil { + return fmt.Errorf("data: %w", err) + } + return nil +} + +func objectProperties(data []byte) (map[string]json.RawMessage, error) { + var properties map[string]json.RawMessage + if err := decodeStrict(data, &properties); err != nil { + return nil, err + } + if properties == nil { + return nil, fmt.Errorf("expected JSON object") + } + return properties, nil +} + +func requireExactProperties(properties map[string]json.RawMessage, required []string) error { + allowed := make(map[string]struct{}, len(required)) + for _, property := range required { + allowed[property] = struct{}{} + if _, exists := properties[property]; !exists { + return fmt.Errorf("missing required property %q", property) + } + } + unknown := make([]string, 0) + for property := range properties { + if _, exists := allowed[property]; !exists { + unknown = append(unknown, property) + } + } + if len(unknown) != 0 { + sort.Strings(unknown) + return fmt.Errorf("unknown property %q", unknown[0]) + } + return nil +} + +func decodeStrict(data []byte, destination any) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(destination); err != nil { + return sanitizeJSONError(err) + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return fmt.Errorf("multiple JSON values") + } + return sanitizeJSONError(err) + } + return nil +} + +func sanitizeJSONError(err error) error { + message := err.Error() + if index := strings.Index(message, " looking for beginning of value"); index >= 0 { + return fmt.Errorf("invalid JSON syntax%s", message[index:]) + } + return err +} diff --git a/directory/import/service/importer.go b/directory/import/service/importer.go new file mode 100644 index 000000000..8e56a6d22 --- /dev/null +++ b/directory/import/service/importer.go @@ -0,0 +1,130 @@ +package service + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + + "github.com/indexdata/crosslink/directory/import/model" +) + +const maxRecordBytes = 1 << 20 + +var ErrRecordTooLarge = errors.New("import record exceeds 1 MiB limit") + +type Repository interface { + ImportEntry(context.Context, model.EntryAggregate, model.ConflictPolicy) (model.RepoResult, error) + ImportTier(context.Context, model.TierAggregate, model.ConflictPolicy) (model.RepoResult, error) + ImportNetwork(context.Context, model.NetworkAggregate, model.ConflictPolicy) (model.RepoResult, error) +} + +type Importer struct { + repository Repository + maxRecordBytes int +} + +func New(repository Repository) *Importer { + return &Importer{repository: repository, maxRecordBytes: maxRecordBytes} +} + +func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, input io.Reader) (model.ImportResult, error) { + result := model.ImportResult{Errors: make([]model.ImportItemError, 0)} + reader := bufio.NewReaderSize(input, i.maxRecordBytes+2) + var recordNumber int32 + + for { + line, readErr := reader.ReadSlice('\n') + if errors.Is(readErr, bufio.ErrBufferFull) { + return result, ErrRecordTooLarge + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return result, fmt.Errorf("read import stream: %w", readErr) + } + + line = bytes.TrimSuffix(line, []byte{'\n'}) + line = bytes.TrimSuffix(line, []byte{'\r'}) + if len(line) > i.maxRecordBytes { + return result, ErrRecordTooLarge + } + if len(bytes.TrimSpace(line)) != 0 { + recordNumber++ + i.importRecord(ctx, policy, recordNumber, line, &result) + } + + if errors.Is(readErr, io.EOF) { + return result, nil + } + } +} + +func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy, line int32, data []byte, result *model.ImportResult) { + record, err := decodeRecord(data) + if err != nil { + incrementFailed(result, record.recordType) + appendError(result, line, record.recordType, record.key, err.Error()) + return + } + + var repoResult model.RepoResult + switch record.recordType { + case "entry": + repoResult, err = i.repository.ImportEntry(ctx, *record.entry, policy) + case "tier": + repoResult, err = i.repository.ImportTier(ctx, *record.tier, policy) + case "network": + repoResult, err = i.repository.ImportNetwork(ctx, *record.network, policy) + } + if err != nil { + incrementFailed(result, record.recordType) + appendError(result, line, record.recordType, record.key, err.Error()) + return + } + + switch repoResult.Outcome { + case model.OutcomeImported: + section(result, record.recordType).Imported++ + case model.OutcomeSkipped: + section(result, record.recordType).Skipped++ + diagnostic := repoResult.Diagnostic + if diagnostic == "" { + diagnostic = "record skipped because its business key already exists" + } + appendError(result, line, record.recordType, record.key, diagnostic) + default: + incrementFailed(result, record.recordType) + appendError(result, line, record.recordType, record.key, "repository returned an invalid import outcome") + } +} + +func incrementFailed(result *model.ImportResult, recordType string) { + if target := section(result, recordType); target != nil { + target.Failed++ + } +} + +func section(result *model.ImportResult, recordType string) *model.ImportSectionResult { + switch recordType { + case "entry": + return &result.Entries + case "tier": + return &result.Tiers + case "network": + return &result.Networks + default: + return nil + } +} + +func appendError(result *model.ImportResult, line int32, recordType, key, message string) { + item := model.ImportItemError{Line: line, Error: message} + if recordType != "" { + item.Type = &recordType + } + if key != "" { + item.Key = &key + } + result.Errors = append(result.Errors, item) +} diff --git a/directory/import/service/importer_test.go b/directory/import/service/importer_test.go new file mode 100644 index 000000000..34c7ffa7d --- /dev/null +++ b/directory/import/service/importer_test.go @@ -0,0 +1,184 @@ +package service + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/indexdata/crosslink/directory/import/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type recordingRepo struct { + entryCalls int + tierCalls int + networkCalls int + result model.RepoResult + err error +} + +func (r *recordingRepo) ImportEntry(context.Context, model.EntryAggregate, model.ConflictPolicy) (model.RepoResult, error) { + r.entryCalls++ + return r.result, r.err +} + +func (r *recordingRepo) ImportTier(context.Context, model.TierAggregate, model.ConflictPolicy) (model.RepoResult, error) { + r.tierCalls++ + return r.result, r.err +} + +func (r *recordingRepo) ImportNetwork(context.Context, model.NetworkAggregate, model.ConflictPolicy) (model.RepoResult, error) { + r.networkCalls++ + return r.result, r.err +} + +func TestImportDispatchesAllAggregateTypes(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + input := strings.Join([]string{validEntryRecord(), validTierRecord(), validNetworkRecord()}, "\n") + + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Entries) + assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Tiers) + assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Networks) + assert.Empty(t, result.Errors) + assert.Equal(t, 1, repo.entryCalls) + assert.Equal(t, 1, repo.tierCalls) + assert.Equal(t, 1, repo.networkCalls) +} + +func TestImportContinuesAfterMalformedRecord(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + input := "{bad json}\n\n" + validEntryRecord() + "\n" + + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + + require.NoError(t, err) + assert.Zero(t, result.Entries.Failed) + assert.Equal(t, int32(1), result.Entries.Imported) + require.Len(t, result.Errors, 1) + assert.Equal(t, int32(1), result.Errors[0].Line) + assert.Nil(t, result.Errors[0].Type) + assert.NotContains(t, result.Errors[0].Error, "{bad json}") + assert.Equal(t, 1, repo.entryCalls) +} + +func TestImportContinuesAfterSemanticFailure(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + badTier := strings.Replace(validTierRecord(), `"level":"standard"`, `"level":"invalid"`, 1) + + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(badTier+"\n"+validTierRecord())) + + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Imported: 1, Failed: 1}, result.Tiers) + require.Len(t, result.Errors, 1) + assert.Equal(t, int32(1), result.Errors[0].Line) + require.NotNil(t, result.Errors[0].Type) + assert.Equal(t, "tier", *result.Errors[0].Type) + assert.Equal(t, 1, repo.tierCalls) +} + +func TestImportBlankLinesDoNotIncrementRecordNumber(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + badTier := strings.Replace(validTierRecord(), `"level":"standard"`, `"level":"invalid"`, 1) + input := "\n" + validTierRecord() + "\n\r\n" + badTier + + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + + require.NoError(t, err) + require.Len(t, result.Errors, 1) + require.Equal(t, int32(2), result.Errors[0].Line) +} + +func TestImportReturnsFatalReaderError(t *testing.T) { + fatal := errors.New("transport failed") + result, err := New(&recordingRepo{}).Import(context.Background(), model.ConflictPolicyFail, errorReader{err: fatal}) + + require.ErrorIs(t, err, fatal) + require.Empty(t, result.Errors) +} + +func TestImportRejectsMissingAndUnknownProperties(t *testing.T) { + tests := map[string]string{ + "missing entry field": strings.Replace(validEntryRecord(), `,"timeZone":null`, "", 1), + "unknown envelope": strings.Replace(validTierRecord(), `"type":"tier"`, `"type":"tier","secret":"do-not-echo"`, 1), + "null collection": strings.Replace(validEntryRecord(), `"endpoints":[]`, `"endpoints":null`, 1), + "missing child field": strings.Replace( + strings.Replace(validEntryRecord(), `"endpoints":[]`, `"endpoints":[{"name":"ISO","type":"ISO18626","address":"https://example.test"}]`, 1), + `,"address":"https://example.test"`, "", 1), + "missing config field": strings.Replace( + strings.Replace(validEntryRecord(), `"illConfig":null`, validILLConfig(), 1), + `,"supplierPatronPattern":null`, "", 1), + } + for name, record := range tests { + t.Run(name, func(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + require.NoError(t, err) + require.Len(t, result.Errors, 1) + assert.NotContains(t, result.Errors[0].Error, "do-not-echo") + assert.Zero(t, repo.entryCalls+repo.tierCalls+repo.networkCalls) + }) + } +} + +func validILLConfig() string { + return `"illConfig":{"iso18626Url":null,"iso18626Vendor":null,"lendersOfLastResort":[],"includeRequestingAgencyInfo":null,"includeSupplierInfo":null,"includeReturnInfo":null,"includeVendorNote":null,"useOfferedCosts":null,"noteFieldSeparator":null,"supplierPatronPattern":null,"duplicateCheckWindowHours":null}` +} + +func TestImportAccountsForSkippedAndRepositoryFailures(t *testing.T) { + t.Run("skipped", func(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeSkipped, Diagnostic: "entry already exists"}} + result, err := New(repo).Import(context.Background(), model.ConflictPolicySkip, strings.NewReader(validEntryRecord())) + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Skipped: 1}, result.Entries) + require.Len(t, result.Errors, 1) + assert.Equal(t, "entry already exists", result.Errors[0].Error) + }) + + t.Run("failed", func(t *testing.T) { + repo := &recordingRepo{err: errors.New("entry parent does not exist")} + result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(validEntryRecord())) + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Failed: 1}, result.Entries) + require.Len(t, result.Errors, 1) + assert.Equal(t, "entry parent does not exist", result.Errors[0].Error) + }) +} + +func TestImportRecordLimitIsExact(t *testing.T) { + record := validTierRecord() + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + importer := New(repo) + importer.maxRecordBytes = len(record) + + result, err := importer.Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record+"\r\n")) + require.NoError(t, err) + assert.Equal(t, int32(1), result.Tiers.Imported) + + repo = &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + importer = New(repo) + importer.maxRecordBytes = len(record) - 1 + _, err = importer.Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record+"\n")) + require.ErrorIs(t, err, ErrRecordTooLarge) + assert.Zero(t, repo.tierCalls) +} + +func validEntryRecord() string { + return `{"type":"entry","key":{"authority":"isil","symbol":"abc"},"data":{"name":"Library","type":"Institution","parent":null,"description":null,"organizationId":null,"contactName":null,"email":null,"fromEmail":null,"tenant":null,"vendor":null,"phoneNumber":null,"lmsLocationCode":null,"hrid":null,"timeZone":null,"symbols":[{"authority":"isil","symbol":"abc"}],"endpoints":[],"addresses":[],"closures":[],"lmsConfig":null,"catalogConfig":null,"illConfig":null,"holdingsPolicy":null}}` +} + +func validTierRecord() string { + return `{"type":"tier","key":{"consortium":{"authority":"isil","symbol":"con"},"name":"Primary"},"data":{"level":"standard","type":"loan","cost":0,"entries":[]}}` +} + +func validNetworkRecord() string { + return `{"type":"network","key":{"consortium":{"authority":"isil","symbol":"con"},"name":"Main"},"data":{"priority":1,"reciprocal":null,"entries":[]}}` +} + +type errorReader struct{ err error } + +func (r errorReader) Read([]byte) (int, error) { return 0, r.err } diff --git a/directory/migrations/006_import_business_keys.down.sql b/directory/migrations/006_import_business_keys.down.sql new file mode 100644 index 000000000..b4f180046 --- /dev/null +++ b/directory/migrations/006_import_business_keys.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE networks DROP CONSTRAINT networks_consortium_name_unique; +ALTER TABLE tiers DROP CONSTRAINT tiers_consortium_name_unique; + +ALTER TABLE networks ALTER COLUMN name DROP NOT NULL; +ALTER TABLE tiers ALTER COLUMN name DROP NOT NULL; diff --git a/directory/migrations/006_import_business_keys.up.sql b/directory/migrations/006_import_business_keys.up.sql new file mode 100644 index 000000000..73e645eed --- /dev/null +++ b/directory/migrations/006_import_business_keys.up.sql @@ -0,0 +1,29 @@ +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM tiers WHERE name IS NULL OR btrim(name) = '') THEN + RAISE EXCEPTION 'cannot add tier business key: tiers contain null or blank names'; + END IF; + IF EXISTS (SELECT 1 FROM networks WHERE name IS NULL OR btrim(name) = '') THEN + RAISE EXCEPTION 'cannot add network business key: networks contain null or blank names'; + END IF; + IF EXISTS ( + SELECT 1 FROM tiers GROUP BY consortium, name HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'cannot add tier business key: duplicate consortium/name pairs exist'; + END IF; + IF EXISTS ( + SELECT 1 FROM networks GROUP BY consortium, name HAVING count(*) > 1 + ) THEN + RAISE EXCEPTION 'cannot add network business key: duplicate consortium/name pairs exist'; + END IF; +END +$$; + +ALTER TABLE tiers ALTER COLUMN name SET NOT NULL; +ALTER TABLE networks ALTER COLUMN name SET NOT NULL; + +ALTER TABLE tiers + ADD CONSTRAINT tiers_consortium_name_unique UNIQUE (consortium, name); + +ALTER TABLE networks + ADD CONSTRAINT networks_consortium_name_unique UNIQUE (consortium, name); diff --git a/directory/oapi-codegen.yaml b/directory/oapi-codegen.yaml index 9e48520c1..fe2f6c08b 100644 --- a/directory/oapi-codegen.yaml +++ b/directory/oapi-codegen.yaml @@ -7,3 +7,4 @@ generate: output: api/directory.gen.go output-options: nullable-type: true + skip-prune: true diff --git a/directory/query.sql b/directory/query.sql index cf2997e2b..19c706a94 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -475,3 +475,40 @@ WHERE entry = @entry; -- name: DeleteHoldingsPolicyByEntry :exec DELETE FROM holdings_policies WHERE entry = @entry; + +-- name: LockTierByBusinessKey :one +SELECT * FROM tiers +WHERE consortium = @consortium AND name = @name +FOR UPDATE; + +-- name: UpdateImportedTier :exec +UPDATE tiers +SET level = @level, type = @type, cost = @cost +WHERE id = @id; + +-- name: DeleteEntryTiersByTier :exec +DELETE FROM entry_tiers WHERE tier = @tier; + +-- name: LockNetworkByBusinessKey :one +SELECT * FROM networks +WHERE consortium = @consortium AND name = @name +FOR UPDATE; + +-- name: UpdateImportedNetwork :exec +UPDATE networks +SET priority = @priority, reciprocal = @reciprocal +WHERE id = @id; + +-- name: DeleteEntryNetworksByNetwork :exec +DELETE FROM entry_networks WHERE network = @network; + +-- name: DeleteClosuresByEntry :exec +DELETE FROM closures WHERE entry = @entry; + +-- name: WouldCreateEntryCycle :one +WITH RECURSIVE descendants AS ( + SELECT id FROM entries WHERE parent = @child + UNION ALL + SELECT e.id FROM entries e JOIN descendants d ON e.parent = d.id +) +SELECT @parent::uuid = @child::uuid OR EXISTS (SELECT 1 FROM descendants WHERE id = @parent); diff --git a/directory/sqlc.yaml b/directory/sqlc.yaml index dce949f81..cda34f46a 100644 --- a/directory/sqlc.yaml +++ b/directory/sqlc.yaml @@ -10,6 +10,14 @@ sql: sql_package: "pgx/v5" emit_pointers_for_null_types: true overrides: + - column: "tiers.name" + go_type: + type: "string" + pointer: true + - column: "networks.name" + go_type: + type: "string" + pointer: true - db_type: "uuid" go_type: import: "github.com/google/uuid" @@ -19,4 +27,4 @@ sql: go_type: import: "github.com/google/uuid" type: "UUID" - pointer: true \ No newline at end of file + pointer: true diff --git a/directory/test/import_test.go b/directory/test/import_test.go new file mode 100644 index 000000000..33048b9ec --- /dev/null +++ b/directory/test/import_test.go @@ -0,0 +1,204 @@ +package test + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/google/uuid" + "github.com/indexdata/crosslink/directory/api" + "github.com/indexdata/crosslink/directory/auth" + "github.com/stretchr/testify/require" +) + +func TestImportOrderedAggregates(t *testing.T) { + resetImportState(t) + consortium := symbolObject("ISIL", "CON") + institution := symbolObject("ISIL", "INST") + branch := symbolObject("ISIL", "BRANCH") + records := []any{ + entryImportRecord(consortium, "Consortium", nil, "Consortium"), + entryImportRecord(institution, "Institution", consortium, "Institution"), + entryImportRecord(branch, "Branch", institution, "Branch"), + map[string]any{"type": "tier", "key": map[string]any{"consortium": consortium, "name": "Loan"}, "data": map[string]any{"level": "standard", "type": "loan", "cost": 1.5, "entries": []any{institution, branch}}}, + map[string]any{"type": "network", "key": map[string]any{"consortium": consortium, "name": "Main"}, "data": map[string]any{"priority": 1, "reciprocal": true, "entries": []any{institution}}}, + } + + response, result := importRequest(t, records, "", standardHeaders) + + require.Equal(t, http.StatusOK, response.StatusCode) + require.Equal(t, api.ImportSectionResult{Imported: 3}, result.Entries) + require.Equal(t, api.ImportSectionResult{Imported: 1}, result.Tiers) + require.Equal(t, api.ImportSectionResult{Imported: 1}, result.Networks) + require.Empty(t, result.Errors) + + consortiumID := importedEntryID(t, "ISIL", "CON") + institutionID := importedEntryID(t, "ISIL", "INST") + branchID := importedEntryID(t, "ISIL", "BRANCH") + require.NotEqual(t, uuid.Nil, consortiumID) + require.NotEqual(t, consortiumID, institutionID) + require.NotEqual(t, institutionID, branchID) + var institutionParent, branchParent uuid.UUID + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT parent FROM entries WHERE id=$1`, institutionID).Scan(&institutionParent)) + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT parent FROM entries WHERE id=$1`, branchID).Scan(&branchParent)) + require.Equal(t, consortiumID, institutionParent) + require.Equal(t, institutionID, branchParent) + + for query, expected := range map[string]int{ + `SELECT count(*) FROM service_endpoints WHERE entry=$1`: 1, + `SELECT count(*) FROM addresses WHERE entry=$1`: 1, + `SELECT count(*) FROM closures WHERE entry=$1`: 1, + `SELECT count(*) FROM lms_configs WHERE entry=$1`: 1, + } { + var count int + require.NoError(t, dbpool.QueryRow(context.Background(), query, consortiumID).Scan(&count)) + require.Equal(t, expected, count) + } + var tierAssignments, networkAssignments int + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT count(*) FROM entry_tiers`).Scan(&tierAssignments)) + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT count(*) FROM entry_networks`).Scan(&networkAssignments)) + require.Equal(t, 2, tierAssignments) + require.Equal(t, 1, networkAssignments) +} + +func TestImportPartialCommitAndConflictPolicies(t *testing.T) { + resetImportState(t) + consortium := symbolObject("ISIL", "CON") + validInstitution := symbolObject("ISIL", "VALID") + missing := symbolObject("ISIL", "MISSING") + records := []any{ + entryImportRecord(consortium, "Consortium", nil, "Consortium"), + entryImportRecord(symbolObject("ISIL", "BAD"), "Bad", missing, "Institution"), + entryImportRecord(validInstitution, "Valid", consortium, "Institution"), + } + + response, result := importRequest(t, records, "", standardHeaders) + require.Equal(t, http.StatusOK, response.StatusCode) + require.Equal(t, api.ImportSectionResult{Imported: 2, Failed: 1}, result.Entries) + require.Len(t, result.Errors, 1) + require.Equal(t, int32(2), result.Errors[0].Line) + require.Equal(t, 2, importedEntryCount(t)) + + originalID := importedEntryID(t, "ISIL", "VALID") + _, failed := importRequest(t, []any{entryImportRecord(validInstitution, "Failed", consortium, "Institution")}, "fail", standardHeaders) + require.Equal(t, api.ImportSectionResult{Failed: 1}, failed.Entries) + _, skipped := importRequest(t, []any{entryImportRecord(validInstitution, "Skipped", consortium, "Institution")}, "skip", standardHeaders) + require.Equal(t, api.ImportSectionResult{Skipped: 1}, skipped.Entries) + require.Len(t, skipped.Errors, 1) + _, updated := importRequest(t, []any{entryImportRecord(validInstitution, "Updated", consortium, "Institution")}, "update", standardHeaders) + require.Equal(t, api.ImportSectionResult{Imported: 1}, updated.Entries) + require.Equal(t, originalID, importedEntryID(t, "ISIL", "VALID")) + var name string + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT name FROM entries WHERE id=$1`, originalID).Scan(&name)) + require.Equal(t, "Updated", name) +} + +func TestImportRequiresConsortialAdmin(t *testing.T) { + for name, permission := range map[string]string{ + "institution": "directory.institution.all", + "system": "directory.system.all", + "public": "directory.public.all", + } { + t.Run(name, func(t *testing.T) { + resetImportState(t) + headers := map[string]string{auth.FolioPermissionsHeader: `[` + mustJSON(t, permission) + `]`} + response, _ := importRequest(t, []any{entryImportRecord(symbolObject("ISIL", "CON"), "Consortium", nil, "Consortium")}, "", headers) + require.Equal(t, http.StatusUnauthorized, response.StatusCode) + require.Zero(t, importedEntryCount(t)) + }) + } +} + +func importRequest(t *testing.T, records []any, policy string, headers map[string]string) (*http.Response, api.ImportResult) { + t.Helper() + var body bytes.Buffer + encoder := json.NewEncoder(&body) + for _, record := range records { + require.NoError(t, encoder.Encode(record)) + } + path := appImportPath(policy) + request := httptest.NewRequest(http.MethodPost, path, &body) + request.Header.Set("Content-Type", "application/x-ndjson") + for key, value := range headers { + request.Header.Set(key, value) + } + recorder := httptest.NewRecorder() + handler.ServeHTTP(recorder, request) + response := recorder.Result() + data, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + var result api.ImportResult + if response.StatusCode == http.StatusOK { + require.NoError(t, json.Unmarshal(data, &result), string(data)) + } + return response, result +} + +func appImportPath(policy string) string { + path := "/directory/import" + if policy != "" { + path += "?conflictPolicy=" + url.QueryEscape(policy) + } + return path +} + +func entryImportRecord(key map[string]any, name string, parent map[string]any, entryType string) map[string]any { + data := map[string]any{ + "name": name, "type": entryType, "parent": parent, "description": nil, "organizationId": nil, + "contactName": nil, "email": nil, "fromEmail": nil, "tenant": nil, "vendor": nil, "phoneNumber": nil, + "lmsLocationCode": nil, "hrid": nil, "timeZone": nil, "symbols": []any{key}, + "endpoints": []any{}, "addresses": []any{}, "closures": []any{}, "lmsConfig": nil, + "catalogConfig": nil, "illConfig": nil, "holdingsPolicy": nil, + } + if entryType == "Consortium" { + data["endpoints"] = []any{map[string]any{"name": "ISO", "type": "ISO18626", "address": "https://example.test/ill"}} + data["addresses"] = []any{map[string]any{"type": "Default", "addressComponents": []any{map[string]any{"seq": 1, "type": "Locality", "value": "Riga"}}}} + data["closures"] = []any{map[string]any{"startDate": "2026-12-24", "endDate": "2026-12-26", "reason": "Holiday"}} + data["lmsConfig"] = map[string]any{ + "address": "https://example.test/ncip", "fromAgency": "FROM", "fromAgencyAuthentication": "credential-value", + "toAgency": nil, "lookupUserEnabled": nil, "acceptItemEnabled": nil, "checkInItemEnabled": nil, + "checkOutItemEnabled": nil, "itemLocation": nil, "requestItemRequestType": nil, + "requestItemRequestScopeType": nil, "requestItemBibIdCode": nil, "requestItemEnabled": nil, + "requestItemPickupLocationEnabled": nil, "requesterPickupLocation": nil, "supplierPickupLocation": nil, + "requesterPatronPattern": nil, + } + } + return map[string]any{"type": "entry", "key": key, "data": data} +} + +func symbolObject(authority, symbol string) map[string]any { + return map[string]any{"authority": authority, "symbol": symbol} +} + +func resetImportState(t *testing.T) { + t.Helper() + _, err := dbpool.Exec(context.Background(), `TRUNCATE entries CASCADE`) + require.NoError(t, err) +} + +func importedEntryID(t *testing.T, authority, symbol string) uuid.UUID { + t.Helper() + var id uuid.UUID + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT owner FROM symbols WHERE authority=$1 AND symbol=$2`, authority, symbol).Scan(&id)) + return id +} + +func importedEntryCount(t *testing.T) int { + t.Helper() + var count int + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT count(*) FROM entries`).Scan(&count)) + return count +} + +func mustJSON(t *testing.T, value string) string { + t.Helper() + data, err := json.Marshal(value) + require.NoError(t, err) + return string(data) +} From c49a5c80b125e9d83e100df04ba223afc16cd709 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Thu, 10 Sep 2026 14:08:23 +0300 Subject: [PATCH 02/18] ILLDEV-484 Improve directory import --- broker/adapter/api_directory.go | 16 +- broker/test/adapter/api_directory_test.go | 7 - directory/api.yaml | 30 ++- directory/api/import_contract_test.go | 6 + directory/api/networks.go | 10 +- directory/api/tiers.go | 10 +- directory/api/tiers_networks_test.go | 28 ++ directory/api/util.go | 6 + directory/app/app.go | 10 +- directory/import/db/entry.go | 6 + directory/import/db/repo_test.go | 190 ++++++++++++++ directory/import/db/tier_network.go | 11 +- directory/import/service/decode.go | 242 +++--------------- directory/import/service/importer.go | 26 +- directory/import/service/importer_test.go | 80 +++++- ....sql => 007_import_business_keys.down.sql} | 2 + ...up.sql => 007_import_business_keys.up.sql} | 10 +- directory/query.sql | 3 + directory/sqlc.yaml | 8 - directory/test/networks_test.go | 24 ++ directory/test/tiers_test.go | 24 ++ 21 files changed, 486 insertions(+), 263 deletions(-) create mode 100644 directory/api/tiers_networks_test.go rename directory/migrations/{006_import_business_keys.down.sql => 007_import_business_keys.down.sql} (67%) rename directory/migrations/{006_import_business_keys.up.sql => 007_import_business_keys.up.sql} (72%) diff --git a/broker/adapter/api_directory.go b/broker/adapter/api_directory.go index 6b231a045..3c21fec44 100644 --- a/broker/adapter/api_directory.go +++ b/broker/adapter/api_directory.go @@ -444,12 +444,10 @@ func getPeerNetworks(peerData dirapi.Entry) map[string]Network { networks := map[string]Network{} if peerData.Networks != nil { for _, n := range *peerData.Networks { - if n.Name != nil { - networks[*n.Name] = Network{ - Name: *n.Name, - Priority: int(n.Priority), - Reciprocal: n.Reciprocal, - } + networks[n.Name] = Network{ + Name: n.Name, + Priority: int(n.Priority), + Reciprocal: n.Reciprocal, } } } @@ -460,12 +458,8 @@ func getPeerTiers(peerData dirapi.Entry) []Tier { tiers := []Tier{} if peerData.Tiers != nil { for _, t := range *peerData.Tiers { - name := "" - if t.Name != nil { - name = *t.Name - } tiers = append(tiers, Tier{ - Name: name, + Name: t.Name, Level: string(t.Level), Type: string(t.Type), Cost: t.Cost, diff --git a/broker/test/adapter/api_directory_test.go b/broker/test/adapter/api_directory_test.go index a58979dac..be375d82f 100644 --- a/broker/test/adapter/api_directory_test.go +++ b/broker/test/adapter/api_directory_test.go @@ -41,10 +41,6 @@ func boolPtr(v bool) *bool { return &v } -func stringPtr(v string) *string { - return &v -} - func withNetworkReciprocal(entry dirapi.Entry, reciprocal *bool) dirapi.Entry { if entry.Networks == nil { return entry @@ -916,9 +912,6 @@ func TestCompareSuppliers(t *testing.T) { assert.False(t, suppliers[1].Local) } -func strPtr(i string) *string { - return &i -} func TestFilterAndSortAppliesHoldingsPolicy(t *testing.T) { appCtx := createLookupCtx() ad := createDirectoryAdapter("") diff --git a/directory/api.yaml b/directory/api.yaml index 60b66a8c5..f66f594a5 100644 --- a/directory/api.yaml +++ b/directory/api.yaml @@ -570,6 +570,12 @@ paths: text/plain: schema: type: string + '409': + description: A tier with the same name already exists in the consortium + content: + text/plain: + schema: + type: string '500': description: Internal server error content: @@ -730,6 +736,12 @@ paths: text/plain: schema: type: string + '409': + description: A network with the same name already exists in the consortium + content: + text/plain: + schema: + type: string '500': description: Internal server error content: @@ -1940,12 +1952,14 @@ components: $ref: '#/components/schemas/Tier' lmsConfig: description: Configuration for LMS (Library Management System) integration via NCIP protocol - $ref: '#/components/schemas/LmsConfig' + allOf: + - $ref: '#/components/schemas/LmsConfig' catalogConfig: $ref: '#/components/schemas/CatalogConfig' holdingsPolicy: description: Policy that determines whether and in which order holdings may supply an item. - $ref: '#/components/schemas/HoldingsPolicy' + allOf: + - $ref: '#/components/schemas/HoldingsPolicy' IllConfig: type: object properties: @@ -1954,7 +1968,8 @@ components: description: URL of the ISO18626 service. When set, used instead of an ISO18626 service from endpoints. iso18626Vendor: description: ISO18626 service vendor. Used only when iso18626Url is non-empty. - $ref: '#/components/schemas/EntryVendor' + allOf: + - $ref: '#/components/schemas/EntryVendor' lendersOfLastResort: type: array description: Suppliers appended to the end of the rota when no regular supplier can fill the request. @@ -2387,7 +2402,8 @@ components: default: "INST-{requesterSymbol}" patronProfiles: description: Patron profiles used to determine whether a user returned by NCIP LookupUser is eligible to create ILL requests. - $ref: '#/components/schemas/PatronProfiles' + allOf: + - $ref: '#/components/schemas/PatronProfiles' PatronProfiles: type: array @@ -2648,6 +2664,7 @@ components: required: - id - consortium + - name properties: id: readOnly: true @@ -2659,6 +2676,8 @@ components: format: uuid name: type: string + minLength: 1 + pattern: '.*\S.*' reciprocal: type: boolean @@ -2689,6 +2708,7 @@ components: required: - id - consortium + - name - level - type - cost @@ -2703,6 +2723,8 @@ components: format: uuid name: type: string + minLength: 1 + pattern: '.*\S.*' level: type: string enum: diff --git a/directory/api/import_contract_test.go b/directory/api/import_contract_test.go index bf4e19794..b73fef388 100644 --- a/directory/api/import_contract_test.go +++ b/directory/api/import_contract_test.go @@ -1,14 +1,20 @@ package api import ( + "context" "testing" + "github.com/getkin/kin-openapi/openapi3" "github.com/stretchr/testify/require" ) func TestImportOpenAPIContract(t *testing.T) { spec, err := GetSpec() require.NoError(t, err) + require.NoError(t, spec.Validate(context.Background())) + sourceSpec, err := openapi3.NewLoader().LoadFromFile("../api.yaml") + require.NoError(t, err) + require.NoError(t, sourceSpec.Validate(context.Background())) operation := spec.Paths.Find("/import") require.NotNil(t, operation) require.NotNil(t, operation.Post) diff --git a/directory/api/networks.go b/directory/api/networks.go index d0250e310..4fc1d4475 100644 --- a/directory/api/networks.go +++ b/directory/api/networks.go @@ -3,9 +3,11 @@ package api import ( "context" "errors" + "log/slog" + "strings" + "github.com/indexdata/crosslink/directory/auth" "github.com/indexdata/crosslink/directory/db" - "log/slog" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -22,6 +24,9 @@ func (a ApiImpl) AddNetwork(ctx context.Context, request AddNetworkRequestObject if request.Body == nil || request.Body.Consortium == uuid.Nil { return AddNetwork400TextResponse("You must provide a consortium"), nil } + if strings.TrimSpace(request.Body.Name) == "" { + return AddNetwork400TextResponse("You must provide a network name"), nil + } consortium, err := a.queries.EntryById(ctx, request.Body.Consortium) if err != nil { @@ -54,6 +59,9 @@ func (a ApiImpl) AddNetwork(ctx context.Context, request AddNetworkRequestObject }) if err != nil { + if isUniqueConstraintViolation(err, "networks_consortium_name_unique") { + return AddNetwork409TextResponse("A network with this name already exists in the consortium"), nil + } slog.ErrorContext(ctx, "failed to create network", "error", err, "name", request.Body.Name) return AddNetwork500TextResponse("Error creating network"), nil } diff --git a/directory/api/tiers.go b/directory/api/tiers.go index 3be6856e5..9828c5920 100644 --- a/directory/api/tiers.go +++ b/directory/api/tiers.go @@ -3,9 +3,11 @@ package api import ( "context" "errors" + "log/slog" + "strings" + "github.com/indexdata/crosslink/directory/auth" "github.com/indexdata/crosslink/directory/db" - "log/slog" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -22,6 +24,9 @@ func (a ApiImpl) AddTier(ctx context.Context, request AddTierRequestObject) (Add if request.Body == nil || request.Body.Consortium == uuid.Nil { return AddTier400TextResponse("You must provide a consortium"), nil } + if strings.TrimSpace(request.Body.Name) == "" { + return AddTier400TextResponse("You must provide a tier name"), nil + } consortium, err := a.queries.EntryById(ctx, request.Body.Consortium) if err != nil { @@ -75,6 +80,9 @@ func (a ApiImpl) AddTier(ctx context.Context, request AddTierRequestObject) (Add }) if err != nil { + if isUniqueConstraintViolation(err, "tiers_consortium_name_unique") { + return AddTier409TextResponse("A tier with this name already exists in the consortium"), nil + } slog.ErrorContext(ctx, "failed to create tier", "error", err, "name", request.Body.Name) return AddTier500TextResponse("Error creating tier"), nil } diff --git a/directory/api/tiers_networks_test.go b/directory/api/tiers_networks_test.go new file mode 100644 index 000000000..669445db6 --- /dev/null +++ b/directory/api/tiers_networks_test.go @@ -0,0 +1,28 @@ +package api + +import ( + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestAddTierRejectsWhitespaceOnlyName(t *testing.T) { + impl := NewApiImpl(nil, nil, nil) + response, err := impl.AddTier(consortialAdminContext(t), AddTierRequestObject{ + Body: &Tier{Consortium: uuid.New(), Name: " \t\n"}, + }) + + require.NoError(t, err) + require.IsType(t, AddTier400TextResponse(""), response) +} + +func TestAddNetworkRejectsWhitespaceOnlyName(t *testing.T) { + impl := NewApiImpl(nil, nil, nil) + response, err := impl.AddNetwork(consortialAdminContext(t), AddNetworkRequestObject{ + Body: &Network{Consortium: uuid.New(), Name: " \t\n"}, + }) + + require.NoError(t, err) + require.IsType(t, AddNetwork400TextResponse(""), response) +} diff --git a/directory/api/util.go b/directory/api/util.go index a5873ba77..9efffd873 100644 --- a/directory/api/util.go +++ b/directory/api/util.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgtype" "github.com/oapi-codegen/nullable" "github.com/oapi-codegen/runtime/types" @@ -60,6 +61,11 @@ func derefOrDefaultPtr[T any](ptr *T, defaultValue *T) *T { return defaultValue } +func isUniqueConstraintViolation(err error, constraint string) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23505" && pgErr.ConstraintName == constraint +} + // Returns true if there is a struct in slice that has a prop with the given name // that either is equal to value or is a point to it // TODO: we could avoid reflection if we could somehow add a method to generated types diff --git a/directory/app/app.go b/directory/app/app.go index 3b154dc04..a71af9459 100644 --- a/directory/app/app.go +++ b/directory/app/app.go @@ -62,10 +62,18 @@ func InitHandler(ctx context.Context, dbpool *pgxpool.Pool) http.Handler { slog.ErrorContext(ctx, "Error loading API spec", "error", err) os.Exit(1) } + if err := swagger.Validate(ctx); err != nil { + slog.ErrorContext(ctx, "Invalid API spec", "error", err) + os.Exit(1) + } queries := db.New(dbpool) importRepo := importdb.New(dbpool) - importer := importservice.New(importRepo) + importer, err := importservice.New(importRepo, swagger) + if err != nil { + slog.ErrorContext(ctx, "Invalid import schemas", "error", err) + os.Exit(1) + } impl := api.NewApiImpl(dbpool, queries, importer) si := api.NewStrictHandler(impl, nil) m := http.NewServeMux() diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 6a0996e53..87ed2bc7a 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -26,6 +26,12 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg } defer func() { _ = tx.Rollback(ctx) }() + if err := queries.LockEntryImportKey(ctx, db.LockEntryImportKeyParams{ + Authority: aggregate.Key.Authority, + Symbol: aggregate.Key.Symbol, + }); err != nil { + return model.RepoResult{}, fmt.Errorf("lock entry %s", key) + } existing, lookupErr := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{ Authority: aggregate.Key.Authority, Symbol: aggregate.Key.Symbol, diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 72e0b1b16..3e1138428 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "sync" "testing" "time" @@ -60,8 +61,22 @@ func TestImportBusinessKeyConstraints(t *testing.T) { VALUES ($1, 'Loan', 'standard', 'loan', 0), ($1, 'Loan', 'standard', 'loan', 0)`, consortiumID) requirePgCode(t, err, "23505") + _, err = testPool.Exec(ctx, ` + INSERT INTO networks (consortium, name, priority) + VALUES ($1, 'Main', 0), ($1, 'Main', 0)`, consortiumID) + requirePgCode(t, err, "23505") + + _, err = testPool.Exec(ctx, `INSERT INTO tiers (consortium, name, level, type, cost) VALUES ($1, NULL, 'standard', 'loan', 0)`, consortiumID) + requirePgCode(t, err, "23502") + _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name, priority) VALUES ($1, NULL, 0)`, consortiumID) requirePgCode(t, err, "23502") + + _, err = testPool.Exec(ctx, `INSERT INTO tiers (consortium, name, level, type, cost) VALUES ($1, E'\t\n', 'standard', 'loan', 0)`, consortiumID) + requirePgCode(t, err, "23514") + + _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name, priority) VALUES ($1, E'\t\n', 0)`, consortiumID) + requirePgCode(t, err, "23514") } func TestImportEntryCreatesCompleteAggregateWithGeneratedIDs(t *testing.T) { @@ -132,6 +147,38 @@ func TestImportEntryConflictPoliciesAndUpdateFullSynchronization(t *testing.T) { require.NotEqual(t, originalEndpointID, replacementEndpointID) } +func TestConcurrentImportEntrySkipHonorsConflictPolicyForMissingKey(t *testing.T) { + resetImportDatabase(t) + newAggregate := func() model.EntryAggregate { return minimalEntryAggregate("concurrent", "Institution") } + results, errs := concurrentlyImportEntry(t, newAggregate, model.ConflictPolicySkip, 8) + + var imported, skipped int + for index, err := range errs { + require.NoError(t, err) + switch results[index].Outcome { + case model.OutcomeImported: + imported++ + case model.OutcomeSkipped: + skipped++ + } + } + require.Equal(t, 1, imported) + require.Equal(t, 7, skipped) + require.Equal(t, 1, entryCount(t)) +} + +func TestConcurrentImportEntryUpdateHonorsConflictPolicyForMissingKey(t *testing.T) { + resetImportDatabase(t) + newAggregate := func() model.EntryAggregate { return minimalEntryAggregate("concurrent", "Institution") } + results, errs := concurrentlyImportEntry(t, newAggregate, model.ConflictPolicyUpdate, 8) + + for index, err := range errs { + require.NoError(t, err) + require.Equal(t, model.OutcomeImported, results[index].Outcome) + } + require.Equal(t, 1, entryCount(t)) +} + func TestImportEntryRejectsInvalidHierarchy(t *testing.T) { resetImportDatabase(t) repo := importdb.New(testPool) @@ -234,6 +281,34 @@ func TestImportTierConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) } +func TestConcurrentImportTierSkipHonorsConflictPolicyForMissingKey(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + newAggregate := func() model.TierAggregate { + return model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Concurrent"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{}}, + } + } + results, errs := concurrentlyImportTier(repo, newAggregate, model.ConflictPolicySkip, 8) + + requireImportOutcomes(t, results, errs, 1, 7) + require.Equal(t, 1, aggregateCount(t, "tiers")) +} + +func TestConcurrentImportTierUpdateHonorsConflictPolicyForMissingKey(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + newAggregate := func() model.TierAggregate { + return model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Concurrent"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{}}, + } + } + results, errs := concurrentlyImportTier(repo, newAggregate, model.ConflictPolicyUpdate, 8) + + requireImportOutcomes(t, results, errs, 8, 0) + require.Equal(t, 1, aggregateCount(t, "tiers")) +} + func TestImportTierRollsBackWhenMemberIsMissing(t *testing.T) { repo, consortium, first, _ := importRepoFixture(t) missing := model.SymbolRef{Authority: "ISIL", Symbol: "MISSING"} @@ -278,6 +353,34 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.Equal(t, []model.SymbolRef{second}, networkAssignments(t, id)) } +func TestConcurrentImportNetworkSkipHonorsConflictPolicyForMissingKey(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + newAggregate := func() model.NetworkAggregate { + return model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Concurrent"}, + Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{}}, + } + } + results, errs := concurrentlyImportNetwork(repo, newAggregate, model.ConflictPolicySkip, 8) + + requireImportOutcomes(t, results, errs, 1, 7) + require.Equal(t, 1, aggregateCount(t, "networks")) +} + +func TestConcurrentImportNetworkUpdateHonorsConflictPolicyForMissingKey(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + newAggregate := func() model.NetworkAggregate { + return model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Concurrent"}, + Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{}}, + } + } + results, errs := concurrentlyImportNetwork(repo, newAggregate, model.ConflictPolicyUpdate, 8) + + requireImportOutcomes(t, results, errs, 8, 0) + require.Equal(t, 1, aggregateCount(t, "networks")) +} + func TestImportNetworkRejectsNonConsortiumOwner(t *testing.T) { repo, _, first, _ := importRepoFixture(t) aggregate := model.NetworkAggregate{ @@ -400,6 +503,93 @@ func minimalEntryAggregate(symbol, entryType string) model.EntryAggregate { } } +func concurrentlyImportEntry(t *testing.T, newAggregate func() model.EntryAggregate, policy model.ConflictPolicy, count int) ([]model.RepoResult, []error) { + t.Helper() + repo := importdb.New(testPool) + start := make(chan struct{}) + results := make([]model.RepoResult, count) + errs := make([]error, count) + var waitGroup sync.WaitGroup + waitGroup.Add(count) + for index := range count { + go func() { + defer waitGroup.Done() + <-start + results[index], errs[index] = repo.ImportEntry(context.Background(), newAggregate(), policy) + }() + } + close(start) + waitGroup.Wait() + return results, errs +} + +func concurrentlyImportTier(repo *importdb.PgImportRepo, newAggregate func() model.TierAggregate, policy model.ConflictPolicy, count int) ([]model.RepoResult, []error) { + start := make(chan struct{}) + results := make([]model.RepoResult, count) + errs := make([]error, count) + var waitGroup sync.WaitGroup + waitGroup.Add(count) + for index := range count { + go func() { + defer waitGroup.Done() + <-start + results[index], errs[index] = repo.ImportTier(context.Background(), newAggregate(), policy) + }() + } + close(start) + waitGroup.Wait() + return results, errs +} + +func concurrentlyImportNetwork(repo *importdb.PgImportRepo, newAggregate func() model.NetworkAggregate, policy model.ConflictPolicy, count int) ([]model.RepoResult, []error) { + start := make(chan struct{}) + results := make([]model.RepoResult, count) + errs := make([]error, count) + var waitGroup sync.WaitGroup + waitGroup.Add(count) + for index := range count { + go func() { + defer waitGroup.Done() + <-start + results[index], errs[index] = repo.ImportNetwork(context.Background(), newAggregate(), policy) + }() + } + close(start) + waitGroup.Wait() + return results, errs +} + +func requireImportOutcomes(t *testing.T, results []model.RepoResult, errs []error, expectedImported, expectedSkipped int) { + t.Helper() + var imported, skipped int + for index, err := range errs { + require.NoError(t, err) + switch results[index].Outcome { + case model.OutcomeImported: + imported++ + case model.OutcomeSkipped: + skipped++ + } + } + require.Equal(t, expectedImported, imported) + require.Equal(t, expectedSkipped, skipped) +} + +func aggregateCount(t *testing.T, table string) int { + t.Helper() + query := fmt.Sprintf(`SELECT count(*) FROM %s`, table) //nolint:gosec // table names are fixed test constants + var count int + require.NoError(t, testPool.QueryRow(context.Background(), query).Scan(&count)) + return count +} + +func entryCount(t *testing.T) int { + t.Helper() + var count int + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT count(*) FROM entries`).Scan(&count)) + return count +} + func resetImportDatabase(t *testing.T) { t.Helper() _, err := testPool.Exec(context.Background(), `TRUNCATE entries CASCADE`) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index fc30d4b02..23d417f06 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -28,7 +28,7 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre return model.RepoResult{}, err } name := aggregate.Key.Name - existing, lookupErr := queries.LockTierByBusinessKey(ctx, db.LockTierByBusinessKeyParams{Consortium: consortium.ID, Name: &name}) + existing, lookupErr := queries.LockTierByBusinessKey(ctx, db.LockTierByBusinessKeyParams{Consortium: consortium.ID, Name: name}) exists := lookupErr == nil if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { return model.RepoResult{}, fmt.Errorf("resolve tier %s", key) @@ -46,7 +46,7 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre err = queries.UpdateImportedTier(ctx, db.UpdateImportedTierParams{ID: tierID, Level: aggregate.Data.Level, Type: aggregate.Data.Type, Cost: aggregate.Data.Cost}) } else { var created db.Tier - created, err = queries.CreateTier(ctx, db.CreateTierParams{Name: &name, Consortium: consortium.ID, Level: aggregate.Data.Level, Type: aggregate.Data.Type, Cost: aggregate.Data.Cost}) + created, err = queries.CreateTier(ctx, db.CreateTierParams{Name: name, Consortium: consortium.ID, Level: aggregate.Data.Level, Type: aggregate.Data.Type, Cost: aggregate.Data.Cost}) tierID = created.ID } if err != nil { @@ -77,7 +77,7 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{}, err } name := aggregate.Key.Name - existing, lookupErr := queries.LockNetworkByBusinessKey(ctx, db.LockNetworkByBusinessKeyParams{Consortium: consortium.ID, Name: &name}) + existing, lookupErr := queries.LockNetworkByBusinessKey(ctx, db.LockNetworkByBusinessKeyParams{Consortium: consortium.ID, Name: name}) exists := lookupErr == nil if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { return model.RepoResult{}, fmt.Errorf("resolve network %s", key) @@ -95,7 +95,7 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ err = queries.UpdateImportedNetwork(ctx, db.UpdateImportedNetworkParams{ID: networkID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) } else { var created db.Network - created, err = queries.CreateNetwork(ctx, db.CreateNetworkParams{Name: &name, Consortium: consortium.ID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) + created, err = queries.CreateNetwork(ctx, db.CreateNetworkParams{Name: name, Consortium: consortium.ID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) networkID = created.ID } if err != nil { @@ -110,6 +110,9 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{Outcome: model.OutcomeImported}, nil } +// resolveConsortium locks the consortium entry for the transaction. Besides +// protecting the reference, this serializes missing tier and network business +// keys within a consortium before their lookup-and-create flows. func resolveConsortium(ctx context.Context, queries *db.Queries, key model.SymbolRef) (db.Entry, error) { entry, err := resolveEntry(ctx, queries, key) if err != nil { diff --git a/directory/import/service/decode.go b/directory/import/service/decode.go index 93b083027..20ecce2b0 100644 --- a/directory/import/service/decode.go +++ b/directory/import/service/decode.go @@ -5,9 +5,9 @@ import ( "encoding/json" "fmt" "io" - "sort" "strings" + "github.com/getkin/kin-openapi/openapi3" "github.com/indexdata/crosslink/directory/import/model" ) @@ -25,21 +25,7 @@ type decodedRecord struct { network *model.NetworkAggregate } -var requiredProperties = map[string][]string{ - "envelope": {"type", "key", "data"}, - "entryKey": {"authority", "symbol"}, - "entryData": { - "name", "type", "parent", "description", "organizationId", "contactName", "email", "fromEmail", - "tenant", "vendor", "phoneNumber", "lmsLocationCode", "hrid", "timeZone", "symbols", "endpoints", - "addresses", "closures", "lmsConfig", "catalogConfig", "illConfig", "holdingsPolicy", - }, - "tierKey": {"consortium", "name"}, - "tierData": {"level", "type", "cost", "entries"}, - "networkKey": {"consortium", "name"}, - "networkData": {"priority", "reciprocal", "entries"}, -} - -func decodeRecord(data []byte) (decodedRecord, error) { +func decodeRecord(data []byte, schemas map[string]*openapi3.Schema) (decodedRecord, error) { var record decodedRecord properties, err := objectProperties(data) @@ -49,23 +35,23 @@ func decodeRecord(data []byte) (decodedRecord, error) { if rawType, exists := properties["type"]; exists { _ = json.Unmarshal(rawType, &record.recordType) } - if err := requireExactProperties(properties, requiredProperties["envelope"]); err != nil { - return record, fmt.Errorf("invalid import envelope: %w", err) + schema, exists := schemas[record.recordType] + if !exists { + return record, fmt.Errorf("unknown import record type") + } + if err := validateAgainstSchema(data, schema); err != nil { + return record, fmt.Errorf("invalid %s aggregate: %w", record.recordType, err) } var env envelope if err := decodeStrict(data, &env); err != nil { return record, fmt.Errorf("invalid import envelope: %w", err) } - record.recordType = env.Type switch env.Type { case "entry": var aggregate model.EntryAggregate - if err := validateEntryShape(env.Key, env.Data); err != nil { - return record, fmt.Errorf("invalid entry aggregate: %w", err) - } - if err := decodeAggregate(env.Key, env.Data, requiredProperties["entryKey"], requiredProperties["entryData"], &aggregate.Key, &aggregate.Data); err != nil { + if err := decodeAggregate(env.Key, env.Data, &aggregate.Key, &aggregate.Data); err != nil { return record, fmt.Errorf("invalid entry aggregate: %w", err) } record.entry = &aggregate @@ -78,10 +64,7 @@ func decodeRecord(data []byte) (decodedRecord, error) { } case "tier": var aggregate model.TierAggregate - if err := validateMembershipShape(env.Key, env.Data, "tier"); err != nil { - return record, fmt.Errorf("invalid tier aggregate: %w", err) - } - if err := decodeAggregate(env.Key, env.Data, requiredProperties["tierKey"], requiredProperties["tierData"], &aggregate.Key, &aggregate.Data); err != nil { + if err := decodeAggregate(env.Key, env.Data, &aggregate.Key, &aggregate.Data); err != nil { return record, fmt.Errorf("invalid tier aggregate: %w", err) } record.tier = &aggregate @@ -94,10 +77,7 @@ func decodeRecord(data []byte) (decodedRecord, error) { } case "network": var aggregate model.NetworkAggregate - if err := validateMembershipShape(env.Key, env.Data, "network"); err != nil { - return record, fmt.Errorf("invalid network aggregate: %w", err) - } - if err := decodeAggregate(env.Key, env.Data, requiredProperties["networkKey"], requiredProperties["networkData"], &aggregate.Key, &aggregate.Data); err != nil { + if err := decodeAggregate(env.Key, env.Data, &aggregate.Key, &aggregate.Data); err != nil { return record, fmt.Errorf("invalid network aggregate: %w", err) } record.network = &aggregate @@ -109,175 +89,38 @@ func decodeRecord(data []byte) (decodedRecord, error) { return record, validationErr } default: - return record, fmt.Errorf("unknown import record type: %s", env.Type) + return record, fmt.Errorf("unknown import record type") } return record, nil } -func validateEntryShape(keyJSON, dataJSON []byte) error { - if err := validateObject(keyJSON, requiredProperties["entryKey"], nil); err != nil { - return fmt.Errorf("key: %w", err) - } - return validateObject(dataJSON, requiredProperties["entryData"], func(properties map[string]json.RawMessage) error { - if err := validateNullableObject(properties["parent"], []string{"authority", "symbol"}, nil); err != nil { - return fmt.Errorf("parent: %w", err) - } - if err := validateObjectArray(properties["symbols"], []string{"authority", "symbol"}, nil); err != nil { - return fmt.Errorf("symbols: %w", err) - } - if err := validateObjectArray(properties["endpoints"], []string{"name", "type", "address"}, nil); err != nil { - return fmt.Errorf("endpoints: %w", err) - } - if err := validateObjectArray(properties["addresses"], []string{"type", "addressComponents"}, func(address map[string]json.RawMessage) error { - return validateObjectArray(address["addressComponents"], []string{"seq", "type", "value"}, nil) - }); err != nil { - return fmt.Errorf("addresses: %w", err) - } - if err := validateObjectArray(properties["closures"], []string{"startDate", "endDate", "reason"}, nil); err != nil { - return fmt.Errorf("closures: %w", err) - } - if err := validateNullableObject(properties["lmsConfig"], []string{ - "address", "fromAgency", "fromAgencyAuthentication", "toAgency", "lookupUserEnabled", "acceptItemEnabled", - "checkInItemEnabled", "checkOutItemEnabled", "itemLocation", "requestItemRequestType", "requestItemRequestScopeType", - "requestItemBibIdCode", "requestItemEnabled", "requestItemPickupLocationEnabled", "requesterPickupLocation", - "supplierPickupLocation", "requesterPatronPattern", - }, nil); err != nil { - return fmt.Errorf("lmsConfig: %w", err) - } - if err := validateCatalogShape(properties["catalogConfig"]); err != nil { - return fmt.Errorf("catalogConfig: %w", err) - } - if err := validateNullableObject(properties["illConfig"], []string{ - "iso18626Url", "iso18626Vendor", "lendersOfLastResort", "includeRequestingAgencyInfo", "includeSupplierInfo", - "includeReturnInfo", "includeVendorNote", "useOfferedCosts", "noteFieldSeparator", "supplierPatronPattern", - "duplicateCheckWindowHours", - }, func(config map[string]json.RawMessage) error { - return validateObjectArray(config["lendersOfLastResort"], []string{"authority", "symbol"}, nil) - }); err != nil { - return fmt.Errorf("illConfig: %w", err) - } - if err := validateHoldingsPolicyShape(properties["holdingsPolicy"]); err != nil { - return fmt.Errorf("holdingsPolicy: %w", err) - } - return nil - }) -} - -func validateMembershipShape(keyJSON, dataJSON []byte, aggregateType string) error { - keyRequired := requiredProperties[aggregateType+"Key"] - dataRequired := requiredProperties[aggregateType+"Data"] - if err := validateObject(keyJSON, keyRequired, func(key map[string]json.RawMessage) error { - return validateObject(key["consortium"], []string{"authority", "symbol"}, nil) - }); err != nil { - return fmt.Errorf("key: %w", err) - } - if err := validateObject(dataJSON, dataRequired, func(data map[string]json.RawMessage) error { - return validateObjectArray(data["entries"], []string{"authority", "symbol"}, nil) - }); err != nil { - return fmt.Errorf("data: %w", err) +func validateAgainstSchema(data []byte, schema *openapi3.Schema) error { + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.UseNumber() + var value any + if err := decoder.Decode(&value); err != nil { + return sanitizeJSONError(err) } - return nil -} - -func validateCatalogShape(raw json.RawMessage) error { - return validateNullableObject(raw, []string{"metadataUpdateMode", "sru", "zoom", "queryConfig", "holdingsFormat", "metadataFormat"}, func(config map[string]json.RawMessage) error { - checks := []struct { - name string - properties []string - nested func(map[string]json.RawMessage) error - }{ - {"sru", []string{"address", "recordSchema"}, nil}, - {"zoom", []string{"address", "options"}, nil}, - {"queryConfig", []string{"type", "identifier", "isbn", "issn", "title"}, nil}, - {"holdingsFormat", []string{"marc", "marc21plus1", "opac", "reservoir"}, func(format map[string]json.RawMessage) error { - return validateNullableObject(format["marc"], []string{"callNumberSubField", "itemIdSubField", "locationSubField", "mainField", "restrictedSubField", "shelvingLocationSubField"}, nil) - }}, - {"metadataFormat", []string{"marc21"}, func(format map[string]json.RawMessage) error { - return validateNullableObject(format["marc21"], []string{"author", "edition", "identifier", "isbn", "issn", "subtitle", "title"}, nil) - }}, - } - for _, check := range checks { - if err := validateNullableObject(config[check.name], check.properties, check.nested); err != nil { - return fmt.Errorf("%s: %w", check.name, err) + if err := schema.VisitJSON(value, + openapi3.EnableFormatValidation(), + openapi3.SetSchemaErrorMessageCustomizer(func(err *openapi3.SchemaError) string { + path := strings.Join(err.JSONPointer(), ".") + if path == "" { + path = "record" } - } - return nil - }) -} - -func validateHoldingsPolicyShape(raw json.RawMessage) error { - return validateNullableObject(raw, []string{"locations", "shelvingLocations", "locationPolicies", "itemLoanPolicies"}, func(policy map[string]json.RawMessage) error { - if err := validateObjectArray(policy["locations"], []string{"code", "name", "supplyPreference"}, nil); err != nil { - return fmt.Errorf("locations: %w", err) - } - if err := validateObjectArray(policy["shelvingLocations"], []string{"code", "name", "supplyPreference"}, nil); err != nil { - return fmt.Errorf("shelvingLocations: %w", err) - } - if err := validateObjectArray(policy["locationPolicies"], []string{"locationCode", "shelvingLocationCode", "supplyPreference"}, nil); err != nil { - return fmt.Errorf("locationPolicies: %w", err) - } - if err := validateObjectArray(policy["itemLoanPolicies"], []string{"code", "name", "lendable"}, nil); err != nil { - return fmt.Errorf("itemLoanPolicies: %w", err) - } - return nil - }) -} - -func validateNullableObject(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - return nil - } - return validateObject(raw, required, nested) -} - -func validateObject(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { - properties, err := objectProperties(raw) - if err != nil { - return err - } - if err := requireExactProperties(properties, required); err != nil { + reason := err.Reason + if reason == "" { + reason = "does not match the schema" + } + return path + ": " + reason + }), + ); err != nil { return err } - if nested != nil { - return nested(properties) - } return nil } -func validateObjectArray(raw json.RawMessage, required []string, nested func(map[string]json.RawMessage) error) error { - if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - return fmt.Errorf("must be an array, not null") - } - var values []json.RawMessage - if err := decodeStrict(raw, &values); err != nil { - return err - } - if values == nil { - return fmt.Errorf("must be an array") - } - for index, value := range values { - if err := validateObject(value, required, nested); err != nil { - return fmt.Errorf("item %d: %w", index+1, err) - } - } - return nil -} - -func decodeAggregate(keyJSON, dataJSON []byte, keyProperties, dataProperties []string, key, data any) error { - keyObject, err := objectProperties(keyJSON) - if err != nil { - return fmt.Errorf("key must be an object: %w", err) - } - if err := requireExactProperties(keyObject, keyProperties); err != nil { - return fmt.Errorf("key: %w", err) - } - dataObject, err := objectProperties(dataJSON) - if err != nil { - return fmt.Errorf("data must be an object: %w", err) - } - if err := requireExactProperties(dataObject, dataProperties); err != nil { - return fmt.Errorf("data: %w", err) - } +func decodeAggregate(keyJSON, dataJSON []byte, key, data any) error { if err := decodeStrict(keyJSON, key); err != nil { return fmt.Errorf("key: %w", err) } @@ -298,27 +141,6 @@ func objectProperties(data []byte) (map[string]json.RawMessage, error) { return properties, nil } -func requireExactProperties(properties map[string]json.RawMessage, required []string) error { - allowed := make(map[string]struct{}, len(required)) - for _, property := range required { - allowed[property] = struct{}{} - if _, exists := properties[property]; !exists { - return fmt.Errorf("missing required property %q", property) - } - } - unknown := make([]string, 0) - for property := range properties { - if _, exists := allowed[property]; !exists { - unknown = append(unknown, property) - } - } - if len(unknown) != 0 { - sort.Strings(unknown) - return fmt.Errorf("unknown property %q", unknown[0]) - } - return nil -} - func decodeStrict(data []byte, destination any) error { decoder := json.NewDecoder(bytes.NewReader(data)) decoder.DisallowUnknownFields() diff --git a/directory/import/service/importer.go b/directory/import/service/importer.go index 8e56a6d22..40e90b8dc 100644 --- a/directory/import/service/importer.go +++ b/directory/import/service/importer.go @@ -8,6 +8,7 @@ import ( "fmt" "io" + "github.com/getkin/kin-openapi/openapi3" "github.com/indexdata/crosslink/directory/import/model" ) @@ -23,11 +24,30 @@ type Repository interface { type Importer struct { repository Repository + recordSchemas map[string]*openapi3.Schema maxRecordBytes int } -func New(repository Repository) *Importer { - return &Importer{repository: repository, maxRecordBytes: maxRecordBytes} +func New(repository Repository, spec *openapi3.T) (*Importer, error) { + schemas := make(map[string]*openapi3.Schema, 3) + for _, recordSchema := range []struct { + recordType string + componentName string + }{ + {recordType: "entry", componentName: "ImportEntryRecord"}, + {recordType: "tier", componentName: "ImportTierRecord"}, + {recordType: "network", componentName: "ImportNetworkRecord"}, + } { + var schemaRef *openapi3.SchemaRef + if spec != nil && spec.Components != nil { + schemaRef = spec.Components.Schemas[recordSchema.componentName] + } + if schemaRef == nil || schemaRef.Value == nil { + return nil, fmt.Errorf("OpenAPI component schema %q is missing", recordSchema.componentName) + } + schemas[recordSchema.recordType] = schemaRef.Value + } + return &Importer{repository: repository, recordSchemas: schemas, maxRecordBytes: maxRecordBytes}, nil } func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, input io.Reader) (model.ImportResult, error) { @@ -61,7 +81,7 @@ func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, inpu } func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy, line int32, data []byte, result *model.ImportResult) { - record, err := decodeRecord(data) + record, err := decodeRecord(data, i.recordSchemas) if err != nil { incrementFailed(result, record.recordType) appendError(result, line, record.recordType, record.key, err.Error()) diff --git a/directory/import/service/importer_test.go b/directory/import/service/importer_test.go index 34c7ffa7d..a21ca777b 100644 --- a/directory/import/service/importer_test.go +++ b/directory/import/service/importer_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/getkin/kin-openapi/openapi3" "github.com/indexdata/crosslink/directory/import/model" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -38,7 +39,7 @@ func TestImportDispatchesAllAggregateTypes(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} input := strings.Join([]string{validEntryRecord(), validTierRecord(), validNetworkRecord()}, "\n") - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) require.NoError(t, err) assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Entries) @@ -54,7 +55,7 @@ func TestImportContinuesAfterMalformedRecord(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} input := "{bad json}\n\n" + validEntryRecord() + "\n" - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) require.NoError(t, err) assert.Zero(t, result.Entries.Failed) @@ -68,9 +69,9 @@ func TestImportContinuesAfterMalformedRecord(t *testing.T) { func TestImportContinuesAfterSemanticFailure(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} - badTier := strings.Replace(validTierRecord(), `"level":"standard"`, `"level":"invalid"`, 1) + badTier := strings.Replace(validTierRecord(), `"name":"Primary"`, `"name":" "`, 1) - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(badTier+"\n"+validTierRecord())) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(badTier+"\n"+validTierRecord())) require.NoError(t, err) assert.Equal(t, model.ImportSectionResult{Imported: 1, Failed: 1}, result.Tiers) @@ -81,12 +82,30 @@ func TestImportContinuesAfterSemanticFailure(t *testing.T) { assert.Equal(t, 1, repo.tierCalls) } +func TestImportDoesNotEchoSchemaRejectedValues(t *testing.T) { + for name, record := range map[string]string{ + "schema violation": strings.Replace(validTierRecord(), `"level":"standard"`, `"level":"private-secret"`, 1), + "unknown record type": strings.Replace(validTierRecord(), `"type":"tier"`, `"type":"private-secret"`, 1), + } { + t.Run(name, func(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + + require.NoError(t, err) + require.Len(t, result.Errors, 1) + assert.NotContains(t, result.Errors[0].Error, "private-secret") + assert.Zero(t, repo.tierCalls) + }) + } +} + func TestImportBlankLinesDoNotIncrementRecordNumber(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} badTier := strings.Replace(validTierRecord(), `"level":"standard"`, `"level":"invalid"`, 1) input := "\n" + validTierRecord() + "\n\r\n" + badTier - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) require.NoError(t, err) require.Len(t, result.Errors, 1) @@ -95,7 +114,7 @@ func TestImportBlankLinesDoNotIncrementRecordNumber(t *testing.T) { func TestImportReturnsFatalReaderError(t *testing.T) { fatal := errors.New("transport failed") - result, err := New(&recordingRepo{}).Import(context.Background(), model.ConflictPolicyFail, errorReader{err: fatal}) + result, err := newTestImporter(t, &recordingRepo{}).Import(context.Background(), model.ConflictPolicyFail, errorReader{err: fatal}) require.ErrorIs(t, err, fatal) require.Empty(t, result.Errors) @@ -116,7 +135,7 @@ func TestImportRejectsMissingAndUnknownProperties(t *testing.T) { for name, record := range tests { t.Run(name, func(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) require.NoError(t, err) require.Len(t, result.Errors, 1) assert.NotContains(t, result.Errors[0].Error, "do-not-echo") @@ -132,7 +151,7 @@ func validILLConfig() string { func TestImportAccountsForSkippedAndRepositoryFailures(t *testing.T) { t.Run("skipped", func(t *testing.T) { repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeSkipped, Diagnostic: "entry already exists"}} - result, err := New(repo).Import(context.Background(), model.ConflictPolicySkip, strings.NewReader(validEntryRecord())) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicySkip, strings.NewReader(validEntryRecord())) require.NoError(t, err) assert.Equal(t, model.ImportSectionResult{Skipped: 1}, result.Entries) require.Len(t, result.Errors, 1) @@ -141,7 +160,7 @@ func TestImportAccountsForSkippedAndRepositoryFailures(t *testing.T) { t.Run("failed", func(t *testing.T) { repo := &recordingRepo{err: errors.New("entry parent does not exist")} - result, err := New(repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(validEntryRecord())) + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(validEntryRecord())) require.NoError(t, err) assert.Equal(t, model.ImportSectionResult{Failed: 1}, result.Entries) require.Len(t, result.Errors, 1) @@ -152,7 +171,7 @@ func TestImportAccountsForSkippedAndRepositoryFailures(t *testing.T) { func TestImportRecordLimitIsExact(t *testing.T) { record := validTierRecord() repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} - importer := New(repo) + importer := newTestImporter(t, repo) importer.maxRecordBytes = len(record) result, err := importer.Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record+"\r\n")) @@ -160,7 +179,7 @@ func TestImportRecordLimitIsExact(t *testing.T) { assert.Equal(t, int32(1), result.Tiers.Imported) repo = &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} - importer = New(repo) + importer = newTestImporter(t, repo) importer.maxRecordBytes = len(record) - 1 _, err = importer.Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record+"\n")) require.ErrorIs(t, err, ErrRecordTooLarge) @@ -182,3 +201,42 @@ func validNetworkRecord() string { type errorReader struct{ err error } func (r errorReader) Read([]byte) (int, error) { return 0, r.err } + +func TestNewRejectsMissingImportRecordSchema(t *testing.T) { + _, err := New(&recordingRepo{}, &openapi3.T{Components: &openapi3.Components{Schemas: openapi3.Schemas{}}}) + + require.ErrorContains(t, err, "ImportEntryRecord") +} + +func TestImportUsesInjectedOpenAPIRecordSchema(t *testing.T) { + spec := loadImportSpec(t) + minimumCost := 1.0 + spec.Components.Schemas["ImportTierData"].Value.Properties["cost"].Value.Min = &minimumCost + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + importer, err := New(repo, spec) + require.NoError(t, err) + + result, err := importer.Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(validTierRecord())) + + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Failed: 1}, result.Tiers) + require.Len(t, result.Errors, 1) + assert.Zero(t, repo.tierCalls) +} + +func newTestImporter(t *testing.T, repository Repository) *Importer { + t.Helper() + spec := loadImportSpec(t) + importer, err := New(repository, spec) + require.NoError(t, err) + return importer +} + +func loadImportSpec(t *testing.T) *openapi3.T { + t.Helper() + loader := openapi3.NewLoader() + spec, err := loader.LoadFromFile("../../api.yaml") + require.NoError(t, err) + require.NoError(t, spec.Validate(context.Background())) + return spec +} diff --git a/directory/migrations/006_import_business_keys.down.sql b/directory/migrations/007_import_business_keys.down.sql similarity index 67% rename from directory/migrations/006_import_business_keys.down.sql rename to directory/migrations/007_import_business_keys.down.sql index b4f180046..d5f86e2aa 100644 --- a/directory/migrations/006_import_business_keys.down.sql +++ b/directory/migrations/007_import_business_keys.down.sql @@ -1,3 +1,5 @@ +ALTER TABLE networks DROP CONSTRAINT networks_name_not_blank; +ALTER TABLE tiers DROP CONSTRAINT tiers_name_not_blank; ALTER TABLE networks DROP CONSTRAINT networks_consortium_name_unique; ALTER TABLE tiers DROP CONSTRAINT tiers_consortium_name_unique; diff --git a/directory/migrations/006_import_business_keys.up.sql b/directory/migrations/007_import_business_keys.up.sql similarity index 72% rename from directory/migrations/006_import_business_keys.up.sql rename to directory/migrations/007_import_business_keys.up.sql index 73e645eed..b59a786f3 100644 --- a/directory/migrations/006_import_business_keys.up.sql +++ b/directory/migrations/007_import_business_keys.up.sql @@ -1,9 +1,9 @@ DO $$ BEGIN - IF EXISTS (SELECT 1 FROM tiers WHERE name IS NULL OR btrim(name) = '') THEN + IF EXISTS (SELECT 1 FROM tiers WHERE name IS NULL OR name !~ '[^[:space:]]') THEN RAISE EXCEPTION 'cannot add tier business key: tiers contain null or blank names'; END IF; - IF EXISTS (SELECT 1 FROM networks WHERE name IS NULL OR btrim(name) = '') THEN + IF EXISTS (SELECT 1 FROM networks WHERE name IS NULL OR name !~ '[^[:space:]]') THEN RAISE EXCEPTION 'cannot add network business key: networks contain null or blank names'; END IF; IF EXISTS ( @@ -27,3 +27,9 @@ ALTER TABLE tiers ALTER TABLE networks ADD CONSTRAINT networks_consortium_name_unique UNIQUE (consortium, name); + +ALTER TABLE tiers + ADD CONSTRAINT tiers_name_not_blank CHECK (name ~ '[^[:space:]]'); + +ALTER TABLE networks + ADD CONSTRAINT networks_name_not_blank CHECK (name ~ '[^[:space:]]'); diff --git a/directory/query.sql b/directory/query.sql index 19c706a94..540e97419 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -19,6 +19,9 @@ SELECT * FROM entries WHERE parent = @parent; -- name: LockConsortiumEntryChanges :exec SELECT pg_advisory_xact_lock(hashtextextended('directoryish:consortium-entry', 0)); +-- name: LockEntryImportKey :exec +SELECT pg_advisory_xact_lock(hashtextextended('directoryish:entry:' || @authority::text || ':' || @symbol::text, 0)); + -- name: CreateEntry :one INSERT INTO entries ( name, description, contact_name, email, from_email, tenant, vendor, phone_number, time_zone, organization_id, type, parent, lms_location_code, hrid diff --git a/directory/sqlc.yaml b/directory/sqlc.yaml index cda34f46a..56c2f14f0 100644 --- a/directory/sqlc.yaml +++ b/directory/sqlc.yaml @@ -10,14 +10,6 @@ sql: sql_package: "pgx/v5" emit_pointers_for_null_types: true overrides: - - column: "tiers.name" - go_type: - type: "string" - pointer: true - - column: "networks.name" - go_type: - type: "string" - pointer: true - db_type: "uuid" go_type: import: "github.com/google/uuid" diff --git a/directory/test/networks_test.go b/directory/test/networks_test.go index 8dcbe5586..63b4e45a5 100644 --- a/directory/test/networks_test.go +++ b/directory/test/networks_test.go @@ -60,6 +60,30 @@ func TestNetworkCases(t *testing.T) { body: `{"name":"Institution Network","consortium":"00000000-0000-0000-0000-000000000002"}`, addlHeaders: consortiumPermissionHeaders, }, + { + name: "POST network without name", + method: http.MethodPost, + endpoint: "/networks", + status: http.StatusBadRequest, + body: `{"consortium":"00000000-0000-0000-0000-000000000004","priority":0}`, + addlHeaders: consortiumPermissionHeaders, + }, + { + name: "POST network with blank name", + method: http.MethodPost, + endpoint: "/networks", + status: http.StatusBadRequest, + body: `{"name":" ","consortium":"00000000-0000-0000-0000-000000000004","priority":0}`, + addlHeaders: consortiumPermissionHeaders, + }, + { + name: "POST network with duplicate name", + method: http.MethodPost, + endpoint: "/networks", + status: http.StatusConflict, + body: `{"name":"The Ultimate Network","consortium":"00000000-0000-0000-0000-000000000004","priority":0}`, + addlHeaders: consortiumPermissionHeaders, + }, { name: "DELETE network", method: http.MethodDelete, diff --git a/directory/test/tiers_test.go b/directory/test/tiers_test.go index f0a92094f..b8ce54860 100644 --- a/directory/test/tiers_test.go +++ b/directory/test/tiers_test.go @@ -59,6 +59,30 @@ func TestTierCases(t *testing.T) { body: `{"name":"Institution Tier","consortium":"00000000-0000-0000-0000-000000000002","level":"standard","type":"loan"}`, addlHeaders: consortiumPermissionHeaders, }, + { + name: "POST tier without name", + method: http.MethodPost, + endpoint: "/tiers", + status: http.StatusBadRequest, + body: `{"consortium":"00000000-0000-0000-0000-000000000004","level":"standard","type":"loan","cost":0}`, + addlHeaders: consortiumPermissionHeaders, + }, + { + name: "POST tier with blank name", + method: http.MethodPost, + endpoint: "/tiers", + status: http.StatusBadRequest, + body: `{"name":" ","consortium":"00000000-0000-0000-0000-000000000004","level":"standard","type":"loan","cost":0}`, + addlHeaders: consortiumPermissionHeaders, + }, + { + name: "POST tier with duplicate name", + method: http.MethodPost, + endpoint: "/tiers", + status: http.StatusConflict, + body: `{"name":"Top Tier","consortium":"00000000-0000-0000-0000-000000000004","level":"standard","type":"loan","cost":0}`, + addlHeaders: consortiumPermissionHeaders, + }, { name: "DELETE tier", method: http.MethodDelete, From 8d4d26ab5492ed4e9ad707fb90bc190d469ae9db Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Fri, 11 Sep 2026 12:58:13 +0300 Subject: [PATCH 03/18] ILLDEV-484 Fix copilot comments --- directory/api.yaml | 13 +- directory/api/import.go | 9 +- directory/api/import_test.go | 6 +- directory/import/db/entry.go | 208 ++++++++++++- directory/import/db/entry_lock_test.go | 31 ++ directory/import/db/repo.go | 12 - directory/import/db/repo_test.go | 345 ++++++++++++++++++++++ directory/import/db/tier_network.go | 132 +++++++-- directory/import/model/config.go | 41 +-- directory/import/model/models.go | 9 +- directory/import/service/importer.go | 29 +- directory/import/service/importer_test.go | 98 +++++- directory/query.sql | 3 + directory/test/import_test.go | 2 +- 14 files changed, 847 insertions(+), 91 deletions(-) create mode 100644 directory/import/db/entry_lock_test.go diff --git a/directory/api.yaml b/directory/api.yaml index f66f594a5..f4d7dda3e 100644 --- a/directory/api.yaml +++ b/directory/api.yaml @@ -1411,14 +1411,20 @@ components: ImportResult: type: object additionalProperties: false - required: [entries, tiers, networks, errors] + required: [entries, tiers, networks, errors, errorsOmitted] properties: entries: { $ref: '#/components/schemas/ImportSectionResult' } tiers: { $ref: '#/components/schemas/ImportSectionResult' } networks: { $ref: '#/components/schemas/ImportSectionResult' } errors: type: array + maxItems: 1000 items: { $ref: '#/components/schemas/ImportItemError' } + errorsOmitted: + type: integer + format: int32 + minimum: 0 + description: Number of additional error details omitted from the response. ImportSymbolRef: type: object @@ -1491,7 +1497,7 @@ components: ImportLmsConfig: type: object additionalProperties: false - required: [address, fromAgency, fromAgencyAuthentication, toAgency, lookupUserEnabled, acceptItemEnabled, checkInItemEnabled, checkOutItemEnabled, itemLocation, requestItemRequestType, requestItemRequestScopeType, requestItemBibIdCode, requestItemEnabled, requestItemPickupLocationEnabled, requesterPickupLocation, supplierPickupLocation, requesterPatronPattern] + required: [address, fromAgency, fromAgencyAuthentication, toAgency, lookupUserEnabled, acceptItemEnabled, checkInItemEnabled, checkOutItemEnabled, itemLocation, requestItemRequestType, requestItemRequestScopeType, requestItemBibIdCode, requestItemEnabled, requestItemPickupLocationEnabled, requesterPickupLocation, supplierPickupLocation, requesterPatronPattern, patronProfiles] properties: address: { type: string } fromAgency: { type: string } @@ -1510,6 +1516,9 @@ components: requesterPickupLocation: { type: string, nullable: true } supplierPickupLocation: { type: string, nullable: true } requesterPatronPattern: { type: string, nullable: true } + patronProfiles: + allOf: [{ $ref: '#/components/schemas/PatronProfiles' }] + nullable: true ImportIllConfig: type: object diff --git a/directory/api/import.go b/directory/api/import.go index dce9b6fc7..176492aaf 100644 --- a/directory/api/import.go +++ b/directory/api/import.go @@ -53,10 +53,11 @@ func mapImportResult(result model.ImportResult) ImportResult { errors = append(errors, item) } return ImportResult{ - Entries: mapImportSection(result.Entries), - Tiers: mapImportSection(result.Tiers), - Networks: mapImportSection(result.Networks), - Errors: errors, + Entries: mapImportSection(result.Entries), + Tiers: mapImportSection(result.Tiers), + Networks: mapImportSection(result.Networks), + Errors: errors, + ErrorsOmitted: result.ErrorsOmitted, } } diff --git a/directory/api/import_test.go b/directory/api/import_test.go index 5c3c4c918..6498b7db0 100644 --- a/directory/api/import_test.go +++ b/directory/api/import_test.go @@ -32,8 +32,9 @@ func (i *recordingAggregateImporter) Import(_ context.Context, policy model.Conf func TestPostImportDefaultsPolicyAndMapsResult(t *testing.T) { importer := &recordingAggregateImporter{result: model.ImportResult{ - Entries: model.ImportSectionResult{Imported: 1}, - Errors: []model.ImportItemError{}, + Entries: model.ImportSectionResult{Imported: 1}, + Errors: []model.ImportItemError{}, + ErrorsOmitted: 7, }} impl := NewApiImpl(nil, nil, importer) @@ -45,6 +46,7 @@ func TestPostImportDefaultsPolicyAndMapsResult(t *testing.T) { mapped := ImportResult(response.(PostImport200JSONResponse)) require.Equal(t, int32(1), mapped.Entries.Imported) require.Empty(t, mapped.Errors) + require.Equal(t, int32(7), mapped.ErrorsOmitted) } func TestPostImportRejectsUnauthorizedCallerBeforeReadingBody(t *testing.T) { diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 87ed2bc7a..857c48cb1 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -1,10 +1,12 @@ package importdb import ( + "bytes" "context" "encoding/json" "errors" "fmt" + "sort" "time" "github.com/google/uuid" @@ -15,10 +17,27 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) +const maxImportLockAttempts = 3 + +var errImportEntryMappingChanged = errors.New("entry symbol mapping changed while acquiring import locks") + func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { if err := aggregate.NormalizeAndValidate(); err != nil { return model.RepoResult{}, err } + for range maxImportLockAttempts { + result, err := r.importEntryAttempt(ctx, aggregate, policy) + if !errors.Is(err, errImportEntryMappingChanged) { + return result, err + } + if err := ctx.Err(); err != nil { + return model.RepoResult{}, err + } + } + return model.RepoResult{}, fmt.Errorf("import entry %s: hierarchy changed repeatedly", aggregate.Key.String()) +} + +func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { key := aggregate.Key.String() tx, queries, err := r.begin(ctx) if err != nil { @@ -32,7 +51,7 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg }); err != nil { return model.RepoResult{}, fmt.Errorf("lock entry %s", key) } - existing, lookupErr := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{ + existing, lookupErr := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{ Authority: aggregate.Key.Authority, Symbol: aggregate.Key.Symbol, }) @@ -41,12 +60,59 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg return model.RepoResult{}, fmt.Errorf("resolve entry %s", key) } if exists && policy != model.ConflictPolicyUpdate { + if _, err := lockEntryRows(ctx, queries, existing.ID); err != nil { + return model.RepoResult{}, fmt.Errorf("lock entry %s: %w", key, err) + } + if err := lockEntryMappings(ctx, queries, entryMapping{ref: aggregate.Key, expectedOwner: &existing.ID}); err != nil { + return model.RepoResult{}, fmt.Errorf("revalidate entry %s: %w", key, err) + } return conflictResult("entry", key, policy) } if !exists && policy != model.ConflictPolicyFail && policy != model.ConflictPolicySkip && policy != model.ConflictPolicyUpdate { return model.RepoResult{}, fmt.Errorf("invalid conflict policy") } + parent, err := resolveParent(ctx, queries, aggregate.Data.Parent) + if err != nil { + return model.RepoResult{}, err + } + lenders, err := resolveLenders(ctx, queries, aggregate.Data.ILLConfig) + if err != nil { + return model.RepoResult{}, err + } + var owner *db.Entry + if exists { + owner = &existing + } + entryIDs := entryLockIDs(owner, parent, lenders) + lockedEntries, err := lockEntryRows(ctx, queries, entryIDs...) + if err != nil { + return model.RepoResult{}, fmt.Errorf("lock entry hierarchy: %w", err) + } + mappings := []entryMapping{{ref: aggregate.Key, expectedOwner: entryIDPointer(existing, exists)}} + if aggregate.Data.Parent != nil { + mappings = append(mappings, entryMapping{ref: *aggregate.Data.Parent, expectedOwner: &parent.ID}) + } + if aggregate.Data.ILLConfig != nil { + for index, lender := range aggregate.Data.ILLConfig.LendersOfLastResort { + mappings = append(mappings, entryMapping{ref: lender, expectedOwner: &lenders[index].ID}) + } + } + if err := lockEntryMappings(ctx, queries, mappings...); err != nil { + return model.RepoResult{}, fmt.Errorf("revalidate entry hierarchy: %w", err) + } + if exists { + existing = lockedEntries[existing.ID] + } + var parentID *uuid.UUID + if parent != nil { + lockedParent := lockedEntries[parent.ID] + if valid, reason := domain.ValidParentForType(aggregate.Data.Type, lockedParent.Type); !valid { + return model.RepoResult{}, fmt.Errorf("invalid parent %s: %s", aggregate.Data.Parent.String(), reason) + } + parentID = &lockedParent.ID + } + if aggregate.Data.Type == "Consortium" || (exists && existing.Type == "Consortium") { if err := queries.LockConsortiumEntryChanges(ctx); err != nil { return model.RepoResult{}, fmt.Errorf("lock consortium entry changes") @@ -62,10 +128,6 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg } } - parentID, err := resolveParent(ctx, queries, aggregate.Data.Parent, aggregate.Data.Type) - if err != nil { - return model.RepoResult{}, err - } if exists { if err := validateEntryUpdateHierarchy(ctx, queries, existing, aggregate.Data.Type, parentID); err != nil { return model.RepoResult{}, err @@ -85,21 +147,139 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg return model.RepoResult{Outcome: model.OutcomeImported}, nil } -func resolveParent(ctx context.Context, queries *db.Queries, parent *model.SymbolRef, entryType string) (*uuid.UUID, error) { +func entryIDPointer(entry db.Entry, exists bool) *uuid.UUID { + if !exists { + return nil + } + return &entry.ID +} + +type entryMapping struct { + ref model.SymbolRef + expectedOwner *uuid.UUID +} + +func lockEntryMappings(ctx context.Context, queries *db.Queries, mappings ...entryMapping) error { + sort.Slice(mappings, func(i, j int) bool { + if mappings[i].ref.Authority != mappings[j].ref.Authority { + return mappings[i].ref.Authority < mappings[j].ref.Authority + } + return mappings[i].ref.Symbol < mappings[j].ref.Symbol + }) + locked := make(map[string]*uuid.UUID, len(mappings)) + for _, mapping := range mappings { + key := mapping.ref.String() + if expectedOwner, exists := locked[key]; exists { + if !sameEntryID(expectedOwner, mapping.expectedOwner) { + return errImportEntryMappingChanged + } + continue + } + symbol, err := queries.SymbolByAuthorityAndSymbolForUpdate(ctx, db.SymbolByAuthorityAndSymbolForUpdateParams{ + Authority: mapping.ref.Authority, + Symbol: mapping.ref.Symbol, + }) + if errors.Is(err, pgx.ErrNoRows) { + if mapping.expectedOwner != nil { + return errImportEntryMappingChanged + } + locked[key] = nil + continue + } + if err != nil { + return err + } + if mapping.expectedOwner == nil || symbol.Owner != *mapping.expectedOwner { + return errImportEntryMappingChanged + } + owner := symbol.Owner + locked[key] = &owner + } + return nil +} + +func sameEntryID(first, second *uuid.UUID) bool { + if first == nil || second == nil { + return first == nil && second == nil + } + return *first == *second +} + +func resolveParent(ctx context.Context, queries *db.Queries, parent *model.SymbolRef) (*db.Entry, error) { if parent == nil { return nil, nil } - entry, err := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{Authority: parent.Authority, Symbol: parent.Symbol}) + entry, err := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{Authority: parent.Authority, Symbol: parent.Symbol}) if errors.Is(err, pgx.ErrNoRows) { return nil, fmt.Errorf("parent %s does not exist", parent.String()) } if err != nil { return nil, fmt.Errorf("resolve parent %s", parent.String()) } - if valid, reason := domain.ValidParentForType(entryType, entry.Type); !valid { - return nil, fmt.Errorf("invalid parent %s: %s", parent.String(), reason) + return &entry, nil +} + +func resolveLenders(ctx context.Context, queries *db.Queries, config *model.ILLConfig) ([]db.Entry, error) { + if config == nil { + return nil, nil + } + lenders := make([]db.Entry, 0, len(config.LendersOfLastResort)) + for _, lender := range config.LendersOfLastResort { + entry, err := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{Authority: lender.Authority, Symbol: lender.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + return nil, fmt.Errorf("lender of last resort %s does not exist", lender.String()) + } + if err != nil { + return nil, fmt.Errorf("resolve lender of last resort %s", lender.String()) + } + lenders = append(lenders, entry) + } + return lenders, nil +} + +func entryLockIDs(owner, parent *db.Entry, lenders []db.Entry) []uuid.UUID { + ids := make([]uuid.UUID, 0, 2+len(lenders)) + if owner != nil { + ids = append(ids, owner.ID) + } + if parent != nil { + ids = append(ids, parent.ID) + } + for _, lender := range lenders { + ids = append(ids, lender.ID) + } + return orderedUniqueEntryIDs(ids...) +} + +func lockEntryRows(ctx context.Context, queries *db.Queries, ids ...uuid.UUID) (map[uuid.UUID]db.Entry, error) { + entries := make(map[uuid.UUID]db.Entry, len(ids)) + for _, id := range orderedUniqueEntryIDs(ids...) { + entry, err := queries.EntryByIdForUpdate(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return nil, errImportEntryMappingChanged + } + if err != nil { + return nil, err + } + entries[id] = entry + } + return entries, nil +} + +func orderedUniqueEntryIDs(ids ...uuid.UUID) []uuid.UUID { + unique := make(map[uuid.UUID]struct{}, len(ids)) + ordered := make([]uuid.UUID, 0, len(ids)) + for _, id := range ids { + if _, exists := unique[id]; exists { + continue + } + unique[id] = struct{}{} + ordered = append(ordered, id) } - return &entry.ID, nil + sort.Slice(ordered, func(i, j int) bool { + return bytes.Compare(ordered[i][:], ordered[j][:]) < 0 + }) + return ordered } func validateEntryUpdateHierarchy(ctx context.Context, queries *db.Queries, existing db.Entry, resultingType string, parentID *uuid.UUID) error { @@ -197,6 +377,10 @@ func replaceEntryConfigs(ctx context.Context, queries *db.Queries, entryID uuid. } if data.LMSConfig != nil { cfg := data.LMSConfig + var patronProfiles []byte + if cfg.PatronProfiles != nil { + patronProfiles, _ = json.Marshal(cfg.PatronProfiles) + } if _, err := queries.UpsertLMSConfig(ctx, db.UpsertLMSConfigParams{ Entry: &entryID, Address: cfg.Address, FromAgency: cfg.FromAgency, FromAgencyAuthentication: cfg.FromAgencyAuthentication, ToAgency: cfg.ToAgency, LookupUserEnabled: cfg.LookupUserEnabled, AcceptItemEnabled: cfg.AcceptItemEnabled, @@ -205,6 +389,7 @@ func replaceEntryConfigs(ctx context.Context, queries *db.Queries, entryID uuid. RequestItemBibCode: cfg.RequestItemBibIDCode, RequestItemEnabled: cfg.RequestItemEnabled, RequestItemPickupLocationEnabled: cfg.RequestItemPickupLocationEnabled, RequesterPickupLocation: cfg.RequesterPickupLocation, SupplierPickupLocation: cfg.SupplierPickupLocation, RequesterPatronPattern: cfg.RequesterPatronPattern, + PatronProfiles: patronProfiles, }); err != nil { return err } @@ -273,9 +458,6 @@ func replaceILLConfig(ctx context.Context, queries *db.Queries, entryID uuid.UUI } lenders := make([]string, 0, len(config.LendersOfLastResort)) for _, lender := range config.LendersOfLastResort { - if _, err := resolveEntry(ctx, queries, lender); err != nil { - return fmt.Errorf("lender of last resort %s does not exist", lender.String()) - } lenders = append(lenders, lender.String()) } _, err := queries.UpsertIllConfig(ctx, db.UpsertIllConfigParams{ diff --git a/directory/import/db/entry_lock_test.go b/directory/import/db/entry_lock_test.go new file mode 100644 index 000000000..7ec5672fd --- /dev/null +++ b/directory/import/db/entry_lock_test.go @@ -0,0 +1,31 @@ +package importdb + +import ( + "testing" + + "github.com/google/uuid" + "github.com/indexdata/crosslink/directory/db" + "github.com/stretchr/testify/require" +) + +func TestOrderedUniqueEntryIDsSortsAndDeduplicates(t *testing.T) { + first := uuid.MustParse("00000000-0000-0000-0000-000000000001") + second := uuid.MustParse("00000000-0000-0000-0000-000000000002") + + require.Equal(t, []uuid.UUID{first, second}, orderedUniqueEntryIDs(second, first, second)) +} + +func TestEntryLockIDsIncludesOwnerParentAndLenders(t *testing.T) { + ownerID := uuid.MustParse("00000000-0000-0000-0000-000000000004") + parentID := uuid.MustParse("00000000-0000-0000-0000-000000000003") + firstLenderID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + secondLenderID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + owner := db.Entry{ID: ownerID} + parent := db.Entry{ID: parentID} + lenders := []db.Entry{{ID: firstLenderID}, {ID: secondLenderID}, {ID: firstLenderID}} + + require.Equal(t, + []uuid.UUID{secondLenderID, firstLenderID, parentID, ownerID}, + entryLockIDs(&owner, &parent, lenders), + ) +} diff --git a/directory/import/db/repo.go b/directory/import/db/repo.go index f1d058b8d..1d494a93f 100644 --- a/directory/import/db/repo.go +++ b/directory/import/db/repo.go @@ -2,7 +2,6 @@ package importdb import ( "context" - "errors" "fmt" "github.com/indexdata/crosslink/directory/db" @@ -40,17 +39,6 @@ func conflictResult(resource, key string, policy model.ConflictPolicy) (model.Re } } -func resolveEntry(ctx context.Context, queries *db.Queries, key model.SymbolRef) (db.Entry, error) { - entry, err := queries.EntryBySymbolForUpdate(ctx, db.EntryBySymbolForUpdateParams{Authority: key.Authority, Symbol: key.Symbol}) - if errors.Is(err, pgx.ErrNoRows) { - return db.Entry{}, fmt.Errorf("entry %s does not exist", key.String()) - } - if err != nil { - return db.Entry{}, fmt.Errorf("resolve entry %s", key.String()) - } - return entry, nil -} - func persistenceError(resource, key string, err error) error { if err == nil { return nil diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 3e1138428..6c3d087f0 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -2,8 +2,10 @@ package importdb_test import ( "context" + "errors" "fmt" "os" + "strings" "sync" "testing" "time" @@ -147,6 +149,25 @@ func TestImportEntryConflictPoliciesAndUpdateFullSynchronization(t *testing.T) { require.NotEqual(t, originalEndpointID, replacementEndpointID) } +func TestImportEntryUpdateReplacesLMSPatronProfiles(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + aggregate := completeEntryAggregate("CON") + initialProfiles := []model.PatronProfile{{Code: stringPointer("STAFF"), CanCreateRequests: true}} + aggregate.Data.LMSConfig.PatronProfiles = &initialProfiles + + _, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + entryID := entryIDBySymbol(t, aggregate.Key) + require.JSONEq(t, `[{"code":"STAFF","canCreateRequests":true}]`, lmsPatronProfiles(t, entryID)) + + replacementProfiles := []model.PatronProfile{{Name: stringPointer("Blocked"), CanCreateRequests: false}} + aggregate.Data.LMSConfig.PatronProfiles = &replacementProfiles + _, err = repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyUpdate) + require.NoError(t, err) + require.JSONEq(t, `[{"name":"Blocked","canCreateRequests":false}]`, lmsPatronProfiles(t, entryID)) +} + func TestConcurrentImportEntrySkipHonorsConflictPolicyForMissingKey(t *testing.T) { resetImportDatabase(t) newAggregate := func() model.EntryAggregate { return minimalEntryAggregate("concurrent", "Institution") } @@ -179,6 +200,217 @@ func TestConcurrentImportEntryUpdateHonorsConflictPolicyForMissingKey(t *testing require.Equal(t, 1, entryCount(t)) } +func TestConcurrentImportEntryOpposingParentsDoNotDeadlock(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + const pairCount = 12 + type importPair struct { + first model.EntryAggregate + second model.EntryAggregate + } + pairs := make([]importPair, pairCount) + for index := range pairCount { + firstID, secondID := uuid.New(), uuid.New() + firstSymbol := fmt.Sprintf("A-%d", index) + secondSymbol := fmt.Sprintf("B-%d", index) + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, $2, 'Institution'), ($3, $4, 'Institution')`, + firstID, "Institution "+firstSymbol, secondID, "Institution "+secondSymbol) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', $2), ($3, 'ISIL', $4)`, + firstID, firstSymbol, secondID, secondSymbol) + require.NoError(t, err) + + first := minimalEntryAggregate(firstSymbol, "Branch") + first.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: secondSymbol} + second := minimalEntryAggregate(secondSymbol, "Branch") + second.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: firstSymbol} + pairs[index] = importPair{first: first, second: second} + } + + repo := importdb.New(testPool) + start := make(chan struct{}) + errs := make([][2]error, pairCount) + var waitGroup sync.WaitGroup + waitGroup.Add(pairCount * 2) + for index := range pairs { + go func() { + defer waitGroup.Done() + <-start + _, errs[index][0] = repo.ImportEntry(ctx, pairs[index].first, model.ConflictPolicyUpdate) + }() + go func() { + defer waitGroup.Done() + <-start + _, errs[index][1] = repo.ImportEntry(ctx, pairs[index].second, model.ConflictPolicyUpdate) + }() + } + close(start) + waitGroup.Wait() + require.NoError(t, ctx.Err()) + + for index, pairErrors := range errs { + var imported int + for _, err := range pairErrors { + if err == nil { + imported++ + continue + } + var pgErr *pgconn.PgError + require.Falsef(t, errors.As(err, &pgErr) && pgErr.Code == "40P01", "pair %d deadlocked: %v", index, err) + require.Truef(t, + strings.Contains(err.Error(), "invalid parent") || strings.Contains(err.Error(), "would create a cycle"), + "pair %d returned an unexpected error: %v", index, err) + } + require.Equalf(t, 1, imported, "pair %d should serialize before hierarchy validation", index) + } +} + +func TestConcurrentImportEntryOpposingLendersDoNotDeadlock(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + const pairCount = 12 + type importPair struct { + first model.EntryAggregate + second model.EntryAggregate + } + pairs := make([]importPair, pairCount) + for index := range pairCount { + firstID, secondID := uuid.New(), uuid.New() + firstSymbol := fmt.Sprintf("LENDER-A-%d", index) + secondSymbol := fmt.Sprintf("LENDER-B-%d", index) + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, $2, 'Institution'), ($3, $4, 'Institution')`, + firstID, "Institution "+firstSymbol, secondID, "Institution "+secondSymbol) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', $2), ($3, 'ISIL', $4)`, + firstID, firstSymbol, secondID, secondSymbol) + require.NoError(t, err) + + first := minimalEntryAggregate(firstSymbol, "Institution") + first.Data.ILLConfig = &model.ILLConfig{LendersOfLastResort: []model.SymbolRef{{Authority: "ISIL", Symbol: secondSymbol}}} + second := minimalEntryAggregate(secondSymbol, "Institution") + second.Data.ILLConfig = &model.ILLConfig{LendersOfLastResort: []model.SymbolRef{{Authority: "ISIL", Symbol: firstSymbol}}} + pairs[index] = importPair{first: first, second: second} + } + + repo := importdb.New(testPool) + start := make(chan struct{}) + errs := make([][2]error, pairCount) + var waitGroup sync.WaitGroup + waitGroup.Add(pairCount * 2) + for index := range pairs { + go func() { + defer waitGroup.Done() + <-start + _, errs[index][0] = repo.ImportEntry(ctx, pairs[index].first, model.ConflictPolicyUpdate) + }() + go func() { + defer waitGroup.Done() + <-start + _, errs[index][1] = repo.ImportEntry(ctx, pairs[index].second, model.ConflictPolicyUpdate) + }() + } + close(start) + waitGroup.Wait() + require.NoError(t, ctx.Err()) + + for index, pairErrors := range errs { + require.NoErrorf(t, pairErrors[0], "first import in pair %d failed", index) + require.NoErrorf(t, pairErrors[1], "second import in pair %d failed", index) + } +} + +func TestImportEntryRetriesWhenParentSymbolChangesBeforeRowLock(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + originalParentID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + ownerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + replacementParentID := uuid.MustParse("00000000-0000-0000-0000-000000000003") + _, err := testPool.Exec(ctx, ` + INSERT INTO entries (id, name, type) VALUES + ($1, 'Original parent', 'Institution'), + ($2, 'Imported branch', 'Branch'), + ($3, 'Replacement parent', 'Institution')`, originalParentID, ownerID, replacementParentID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, ` + INSERT INTO symbols (owner, authority, symbol) VALUES + ($1, 'ISIL', 'PARENT'), + ($2, 'ISIL', 'BRANCH')`, originalParentID, ownerID) + require.NoError(t, err) + + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT id FROM entries WHERE id=$1 FOR UPDATE`, originalParentID) + require.NoError(t, err) + + aggregate := minimalEntryAggregate("BRANCH", "Branch") + aggregate.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "PARENT"} + importDone := make(chan error, 1) + go func() { + _, importErr := importdb.New(testPool).ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) + importDone <- importErr + }() + require.Eventually(t, func() bool { + var waiting bool + err := testPool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE datname=current_database() AND pid <> pg_backend_pid() AND wait_event_type='Lock' + )`).Scan(&waiting) + return err == nil && waiting + }, 2*time.Second, 10*time.Millisecond) + + _, err = testPool.Exec(ctx, `UPDATE symbols SET owner=$1 WHERE authority='ISIL' AND symbol='PARENT'`, replacementParentID) + require.NoError(t, err) + require.NoError(t, blocker.Commit(ctx)) + require.NoError(t, <-importDone) + + var parentID uuid.UUID + require.NoError(t, testPool.QueryRow(ctx, `SELECT parent FROM entries WHERE id=$1`, ownerID).Scan(&parentID)) + require.Equal(t, replacementParentID, parentID) +} + +func TestImportEntryRetriesWhenParentIsDeletedBeforeRowLock(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + parentID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + ownerID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, 'Parent', 'Institution'), ($2, 'Branch', 'Branch')`, parentID, ownerID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', 'PARENT'), ($2, 'ISIL', 'BRANCH')`, parentID, ownerID) + require.NoError(t, err) + + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT id FROM entries WHERE id=$1 FOR UPDATE`, parentID) + require.NoError(t, err) + + aggregate := minimalEntryAggregate("BRANCH", "Branch") + aggregate.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "PARENT"} + importDone := make(chan error, 1) + go func() { + _, importErr := importdb.New(testPool).ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) + importDone <- importErr + }() + require.Eventually(t, func() bool { + var waiting bool + err := testPool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE datname=current_database() AND pid <> pg_backend_pid() AND wait_event_type='Lock' + )`).Scan(&waiting) + return err == nil && waiting + }, 2*time.Second, 10*time.Millisecond) + + _, err = blocker.Exec(ctx, `DELETE FROM entries WHERE id=$1`, parentID) + require.NoError(t, err) + require.NoError(t, blocker.Commit(ctx)) + require.ErrorContains(t, <-importDone, "parent ISIL:PARENT does not exist") +} + func TestImportEntryRejectsInvalidHierarchy(t *testing.T) { resetImportDatabase(t) repo := importdb.New(testPool) @@ -281,6 +513,54 @@ func TestImportTierConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) } +func TestConcurrentEntryAndTierImportsUseSameEntryLockOrder(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + memberID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + consortiumID := uuid.MustParse("ffffffff-ffff-ffff-ffff-ffffffffffff") + tierID := uuid.New() + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, 'Member', 'Institution'), ($2, 'Consortium', 'Consortium')`, memberID, consortiumID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', 'MEMBER'), ($2, 'ISIL', 'CON')`, memberID, consortiumID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO tiers (id, consortium, name, level, type, cost) VALUES ($1, $2, 'Loan', 'standard', 'loan', 0)`, tierID, consortiumID) + require.NoError(t, err) + + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT id FROM tiers WHERE id=$1 FOR UPDATE`, tierID) + require.NoError(t, err) + + repo := importdb.New(testPool) + consortium := model.SymbolRef{Authority: "ISIL", Symbol: "CON"} + member := model.SymbolRef{Authority: "ISIL", Symbol: "MEMBER"} + tier := model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Loan"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{member}}, + } + tierDone := make(chan error, 1) + go func() { + _, importErr := repo.ImportTier(ctx, tier, model.ConflictPolicyUpdate) + tierDone <- importErr + }() + waitForDatabaseLockWaiters(t, ctx, 1) + + entry := minimalEntryAggregate("MEMBER", "Institution") + entry.Data.Parent = &consortium + entryDone := make(chan error, 1) + go func() { + _, importErr := repo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) + entryDone <- importErr + }() + waitForDatabaseLockWaiters(t, ctx, 2) + + require.NoError(t, blocker.Commit(ctx)) + require.NoError(t, <-tierDone) + require.NoError(t, <-entryDone) +} + func TestConcurrentImportTierSkipHonorsConflictPolicyForMissingKey(t *testing.T) { repo, consortium, _, _ := importRepoFixture(t) newAggregate := func() model.TierAggregate { @@ -353,6 +633,54 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.Equal(t, []model.SymbolRef{second}, networkAssignments(t, id)) } +func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + memberID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + consortiumID := uuid.MustParse("ffffffff-ffff-ffff-ffff-ffffffffffff") + networkID := uuid.New() + _, err := testPool.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, 'Member', 'Institution'), ($2, 'Consortium', 'Consortium')`, memberID, consortiumID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', 'MEMBER'), ($2, 'ISIL', 'CON')`, memberID, consortiumID) + require.NoError(t, err) + _, err = testPool.Exec(ctx, `INSERT INTO networks (id, consortium, name, priority) VALUES ($1, $2, 'Main', 0)`, networkID, consortiumID) + require.NoError(t, err) + + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT id FROM networks WHERE id=$1 FOR UPDATE`, networkID) + require.NoError(t, err) + + repo := importdb.New(testPool) + consortium := model.SymbolRef{Authority: "ISIL", Symbol: "CON"} + member := model.SymbolRef{Authority: "ISIL", Symbol: "MEMBER"} + network := model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Main"}, + Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{member}}, + } + networkDone := make(chan error, 1) + go func() { + _, importErr := repo.ImportNetwork(ctx, network, model.ConflictPolicyUpdate) + networkDone <- importErr + }() + waitForDatabaseLockWaiters(t, ctx, 1) + + entry := minimalEntryAggregate("MEMBER", "Institution") + entry.Data.Parent = &consortium + entryDone := make(chan error, 1) + go func() { + _, importErr := repo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) + entryDone <- importErr + }() + waitForDatabaseLockWaiters(t, ctx, 2) + + require.NoError(t, blocker.Commit(ctx)) + require.NoError(t, <-networkDone) + require.NoError(t, <-entryDone) +} + func TestConcurrentImportNetworkSkipHonorsConflictPolicyForMissingKey(t *testing.T) { repo, consortium, _, _ := importRepoFixture(t) newAggregate := func() model.NetworkAggregate { @@ -463,6 +791,13 @@ func assertTierDoesNotExist(t *testing.T, consortium model.SymbolRef, name strin require.Zero(t, count) } +func lmsPatronProfiles(t *testing.T, entryID uuid.UUID) string { + t.Helper() + var profiles []byte + require.NoError(t, testPool.QueryRow(context.Background(), `SELECT patron_profiles FROM lms_configs WHERE entry=$1`, entryID).Scan(&profiles)) + return string(profiles) +} + func completeEntryAggregate(symbol string) model.EntryAggregate { aggregate := minimalEntryAggregate(symbol, "Consortium") text := "value" @@ -523,6 +858,16 @@ func concurrentlyImportEntry(t *testing.T, newAggregate func() model.EntryAggreg return results, errs } +func waitForDatabaseLockWaiters(t *testing.T, ctx context.Context, minimum int) { + t.Helper() + require.Eventually(t, func() bool { + var count int + err := testPool.QueryRow(ctx, `SELECT count(*) FROM pg_stat_activity + WHERE datname=current_database() AND pid <> pg_backend_pid() AND wait_event_type='Lock'`).Scan(&count) + return err == nil && count >= minimum + }, 2*time.Second, 10*time.Millisecond) +} + func concurrentlyImportTier(repo *importdb.PgImportRepo, newAggregate func() model.TierAggregate, policy model.ConflictPolicy, count int) ([]model.RepoResult, []error) { start := make(chan struct{}) results := make([]model.RepoResult, count) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index 23d417f06..58001a36e 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "sort" "github.com/google/uuid" "github.com/indexdata/crosslink/directory/db" @@ -17,13 +16,26 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + for range maxImportLockAttempts { + result, err := r.importTierAttempt(ctx, aggregate, policy, key) + if !errors.Is(err, errImportEntryMappingChanged) { + return result, err + } + if err := ctx.Err(); err != nil { + return model.RepoResult{}, err + } + } + return model.RepoResult{}, fmt.Errorf("import tier %s: entry mappings changed repeatedly", key) +} + +func (r *PgImportRepo) importTierAttempt(ctx context.Context, aggregate model.TierAggregate, policy model.ConflictPolicy, key string) (model.RepoResult, error) { tx, queries, err := r.begin(ctx) if err != nil { return model.RepoResult{}, err } defer func() { _ = tx.Rollback(ctx) }() - consortium, err := resolveConsortium(ctx, queries, aggregate.Key.Consortium) + consortium, assignments, err := resolveAndLockAssignments(ctx, queries, aggregate.Key.Consortium, aggregate.Data.Entries) if err != nil { return model.RepoResult{}, err } @@ -39,6 +51,10 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre if !exists && !validPolicy(policy) { return model.RepoResult{}, fmt.Errorf("invalid conflict policy") } + assignmentEntries, err := requireAssignmentEntries(assignments) + if err != nil { + return model.RepoResult{}, err + } var tierID uuid.UUID if exists { @@ -52,7 +68,7 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre if err != nil { return model.RepoResult{}, persistenceError("tier", key, err) } - if err := replaceTierAssignments(ctx, queries, tierID, aggregate.Data.Entries); err != nil { + if err := replaceTierAssignments(ctx, queries, tierID, assignmentEntries); err != nil { return model.RepoResult{}, err } if err := tx.Commit(ctx); err != nil { @@ -66,13 +82,26 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name + for range maxImportLockAttempts { + result, err := r.importNetworkAttempt(ctx, aggregate, policy, key) + if !errors.Is(err, errImportEntryMappingChanged) { + return result, err + } + if err := ctx.Err(); err != nil { + return model.RepoResult{}, err + } + } + return model.RepoResult{}, fmt.Errorf("import network %s: entry mappings changed repeatedly", key) +} + +func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model.NetworkAggregate, policy model.ConflictPolicy, key string) (model.RepoResult, error) { tx, queries, err := r.begin(ctx) if err != nil { return model.RepoResult{}, err } defer func() { _ = tx.Rollback(ctx) }() - consortium, err := resolveConsortium(ctx, queries, aggregate.Key.Consortium) + consortium, assignments, err := resolveAndLockAssignments(ctx, queries, aggregate.Key.Consortium, aggregate.Data.Entries) if err != nil { return model.RepoResult{}, err } @@ -88,6 +117,10 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ if !exists && !validPolicy(policy) { return model.RepoResult{}, fmt.Errorf("invalid conflict policy") } + assignmentEntries, err := requireAssignmentEntries(assignments) + if err != nil { + return model.RepoResult{}, err + } var networkID uuid.UUID if exists { @@ -101,7 +134,7 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ if err != nil { return model.RepoResult{}, persistenceError("network", key, err) } - if err := replaceNetworkAssignments(ctx, queries, networkID, aggregate.Data.Entries); err != nil { + if err := replaceNetworkAssignments(ctx, queries, networkID, assignmentEntries); err != nil { return model.RepoResult{}, err } if err := tx.Commit(ctx); err != nil { @@ -110,29 +143,74 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{Outcome: model.OutcomeImported}, nil } -// resolveConsortium locks the consortium entry for the transaction. Besides -// protecting the reference, this serializes missing tier and network business -// keys within a consortium before their lookup-and-create flows. -func resolveConsortium(ctx context.Context, queries *db.Queries, key model.SymbolRef) (db.Entry, error) { - entry, err := resolveEntry(ctx, queries, key) +type resolvedAssignment struct { + ref model.SymbolRef + entry *db.Entry +} + +func resolveAndLockAssignments(ctx context.Context, queries *db.Queries, consortiumRef model.SymbolRef, refs []model.SymbolRef) (db.Entry, []resolvedAssignment, error) { + consortium, err := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{Authority: consortiumRef.Authority, Symbol: consortiumRef.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + return db.Entry{}, nil, fmt.Errorf("consortium %s does not exist", consortiumRef.String()) + } + if err != nil { + return db.Entry{}, nil, fmt.Errorf("resolve consortium %s", consortiumRef.String()) + } + + assignments := make([]resolvedAssignment, 0, len(refs)) + entryIDs := []uuid.UUID{consortium.ID} + mappings := []entryMapping{{ref: consortiumRef, expectedOwner: &consortium.ID}} + for _, ref := range refs { + entry, err := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{Authority: ref.Authority, Symbol: ref.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + assignments = append(assignments, resolvedAssignment{ref: ref}) + mappings = append(mappings, entryMapping{ref: ref}) + continue + } + if err != nil { + return db.Entry{}, nil, fmt.Errorf("resolve entry %s", ref.String()) + } + assignments = append(assignments, resolvedAssignment{ref: ref, entry: &entry}) + entryIDs = append(entryIDs, entry.ID) + mappings = append(mappings, entryMapping{ref: ref, expectedOwner: &entry.ID}) + } + + lockedEntries, err := lockEntryRows(ctx, queries, entryIDs...) if err != nil { - return db.Entry{}, fmt.Errorf("consortium %s does not exist", key.String()) + return db.Entry{}, nil, fmt.Errorf("lock assignment entries: %w", err) + } + if err := lockEntryMappings(ctx, queries, mappings...); err != nil { + return db.Entry{}, nil, fmt.Errorf("revalidate assignment entries: %w", err) + } + consortium = lockedEntries[consortium.ID] + if consortium.Type != "Consortium" { + return db.Entry{}, nil, fmt.Errorf("entry %s is not a consortium", consortiumRef.String()) + } + for index := range assignments { + if assignments[index].entry != nil { + entry := lockedEntries[assignments[index].entry.ID] + assignments[index].entry = &entry + } } - if entry.Type != "Consortium" { - return db.Entry{}, fmt.Errorf("entry %s is not a consortium", key.String()) + return consortium, assignments, nil +} + +func requireAssignmentEntries(assignments []resolvedAssignment) ([]db.Entry, error) { + entries := make([]db.Entry, 0, len(assignments)) + for _, assignment := range assignments { + if assignment.entry == nil { + return nil, fmt.Errorf("entry %s does not exist", assignment.ref.String()) + } + entries = append(entries, *assignment.entry) } - return entry, nil + return entries, nil } -func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uuid.UUID, refs []model.SymbolRef) error { +func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uuid.UUID, entries []db.Entry) error { if err := queries.DeleteEntryTiersByTier(ctx, tierID); err != nil { return fmt.Errorf("replace tier assignments") } - for _, ref := range sortedRefs(refs) { - entry, err := resolveEntry(ctx, queries, ref) - if err != nil { - return err - } + for _, entry := range entries { if _, err := queries.CreateEntryTier(ctx, db.CreateEntryTierParams{Entry: entry.ID, Tier: tierID}); err != nil { return fmt.Errorf("replace tier assignments") } @@ -140,15 +218,11 @@ func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uui return nil } -func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, networkID uuid.UUID, refs []model.SymbolRef) error { +func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, networkID uuid.UUID, entries []db.Entry) error { if err := queries.DeleteEntryNetworksByNetwork(ctx, networkID); err != nil { return fmt.Errorf("replace network assignments") } - for _, ref := range sortedRefs(refs) { - entry, err := resolveEntry(ctx, queries, ref) - if err != nil { - return err - } + for _, entry := range entries { if _, err := queries.CreateEntryNetwork(ctx, db.CreateEntryNetworkParams{Entry: entry.ID, Network: networkID}); err != nil { return fmt.Errorf("replace network assignments") } @@ -156,12 +230,6 @@ func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, network return nil } -func sortedRefs(refs []model.SymbolRef) []model.SymbolRef { - result := append([]model.SymbolRef(nil), refs...) - sort.Slice(result, func(i, j int) bool { return result[i].String() < result[j].String() }) - return result -} - func validPolicy(policy model.ConflictPolicy) bool { return policy == model.ConflictPolicyFail || policy == model.ConflictPolicySkip || policy == model.ConflictPolicyUpdate } diff --git a/directory/import/model/config.go b/directory/import/model/config.go index 8da1d3cca..1e54b786c 100644 --- a/directory/import/model/config.go +++ b/directory/import/model/config.go @@ -3,23 +3,30 @@ package model import "fmt" type LMSConfig struct { - Address string `json:"address"` - FromAgency string `json:"fromAgency"` - FromAgencyAuthentication *string `json:"fromAgencyAuthentication"` - ToAgency *string `json:"toAgency"` - LookupUserEnabled *bool `json:"lookupUserEnabled"` - AcceptItemEnabled *bool `json:"acceptItemEnabled"` - CheckInItemEnabled *bool `json:"checkInItemEnabled"` - CheckOutItemEnabled *bool `json:"checkOutItemEnabled"` - ItemLocation *string `json:"itemLocation"` - RequestItemRequestType *string `json:"requestItemRequestType"` - RequestItemRequestScopeType *string `json:"requestItemRequestScopeType"` - RequestItemBibIDCode *string `json:"requestItemBibIdCode"` - RequestItemEnabled *bool `json:"requestItemEnabled"` - RequestItemPickupLocationEnabled *bool `json:"requestItemPickupLocationEnabled"` - RequesterPickupLocation *string `json:"requesterPickupLocation"` - SupplierPickupLocation *string `json:"supplierPickupLocation"` - RequesterPatronPattern *string `json:"requesterPatronPattern"` + Address string `json:"address"` + FromAgency string `json:"fromAgency"` + FromAgencyAuthentication *string `json:"fromAgencyAuthentication"` + ToAgency *string `json:"toAgency"` + LookupUserEnabled *bool `json:"lookupUserEnabled"` + AcceptItemEnabled *bool `json:"acceptItemEnabled"` + CheckInItemEnabled *bool `json:"checkInItemEnabled"` + CheckOutItemEnabled *bool `json:"checkOutItemEnabled"` + ItemLocation *string `json:"itemLocation"` + RequestItemRequestType *string `json:"requestItemRequestType"` + RequestItemRequestScopeType *string `json:"requestItemRequestScopeType"` + RequestItemBibIDCode *string `json:"requestItemBibIdCode"` + RequestItemEnabled *bool `json:"requestItemEnabled"` + RequestItemPickupLocationEnabled *bool `json:"requestItemPickupLocationEnabled"` + RequesterPickupLocation *string `json:"requesterPickupLocation"` + SupplierPickupLocation *string `json:"supplierPickupLocation"` + RequesterPatronPattern *string `json:"requesterPatronPattern"` + PatronProfiles *[]PatronProfile `json:"patronProfiles"` +} + +type PatronProfile struct { + Code *string `json:"code,omitempty"` + Name *string `json:"name,omitempty"` + CanCreateRequests bool `json:"canCreateRequests"` } type ILLConfig struct { diff --git a/directory/import/model/models.go b/directory/import/model/models.go index f49ee07ca..95d5ccdef 100644 --- a/directory/import/model/models.go +++ b/directory/import/model/models.go @@ -53,10 +53,11 @@ type ImportItemError struct { } type ImportResult struct { - Entries ImportSectionResult `json:"entries"` - Tiers ImportSectionResult `json:"tiers"` - Networks ImportSectionResult `json:"networks"` - Errors []ImportItemError `json:"errors"` + Entries ImportSectionResult `json:"entries"` + Tiers ImportSectionResult `json:"tiers"` + Networks ImportSectionResult `json:"networks"` + Errors []ImportItemError `json:"errors"` + ErrorsOmitted int32 `json:"errorsOmitted"` } type SymbolRef struct { diff --git a/directory/import/service/importer.go b/directory/import/service/importer.go index 40e90b8dc..7b8a13a68 100644 --- a/directory/import/service/importer.go +++ b/directory/import/service/importer.go @@ -7,12 +7,17 @@ import ( "errors" "fmt" "io" + "unicode/utf8" "github.com/getkin/kin-openapi/openapi3" "github.com/indexdata/crosslink/directory/import/model" ) -const maxRecordBytes = 1 << 20 +const ( + maxRecordBytes = 1 << 20 + maxRetainedErrorDetails = 1000 + maxRetainedErrorFieldBytes = 1024 +) var ErrRecordTooLarge = errors.New("import record exceeds 1 MiB limit") @@ -139,12 +144,30 @@ func section(result *model.ImportResult, recordType string) *model.ImportSection } func appendError(result *model.ImportResult, line int32, recordType, key, message string) { - item := model.ImportItemError{Line: line, Error: message} - if recordType != "" { + if len(result.Errors) >= maxRetainedErrorDetails { + result.ErrorsOmitted++ + return + } + item := model.ImportItemError{Line: line, Error: truncateErrorField(message)} + switch recordType { + case "entry", "tier", "network": item.Type = &recordType } if key != "" { + key = truncateErrorField(key) item.Key = &key } result.Errors = append(result.Errors, item) } + +func truncateErrorField(value string) string { + if len(value) <= maxRetainedErrorFieldBytes { + return value + } + const suffix = "..." + end := maxRetainedErrorFieldBytes - len(suffix) + for end > 0 && !utf8.RuneStart(value[end]) { + end-- + } + return value[:end] + suffix +} diff --git a/directory/import/service/importer_test.go b/directory/import/service/importer_test.go index a21ca777b..331e7d44f 100644 --- a/directory/import/service/importer_test.go +++ b/directory/import/service/importer_test.go @@ -16,12 +16,14 @@ type recordingRepo struct { entryCalls int tierCalls int networkCalls int + entry *model.EntryAggregate result model.RepoResult err error } -func (r *recordingRepo) ImportEntry(context.Context, model.EntryAggregate, model.ConflictPolicy) (model.RepoResult, error) { +func (r *recordingRepo) ImportEntry(_ context.Context, aggregate model.EntryAggregate, _ model.ConflictPolicy) (model.RepoResult, error) { r.entryCalls++ + r.entry = &aggregate return r.result, r.err } @@ -131,6 +133,9 @@ func TestImportRejectsMissingAndUnknownProperties(t *testing.T) { "missing config field": strings.Replace( strings.Replace(validEntryRecord(), `"illConfig":null`, validILLConfig(), 1), `,"supplierPatronPattern":null`, "", 1), + "missing patron profiles": strings.Replace( + strings.Replace(validEntryRecord(), `"lmsConfig":null`, validLMSConfig(), 1), + `,"patronProfiles":[{"code":"STAFF","canCreateRequests":true}]`, "", 1), } for name, record := range tests { t.Run(name, func(t *testing.T) { @@ -144,6 +149,45 @@ func TestImportRejectsMissingAndUnknownProperties(t *testing.T) { } } +func TestImportAcceptsLMSPatronProfiles(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + record := strings.Replace(validEntryRecord(), `"lmsConfig":null`, validLMSConfig(), 1) + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Entries) + assert.Empty(t, result.Errors) + assert.Equal(t, 1, repo.entryCalls) + require.NotNil(t, repo.entry) + require.NotNil(t, repo.entry.Data.LMSConfig) + require.NotNil(t, repo.entry.Data.LMSConfig.PatronProfiles) + profiles := *repo.entry.Data.LMSConfig.PatronProfiles + require.Len(t, profiles, 1) + require.NotNil(t, profiles[0].Code) + require.Equal(t, "STAFF", *profiles[0].Code) + require.True(t, profiles[0].CanCreateRequests) +} + +func TestImportAcceptsNullLMSPatronProfiles(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + lmsConfig := strings.Replace(validLMSConfig(), `"patronProfiles":[{"code":"STAFF","canCreateRequests":true}]`, `"patronProfiles":null`, 1) + record := strings.Replace(validEntryRecord(), `"lmsConfig":null`, lmsConfig, 1) + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + + require.NoError(t, err) + assert.Equal(t, model.ImportSectionResult{Imported: 1}, result.Entries) + assert.Empty(t, result.Errors) + require.NotNil(t, repo.entry) + require.NotNil(t, repo.entry.Data.LMSConfig) + require.Nil(t, repo.entry.Data.LMSConfig.PatronProfiles) +} + +func validLMSConfig() string { + return `"lmsConfig":{"address":"https://example.test/ncip","fromAgency":"FROM","fromAgencyAuthentication":null,"toAgency":null,"lookupUserEnabled":true,"acceptItemEnabled":true,"checkInItemEnabled":true,"checkOutItemEnabled":true,"itemLocation":null,"requestItemRequestType":null,"requestItemRequestScopeType":null,"requestItemBibIdCode":null,"requestItemEnabled":true,"requestItemPickupLocationEnabled":true,"requesterPickupLocation":null,"supplierPickupLocation":null,"requesterPatronPattern":null,"patronProfiles":[{"code":"STAFF","canCreateRequests":true}]}` +} + func validILLConfig() string { return `"illConfig":{"iso18626Url":null,"iso18626Vendor":null,"lendersOfLastResort":[],"includeRequestingAgencyInfo":null,"includeSupplierInfo":null,"includeReturnInfo":null,"includeVendorNote":null,"useOfferedCosts":null,"noteFieldSeparator":null,"supplierPatronPattern":null,"duplicateCheckWindowHours":null}` } @@ -168,6 +212,58 @@ func TestImportAccountsForSkippedAndRepositoryFailures(t *testing.T) { }) } +func TestImportCapsFailureDetailsWhilePreservingCounters(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + badTier := strings.Replace(validTierRecord(), `"name":"Primary"`, `"name":" "`, 1) + input := strings.Repeat(badTier+"\n", 1005) + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + + require.NoError(t, err) + require.Equal(t, model.ImportSectionResult{Failed: 1005}, result.Tiers) + require.Len(t, result.Errors, 1000) + require.Equal(t, int32(5), result.ErrorsOmitted) + require.Zero(t, repo.tierCalls) +} + +func TestImportCapsSkippedDetailsWhilePreservingCounters(t *testing.T) { + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeSkipped, Diagnostic: "entry already exists"}} + input := strings.Repeat(validEntryRecord()+"\n", 1005) + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicySkip, strings.NewReader(input)) + + require.NoError(t, err) + require.Equal(t, model.ImportSectionResult{Skipped: 1005}, result.Entries) + require.Len(t, result.Errors, 1000) + require.Equal(t, int32(5), result.ErrorsOmitted) + require.Equal(t, 1005, repo.entryCalls) +} + +func TestImportCapsRetainedErrorFieldSizes(t *testing.T) { + repo := &recordingRepo{err: errors.New(strings.Repeat("failure", 1000))} + longSymbol := strings.Repeat("x", 5000) + record := strings.ReplaceAll(validEntryRecord(), `"symbol":"abc"`, `"symbol":"`+longSymbol+`"`) + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + + require.NoError(t, err) + require.Len(t, result.Errors, 1) + require.NotNil(t, result.Errors[0].Key) + require.LessOrEqual(t, len(*result.Errors[0].Key), 1024) + require.LessOrEqual(t, len(result.Errors[0].Error), 1024) +} + +func TestImportDoesNotRetainUnknownRecordType(t *testing.T) { + unknownType := strings.Repeat("x", 5000) + record := `{"type":"` + unknownType + `","key":{},"data":{}}` + + result, err := newTestImporter(t, &recordingRepo{}).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(record)) + + require.NoError(t, err) + require.Len(t, result.Errors, 1) + require.Nil(t, result.Errors[0].Type) +} + func TestImportRecordLimitIsExact(t *testing.T) { record := validTierRecord() repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} diff --git a/directory/query.sql b/directory/query.sql index 540e97419..b78f7e44f 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -10,6 +10,9 @@ SELECT e.* FROM entries e, symbols s WHERE e.id = s.owner AND s.authority = @aut -- name: EntryBySymbol :one SELECT e.* FROM entries e, symbols s WHERE e.id = s.owner AND s.authority = @authority AND s.symbol = @symbol LIMIT 1; +-- name: SymbolByAuthorityAndSymbolForUpdate :one +SELECT * FROM symbols WHERE authority = @authority AND symbol = @symbol LIMIT 1 FOR UPDATE; + -- name: GetConsortialEntry :one SELECT * FROM entries WHERE type = 'Consortium' LIMIT 1; diff --git a/directory/test/import_test.go b/directory/test/import_test.go index 33048b9ec..c3df70781 100644 --- a/directory/test/import_test.go +++ b/directory/test/import_test.go @@ -166,7 +166,7 @@ func entryImportRecord(key map[string]any, name string, parent map[string]any, e "checkOutItemEnabled": nil, "itemLocation": nil, "requestItemRequestType": nil, "requestItemRequestScopeType": nil, "requestItemBibIdCode": nil, "requestItemEnabled": nil, "requestItemPickupLocationEnabled": nil, "requesterPickupLocation": nil, "supplierPickupLocation": nil, - "requesterPatronPattern": nil, + "requesterPatronPattern": nil, "patronProfiles": nil, } } return map[string]any{"type": "entry", "key": key, "data": data} From c0b43e26692c2b2bc25da1e9bf1e544bd3620422 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Fri, 11 Sep 2026 13:21:22 +0300 Subject: [PATCH 04/18] ILLDEV-484 Fix after rebase --- broker/test/adapter/api_directory_test.go | 18 ++++---- directory/api.yaml | 14 ++++-- directory/import/db/repo_test.go | 46 ++++++++++++------- directory/import/db/tier_network.go | 21 ++++++--- directory/import/model/models.go | 23 ++++++++-- directory/import/model/models_test.go | 6 +-- directory/import/service/importer_test.go | 2 +- ....sql => 008_import_business_keys.down.sql} | 0 ...up.sql => 008_import_business_keys.up.sql} | 0 directory/query.sql | 2 +- directory/test/import_test.go | 12 ++++- 11 files changed, 98 insertions(+), 46 deletions(-) rename directory/migrations/{007_import_business_keys.down.sql => 008_import_business_keys.down.sql} (100%) rename directory/migrations/{007_import_business_keys.up.sql => 008_import_business_keys.up.sql} (100%) diff --git a/broker/test/adapter/api_directory_test.go b/broker/test/adapter/api_directory_test.go index be375d82f..937662677 100644 --- a/broker/test/adapter/api_directory_test.go +++ b/broker/test/adapter/api_directory_test.go @@ -770,20 +770,20 @@ func TestFilterAndSortUsesCompatibleNetworkPriority(t *testing.T) { appCtx := createLookupCtx() ad := createDirectoryAdapter("") requesterNetworks := []dirapi.EntryNetworkDetails{ - {Name: stringPtr("Reciprocal"), Priority: 1, Reciprocal: boolPtr(true)}, - {Name: stringPtr("Paid Low"), Priority: 5, Reciprocal: boolPtr(false)}, - {Name: stringPtr("Paid High"), Priority: 3, Reciprocal: boolPtr(false)}, + {Name: "Reciprocal", Priority: 1, Reciprocal: boolPtr(true)}, + {Name: "Paid Low", Priority: 5, Reciprocal: boolPtr(false)}, + {Name: "Paid High", Priority: 3, Reciprocal: boolPtr(false)}, } paidTier := []dirapi.Tier{ - {Name: stringPtr("Paid Core Loan"), Level: dirapi.Standard, Type: dirapi.Loan, Cost: 34.4}, + {Name: "Paid Core Loan", Level: dirapi.Standard, Type: dirapi.Loan, Cost: 34.4}, } requesterData := dirapi.Entry{Name: "Requester", Networks: &requesterNetworks} supplierANetworks := []dirapi.EntryNetworkDetails{ - {Name: stringPtr("Reciprocal"), Priority: 1, Reciprocal: boolPtr(true)}, - {Name: stringPtr("Paid Low"), Priority: -10, Reciprocal: boolPtr(false)}, + {Name: "Reciprocal", Priority: 1, Reciprocal: boolPtr(true)}, + {Name: "Paid Low", Priority: -10, Reciprocal: boolPtr(false)}, } supplierBNetworks := []dirapi.EntryNetworkDetails{ - {Name: stringPtr("Paid High"), Priority: 99, Reciprocal: boolPtr(false)}, + {Name: "Paid High", Priority: 99, Reciprocal: boolPtr(false)}, } entries := []adapter.Supplier{ {PeerId: "A", Symbol: "A", CustomData: dirapi.Entry{Name: "Supplier A", Networks: &supplierANetworks, Tiers: &paidTier}}, @@ -915,8 +915,8 @@ func TestCompareSuppliers(t *testing.T) { func TestFilterAndSortAppliesHoldingsPolicy(t *testing.T) { appCtx := createLookupCtx() ad := createDirectoryAdapter("") - networks := []dirapi.EntryNetworkDetails{{Name: strPtr("Reciprocal"), Priority: 1}} - tiers := []dirapi.Tier{{Name: strPtr("Core Loan"), Level: "Core", Type: "Loan", Cost: 0}} + networks := []dirapi.EntryNetworkDetails{{Name: "Reciprocal", Priority: 1}} + tiers := []dirapi.Tier{{Name: "Core Loan", Level: "Core", Type: "Loan", Cost: 0}} customData := dirapi.Entry{ Name: "Supplier", Networks: &networks, Tiers: &tiers, HoldingsPolicy: &dirapi.HoldingsPolicy{ diff --git a/directory/api.yaml b/directory/api.yaml index f4d7dda3e..362357a3a 100644 --- a/directory/api.yaml +++ b/directory/api.yaml @@ -1434,6 +1434,15 @@ components: authority: { type: string, minLength: 1 } symbol: { type: string, minLength: 1 } + ImportNetworkAssignment: + type: object + additionalProperties: false + required: [authority, symbol, priority] + properties: + authority: { type: string, minLength: 1 } + symbol: { type: string, minLength: 1 } + priority: { type: integer, format: int32 } + ImportEntryKey: $ref: '#/components/schemas/ImportSymbolRef' @@ -1731,13 +1740,12 @@ components: ImportNetworkData: type: object additionalProperties: false - required: [priority, reciprocal, entries] + required: [reciprocal, entries] properties: - priority: { type: integer, format: int32 } reciprocal: { type: boolean, nullable: true } entries: type: array - items: { $ref: '#/components/schemas/ImportSymbolRef' } + items: { $ref: '#/components/schemas/ImportNetworkAssignment' } ImportEntryRecord: type: object diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 6c3d087f0..59f41a06d 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -64,20 +64,20 @@ func TestImportBusinessKeyConstraints(t *testing.T) { requirePgCode(t, err, "23505") _, err = testPool.Exec(ctx, ` - INSERT INTO networks (consortium, name, priority) - VALUES ($1, 'Main', 0), ($1, 'Main', 0)`, consortiumID) + INSERT INTO networks (consortium, name) + VALUES ($1, 'Main'), ($1, 'Main')`, consortiumID) requirePgCode(t, err, "23505") _, err = testPool.Exec(ctx, `INSERT INTO tiers (consortium, name, level, type, cost) VALUES ($1, NULL, 'standard', 'loan', 0)`, consortiumID) requirePgCode(t, err, "23502") - _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name, priority) VALUES ($1, NULL, 0)`, consortiumID) + _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name) VALUES ($1, NULL)`, consortiumID) requirePgCode(t, err, "23502") _, err = testPool.Exec(ctx, `INSERT INTO tiers (consortium, name, level, type, cost) VALUES ($1, E'\t\n', 'standard', 'loan', 0)`, consortiumID) requirePgCode(t, err, "23514") - _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name, priority) VALUES ($1, E'\t\n', 0)`, consortiumID) + _, err = testPool.Exec(ctx, `INSERT INTO networks (consortium, name) VALUES ($1, E'\t\n')`, consortiumID) requirePgCode(t, err, "23514") } @@ -608,7 +608,7 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) reciprocal := true aggregate := model.NetworkAggregate{ Key: model.NetworkKey{Consortium: consortium, Name: "Main"}, - Data: model.NetworkData{Priority: 1, Reciprocal: &reciprocal, Entries: []model.SymbolRef{first}}, + Data: model.NetworkData{Reciprocal: &reciprocal, Entries: []model.NetworkAssignment{{SymbolRef: first, Priority: 1}}}, } result, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) @@ -616,7 +616,7 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.Equal(t, model.OutcomeImported, result.Outcome) id := networkIDByKey(t, consortium, "Main") require.NotEqual(t, uuid.Nil, id) - require.Equal(t, []model.SymbolRef{first}, networkAssignments(t, id)) + require.Equal(t, []model.NetworkAssignment{{SymbolRef: first, Priority: 1}}, networkAssignments(t, id)) _, err = repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) require.ErrorContains(t, err, "already exists") @@ -624,13 +624,12 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.NoError(t, err) require.Equal(t, model.OutcomeSkipped, skipped.Outcome) - aggregate.Data.Priority = 2 aggregate.Data.Reciprocal = nil - aggregate.Data.Entries = []model.SymbolRef{second} + aggregate.Data.Entries = []model.NetworkAssignment{{SymbolRef: second, Priority: 2}} _, err = repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyUpdate) require.NoError(t, err) require.Equal(t, id, networkIDByKey(t, consortium, "Main")) - require.Equal(t, []model.SymbolRef{second}, networkAssignments(t, id)) + require.Equal(t, []model.NetworkAssignment{{SymbolRef: second, Priority: 2}}, networkAssignments(t, id)) } func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { @@ -644,7 +643,7 @@ func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { require.NoError(t, err) _, err = testPool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', 'MEMBER'), ($2, 'ISIL', 'CON')`, memberID, consortiumID) require.NoError(t, err) - _, err = testPool.Exec(ctx, `INSERT INTO networks (id, consortium, name, priority) VALUES ($1, $2, 'Main', 0)`, networkID, consortiumID) + _, err = testPool.Exec(ctx, `INSERT INTO networks (id, consortium, name) VALUES ($1, $2, 'Main')`, networkID, consortiumID) require.NoError(t, err) blocker, err := testPool.Begin(ctx) @@ -658,7 +657,7 @@ func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { member := model.SymbolRef{Authority: "ISIL", Symbol: "MEMBER"} network := model.NetworkAggregate{ Key: model.NetworkKey{Consortium: consortium, Name: "Main"}, - Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{member}}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{{SymbolRef: member, Priority: 1}}}, } networkDone := make(chan error, 1) go func() { @@ -686,7 +685,7 @@ func TestConcurrentImportNetworkSkipHonorsConflictPolicyForMissingKey(t *testing newAggregate := func() model.NetworkAggregate { return model.NetworkAggregate{ Key: model.NetworkKey{Consortium: consortium, Name: "Concurrent"}, - Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{}}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, } } results, errs := concurrentlyImportNetwork(repo, newAggregate, model.ConflictPolicySkip, 8) @@ -700,7 +699,7 @@ func TestConcurrentImportNetworkUpdateHonorsConflictPolicyForMissingKey(t *testi newAggregate := func() model.NetworkAggregate { return model.NetworkAggregate{ Key: model.NetworkKey{Consortium: consortium, Name: "Concurrent"}, - Data: model.NetworkData{Priority: 1, Entries: []model.SymbolRef{}}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, } } results, errs := concurrentlyImportNetwork(repo, newAggregate, model.ConflictPolicyUpdate, 8) @@ -713,7 +712,7 @@ func TestImportNetworkRejectsNonConsortiumOwner(t *testing.T) { repo, _, first, _ := importRepoFixture(t) aggregate := model.NetworkAggregate{ Key: model.NetworkKey{Consortium: first, Name: "Main"}, - Data: model.NetworkData{Entries: []model.SymbolRef{}}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, } _, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) @@ -762,9 +761,24 @@ func tierAssignments(t *testing.T, id uuid.UUID) []model.SymbolRef { return aggregateAssignments(t, "entry_tiers", "tier", id) } -func networkAssignments(t *testing.T, id uuid.UUID) []model.SymbolRef { +func networkAssignments(t *testing.T, id uuid.UUID) []model.NetworkAssignment { t.Helper() - return aggregateAssignments(t, "entry_networks", "network", id) + rows, err := testPool.Query(context.Background(), ` + SELECT s.authority, s.symbol, en.priority + FROM entry_networks en + JOIN symbols s ON s.owner=en.entry + WHERE en.network=$1 + ORDER BY s.authority, s.symbol`, id) + require.NoError(t, err) + defer rows.Close() + var result []model.NetworkAssignment + for rows.Next() { + var assignment model.NetworkAssignment + require.NoError(t, rows.Scan(&assignment.Authority, &assignment.Symbol, &assignment.Priority)) + result = append(result, assignment) + } + require.NoError(t, rows.Err()) + return result } func aggregateAssignments(t *testing.T, table, aggregateColumn string, id uuid.UUID) []model.SymbolRef { diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index 58001a36e..fb6274aaa 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -101,7 +101,11 @@ func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model } defer func() { _ = tx.Rollback(ctx) }() - consortium, assignments, err := resolveAndLockAssignments(ctx, queries, aggregate.Key.Consortium, aggregate.Data.Entries) + refs := make([]model.SymbolRef, len(aggregate.Data.Entries)) + for index, assignment := range aggregate.Data.Entries { + refs[index] = assignment.SymbolRef + } + consortium, assignments, err := resolveAndLockAssignments(ctx, queries, aggregate.Key.Consortium, refs) if err != nil { return model.RepoResult{}, err } @@ -125,16 +129,16 @@ func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model var networkID uuid.UUID if exists { networkID = existing.ID - err = queries.UpdateImportedNetwork(ctx, db.UpdateImportedNetworkParams{ID: networkID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) + err = queries.UpdateImportedNetwork(ctx, db.UpdateImportedNetworkParams{ID: networkID, Reciprocal: aggregate.Data.Reciprocal}) } else { var created db.Network - created, err = queries.CreateNetwork(ctx, db.CreateNetworkParams{Name: name, Consortium: consortium.ID, Priority: aggregate.Data.Priority, Reciprocal: aggregate.Data.Reciprocal}) + created, err = queries.CreateNetwork(ctx, db.CreateNetworkParams{Name: name, Consortium: consortium.ID, Reciprocal: aggregate.Data.Reciprocal}) networkID = created.ID } if err != nil { return model.RepoResult{}, persistenceError("network", key, err) } - if err := replaceNetworkAssignments(ctx, queries, networkID, assignmentEntries); err != nil { + if err := replaceNetworkAssignments(ctx, queries, networkID, assignmentEntries, aggregate.Data.Entries); err != nil { return model.RepoResult{}, err } if err := tx.Commit(ctx); err != nil { @@ -218,12 +222,15 @@ func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uui return nil } -func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, networkID uuid.UUID, entries []db.Entry) error { +func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, networkID uuid.UUID, entries []db.Entry, assignments []model.NetworkAssignment) error { + if len(entries) != len(assignments) { + return fmt.Errorf("replace network assignments: entry count mismatch") + } if err := queries.DeleteEntryNetworksByNetwork(ctx, networkID); err != nil { return fmt.Errorf("replace network assignments") } - for _, entry := range entries { - if _, err := queries.CreateEntryNetwork(ctx, db.CreateEntryNetworkParams{Entry: entry.ID, Network: networkID}); err != nil { + for index, entry := range entries { + if _, err := queries.CreateEntryNetwork(ctx, db.CreateEntryNetworkParams{Entry: entry.ID, Network: networkID, Priority: assignments[index].Priority}); err != nil { return fmt.Errorf("replace network assignments") } } diff --git a/directory/import/model/models.go b/directory/import/model/models.go index 95d5ccdef..3132aafeb 100644 --- a/directory/import/model/models.go +++ b/directory/import/model/models.go @@ -229,10 +229,14 @@ type NetworkKey struct { Name string `json:"name"` } +type NetworkAssignment struct { + SymbolRef + Priority int32 `json:"priority"` +} + type NetworkData struct { - Priority int32 `json:"priority"` - Reciprocal *bool `json:"reciprocal"` - Entries []SymbolRef `json:"entries"` + Reciprocal *bool `json:"reciprocal"` + Entries []NetworkAssignment `json:"entries"` } type NetworkAggregate struct { @@ -247,7 +251,18 @@ func (a *NetworkAggregate) NormalizeAndValidate() error { if strings.TrimSpace(a.Key.Name) == "" { return fmt.Errorf("network name is required") } - return normalizeUniqueRefs(a.Data.Entries, "network", func(refs []SymbolRef) { a.Data.Entries = refs }) + seen := make(map[string]struct{}, len(a.Data.Entries)) + for index := range a.Data.Entries { + if err := a.Data.Entries[index].NormalizeAndValidate(); err != nil { + return fmt.Errorf("network entry %d: %w", index+1, err) + } + key := a.Data.Entries[index].String() + if _, exists := seen[key]; exists { + return fmt.Errorf("duplicate network entry %s", key) + } + seen[key] = struct{}{} + } + return nil } func normalizeUniqueRefs(refs []SymbolRef, resource string, assign func([]SymbolRef)) error { diff --git a/directory/import/model/models_test.go b/directory/import/model/models_test.go index 7573036ec..ca0f70aa4 100644 --- a/directory/import/model/models_test.go +++ b/directory/import/model/models_test.go @@ -56,9 +56,9 @@ func TestTierAggregateRejectsInvalidEnum(t *testing.T) { func TestNetworkAggregateRejectsDuplicateEntries(t *testing.T) { aggregate := NetworkAggregate{ Key: NetworkKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Main"}, - Data: NetworkData{Entries: []SymbolRef{ - {Authority: "isil", Symbol: "lib"}, - {Authority: "ISIL", Symbol: "LIB"}, + Data: NetworkData{Entries: []NetworkAssignment{ + {SymbolRef: SymbolRef{Authority: "isil", Symbol: "lib"}, Priority: 1}, + {SymbolRef: SymbolRef{Authority: "ISIL", Symbol: "LIB"}, Priority: 2}, }}, } diff --git a/directory/import/service/importer_test.go b/directory/import/service/importer_test.go index 331e7d44f..2805a0847 100644 --- a/directory/import/service/importer_test.go +++ b/directory/import/service/importer_test.go @@ -291,7 +291,7 @@ func validTierRecord() string { } func validNetworkRecord() string { - return `{"type":"network","key":{"consortium":{"authority":"isil","symbol":"con"},"name":"Main"},"data":{"priority":1,"reciprocal":null,"entries":[]}}` + return `{"type":"network","key":{"consortium":{"authority":"isil","symbol":"con"},"name":"Main"},"data":{"reciprocal":null,"entries":[]}}` } type errorReader struct{ err error } diff --git a/directory/migrations/007_import_business_keys.down.sql b/directory/migrations/008_import_business_keys.down.sql similarity index 100% rename from directory/migrations/007_import_business_keys.down.sql rename to directory/migrations/008_import_business_keys.down.sql diff --git a/directory/migrations/007_import_business_keys.up.sql b/directory/migrations/008_import_business_keys.up.sql similarity index 100% rename from directory/migrations/007_import_business_keys.up.sql rename to directory/migrations/008_import_business_keys.up.sql diff --git a/directory/query.sql b/directory/query.sql index b78f7e44f..2c083e01f 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -502,7 +502,7 @@ FOR UPDATE; -- name: UpdateImportedNetwork :exec UPDATE networks -SET priority = @priority, reciprocal = @reciprocal +SET reciprocal = @reciprocal WHERE id = @id; -- name: DeleteEntryNetworksByNetwork :exec diff --git a/directory/test/import_test.go b/directory/test/import_test.go index c3df70781..7d972f41e 100644 --- a/directory/test/import_test.go +++ b/directory/test/import_test.go @@ -26,7 +26,10 @@ func TestImportOrderedAggregates(t *testing.T) { entryImportRecord(institution, "Institution", consortium, "Institution"), entryImportRecord(branch, "Branch", institution, "Branch"), map[string]any{"type": "tier", "key": map[string]any{"consortium": consortium, "name": "Loan"}, "data": map[string]any{"level": "standard", "type": "loan", "cost": 1.5, "entries": []any{institution, branch}}}, - map[string]any{"type": "network", "key": map[string]any{"consortium": consortium, "name": "Main"}, "data": map[string]any{"priority": 1, "reciprocal": true, "entries": []any{institution}}}, + map[string]any{"type": "network", "key": map[string]any{"consortium": consortium, "name": "Main"}, "data": map[string]any{"reciprocal": true, "entries": []any{ + map[string]any{"authority": "ISIL", "symbol": "INST", "priority": 7}, + map[string]any{"authority": "ISIL", "symbol": "BRANCH", "priority": 3}, + }}}, } response, result := importRequest(t, records, "", standardHeaders) @@ -63,7 +66,12 @@ func TestImportOrderedAggregates(t *testing.T) { require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT count(*) FROM entry_tiers`).Scan(&tierAssignments)) require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT count(*) FROM entry_networks`).Scan(&networkAssignments)) require.Equal(t, 2, tierAssignments) - require.Equal(t, 1, networkAssignments) + require.Equal(t, 2, networkAssignments) + var institutionPriority, branchPriority int32 + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT priority FROM entry_networks WHERE entry=$1`, institutionID).Scan(&institutionPriority)) + require.NoError(t, dbpool.QueryRow(context.Background(), `SELECT priority FROM entry_networks WHERE entry=$1`, branchID).Scan(&branchPriority)) + require.Equal(t, int32(7), institutionPriority) + require.Equal(t, int32(3), branchPriority) } func TestImportPartialCommitAndConflictPolicies(t *testing.T) { From 681be35b246a8ddbc48496572310517382301c60 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Fri, 11 Sep 2026 15:19:17 +0300 Subject: [PATCH 05/18] ILLDEV-484 Fix copilot comments --- directory/import/db/entry.go | 68 +++++++++++++++++++++-- directory/import/db/repo_test.go | 37 ++++++++++++ directory/import/service/importer.go | 33 +++++++++-- directory/import/service/importer_test.go | 48 ++++++++++++++++ 4 files changed, 176 insertions(+), 10 deletions(-) diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 857c48cb1..35f8b245e 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -45,11 +45,8 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E } defer func() { _ = tx.Rollback(ctx) }() - if err := queries.LockEntryImportKey(ctx, db.LockEntryImportKeyParams{ - Authority: aggregate.Key.Authority, - Symbol: aggregate.Key.Symbol, - }); err != nil { - return model.RepoResult{}, fmt.Errorf("lock entry %s", key) + if err := lockEntryImportKeys(ctx, queries, aggregate.Data.Symbols); err != nil { + return model.RepoResult{}, fmt.Errorf("lock entry %s symbols: %w", key, err) } existing, lookupErr := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{ Authority: aggregate.Key.Authority, @@ -80,16 +77,21 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E if err != nil { return model.RepoResult{}, err } + symbolMappings, symbolOwnerIDs, err := resolveImportSymbolMappings(ctx, queries, aggregate.Key, aggregate.Data.Symbols, existing, exists) + if err != nil { + return model.RepoResult{}, err + } var owner *db.Entry if exists { owner = &existing } entryIDs := entryLockIDs(owner, parent, lenders) + entryIDs = append(entryIDs, symbolOwnerIDs...) lockedEntries, err := lockEntryRows(ctx, queries, entryIDs...) if err != nil { return model.RepoResult{}, fmt.Errorf("lock entry hierarchy: %w", err) } - mappings := []entryMapping{{ref: aggregate.Key, expectedOwner: entryIDPointer(existing, exists)}} + mappings := append([]entryMapping(nil), symbolMappings...) if aggregate.Data.Parent != nil { mappings = append(mappings, entryMapping{ref: *aggregate.Data.Parent, expectedOwner: &parent.ID}) } @@ -101,6 +103,9 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E if err := lockEntryMappings(ctx, queries, mappings...); err != nil { return model.RepoResult{}, fmt.Errorf("revalidate entry hierarchy: %w", err) } + if err := validateImportSymbolOwnership(symbolMappings, entryIDPointer(existing, exists)); err != nil { + return model.RepoResult{}, err + } if exists { existing = lockedEntries[existing.ID] } @@ -159,6 +164,57 @@ type entryMapping struct { expectedOwner *uuid.UUID } +func lockEntryImportKeys(ctx context.Context, queries *db.Queries, refs []model.SymbolRef) error { + ordered := append([]model.SymbolRef(nil), refs...) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].Authority != ordered[j].Authority { + return ordered[i].Authority < ordered[j].Authority + } + return ordered[i].Symbol < ordered[j].Symbol + }) + for index, ref := range ordered { + if index > 0 && ref == ordered[index-1] { + continue + } + if err := queries.LockEntryImportKey(ctx, db.LockEntryImportKeyParams{Authority: ref.Authority, Symbol: ref.Symbol}); err != nil { + return err + } + } + return nil +} + +func resolveImportSymbolMappings(ctx context.Context, queries *db.Queries, key model.SymbolRef, refs []model.SymbolRef, existing db.Entry, exists bool) ([]entryMapping, []uuid.UUID, error) { + mappings := make([]entryMapping, 0, len(refs)) + ownerIDs := make([]uuid.UUID, 0, len(refs)) + for _, ref := range refs { + if ref == key { + mappings = append(mappings, entryMapping{ref: ref, expectedOwner: entryIDPointer(existing, exists)}) + continue + } + entry, err := queries.EntryBySymbol(ctx, db.EntryBySymbolParams{Authority: ref.Authority, Symbol: ref.Symbol}) + if errors.Is(err, pgx.ErrNoRows) { + mappings = append(mappings, entryMapping{ref: ref}) + continue + } + if err != nil { + return nil, nil, fmt.Errorf("resolve entry symbol %s: %w", ref.String(), err) + } + ownerID := entry.ID + mappings = append(mappings, entryMapping{ref: ref, expectedOwner: &ownerID}) + ownerIDs = append(ownerIDs, ownerID) + } + return mappings, ownerIDs, nil +} + +func validateImportSymbolOwnership(mappings []entryMapping, entryID *uuid.UUID) error { + for _, mapping := range mappings { + if mapping.expectedOwner != nil && !sameEntryID(mapping.expectedOwner, entryID) { + return fmt.Errorf("entry symbol %s already belongs to another entry", mapping.ref.String()) + } + } + return nil +} + func lockEntryMappings(ctx context.Context, queries *db.Queries, mappings ...entryMapping) error { sort.Slice(mappings, func(i, j int) bool { if mappings[i].ref.Authority != mappings[j].ref.Authority { diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 59f41a06d..62aa771be 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -200,6 +200,43 @@ func TestConcurrentImportEntryUpdateHonorsConflictPolicyForMissingKey(t *testing require.Equal(t, 1, entryCount(t)) } +func TestImportEntryLocksAndRevalidatesSecondarySymbols(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + repo := importdb.New(testPool) + owner := minimalEntryAggregate("OWNER", "Institution") + _, err := repo.ImportEntry(ctx, owner, model.ConflictPolicyFail) + require.NoError(t, err) + ownerID := entryIDBySymbol(t, owner.Key) + + secondary := model.SymbolRef{Authority: "ISIL", Symbol: "SHARED"} + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('directoryish:entry:ISIL:SHARED', 0))`) + require.NoError(t, err) + + aggregate := minimalEntryAggregate("IMPORTED", "Institution") + aggregate.Data.Symbols = append(aggregate.Data.Symbols, secondary) + importDone := make(chan error, 1) + go func() { + _, importErr := repo.ImportEntry(ctx, aggregate, model.ConflictPolicyFail) + importDone <- importErr + }() + waitForDatabaseLockWaiters(t, ctx, 1) + + _, err = blocker.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, $2, $3)`, ownerID, secondary.Authority, secondary.Symbol) + require.NoError(t, err) + require.NoError(t, blocker.Commit(ctx)) + + importErr := <-importDone + require.ErrorContains(t, importErr, "entry symbol ISIL:SHARED already belongs to another entry") + assertEntryDoesNotExist(t, aggregate.Key) + require.Equal(t, ownerID, entryIDBySymbol(t, secondary)) +} + func TestConcurrentImportEntryOpposingParentsDoNotDeadlock(t *testing.T) { resetImportDatabase(t) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/directory/import/service/importer.go b/directory/import/service/importer.go index 7b8a13a68..17e285bbd 100644 --- a/directory/import/service/importer.go +++ b/directory/import/service/importer.go @@ -61,7 +61,13 @@ func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, inpu var recordNumber int32 for { + if err := ctx.Err(); err != nil { + return result, err + } line, readErr := reader.ReadSlice('\n') + if err := ctx.Err(); err != nil { + return result, err + } if errors.Is(readErr, bufio.ErrBufferFull) { return result, ErrRecordTooLarge } @@ -76,7 +82,9 @@ func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, inpu } if len(bytes.TrimSpace(line)) != 0 { recordNumber++ - i.importRecord(ctx, policy, recordNumber, line, &result) + if err := i.importRecord(ctx, policy, recordNumber, line, &result); err != nil { + return result, err + } } if errors.Is(readErr, io.EOF) { @@ -85,12 +93,15 @@ func (i *Importer) Import(ctx context.Context, policy model.ConflictPolicy, inpu } } -func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy, line int32, data []byte, result *model.ImportResult) { +func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy, line int32, data []byte, result *model.ImportResult) error { record, err := decodeRecord(data, i.recordSchemas) if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } incrementFailed(result, record.recordType) appendError(result, line, record.recordType, record.key, err.Error()) - return + return nil } var repoResult model.RepoResult @@ -103,9 +114,18 @@ func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy repoResult, err = i.repository.ImportNetwork(ctx, *record.network, policy) } if err != nil { + if isContextError(err) { + return err + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } incrementFailed(result, record.recordType) appendError(result, line, record.recordType, record.key, err.Error()) - return + return nil + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr } switch repoResult.Outcome { @@ -122,6 +142,11 @@ func (i *Importer) importRecord(ctx context.Context, policy model.ConflictPolicy incrementFailed(result, record.recordType) appendError(result, line, record.recordType, record.key, "repository returned an invalid import outcome") } + return nil +} + +func isContextError(err error) bool { + return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) } func incrementFailed(result *model.ImportResult, recordType string) { diff --git a/directory/import/service/importer_test.go b/directory/import/service/importer_test.go index 2805a0847..86955f4f6 100644 --- a/directory/import/service/importer_test.go +++ b/directory/import/service/importer_test.go @@ -19,11 +19,15 @@ type recordingRepo struct { entry *model.EntryAggregate result model.RepoResult err error + beforeEntry func() } func (r *recordingRepo) ImportEntry(_ context.Context, aggregate model.EntryAggregate, _ model.ConflictPolicy) (model.RepoResult, error) { r.entryCalls++ r.entry = &aggregate + if r.beforeEntry != nil { + r.beforeEntry() + } return r.result, r.err } @@ -122,6 +126,50 @@ func TestImportReturnsFatalReaderError(t *testing.T) { require.Empty(t, result.Errors) } +func TestImportPropagatesRepositoryCancellation(t *testing.T) { + for name, fatal := range map[string]error{ + "canceled": context.Canceled, + "deadline": context.DeadlineExceeded, + } { + t.Run(name, func(t *testing.T) { + repo := &recordingRepo{err: fatal} + input := validEntryRecord() + "\n" + validEntryRecord() + + result, err := newTestImporter(t, repo).Import(context.Background(), model.ConflictPolicyFail, strings.NewReader(input)) + + require.ErrorIs(t, err, fatal) + require.Equal(t, 1, repo.entryCalls) + require.Zero(t, result.Entries.Failed) + require.Empty(t, result.Errors) + }) + } +} + +func TestImportPropagatesCanceledContextWhenRepositoryMasksCause(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + repo := &recordingRepo{err: errors.New("database operation failed"), beforeEntry: cancel} + input := validEntryRecord() + "\n" + validEntryRecord() + + result, err := newTestImporter(t, repo).Import(ctx, model.ConflictPolicyFail, strings.NewReader(input)) + + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, repo.entryCalls) + require.Zero(t, result.Entries.Failed) + require.Empty(t, result.Errors) +} + +func TestImportStopsBeforeReadingWhenContextAlreadyCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + repo := &recordingRepo{result: model.RepoResult{Outcome: model.OutcomeImported}} + + result, err := newTestImporter(t, repo).Import(ctx, model.ConflictPolicyFail, strings.NewReader(validEntryRecord())) + + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, repo.entryCalls) + require.Zero(t, result.Entries.Imported) +} + func TestImportRejectsMissingAndUnknownProperties(t *testing.T) { tests := map[string]string{ "missing entry field": strings.Replace(validEntryRecord(), `,"timeZone":null`, "", 1), From 1eca5dea5ea88e6fe849283e33aea1c48a88e4e4 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Mon, 14 Sep 2026 12:10:53 +0300 Subject: [PATCH 06/18] ILLDEV-484 Fix copilot comments --- broker/test/adapter/api_directory_test.go | 10 ++- directory/import/db/entry.go | 42 ++++++++--- directory/import/db/entry_lock_test.go | 88 +++++++++++++++++++++++ directory/import/db/repo.go | 2 +- 4 files changed, 127 insertions(+), 15 deletions(-) diff --git a/broker/test/adapter/api_directory_test.go b/broker/test/adapter/api_directory_test.go index 937662677..b7251224a 100644 --- a/broker/test/adapter/api_directory_test.go +++ b/broker/test/adapter/api_directory_test.go @@ -972,15 +972,15 @@ func TestFilterAndSortAppliesHoldingsPolicy(t *testing.T) { } func TestFilterAndSortResolvesHoldingsPolicyForNonMatchingSuppliers(t *testing.T) { - networks := []dirapi.EntryNetworkDetails{{Name: strPtr("Reciprocal"), Priority: 1}} - tiers := []dirapi.Tier{{Name: strPtr("Core Loan"), Level: "Core", Type: "Loan", Cost: 0}} + networks := []dirapi.EntryNetworkDetails{{Name: "Reciprocal", Priority: 1}} + tiers := []dirapi.Tier{{Name: "Core Loan", Level: "Core", Type: "Loan", Cost: 0}} requester := dirapi.Entry{Networks: &networks, Tiers: &tiers} for _, tc := range []struct { name string networks []dirapi.EntryNetworkDetails tiers []dirapi.Tier }{ - {name: "no shared network", networks: []dirapi.EntryNetworkDetails{{Name: strPtr("Other"), Priority: 1}}, tiers: tiers}, + {name: "no shared network", networks: []dirapi.EntryNetworkDetails{{Name: "Other", Priority: 1}}, tiers: tiers}, {name: "no matching tier", networks: networks, tiers: []dirapi.Tier{{Type: "Copy", Cost: 0}}}, {name: "no matching cost", networks: networks, tiers: []dirapi.Tier{{Type: "Loan", Cost: 10}}}, } { @@ -1009,3 +1009,7 @@ func TestFilterAndSortResolvesHoldingsPolicyForNonMatchingSuppliers(t *testing.T }) } } + +func strPtr(s string) *string { + return &s +} diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 35f8b245e..ff90141fc 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -14,6 +14,7 @@ import ( "github.com/indexdata/crosslink/directory/domain" "github.com/indexdata/crosslink/directory/import/model" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgtype" ) @@ -25,16 +26,35 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg if err := aggregate.NormalizeAndValidate(); err != nil { return model.RepoResult{}, err } + return runImportEntryAttempts(ctx, aggregate.Key.String(), func() (model.RepoResult, error) { + return r.importEntryAttempt(ctx, aggregate, policy) + }) +} + +func runImportEntryAttempts(ctx context.Context, key string, attempt func() (model.RepoResult, error)) (model.RepoResult, error) { + var lastErr error for range maxImportLockAttempts { - result, err := r.importEntryAttempt(ctx, aggregate, policy) - if !errors.Is(err, errImportEntryMappingChanged) { + if err := ctx.Err(); err != nil { + return model.RepoResult{}, err + } + result, err := attempt() + if !retryableImportError(err) { return result, err } + lastErr = err if err := ctx.Err(); err != nil { return model.RepoResult{}, err } } - return model.RepoResult{}, fmt.Errorf("import entry %s: hierarchy changed repeatedly", aggregate.Key.String()) + return model.RepoResult{}, fmt.Errorf("import entry %s: transaction conflicted repeatedly: %w", key, lastErr) +} + +func retryableImportError(err error) bool { + if errors.Is(err, errImportEntryMappingChanged) { + return true + } + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && (pgErr.Code == "40P01" || pgErr.Code == "40001") } func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { @@ -54,7 +74,7 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E }) exists := lookupErr == nil if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { - return model.RepoResult{}, fmt.Errorf("resolve entry %s", key) + return model.RepoResult{}, fmt.Errorf("resolve entry %s: %w", key, lookupErr) } if exists && policy != model.ConflictPolicyUpdate { if _, err := lockEntryRows(ctx, queries, existing.ID); err != nil { @@ -120,7 +140,7 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E if aggregate.Data.Type == "Consortium" || (exists && existing.Type == "Consortium") { if err := queries.LockConsortiumEntryChanges(ctx); err != nil { - return model.RepoResult{}, fmt.Errorf("lock consortium entry changes") + return model.RepoResult{}, fmt.Errorf("lock consortium entry changes: %w", err) } } if aggregate.Data.Type == "Consortium" && (!exists || existing.Type != "Consortium") { @@ -129,7 +149,7 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E return model.RepoResult{}, fmt.Errorf("consortium already exists") } if err != nil && !errors.Is(err, pgx.ErrNoRows) { - return model.RepoResult{}, fmt.Errorf("check existing consortium") + return model.RepoResult{}, fmt.Errorf("check existing consortium: %w", err) } } @@ -147,7 +167,7 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E return model.RepoResult{}, persistenceError("entry", key, err) } if err := tx.Commit(ctx); err != nil { - return model.RepoResult{}, fmt.Errorf("commit entry %s import", key) + return model.RepoResult{}, fmt.Errorf("commit entry %s import: %w", key, err) } return model.RepoResult{Outcome: model.OutcomeImported}, nil } @@ -270,7 +290,7 @@ func resolveParent(ctx context.Context, queries *db.Queries, parent *model.Symbo return nil, fmt.Errorf("parent %s does not exist", parent.String()) } if err != nil { - return nil, fmt.Errorf("resolve parent %s", parent.String()) + return nil, fmt.Errorf("resolve parent %s: %w", parent.String(), err) } return &entry, nil } @@ -286,7 +306,7 @@ func resolveLenders(ctx context.Context, queries *db.Queries, config *model.ILLC return nil, fmt.Errorf("lender of last resort %s does not exist", lender.String()) } if err != nil { - return nil, fmt.Errorf("resolve lender of last resort %s", lender.String()) + return nil, fmt.Errorf("resolve lender of last resort %s: %w", lender.String(), err) } lenders = append(lenders, entry) } @@ -342,7 +362,7 @@ func validateEntryUpdateHierarchy(ctx context.Context, queries *db.Queries, exis if parentID != nil { cycle, err := queries.WouldCreateEntryCycle(ctx, db.WouldCreateEntryCycleParams{Child: existing.ID, Parent: *parentID}) if err != nil { - return fmt.Errorf("validate entry hierarchy") + return fmt.Errorf("validate entry hierarchy: %w", err) } if cycle != nil && *cycle { return fmt.Errorf("entry parent would create a cycle") @@ -351,7 +371,7 @@ func validateEntryUpdateHierarchy(ctx context.Context, queries *db.Queries, exis if resultingType != existing.Type { children, err := queries.EntriesByParent(ctx, &existing.ID) if err != nil { - return fmt.Errorf("validate entry children") + return fmt.Errorf("validate entry children: %w", err) } for _, child := range children { if valid, reason := domain.ValidParentForType(child.Type, resultingType); !valid { diff --git a/directory/import/db/entry_lock_test.go b/directory/import/db/entry_lock_test.go index 7ec5672fd..4efe869c8 100644 --- a/directory/import/db/entry_lock_test.go +++ b/directory/import/db/entry_lock_test.go @@ -1,13 +1,101 @@ package importdb import ( + "context" + "errors" + "fmt" "testing" "github.com/google/uuid" "github.com/indexdata/crosslink/directory/db" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" ) +func TestRunImportEntryAttemptsRetriesTransactionConflicts(t *testing.T) { + attempts := 0 + want := model.RepoResult{Outcome: model.OutcomeImported} + + result, err := runImportEntryAttempts(context.Background(), "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + switch attempts { + case 1: + return model.RepoResult{}, &pgconn.PgError{Code: "40P01"} + case 2: + return model.RepoResult{}, fmt.Errorf("commit: %w", &pgconn.PgError{Code: "40001"}) + default: + return want, nil + } + }) + + require.NoError(t, err) + require.Equal(t, want, result) + require.Equal(t, 3, attempts) +} + +func TestRunImportEntryAttemptsStopsForNonRetryableError(t *testing.T) { + attempts := 0 + wantErr := errors.New("invalid entry") + + _, err := runImportEntryAttempts(context.Background(), "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + return model.RepoResult{}, wantErr + }) + + require.ErrorIs(t, err, wantErr) + require.Equal(t, 1, attempts) +} + +func TestRunImportEntryAttemptsStopsWhenContextIsCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + attempts := 0 + + _, err := runImportEntryAttempts(ctx, "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + cancel() + return model.RepoResult{}, &pgconn.PgError{Code: "40P01"} + }) + + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, attempts) +} + +func TestRunImportEntryAttemptsDoesNotStartWithCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + attempts := 0 + + _, err := runImportEntryAttempts(ctx, "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + return model.RepoResult{Outcome: model.OutcomeImported}, nil + }) + + require.ErrorIs(t, err, context.Canceled) + require.Zero(t, attempts) +} + +func TestRetryableImportError(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "entry mapping changed", err: errImportEntryMappingChanged, want: true}, + {name: "deadlock", err: &pgconn.PgError{Code: "40P01"}, want: true}, + {name: "wrapped serialization failure", err: fmt.Errorf("lock entry hierarchy: %w", &pgconn.PgError{Code: "40001"}), want: true}, + {name: "non-retryable PostgreSQL error", err: &pgconn.PgError{Code: "23505"}, want: false}, + {name: "ordinary error", err: errors.New("failed"), want: false}, + {name: "nil", err: nil, want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, retryableImportError(test.err)) + }) + } +} + func TestOrderedUniqueEntryIDsSortsAndDeduplicates(t *testing.T) { first := uuid.MustParse("00000000-0000-0000-0000-000000000001") second := uuid.MustParse("00000000-0000-0000-0000-000000000002") diff --git a/directory/import/db/repo.go b/directory/import/db/repo.go index 1d494a93f..7283d21c1 100644 --- a/directory/import/db/repo.go +++ b/directory/import/db/repo.go @@ -43,5 +43,5 @@ func persistenceError(resource, key string, err error) error { if err == nil { return nil } - return fmt.Errorf("persist %s %s aggregate", resource, key) + return fmt.Errorf("persist %s %s aggregate: %w", resource, key, err) } From 73d9a70f70d8b076f45046846537cc1934e9b626 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Mon, 14 Sep 2026 14:15:33 +0300 Subject: [PATCH 07/18] ILLDEV-484 Fix copilot comments --- directory/import/db/entry.go | 13 ++- directory/import/db/entry_lock_test.go | 3 +- directory/import/db/repo_test.go | 135 +++++++++++++++++++++++++ directory/import/db/tier_network.go | 8 +- directory/import/model/models.go | 20 ++-- directory/import/model/models_test.go | 38 +++++++ 6 files changed, 199 insertions(+), 18 deletions(-) diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index ff90141fc..311f5783d 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -20,6 +20,8 @@ import ( const maxImportLockAttempts = 3 +const entrySymbolUniqueConstraint = "symbols_authority_symbol_key" + var errImportEntryMappingChanged = errors.New("entry symbol mapping changed while acquiring import locks") func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { @@ -54,7 +56,12 @@ func retryableImportError(err error) bool { return true } var pgErr *pgconn.PgError - return errors.As(err, &pgErr) && (pgErr.Code == "40P01" || pgErr.Code == "40001") + if !errors.As(err, &pgErr) { + return false + } + return pgErr.Code == "40P01" || + pgErr.Code == "40001" || + (pgErr.Code == "23505" && pgErr.ConstraintName == entrySymbolUniqueConstraint) } func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { @@ -242,9 +249,9 @@ func lockEntryMappings(ctx context.Context, queries *db.Queries, mappings ...ent } return mappings[i].ref.Symbol < mappings[j].ref.Symbol }) - locked := make(map[string]*uuid.UUID, len(mappings)) + locked := make(map[model.SymbolRef]*uuid.UUID, len(mappings)) for _, mapping := range mappings { - key := mapping.ref.String() + key := mapping.ref if expectedOwner, exists := locked[key]; exists { if !sameEntryID(expectedOwner, mapping.expectedOwner) { return errImportEntryMappingChanged diff --git a/directory/import/db/entry_lock_test.go b/directory/import/db/entry_lock_test.go index 4efe869c8..7b5089810 100644 --- a/directory/import/db/entry_lock_test.go +++ b/directory/import/db/entry_lock_test.go @@ -84,7 +84,8 @@ func TestRetryableImportError(t *testing.T) { {name: "entry mapping changed", err: errImportEntryMappingChanged, want: true}, {name: "deadlock", err: &pgconn.PgError{Code: "40P01"}, want: true}, {name: "wrapped serialization failure", err: fmt.Errorf("lock entry hierarchy: %w", &pgconn.PgError{Code: "40001"}), want: true}, - {name: "non-retryable PostgreSQL error", err: &pgconn.PgError{Code: "23505"}, want: false}, + {name: "symbol key created concurrently", err: &pgconn.PgError{Code: "23505", ConstraintName: "symbols_authority_symbol_key"}, want: true}, + {name: "other unique violation", err: &pgconn.PgError{Code: "23505", ConstraintName: "entries_hrid_key"}, want: false}, {name: "ordinary error", err: errors.New("failed"), want: false}, {name: "nil", err: nil, want: false}, } diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 62aa771be..556a6c760 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -448,6 +448,62 @@ func TestImportEntryRetriesWhenParentIsDeletedBeforeRowLock(t *testing.T) { require.ErrorContains(t, <-importDone, "parent ISIL:PARENT does not exist") } +func TestImportEntryRetriesSymbolCreatedByAPIAndReappliesPolicy(t *testing.T) { + for _, policy := range []model.ConflictPolicy{model.ConflictPolicySkip, model.ConflictPolicyUpdate} { + t.Run(string(policy), func(t *testing.T) { + resetImportDatabase(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + blocker, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('directoryish:consortium-entry', 0))`) + require.NoError(t, err) + + aggregate := minimalEntryAggregate("RACE", "Consortium") + type importOutcome struct { + result model.RepoResult + err error + } + importDone := make(chan importOutcome, 1) + go func() { + result, importErr := importdb.New(testPool).ImportEntry(ctx, aggregate, policy) + importDone <- importOutcome{result: result, err: importErr} + }() + waitForDatabaseLockWaiters(t, ctx, 1) + + apiEntryID := uuid.New() + apiTx, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = apiTx.Rollback(ctx) }() + _, err = apiTx.Exec(ctx, `INSERT INTO entries (id, name, type) VALUES ($1, 'API entry', 'Institution')`, apiEntryID) + require.NoError(t, err) + _, err = apiTx.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'ISIL', 'RACE')`, apiEntryID) + require.NoError(t, err) + require.NoError(t, apiTx.Commit(ctx)) + require.NoError(t, blocker.Commit(ctx)) + + outcome := <-importDone + require.NoError(t, outcome.err) + require.Equal(t, apiEntryID, entryIDBySymbol(t, aggregate.Key)) + if policy == model.ConflictPolicySkip { + require.Equal(t, model.OutcomeSkipped, outcome.result.Outcome) + var name, entryType string + require.NoError(t, testPool.QueryRow(ctx, `SELECT name, type FROM entries WHERE id=$1`, apiEntryID).Scan(&name, &entryType)) + require.Equal(t, "API entry", name) + require.Equal(t, "Institution", entryType) + } else { + require.Equal(t, model.OutcomeImported, outcome.result.Outcome) + var name, entryType string + require.NoError(t, testPool.QueryRow(ctx, `SELECT name, type FROM entries WHERE id=$1`, apiEntryID).Scan(&name, &entryType)) + require.Equal(t, aggregate.Data.Name, name) + require.Equal(t, "Consortium", entryType) + } + }) + } +} + func TestImportEntryRejectsInvalidHierarchy(t *testing.T) { resetImportDatabase(t) repo := importdb.New(testPool) @@ -757,6 +813,85 @@ func TestImportNetworkRejectsNonConsortiumOwner(t *testing.T) { require.ErrorContains(t, err, "is not a consortium") } +func TestBusinessKeyLookupErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + table := resource + "s" + unavailableTable := table + "_unavailable" + _, err := testPool.Exec(context.Background(), fmt.Sprintf("ALTER TABLE %s RENAME TO %s", table, unavailableTable)) //nolint:gosec // fixed test identifiers + require.NoError(t, err) + t.Cleanup(func() { + _, restoreErr := testPool.Exec(context.Background(), fmt.Sprintf("ALTER TABLE %s RENAME TO %s", unavailableTable, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, restoreErr) + }) + + if resource == "tier" { + _, err = repo.ImportTier(context.Background(), model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Missing table"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{}}, + }, model.ConflictPolicyFail) + } else { + _, err = repo.ImportNetwork(context.Background(), model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Missing table"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, + }, model.ConflictPolicyFail) + } + + require.ErrorContains(t, err, "resolve "+resource) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "42P01", pgErr.Code) + }) + } +} + +func TestAggregateCommitErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + table := resource + "s" + trigger := "fail_" + resource + "_import_commit" + _, err := testPool.Exec(context.Background(), ` + CREATE OR REPLACE FUNCTION fail_import_aggregate_commit() RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'forced commit failure' USING ERRCODE = '40001'; + END; + $$ LANGUAGE plpgsql`) + require.NoError(t, err) + _, err = testPool.Exec(context.Background(), fmt.Sprintf(` + CREATE CONSTRAINT TRIGGER %s + AFTER INSERT ON %s + DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW EXECUTE FUNCTION fail_import_aggregate_commit()`, trigger, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, err) + t.Cleanup(func() { + _, cleanupErr := testPool.Exec(context.Background(), fmt.Sprintf("DROP TRIGGER %s ON %s", trigger, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, cleanupErr) + _, cleanupErr = testPool.Exec(context.Background(), `DROP FUNCTION fail_import_aggregate_commit()`) + require.NoError(t, cleanupErr) + }) + + if resource == "tier" { + _, err = repo.ImportTier(context.Background(), model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Commit failure"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{}}, + }, model.ConflictPolicyFail) + } else { + _, err = repo.ImportNetwork(context.Background(), model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Commit failure"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, + }, model.ConflictPolicyFail) + } + + require.ErrorContains(t, err, "commit "+resource) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "40001", pgErr.Code) + }) + } +} + func importRepoFixture(t *testing.T) (*importdb.PgImportRepo, model.SymbolRef, model.SymbolRef, model.SymbolRef) { t.Helper() resetImportDatabase(t) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index fb6274aaa..9cc5263b4 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -43,7 +43,7 @@ func (r *PgImportRepo) importTierAttempt(ctx context.Context, aggregate model.Ti existing, lookupErr := queries.LockTierByBusinessKey(ctx, db.LockTierByBusinessKeyParams{Consortium: consortium.ID, Name: name}) exists := lookupErr == nil if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { - return model.RepoResult{}, fmt.Errorf("resolve tier %s", key) + return model.RepoResult{}, fmt.Errorf("resolve tier %s: %w", key, lookupErr) } if exists && policy != model.ConflictPolicyUpdate { return conflictResult("tier", key, policy) @@ -72,7 +72,7 @@ func (r *PgImportRepo) importTierAttempt(ctx context.Context, aggregate model.Ti return model.RepoResult{}, err } if err := tx.Commit(ctx); err != nil { - return model.RepoResult{}, fmt.Errorf("commit tier %s import", key) + return model.RepoResult{}, fmt.Errorf("commit tier %s import: %w", key, err) } return model.RepoResult{Outcome: model.OutcomeImported}, nil } @@ -113,7 +113,7 @@ func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model existing, lookupErr := queries.LockNetworkByBusinessKey(ctx, db.LockNetworkByBusinessKeyParams{Consortium: consortium.ID, Name: name}) exists := lookupErr == nil if lookupErr != nil && !errors.Is(lookupErr, pgx.ErrNoRows) { - return model.RepoResult{}, fmt.Errorf("resolve network %s", key) + return model.RepoResult{}, fmt.Errorf("resolve network %s: %w", key, lookupErr) } if exists && policy != model.ConflictPolicyUpdate { return conflictResult("network", key, policy) @@ -142,7 +142,7 @@ func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model return model.RepoResult{}, err } if err := tx.Commit(ctx); err != nil { - return model.RepoResult{}, fmt.Errorf("commit network %s import", key) + return model.RepoResult{}, fmt.Errorf("commit network %s import: %w", key, err) } return model.RepoResult{Outcome: model.OutcomeImported}, nil } diff --git a/directory/import/model/models.go b/directory/import/model/models.go index 3132aafeb..1381cd32d 100644 --- a/directory/import/model/models.go +++ b/directory/import/model/models.go @@ -147,18 +147,18 @@ func (a *EntryAggregate) NormalizeAndValidate() error { return fmt.Errorf("parent: %w", err) } } - seen := make(map[string]struct{}, len(a.Data.Symbols)) + seen := make(map[SymbolRef]struct{}, len(a.Data.Symbols)) keyCount := 0 for index := range a.Data.Symbols { if err := a.Data.Symbols[index].NormalizeAndValidate(); err != nil { return fmt.Errorf("symbol %d: %w", index+1, err) } - value := a.Data.Symbols[index].String() + value := a.Data.Symbols[index] if _, exists := seen[value]; exists { - return fmt.Errorf("duplicate entry symbol %s", value) + return fmt.Errorf("duplicate entry symbol %s", value.String()) } seen[value] = struct{}{} - if value == a.Key.String() { + if value == a.Key { keyCount++ } } @@ -251,14 +251,14 @@ func (a *NetworkAggregate) NormalizeAndValidate() error { if strings.TrimSpace(a.Key.Name) == "" { return fmt.Errorf("network name is required") } - seen := make(map[string]struct{}, len(a.Data.Entries)) + seen := make(map[SymbolRef]struct{}, len(a.Data.Entries)) for index := range a.Data.Entries { if err := a.Data.Entries[index].NormalizeAndValidate(); err != nil { return fmt.Errorf("network entry %d: %w", index+1, err) } - key := a.Data.Entries[index].String() + key := a.Data.Entries[index].SymbolRef if _, exists := seen[key]; exists { - return fmt.Errorf("duplicate network entry %s", key) + return fmt.Errorf("duplicate network entry %s", key.String()) } seen[key] = struct{}{} } @@ -266,14 +266,14 @@ func (a *NetworkAggregate) NormalizeAndValidate() error { } func normalizeUniqueRefs(refs []SymbolRef, resource string, assign func([]SymbolRef)) error { - seen := make(map[string]struct{}, len(refs)) + seen := make(map[SymbolRef]struct{}, len(refs)) for index := range refs { if err := refs[index].NormalizeAndValidate(); err != nil { return fmt.Errorf("%s entry %d: %w", resource, index+1, err) } - key := refs[index].String() + key := refs[index] if _, exists := seen[key]; exists { - return fmt.Errorf("duplicate %s entry %s", resource, key) + return fmt.Errorf("duplicate %s entry %s", resource, key.String()) } seen[key] = struct{}{} } diff --git a/directory/import/model/models_test.go b/directory/import/model/models_test.go index ca0f70aa4..7b4a74795 100644 --- a/directory/import/model/models_test.go +++ b/directory/import/model/models_test.go @@ -42,6 +42,16 @@ func TestEntryAggregateRejectsDuplicateNormalizedSymbols(t *testing.T) { require.EqualError(t, err, "duplicate entry symbol ISIL:LIB") } +func TestEntryAggregateAcceptsDistinctSymbolsWithSameDisplayString(t *testing.T) { + first := SymbolRef{Authority: "A:B", Symbol: "C"} + second := SymbolRef{Authority: "A", Symbol: "B:C"} + aggregate := validEntryAggregate() + aggregate.Key = first + aggregate.Data.Symbols = []SymbolRef{first, second} + + require.NoError(t, aggregate.NormalizeAndValidate()) +} + func TestTierAggregateRejectsInvalidEnum(t *testing.T) { aggregate := TierAggregate{ Key: TierKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Loan"}, @@ -53,6 +63,22 @@ func TestTierAggregateRejectsInvalidEnum(t *testing.T) { require.EqualError(t, err, "invalid tier level: instant") } +func TestTierAggregateAcceptsDistinctEntriesWithSameDisplayString(t *testing.T) { + aggregate := TierAggregate{ + Key: TierKey{Consortium: SymbolRef{Authority: "ISIL", Symbol: "CONSORTIUM"}, Name: "Loan"}, + Data: TierData{ + Level: "standard", + Type: "loan", + Entries: []SymbolRef{ + {Authority: "A:B", Symbol: "C"}, + {Authority: "A", Symbol: "B:C"}, + }, + }, + } + + require.NoError(t, aggregate.NormalizeAndValidate()) +} + func TestNetworkAggregateRejectsDuplicateEntries(t *testing.T) { aggregate := NetworkAggregate{ Key: NetworkKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Main"}, @@ -67,6 +93,18 @@ func TestNetworkAggregateRejectsDuplicateEntries(t *testing.T) { require.EqualError(t, err, "duplicate network entry ISIL:LIB") } +func TestNetworkAggregateAcceptsDistinctEntriesWithSameDisplayString(t *testing.T) { + aggregate := NetworkAggregate{ + Key: NetworkKey{Consortium: SymbolRef{Authority: "ISIL", Symbol: "CONSORTIUM"}, Name: "Main"}, + Data: NetworkData{Entries: []NetworkAssignment{ + {SymbolRef: SymbolRef{Authority: "A:B", Symbol: "C"}, Priority: 1}, + {SymbolRef: SymbolRef{Authority: "A", Symbol: "B:C"}, Priority: 2}, + }}, + } + + require.NoError(t, aggregate.NormalizeAndValidate()) +} + func TestEntryAggregateRejectsInvalidClosureRange(t *testing.T) { aggregate := validEntryAggregate() aggregate.Data.Closures = []Closure{{StartDate: "2026-09-03", EndDate: "2026-09-02", Reason: "maintenance"}} From c0cb3681cc55a13e1a522047aba0945a0588ae71 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Mon, 14 Sep 2026 15:59:29 +0300 Subject: [PATCH 08/18] ILLDEV-484 Fix copilot comments --- directory/app/app.go | 3 +- directory/app/import_limit_test.go | 104 ++++++++++++++++++++++++++ directory/app/request_validation.go | 66 ++++++++++++++++ directory/import/db/repo_test.go | 14 ++++ directory/import/model/config.go | 8 +- directory/import/model/models_test.go | 21 ++++++ 6 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 directory/app/request_validation.go diff --git a/directory/app/app.go b/directory/app/app.go index a71af9459..a6e959748 100644 --- a/directory/app/app.go +++ b/directory/app/app.go @@ -14,7 +14,6 @@ import ( _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" - apiValidator "github.com/oapi-codegen/nethttp-middleware" slogctx "github.com/veqryn/slog-context" sloghttp "github.com/veqryn/slog-context/http" pgxUUID "github.com/vgarvardt/pgx-google-uuid/v5" @@ -81,7 +80,7 @@ func InitHandler(ctx context.Context, dbpool *pgxpool.Pool) http.Handler { BaseURL: BasePath, BaseRouter: m, }) - handlerWithValidation := apiValidator.OapiRequestValidator(swagger) + handlerWithValidation := openAPIRequestValidationMiddleware(swagger) handlerWithLogging := httpLoggingMiddleware(handlerWithValidation(h)) handlerWithHelper := enhancedcontext.EnhancedContextMiddleware(handlerWithLogging) handlerWithLimit := ImportBodyLimitMiddleware(MaxImportBodyBytes, handlerWithHelper) diff --git a/directory/app/import_limit_test.go b/directory/app/import_limit_test.go index 38a0a6a5e..b8179e692 100644 --- a/directory/app/import_limit_test.go +++ b/directory/app/import_limit_test.go @@ -8,10 +8,94 @@ import ( "strings" "testing" + "github.com/indexdata/crosslink/directory/api" "github.com/indexdata/crosslink/directory/auth" "github.com/stretchr/testify/require" ) +func TestOpenAPIRequestValidationStreamsImportBody(t *testing.T) { + spec, err := api.GetSpec() + require.NoError(t, err) + body := &countingBody{remaining: MaxImportBodyBytes} + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", body) + request.ContentLength = MaxImportBodyBytes + request.Header.Set("Content-Type", "application/x-ndjson") + response := httptest.NewRecorder() + handler := openAPIRequestValidationMiddleware(spec)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusNoContent, response.Code) + require.EqualValues(t, 1, body.bytesRead) +} + +func TestOpenAPIRequestValidationReplaysCompleteImportBody(t *testing.T) { + spec, err := api.GetSpec() + require.NoError(t, err) + const payload = "first record\nsecond record\n" + var received string + handler := openAPIRequestValidationMiddleware(spec)(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + body, readErr := io.ReadAll(request.Body) + require.NoError(t, readErr) + received = string(body) + w.WriteHeader(http.StatusNoContent) + })) + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", strings.NewReader(payload)) + request.Header.Set("Content-Type", "application/x-ndjson; charset=utf-8") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusNoContent, response.Code) + require.Equal(t, payload, received) +} + +func TestOpenAPIRequestValidationRetainsImportBodyChecks(t *testing.T) { + spec, err := api.GetSpec() + require.NoError(t, err) + for name, testCase := range map[string]struct { + body io.Reader + contentType string + }{ + "missing body": {body: http.NoBody, contentType: "application/x-ndjson"}, + "wrong content type": {body: strings.NewReader("record"), contentType: "application/json"}, + } { + t.Run(name, func(t *testing.T) { + called := false + handler := openAPIRequestValidationMiddleware(spec)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + called = true + })) + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", testCase.body) + request.Header.Set("Content-Type", testCase.contentType) + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusBadRequest, response.Code) + require.False(t, called) + }) + } +} + +func TestOpenAPIRequestValidationStillValidatesOtherRequestBodies(t *testing.T) { + spec, err := api.GetSpec() + require.NoError(t, err) + called := false + handler := openAPIRequestValidationMiddleware(spec)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + called = true + })) + request := httptest.NewRequest(http.MethodPost, BasePath+"/entries", http.NoBody) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusBadRequest, response.Code) + require.False(t, called) +} + func TestImportBodyLimitRejectsKnownAndChunkedOverflow(t *testing.T) { for name, contentLength := range map[string]int64{"known": 129, "chunked": -1} { t.Run(name, func(t *testing.T) { @@ -90,3 +174,23 @@ func (r *trackingReadCloser) Read(data []byte) (int, error) { } func (r *trackingReadCloser) Close() error { return nil } + +type countingBody struct { + remaining int64 + bytesRead int64 +} + +func (r *countingBody) Read(data []byte) (int, error) { + if r.remaining == 0 { + return 0, io.EOF + } + count := min(int64(len(data)), r.remaining) + for index := range int(count) { + data[index] = 'x' + } + r.remaining -= count + r.bytesRead += count + return int(count), nil +} + +func (r *countingBody) Close() error { return nil } diff --git a/directory/app/request_validation.go b/directory/app/request_validation.go new file mode 100644 index 000000000..26ce74490 --- /dev/null +++ b/directory/app/request_validation.go @@ -0,0 +1,66 @@ +package app + +import ( + "bytes" + "errors" + "io" + "mime" + "net/http" + + "github.com/getkin/kin-openapi/openapi3" + apiValidator "github.com/oapi-codegen/nethttp-middleware" +) + +const importContentType = "application/x-ndjson" + +func openAPIRequestValidationMiddleware(spec *openapi3.T) func(http.Handler) http.Handler { + genericValidator := apiValidator.OapiRequestValidatorWithOptions(spec, &apiValidator.Options{ + Skipper: isImportRequest, + }) + return func(next http.Handler) http.Handler { + return validateImportRequest(genericValidator(next)) + } +} + +func validateImportRequest(next http.Handler) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + if !isImportRequest(request) { + next.ServeHTTP(writer, request) + return + } + + contentType, _, err := mime.ParseMediaType(request.Header.Get("Content-Type")) + if err != nil || contentType != importContentType { + http.Error(writer, "invalid Content-Type", http.StatusBadRequest) + return + } + if request.Body == nil || request.Body == http.NoBody { + http.Error(writer, "body is required", http.StatusBadRequest) + return + } + + var firstByte [1]byte + if _, err := io.ReadFull(request.Body, firstByte[:]); err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + http.Error(writer, "body is required", http.StatusBadRequest) + } else { + http.Error(writer, "failed to read request body", http.StatusBadRequest) + } + return + } + request.Body = &prefixedReadCloser{ + Reader: io.MultiReader(bytes.NewReader(firstByte[:]), request.Body), + Closer: request.Body, + } + next.ServeHTTP(writer, request) + }) +} + +func isImportRequest(request *http.Request) bool { + return request.Method == http.MethodPost && request.URL.Path == BasePath+"/import" +} + +type prefixedReadCloser struct { + io.Reader + io.Closer +} diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 556a6c760..648a990d5 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -564,6 +564,20 @@ func TestImportEntryRollsBackAfterLateSymbolConflict(t *testing.T) { assertEntryDoesNotExist(t, aggregate.Key) } +func TestImportEntryRejectsAmbiguousLenderAuthorityBeforeWriting(t *testing.T) { + resetImportDatabase(t) + repo := importdb.New(testPool) + aggregate := minimalEntryAggregate("CON", "Consortium") + aggregate.Data.ILLConfig = &model.ILLConfig{ + LendersOfLastResort: []model.SymbolRef{{Authority: "A:B", Symbol: "C"}}, + } + + _, err := repo.ImportEntry(context.Background(), aggregate, model.ConflictPolicyFail) + + require.ErrorContains(t, err, "lender of last resort 1 authority must not contain ':'") + assertEntryDoesNotExist(t, aggregate.Key) +} + func TestImportEntryAllowsOnlyOneConsortium(t *testing.T) { resetImportDatabase(t) repo := importdb.New(testPool) diff --git a/directory/import/model/config.go b/directory/import/model/config.go index 1e54b786c..6047cdf6b 100644 --- a/directory/import/model/config.go +++ b/directory/import/model/config.go @@ -1,6 +1,9 @@ package model -import "fmt" +import ( + "fmt" + "strings" +) type LMSConfig struct { Address string `json:"address"` @@ -136,6 +139,9 @@ func validateConfigEnums(catalog *CatalogConfig, ill *ILLConfig) error { if err := ill.LendersOfLastResort[index].NormalizeAndValidate(); err != nil { return fmt.Errorf("lender of last resort %d: %w", index+1, err) } + if strings.Contains(ill.LendersOfLastResort[index].Authority, ":") { + return fmt.Errorf("lender of last resort %d authority must not contain ':'", index+1) + } } } if catalog != nil && catalog.MetadataUpdateMode != nil && !oneOf(*catalog.MetadataUpdateMode, "replace", "merge", "none", "auto") { diff --git a/directory/import/model/models_test.go b/directory/import/model/models_test.go index 7b4a74795..94d552e54 100644 --- a/directory/import/model/models_test.go +++ b/directory/import/model/models_test.go @@ -52,6 +52,27 @@ func TestEntryAggregateAcceptsDistinctSymbolsWithSameDisplayString(t *testing.T) require.NoError(t, aggregate.NormalizeAndValidate()) } +func TestEntryAggregateRejectsColonInLenderAuthority(t *testing.T) { + aggregate := validEntryAggregate() + aggregate.Data.ILLConfig = &ILLConfig{ + LendersOfLastResort: []SymbolRef{{Authority: " a:b ", Symbol: " c "}}, + } + + err := aggregate.NormalizeAndValidate() + + require.EqualError(t, err, "lender of last resort 1 authority must not contain ':'") +} + +func TestEntryAggregateAllowsColonInLenderSymbol(t *testing.T) { + aggregate := validEntryAggregate() + aggregate.Data.ILLConfig = &ILLConfig{ + LendersOfLastResort: []SymbolRef{{Authority: " a ", Symbol: " b:c "}}, + } + + require.NoError(t, aggregate.NormalizeAndValidate()) + assert.Equal(t, SymbolRef{Authority: "A", Symbol: "B:C"}, aggregate.Data.ILLConfig.LendersOfLastResort[0]) +} + func TestTierAggregateRejectsInvalidEnum(t *testing.T) { aggregate := TierAggregate{ Key: TierKey{Consortium: SymbolRef{Authority: "isil", Symbol: "consortium"}, Name: "Loan"}, From ea27f3abb2c798a442c0896c3a397159a6b37d52 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Mon, 14 Sep 2026 18:34:51 +0300 Subject: [PATCH 09/18] ILLDEV-484 Fix copilot comments --- directory/import/db/entry.go | 42 ++++- directory/import/db/entry_lock_test.go | 29 ++++ directory/import/db/repo_test.go | 228 ++++++++++++++++++++++--- directory/import/db/tier_network.go | 16 +- directory/query.sql | 3 + directory/test/concurrency_test.go | 104 +++++++++++ 6 files changed, 389 insertions(+), 33 deletions(-) diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 311f5783d..9d3ecd62f 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -18,7 +18,11 @@ import ( "github.com/jackc/pgx/v5/pgtype" ) -const maxImportLockAttempts = 3 +const ( + maxImportEntryLockAttempts = 5 + maxImportMappingAttempts = 3 + importLockRetryBaseWait = 10 * time.Millisecond +) const entrySymbolUniqueConstraint = "symbols_authority_symbol_key" @@ -35,7 +39,7 @@ func (r *PgImportRepo) ImportEntry(ctx context.Context, aggregate model.EntryAgg func runImportEntryAttempts(ctx context.Context, key string, attempt func() (model.RepoResult, error)) (model.RepoResult, error) { var lastErr error - for range maxImportLockAttempts { + for attemptIndex := 0; attemptIndex < maxImportEntryLockAttempts; attemptIndex++ { if err := ctx.Err(); err != nil { return model.RepoResult{}, err } @@ -47,6 +51,11 @@ func runImportEntryAttempts(ctx context.Context, key string, attempt func() (mod if err := ctx.Err(); err != nil { return model.RepoResult{}, err } + if attemptIndex+1 < maxImportEntryLockAttempts { + if err := waitForImportRetry(ctx, attemptIndex); err != nil { + return model.RepoResult{}, err + } + } } return model.RepoResult{}, fmt.Errorf("import entry %s: transaction conflicted repeatedly: %w", key, lastErr) } @@ -61,9 +70,21 @@ func retryableImportError(err error) bool { } return pgErr.Code == "40P01" || pgErr.Code == "40001" || + pgErr.Code == "55P03" || (pgErr.Code == "23505" && pgErr.ConstraintName == entrySymbolUniqueConstraint) } +func waitForImportRetry(ctx context.Context, attempt int) error { + timer := time.NewTimer(importLockRetryBaseWait << attempt) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.EntryAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { key := aggregate.Key.String() tx, queries, err := r.begin(ctx) @@ -114,7 +135,7 @@ func (r *PgImportRepo) importEntryAttempt(ctx context.Context, aggregate model.E } entryIDs := entryLockIDs(owner, parent, lenders) entryIDs = append(entryIDs, symbolOwnerIDs...) - lockedEntries, err := lockEntryRows(ctx, queries, entryIDs...) + lockedEntries, err := lockEntryRowsWithoutWaiting(ctx, queries, entryIDs...) if err != nil { return model.RepoResult{}, fmt.Errorf("lock entry hierarchy: %w", err) } @@ -349,6 +370,21 @@ func lockEntryRows(ctx context.Context, queries *db.Queries, ids ...uuid.UUID) ( return entries, nil } +func lockEntryRowsWithoutWaiting(ctx context.Context, queries *db.Queries, ids ...uuid.UUID) (map[uuid.UUID]db.Entry, error) { + entries := make(map[uuid.UUID]db.Entry, len(ids)) + for _, id := range orderedUniqueEntryIDs(ids...) { + entry, err := queries.EntryByIdForImportUpdate(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return nil, errImportEntryMappingChanged + } + if err != nil { + return nil, err + } + entries[id] = entry + } + return entries, nil +} + func orderedUniqueEntryIDs(ids ...uuid.UUID) []uuid.UUID { unique := make(map[uuid.UUID]struct{}, len(ids)) ordered := make([]uuid.UUID, 0, len(ids)) diff --git a/directory/import/db/entry_lock_test.go b/directory/import/db/entry_lock_test.go index 7b5089810..6146f6328 100644 --- a/directory/import/db/entry_lock_test.go +++ b/directory/import/db/entry_lock_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "testing" + "time" "github.com/google/uuid" "github.com/indexdata/crosslink/directory/db" @@ -75,6 +76,33 @@ func TestRunImportEntryAttemptsDoesNotStartWithCanceledContext(t *testing.T) { require.Zero(t, attempts) } +func TestRunImportEntryAttemptsExhaustsLockUnavailableRetries(t *testing.T) { + attempts := 0 + lockErr := &pgconn.PgError{Code: "55P03"} + + _, err := runImportEntryAttempts(context.Background(), "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + return model.RepoResult{}, lockErr + }) + + require.ErrorIs(t, err, lockErr) + require.Equal(t, 5, attempts) +} + +func TestRunImportEntryAttemptsStopsWhenRetryWaitIsCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + attempts := 0 + + _, err := runImportEntryAttempts(ctx, "ISIL:TEST", func() (model.RepoResult, error) { + attempts++ + time.AfterFunc(time.Millisecond, cancel) + return model.RepoResult{}, &pgconn.PgError{Code: "55P03"} + }) + + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, 1, attempts) +} + func TestRetryableImportError(t *testing.T) { tests := []struct { name string @@ -84,6 +112,7 @@ func TestRetryableImportError(t *testing.T) { {name: "entry mapping changed", err: errImportEntryMappingChanged, want: true}, {name: "deadlock", err: &pgconn.PgError{Code: "40P01"}, want: true}, {name: "wrapped serialization failure", err: fmt.Errorf("lock entry hierarchy: %w", &pgconn.PgError{Code: "40001"}), want: true}, + {name: "entry row lock unavailable", err: fmt.Errorf("lock entry hierarchy: %w", &pgconn.PgError{Code: "55P03"}), want: true}, {name: "symbol key created concurrently", err: &pgconn.PgError{Code: "23505", ConstraintName: "symbols_authority_symbol_key"}, want: true}, {name: "other unique violation", err: &pgconn.PgError{Code: "23505", ConstraintName: "entries_hrid_key"}, want: false}, {name: "ordinary error", err: errors.New("failed"), want: false}, diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 648a990d5..5d3190e26 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -14,6 +14,7 @@ import ( "github.com/indexdata/crosslink/directory/app" importdb "github.com/indexdata/crosslink/directory/import/db" "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" "github.com/stretchr/testify/require" @@ -24,6 +25,24 @@ import ( var testPool *pgxpool.Pool +type lockUnavailableTracer struct { + observed chan struct{} +} + +func (t lockUnavailableTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + return ctx +} + +func (t lockUnavailableTracer) TraceQueryEnd(_ context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + var pgErr *pgconn.PgError + if errors.As(data.Err, &pgErr) && pgErr.Code == "55P03" { + select { + case t.observed <- struct{}{}: + default: + } + } +} + func TestMain(m *testing.M) { ctx := context.Background() container, err := postgres.Run(ctx, "postgres", @@ -385,19 +404,13 @@ func TestImportEntryRetriesWhenParentSymbolChangesBeforeRowLock(t *testing.T) { aggregate := minimalEntryAggregate("BRANCH", "Branch") aggregate.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "PARENT"} + repo, lockUnavailable := importRepoObservingLockUnavailable(t, ctx) importDone := make(chan error, 1) go func() { - _, importErr := importdb.New(testPool).ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) + _, importErr := repo.ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) importDone <- importErr }() - require.Eventually(t, func() bool { - var waiting bool - err := testPool.QueryRow(ctx, `SELECT EXISTS ( - SELECT 1 FROM pg_stat_activity - WHERE datname=current_database() AND pid <> pg_backend_pid() AND wait_event_type='Lock' - )`).Scan(&waiting) - return err == nil && waiting - }, 2*time.Second, 10*time.Millisecond) + waitForLockUnavailable(t, ctx, lockUnavailable) _, err = testPool.Exec(ctx, `UPDATE symbols SET owner=$1 WHERE authority='ISIL' AND symbol='PARENT'`, replacementParentID) require.NoError(t, err) @@ -428,19 +441,13 @@ func TestImportEntryRetriesWhenParentIsDeletedBeforeRowLock(t *testing.T) { aggregate := minimalEntryAggregate("BRANCH", "Branch") aggregate.Data.Parent = &model.SymbolRef{Authority: "ISIL", Symbol: "PARENT"} + repo, lockUnavailable := importRepoObservingLockUnavailable(t, ctx) importDone := make(chan error, 1) go func() { - _, importErr := importdb.New(testPool).ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) + _, importErr := repo.ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) importDone <- importErr }() - require.Eventually(t, func() bool { - var waiting bool - err := testPool.QueryRow(ctx, `SELECT EXISTS ( - SELECT 1 FROM pg_stat_activity - WHERE datname=current_database() AND pid <> pg_backend_pid() AND wait_event_type='Lock' - )`).Scan(&waiting) - return err == nil && waiting - }, 2*time.Second, 10*time.Millisecond) + waitForLockUnavailable(t, ctx, lockUnavailable) _, err = blocker.Exec(ctx, `DELETE FROM entries WHERE id=$1`, parentID) require.NoError(t, err) @@ -656,12 +663,13 @@ func TestConcurrentEntryAndTierImportsUseSameEntryLockOrder(t *testing.T) { entry := minimalEntryAggregate("MEMBER", "Institution") entry.Data.Parent = &consortium + entryRepo, lockUnavailable := importRepoObservingLockUnavailable(t, ctx) entryDone := make(chan error, 1) go func() { - _, importErr := repo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) + _, importErr := entryRepo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) entryDone <- importErr }() - waitForDatabaseLockWaiters(t, ctx, 2) + waitForLockUnavailable(t, ctx, lockUnavailable) require.NoError(t, blocker.Commit(ctx)) require.NoError(t, <-tierDone) @@ -775,12 +783,13 @@ func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { entry := minimalEntryAggregate("MEMBER", "Institution") entry.Data.Parent = &consortium + entryRepo, lockUnavailable := importRepoObservingLockUnavailable(t, ctx) entryDone := make(chan error, 1) go func() { - _, importErr := repo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) + _, importErr := entryRepo.ImportEntry(ctx, entry, model.ConflictPolicyUpdate) entryDone <- importErr }() - waitForDatabaseLockWaiters(t, ctx, 2) + waitForLockUnavailable(t, ctx, lockUnavailable) require.NoError(t, blocker.Commit(ctx)) require.NoError(t, <-networkDone) @@ -860,6 +869,160 @@ func TestBusinessKeyLookupErrorsPreservePostgreSQLCause(t *testing.T) { } } +func TestAssignmentDeleteErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + table := "entry_" + resource + "s" + unavailableTable := table + "_unavailable" + _, err := testPool.Exec(context.Background(), fmt.Sprintf("ALTER TABLE %s RENAME TO %s", table, unavailableTable)) //nolint:gosec // fixed test identifiers + require.NoError(t, err) + t.Cleanup(func() { + _, restoreErr := testPool.Exec(context.Background(), fmt.Sprintf("ALTER TABLE %s RENAME TO %s", unavailableTable, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, restoreErr) + }) + + err = importAggregateWithAssignment(repo, resource, consortium, first) + + require.ErrorContains(t, err, "replace "+resource+" assignments: delete existing assignments") + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "42P01", pgErr.Code) + }) + } +} + +func TestAssignmentCreateErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + table := "entry_" + resource + "s" + trigger := "fail_" + resource + "_assignment_create" + _, err := testPool.Exec(context.Background(), ` + CREATE OR REPLACE FUNCTION fail_import_assignment_create() RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'forced assignment create failure' USING ERRCODE = '23514'; + END; + $$ LANGUAGE plpgsql`) + require.NoError(t, err) + _, err = testPool.Exec(context.Background(), fmt.Sprintf(` + CREATE TRIGGER %s + BEFORE INSERT ON %s + FOR EACH ROW EXECUTE FUNCTION fail_import_assignment_create()`, trigger, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, err) + t.Cleanup(func() { + _, cleanupErr := testPool.Exec(context.Background(), fmt.Sprintf("DROP TRIGGER %s ON %s", trigger, table)) //nolint:gosec // fixed test identifiers + require.NoError(t, cleanupErr) + _, cleanupErr = testPool.Exec(context.Background(), `DROP FUNCTION fail_import_assignment_create()`) + require.NoError(t, cleanupErr) + }) + + err = importAggregateWithAssignment(repo, resource, consortium, first) + + require.ErrorContains(t, err, "replace "+resource+" assignments: create assignment") + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "23514", pgErr.Code) + }) + } +} + +func importAggregateWithAssignment(repo *importdb.PgImportRepo, resource string, consortium, entry model.SymbolRef) error { + if resource == "tier" { + _, err := repo.ImportTier(context.Background(), model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Assignment failure"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{entry}}, + }, model.ConflictPolicyFail) + return err + } + _, err := repo.ImportNetwork(context.Background(), model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Assignment failure"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{{SymbolRef: entry, Priority: 1}}}, + }, model.ConflictPolicyFail) + return err +} + +func TestAssignmentConsortiumLookupErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, _, _ := importRepoFixture(t) + _, err := testPool.Exec(context.Background(), `ALTER TABLE symbols RENAME TO symbols_unavailable`) + require.NoError(t, err) + t.Cleanup(func() { + _, restoreErr := testPool.Exec(context.Background(), `ALTER TABLE symbols_unavailable RENAME TO symbols`) + require.NoError(t, restoreErr) + }) + + if resource == "tier" { + _, err = repo.ImportTier(context.Background(), model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Lookup failure"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{}}, + }, model.ConflictPolicyFail) + } else { + _, err = repo.ImportNetwork(context.Background(), model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Lookup failure"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{}}, + }, model.ConflictPolicyFail) + } + + require.ErrorContains(t, err, "resolve consortium "+consortium.String()) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "42P01", pgErr.Code) + }) + } +} + +func TestAssignmentEntryLookupErrorsPreservePostgreSQLCause(t *testing.T) { + for _, resource := range []string{"tier", "network"} { + t.Run(resource, func(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + _, err := testPool.Exec(context.Background(), `ALTER TABLE symbols RENAME TO symbols_available`) + require.NoError(t, err) + t.Cleanup(func() { + _, cleanupErr := testPool.Exec(context.Background(), `DROP VIEW IF EXISTS symbols`) + require.NoError(t, cleanupErr) + _, cleanupErr = testPool.Exec(context.Background(), `ALTER TABLE symbols_available RENAME TO symbols`) + require.NoError(t, cleanupErr) + _, cleanupErr = testPool.Exec(context.Background(), `DROP FUNCTION fail_assignment_entry_lookup(text, uuid)`) + require.NoError(t, cleanupErr) + }) + _, err = testPool.Exec(context.Background(), ` + CREATE FUNCTION fail_assignment_entry_lookup(symbol_value text, owner_value uuid) RETURNS uuid AS $$ + BEGIN + IF symbol_value = 'FIRST' THEN + RAISE EXCEPTION 'forced assignment entry lookup failure' USING ERRCODE = '42P01'; + END IF; + RETURN owner_value; + END; + $$ LANGUAGE plpgsql`) + require.NoError(t, err) + _, err = testPool.Exec(context.Background(), ` + CREATE VIEW symbols AS + SELECT fail_assignment_entry_lookup(symbol, owner) AS owner, authority, symbol + FROM symbols_available`) + require.NoError(t, err) + + if resource == "tier" { + _, err = repo.ImportTier(context.Background(), model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Assignment lookup failure"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{first}}, + }, model.ConflictPolicyFail) + } else { + _, err = repo.ImportNetwork(context.Background(), model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Assignment lookup failure"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{{SymbolRef: first, Priority: 1}}}, + }, model.ConflictPolicyFail) + } + + require.ErrorContains(t, err, "resolve entry "+first.String()) + var pgErr *pgconn.PgError + require.ErrorAs(t, err, &pgErr) + require.Equal(t, "42P01", pgErr.Code) + }) + } +} + func TestAggregateCommitErrorsPreservePostgreSQLCause(t *testing.T) { for _, resource := range []string{"tier", "network"} { t.Run(resource, func(t *testing.T) { @@ -1068,6 +1231,27 @@ func waitForDatabaseLockWaiters(t *testing.T, ctx context.Context, minimum int) }, 2*time.Second, 10*time.Millisecond) } +func importRepoObservingLockUnavailable(t *testing.T, ctx context.Context) (*importdb.PgImportRepo, <-chan struct{}) { + t.Helper() + observed := make(chan struct{}, 1) + config, err := pgxpool.ParseConfig(app.ConnectionString) + require.NoError(t, err) + config.ConnConfig.Tracer = lockUnavailableTracer{observed: observed} + pool, err := pgxpool.NewWithConfig(ctx, config) + require.NoError(t, err) + t.Cleanup(pool.Close) + return importdb.New(pool), observed +} + +func waitForLockUnavailable(t *testing.T, ctx context.Context, observed <-chan struct{}) { + t.Helper() + select { + case <-observed: + case <-ctx.Done(): + require.FailNow(t, "import did not report a non-waiting row-lock conflict", ctx.Err()) + } +} + func concurrentlyImportTier(repo *importdb.PgImportRepo, newAggregate func() model.TierAggregate, policy model.ConflictPolicy, count int) ([]model.RepoResult, []error) { start := make(chan struct{}) results := make([]model.RepoResult, count) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index 9cc5263b4..59d75b392 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -16,7 +16,7 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name - for range maxImportLockAttempts { + for range maxImportMappingAttempts { result, err := r.importTierAttempt(ctx, aggregate, policy, key) if !errors.Is(err, errImportEntryMappingChanged) { return result, err @@ -82,7 +82,7 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name - for range maxImportLockAttempts { + for range maxImportMappingAttempts { result, err := r.importNetworkAttempt(ctx, aggregate, policy, key) if !errors.Is(err, errImportEntryMappingChanged) { return result, err @@ -158,7 +158,7 @@ func resolveAndLockAssignments(ctx context.Context, queries *db.Queries, consort return db.Entry{}, nil, fmt.Errorf("consortium %s does not exist", consortiumRef.String()) } if err != nil { - return db.Entry{}, nil, fmt.Errorf("resolve consortium %s", consortiumRef.String()) + return db.Entry{}, nil, fmt.Errorf("resolve consortium %s: %w", consortiumRef.String(), err) } assignments := make([]resolvedAssignment, 0, len(refs)) @@ -172,7 +172,7 @@ func resolveAndLockAssignments(ctx context.Context, queries *db.Queries, consort continue } if err != nil { - return db.Entry{}, nil, fmt.Errorf("resolve entry %s", ref.String()) + return db.Entry{}, nil, fmt.Errorf("resolve entry %s: %w", ref.String(), err) } assignments = append(assignments, resolvedAssignment{ref: ref, entry: &entry}) entryIDs = append(entryIDs, entry.ID) @@ -212,11 +212,11 @@ func requireAssignmentEntries(assignments []resolvedAssignment) ([]db.Entry, err func replaceTierAssignments(ctx context.Context, queries *db.Queries, tierID uuid.UUID, entries []db.Entry) error { if err := queries.DeleteEntryTiersByTier(ctx, tierID); err != nil { - return fmt.Errorf("replace tier assignments") + return fmt.Errorf("replace tier assignments: delete existing assignments: %w", err) } for _, entry := range entries { if _, err := queries.CreateEntryTier(ctx, db.CreateEntryTierParams{Entry: entry.ID, Tier: tierID}); err != nil { - return fmt.Errorf("replace tier assignments") + return fmt.Errorf("replace tier assignments: create assignment: %w", err) } } return nil @@ -227,11 +227,11 @@ func replaceNetworkAssignments(ctx context.Context, queries *db.Queries, network return fmt.Errorf("replace network assignments: entry count mismatch") } if err := queries.DeleteEntryNetworksByNetwork(ctx, networkID); err != nil { - return fmt.Errorf("replace network assignments") + return fmt.Errorf("replace network assignments: delete existing assignments: %w", err) } for index, entry := range entries { if _, err := queries.CreateEntryNetwork(ctx, db.CreateEntryNetworkParams{Entry: entry.ID, Network: networkID, Priority: assignments[index].Priority}); err != nil { - return fmt.Errorf("replace network assignments") + return fmt.Errorf("replace network assignments: create assignment: %w", err) } } return nil diff --git a/directory/query.sql b/directory/query.sql index 2c083e01f..d6f6e2d7d 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -4,6 +4,9 @@ SELECT * FROM entries WHERE id = $1 LIMIT 1; -- name: EntryByIdForUpdate :one SELECT * FROM entries WHERE id = $1 LIMIT 1 FOR UPDATE; +-- name: EntryByIdForImportUpdate :one +SELECT * FROM entries WHERE id = $1 LIMIT 1 FOR UPDATE NOWAIT; + -- name: EntryBySymbolForUpdate :one SELECT e.* FROM entries e, symbols s WHERE e.id = s.owner AND s.authority = @authority AND s.symbol = @symbol LIMIT 1 FOR UPDATE OF e; diff --git a/directory/test/concurrency_test.go b/directory/test/concurrency_test.go index 43b6344dc..ec40d4a86 100644 --- a/directory/test/concurrency_test.go +++ b/directory/test/concurrency_test.go @@ -3,9 +3,19 @@ package test import ( "context" "encoding/json" + "errors" "net/http" "sync" "testing" + "time" + + "github.com/indexdata/crosslink/directory/app" + importdb "github.com/indexdata/crosslink/directory/import/db" + "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" ) var consortiumPermissionHeaders = map[string]string{ @@ -13,6 +23,24 @@ var consortiumPermissionHeaders = map[string]string{ "X-Okapi-Permissions": `["directory.consortium.all"]`, } +type lockUnavailableTracer struct { + observed chan struct{} +} + +func (t lockUnavailableTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { + return ctx +} + +func (t lockUnavailableTracer) TraceQueryEnd(_ context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + var pgErr *pgconn.PgError + if errors.As(data.Err, &pgErr) && pgErr.Code == "55P03" { + select { + case t.observed <- struct{}{}: + default: + } + } +} + func TestConcurrency(t *testing.T) { t.Run("ConcurrentEntryPatch", func(t *testing.T) { resetDb() @@ -148,6 +176,82 @@ func TestConcurrency(t *testing.T) { }) } +func TestConcurrentImportDoesNotDeadlockEntryPatch(t *testing.T) { + resetDb() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + parentID := "00000000-0000-0000-0000-000000000001" + childID := "00000000-0000-0000-0000-000000000002" + _, err := dbpool.Exec(ctx, `UPDATE entries SET parent=NULL, type='Institution' WHERE id=$1`, parentID) + require.NoError(t, err) + _, err = dbpool.Exec(ctx, `UPDATE entries SET parent=$1, type='Branch' WHERE id=$2`, parentID, childID) + require.NoError(t, err) + _, err = dbpool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'TEST', 'PARENT')`, parentID) + require.NoError(t, err) + + blocker, err := dbpool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = blocker.Rollback(ctx) }() + _, err = blocker.Exec(ctx, `SELECT id FROM entries WHERE id=$1 FOR UPDATE`, parentID) + require.NoError(t, err) + lockUnavailable := make(chan struct{}, 1) + importPoolConfig, err := pgxpool.ParseConfig(app.ConnectionString) + require.NoError(t, err) + importPoolConfig.ConnConfig.Tracer = lockUnavailableTracer{observed: lockUnavailable} + importPool, err := pgxpool.NewWithConfig(ctx, importPoolConfig) + require.NoError(t, err) + t.Cleanup(importPool.Close) + + parent := model.SymbolRef{Authority: "TEST", Symbol: "PARENT"} + key := model.SymbolRef{Authority: "TEST", Symbol: "ANINST"} + aggregate := model.EntryAggregate{ + Key: key, + Data: model.EntryData{ + Name: "Imported child", + Type: "Branch", + Parent: &parent, + Symbols: []model.SymbolRef{key}, + Endpoints: []model.ServiceEndpoint{}, + Addresses: []model.Address{}, + Closures: []model.Closure{}, + }, + } + importDone := make(chan error, 1) + go func() { + _, importErr := importdb.New(importPool).ImportEntry(ctx, aggregate, model.ConflictPolicyUpdate) + importDone <- importErr + }() + + select { + case <-lockUnavailable: + case <-ctx.Done(): + require.FailNow(t, "import did not report a non-waiting row-lock conflict", ctx.Err()) + } + + patchDone := make(chan *http.Response, 1) + go func() { + response, _ := jsonReq(t, http.MethodPatch, "/entries/by-id/"+childID, + `{"parent":"`+parentID+`"}`, consortiumPermissionHeaders) + patchDone <- response + }() + require.Eventually(t, func() bool { + var waiting bool + queryErr := dbpool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE datname=current_database() + AND pid <> pg_backend_pid() + AND wait_event_type='Lock' + AND query LIKE '%EntryByIdForUpdate%' + AND query NOT LIKE '%EntryByIdForImportUpdate%' + )`).Scan(&waiting) + return queryErr == nil && waiting + }, 2*time.Second, 10*time.Millisecond) + require.NoError(t, blocker.Commit(ctx)) + + require.Equal(t, http.StatusNoContent, (<-patchDone).StatusCode) + require.NoError(t, <-importDone) +} + func assertOneConsortiumWrite(t *testing.T, statuses <-chan int, successStatus int) { t.Helper() successes, rejections := 0, 0 From b66651df4c40ab77403c3ecaabc95e56ff147f5e Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 07:28:54 +0300 Subject: [PATCH 10/18] ILLDEV-484 Fix copilot comments --- directory/import/db/entry.go | 1 - directory/import/db/tier_network.go | 62 +++++++++-- directory/test/concurrency_test.go | 161 ++++++++++++++++++++++++++++ 3 files changed, 216 insertions(+), 8 deletions(-) diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 9d3ecd62f..7e75ac0b8 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -20,7 +20,6 @@ import ( const ( maxImportEntryLockAttempts = 5 - maxImportMappingAttempts = 3 importLockRetryBaseWait = 10 * time.Millisecond ) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index 59d75b392..fe27106e8 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -9,6 +9,7 @@ import ( "github.com/indexdata/crosslink/directory/db" "github.com/indexdata/crosslink/directory/import/model" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" ) func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { @@ -16,16 +17,23 @@ func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggre return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name - for range maxImportMappingAttempts { + var lastErr error + for attemptIndex := 0; attemptIndex < maxImportEntryLockAttempts; attemptIndex++ { result, err := r.importTierAttempt(ctx, aggregate, policy, key) - if !errors.Is(err, errImportEntryMappingChanged) { + if !retryableAssignmentImportError(err) { return result, err } + lastErr = err if err := ctx.Err(); err != nil { return model.RepoResult{}, err } + if attemptIndex+1 < maxImportEntryLockAttempts { + if err := waitForImportRetry(ctx, attemptIndex); err != nil { + return model.RepoResult{}, err + } + } } - return model.RepoResult{}, fmt.Errorf("import tier %s: entry mappings changed repeatedly", key) + return model.RepoResult{}, fmt.Errorf("import tier %s: transaction conflicted repeatedly: %w", key, lastErr) } func (r *PgImportRepo) importTierAttempt(ctx context.Context, aggregate model.TierAggregate, policy model.ConflictPolicy, key string) (model.RepoResult, error) { @@ -82,16 +90,23 @@ func (r *PgImportRepo) ImportNetwork(ctx context.Context, aggregate model.Networ return model.RepoResult{}, err } key := aggregate.Key.Consortium.String() + "/" + aggregate.Key.Name - for range maxImportMappingAttempts { + var lastErr error + for attemptIndex := 0; attemptIndex < maxImportEntryLockAttempts; attemptIndex++ { result, err := r.importNetworkAttempt(ctx, aggregate, policy, key) - if !errors.Is(err, errImportEntryMappingChanged) { + if !retryableAssignmentImportError(err) { return result, err } + lastErr = err if err := ctx.Err(); err != nil { return model.RepoResult{}, err } + if attemptIndex+1 < maxImportEntryLockAttempts { + if err := waitForImportRetry(ctx, attemptIndex); err != nil { + return model.RepoResult{}, err + } + } } - return model.RepoResult{}, fmt.Errorf("import network %s: entry mappings changed repeatedly", key) + return model.RepoResult{}, fmt.Errorf("import network %s: transaction conflicted repeatedly: %w", key, lastErr) } func (r *PgImportRepo) importNetworkAttempt(ctx context.Context, aggregate model.NetworkAggregate, policy model.ConflictPolicy, key string) (model.RepoResult, error) { @@ -179,7 +194,7 @@ func resolveAndLockAssignments(ctx context.Context, queries *db.Queries, consort mappings = append(mappings, entryMapping{ref: ref, expectedOwner: &entry.ID}) } - lockedEntries, err := lockEntryRows(ctx, queries, entryIDs...) + lockedEntries, err := lockAssignmentEntryRows(ctx, queries, entryIDs...) if err != nil { return db.Entry{}, nil, fmt.Errorf("lock assignment entries: %w", err) } @@ -199,6 +214,39 @@ func resolveAndLockAssignments(ctx context.Context, queries *db.Queries, consort return consortium, assignments, nil } +func retryableAssignmentImportError(err error) bool { + if errors.Is(err, errImportEntryMappingChanged) { + return true + } + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + return false + } + return pgErr.Code == "40P01" || pgErr.Code == "40001" || pgErr.Code == "55P03" +} + +func lockAssignmentEntryRows(ctx context.Context, queries *db.Queries, ids ...uuid.UUID) (map[uuid.UUID]db.Entry, error) { + ordered := orderedUniqueEntryIDs(ids...) + entries := make(map[uuid.UUID]db.Entry, len(ordered)) + for index, id := range ordered { + var entry db.Entry + var err error + if index == 0 { + entry, err = queries.EntryByIdForUpdate(ctx, id) + } else { + entry, err = queries.EntryByIdForImportUpdate(ctx, id) + } + if errors.Is(err, pgx.ErrNoRows) { + return nil, errImportEntryMappingChanged + } + if err != nil { + return nil, err + } + entries[id] = entry + } + return entries, nil +} + func requireAssignmentEntries(assignments []resolvedAssignment) ([]db.Entry, error) { entries := make([]db.Entry, 0, len(assignments)) for _, assignment := range assignments { diff --git a/directory/test/concurrency_test.go b/directory/test/concurrency_test.go index ec40d4a86..b3238c235 100644 --- a/directory/test/concurrency_test.go +++ b/directory/test/concurrency_test.go @@ -5,10 +5,12 @@ import ( "encoding/json" "errors" "net/http" + "strings" "sync" "testing" "time" + "github.com/google/uuid" "github.com/indexdata/crosslink/directory/app" importdb "github.com/indexdata/crosslink/directory/import/db" "github.com/indexdata/crosslink/directory/import/model" @@ -27,6 +29,46 @@ type lockUnavailableTracer struct { observed chan struct{} } +type pauseEntryLockTracer struct { + entryID uuid.UUID + locked chan struct{} + release <-chan struct{} + conflict chan<- string + once sync.Once +} + +type pauseEntryLockContextKey struct{} + +func (t *pauseEntryLockTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + if strings.Contains(data.SQL, "-- name: EntryByIdForUpdate") && + !strings.Contains(data.SQL, "-- name: EntryByIdForImportUpdate") && len(data.Args) > 0 { + if entryID, ok := data.Args[0].(uuid.UUID); ok && entryID == t.entryID { + return context.WithValue(ctx, pauseEntryLockContextKey{}, true) + } + } + return ctx +} + +func (t *pauseEntryLockTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + var pgErr *pgconn.PgError + if errors.As(data.Err, &pgErr) { + select { + case t.conflict <- pgErr.Code: + default: + } + } + if data.Err != nil || ctx.Value(pauseEntryLockContextKey{}) != true { + return + } + t.once.Do(func() { + close(t.locked) + select { + case <-t.release: + case <-ctx.Done(): + } + }) +} + func (t lockUnavailableTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context { return ctx } @@ -252,6 +294,125 @@ func TestConcurrentImportDoesNotDeadlockEntryPatch(t *testing.T) { require.NoError(t, <-importDone) } +func TestConcurrentAssignmentImportsDoNotDeadlockEntryPatch(t *testing.T) { + for _, testCase := range []struct { + name string + importRun func(context.Context, *importdb.PgImportRepo, model.SymbolRef, model.SymbolRef) error + }{ + { + name: "tier", + importRun: func(ctx context.Context, repo *importdb.PgImportRepo, consortium, member model.SymbolRef) error { + _, err := repo.ImportTier(ctx, model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Concurrent lock"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{member}}, + }, model.ConflictPolicyUpdate) + return err + }, + }, + { + name: "network", + importRun: func(ctx context.Context, repo *importdb.PgImportRepo, consortium, member model.SymbolRef) error { + _, err := repo.ImportNetwork(ctx, model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Concurrent lock"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{{ + SymbolRef: member, + Priority: 1, + }}}, + }, model.ConflictPolicyUpdate) + return err + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + resetDb() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + consortiumID := uuid.MustParse("00000000-0000-0000-0000-000000000001") + memberID := uuid.MustParse("00000000-0000-0000-0000-000000000002") + _, err := dbpool.Exec(ctx, `UPDATE entries SET parent=NULL, type='Institution'`) + require.NoError(t, err) + _, err = dbpool.Exec(ctx, `UPDATE entries SET type='Consortium' WHERE id=$1`, consortiumID) + require.NoError(t, err) + _, err = dbpool.Exec(ctx, `UPDATE entries SET parent=$1, type='Institution' WHERE id=$2`, consortiumID, memberID) + require.NoError(t, err) + _, err = dbpool.Exec(ctx, `INSERT INTO symbols (owner, authority, symbol) VALUES ($1, 'TEST', 'LOCK-CON')`, consortiumID) + require.NoError(t, err) + + firstLocked := make(chan struct{}) + resumeImport := make(chan struct{}) + var resumeOnce sync.Once + importConflict := make(chan string, 1) + tracer := &pauseEntryLockTracer{ + entryID: consortiumID, locked: firstLocked, release: resumeImport, conflict: importConflict, + } + importPoolConfig, err := pgxpool.ParseConfig(app.ConnectionString) + require.NoError(t, err) + importPoolConfig.ConnConfig.Tracer = tracer + importPool, err := pgxpool.NewWithConfig(ctx, importPoolConfig) + require.NoError(t, err) + t.Cleanup(importPool.Close) + t.Cleanup(func() { resumeOnce.Do(func() { close(resumeImport) }) }) + + consortium := model.SymbolRef{Authority: "TEST", Symbol: "LOCK-CON"} + member := model.SymbolRef{Authority: "TEST", Symbol: "ANINST"} + importDone := make(chan error, 1) + go func() { + importDone <- testCase.importRun(ctx, importdb.New(importPool), consortium, member) + }() + + select { + case <-firstLocked: + case <-ctx.Done(): + require.FailNow(t, "import did not acquire its first entry lock", ctx.Err()) + } + + type patchResult struct { + response *http.Response + body string + } + patchDone := make(chan patchResult, 1) + go func() { + response, body := jsonReq(t, http.MethodPatch, "/entries/by-id/"+memberID.String(), + `{"parent":"`+consortiumID.String()+`"}`, consortiumPermissionHeaders) + patchDone <- patchResult{response: response, body: body} + }() + require.Eventually(t, func() bool { + var waiting bool + queryErr := dbpool.QueryRow(ctx, `SELECT EXISTS ( + SELECT 1 FROM pg_stat_activity + WHERE datname=current_database() + AND pid <> pg_backend_pid() + AND wait_event_type='Lock' + AND query LIKE '%EntryByIdForUpdate%' + AND query NOT LIKE '%EntryByIdForImportUpdate%' + )`).Scan(&waiting) + return queryErr == nil && waiting + }, 2*time.Second, 10*time.Millisecond) + resumeOnce.Do(func() { close(resumeImport) }) + select { + case code := <-importConflict: + require.Equal(t, "55P03", code, "import must avoid waiting on its second entry lock") + case <-ctx.Done(): + require.FailNow(t, "import did not report a non-waiting second-lock conflict", ctx.Err()) + } + + var patch patchResult + select { + case patch = <-patchDone: + case <-ctx.Done(): + require.FailNow(t, "entry patch did not finish", ctx.Err()) + } + require.Equal(t, http.StatusNoContent, patch.response.StatusCode, patch.body) + select { + case importErr := <-importDone: + require.NoError(t, importErr) + case <-ctx.Done(): + require.FailNow(t, "assignment import did not finish", ctx.Err()) + } + }) + } +} + func assertOneConsortiumWrite(t *testing.T, statuses <-chan int, successStatus int) { t.Helper() successes, rejections := 0, 0 From 7eccb73e4e5e233ba83f70b0cad6405e7760e507 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 09:07:35 +0300 Subject: [PATCH 11/18] ILLDEV-484 Fix copilot comments --- directory/import/db/repo_test.go | 90 +++++++++++++++++++++++++++++++- directory/query.sql | 6 ++- 2 files changed, 93 insertions(+), 3 deletions(-) diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 5d3190e26..847a6fa2f 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -12,6 +12,7 @@ import ( "github.com/google/uuid" "github.com/indexdata/crosslink/directory/app" + "github.com/indexdata/crosslink/directory/db" importdb "github.com/indexdata/crosslink/directory/import/db" "github.com/indexdata/crosslink/directory/import/model" "github.com/jackc/pgx/v5" @@ -100,6 +101,31 @@ func TestImportBusinessKeyConstraints(t *testing.T) { requirePgCode(t, err, "23514") } +func TestEntryImportAdvisoryLockDistinguishesAmbiguousSymbolRefs(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + firstTx, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = firstTx.Rollback(context.Background()) }() + require.NoError(t, db.New(firstTx).LockEntryImportKey(ctx, db.LockEntryImportKeyParams{ + Authority: "A:B", + Symbol: "C", + })) + + secondTx, err := testPool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = secondTx.Rollback(context.Background()) }() + lockCtx, cancelLock := context.WithTimeout(ctx, time.Second) + defer cancelLock() + + err = db.New(secondTx).LockEntryImportKey(lockCtx, db.LockEntryImportKeyParams{ + Authority: "A", + Symbol: "B:C", + }) + + require.NoError(t, err) +} + func TestImportEntryCreatesCompleteAggregateWithGeneratedIDs(t *testing.T) { resetImportDatabase(t) repo := importdb.New(testPool) @@ -234,8 +260,10 @@ func TestImportEntryLocksAndRevalidatesSecondarySymbols(t *testing.T) { blocker, err := testPool.Begin(ctx) require.NoError(t, err) defer func() { _ = blocker.Rollback(ctx) }() - _, err = blocker.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended('directoryish:entry:ISIL:SHARED', 0))`) - require.NoError(t, err) + require.NoError(t, db.New(blocker).LockEntryImportKey(ctx, db.LockEntryImportKeyParams{ + Authority: secondary.Authority, + Symbol: secondary.Symbol, + })) aggregate := minimalEntryAggregate("IMPORTED", "Institution") aggregate.Data.Symbols = append(aggregate.Data.Symbols, secondary) @@ -627,6 +655,32 @@ func TestImportTierConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) } +func TestExistingTierConflictPolicyPrecedesMissingAssignmentValidation(t *testing.T) { + for _, policy := range []model.ConflictPolicy{model.ConflictPolicySkip, model.ConflictPolicyFail} { + t.Run(string(policy), func(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + aggregate := model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Loan"}, + Data: model.TierData{Level: "standard", Type: "loan", Entries: []model.SymbolRef{first}}, + } + _, err := repo.ImportTier(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + aggregate.Data.Entries = []model.SymbolRef{{Authority: "ISIL", Symbol: "MISSING"}} + + result, err := repo.ImportTier(context.Background(), aggregate, policy) + + if policy == model.ConflictPolicySkip { + require.NoError(t, err) + require.Equal(t, model.OutcomeSkipped, result.Outcome) + require.Contains(t, result.Diagnostic, "already exists") + return + } + require.ErrorContains(t, err, "already exists") + require.NotContains(t, err.Error(), "does not exist") + }) + } +} + func TestConcurrentEntryAndTierImportsUseSameEntryLockOrder(t *testing.T) { resetImportDatabase(t) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) @@ -747,6 +801,38 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.Equal(t, []model.NetworkAssignment{{SymbolRef: second, Priority: 2}}, networkAssignments(t, id)) } +func TestExistingNetworkConflictPolicyPrecedesMissingAssignmentValidation(t *testing.T) { + for _, policy := range []model.ConflictPolicy{model.ConflictPolicySkip, model.ConflictPolicyFail} { + t.Run(string(policy), func(t *testing.T) { + repo, consortium, first, _ := importRepoFixture(t) + aggregate := model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Main"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{{ + SymbolRef: first, + Priority: 1, + }}}, + } + _, err := repo.ImportNetwork(context.Background(), aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + aggregate.Data.Entries = []model.NetworkAssignment{{ + SymbolRef: model.SymbolRef{Authority: "ISIL", Symbol: "MISSING"}, + Priority: 1, + }} + + result, err := repo.ImportNetwork(context.Background(), aggregate, policy) + + if policy == model.ConflictPolicySkip { + require.NoError(t, err) + require.Equal(t, model.OutcomeSkipped, result.Outcome) + require.Contains(t, result.Diagnostic, "already exists") + return + } + require.ErrorContains(t, err, "already exists") + require.NotContains(t, err.Error(), "does not exist") + }) + } +} + func TestConcurrentEntryAndNetworkImportsUseSameEntryLockOrder(t *testing.T) { resetImportDatabase(t) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/directory/query.sql b/directory/query.sql index d6f6e2d7d..62b974115 100644 --- a/directory/query.sql +++ b/directory/query.sql @@ -26,7 +26,11 @@ SELECT * FROM entries WHERE parent = @parent; SELECT pg_advisory_xact_lock(hashtextextended('directoryish:consortium-entry', 0)); -- name: LockEntryImportKey :exec -SELECT pg_advisory_xact_lock(hashtextextended('directoryish:entry:' || @authority::text || ':' || @symbol::text, 0)); +SELECT pg_advisory_xact_lock(hashtextextended( + 'directoryish:entry:' || octet_length(@authority::text)::text || ':' || @authority::text || + ':' || octet_length(@symbol::text)::text || ':' || @symbol::text, + 0 +)); -- name: CreateEntry :one INSERT INTO entries ( From 742ae15af59c88036ea148addc88852356cf1e48 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 09:48:52 +0300 Subject: [PATCH 12/18] ILLDEV-484 Fix copilot comments --- directory/app/import_limit.go | 2 +- directory/import/db/entry.go | 12 +++++-- directory/import/db/entry_lock_test.go | 47 ++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/directory/app/import_limit.go b/directory/app/import_limit.go index c7ad99041..4ece394b1 100644 --- a/directory/app/import_limit.go +++ b/directory/app/import_limit.go @@ -6,7 +6,7 @@ import ( "github.com/indexdata/crosslink/directory/auth" ) -const MaxImportBodyBytes int64 = 2 << 30 +const MaxImportBodyBytes int64 = 1 << 30 func ImportBodyLimitMiddleware(maxBytes int64, next http.Handler) http.Handler { return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { diff --git a/directory/import/db/entry.go b/directory/import/db/entry.go index 7e75ac0b8..6bb4d6f15 100644 --- a/directory/import/db/entry.go +++ b/directory/import/db/entry.go @@ -477,9 +477,15 @@ func replaceEntryChildren(ctx context.Context, queries *db.Queries, entryID uuid if err := queries.DeleteClosuresByEntry(ctx, entryID); err != nil { return err } - for _, closure := range data.Closures { - start, _ := time.Parse(time.DateOnly, closure.StartDate) - end, _ := time.Parse(time.DateOnly, closure.EndDate) + for index, closure := range data.Closures { + start, err := time.Parse(time.DateOnly, closure.StartDate) + if err != nil { + return fmt.Errorf("parse closure %d startDate: %w", index+1, err) + } + end, err := time.Parse(time.DateOnly, closure.EndDate) + if err != nil { + return fmt.Errorf("parse closure %d endDate: %w", index+1, err) + } if _, err := queries.CreateClosure(ctx, db.CreateClosureParams{ Entry: entryID, StartDate: pgtype.Timestamp{Time: start, Valid: true}, EndDate: pgtype.Timestamp{Time: end, Valid: true}, Reason: closure.Reason, }); err != nil { diff --git a/directory/import/db/entry_lock_test.go b/directory/import/db/entry_lock_test.go index 6146f6328..d9ff5ea8c 100644 --- a/directory/import/db/entry_lock_test.go +++ b/directory/import/db/entry_lock_test.go @@ -10,10 +10,57 @@ import ( "github.com/google/uuid" "github.com/indexdata/crosslink/directory/db" "github.com/indexdata/crosslink/directory/import/model" + "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/stretchr/testify/require" ) +var errClosurePersisted = errors.New("closure reached persistence") + +type closurePersistenceStub struct{} + +func (closurePersistenceStub) Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error) { + return pgconn.CommandTag{}, nil +} + +func (closurePersistenceStub) Query(context.Context, string, ...interface{}) (pgx.Rows, error) { + return nil, errClosurePersisted +} + +func (closurePersistenceStub) QueryRow(context.Context, string, ...interface{}) pgx.Row { + return closurePersistenceRow{} +} + +type closurePersistenceRow struct{} + +func (closurePersistenceRow) Scan(...interface{}) error { + return errClosurePersisted +} + +func TestReplaceEntryChildrenReturnsInvalidClosureStartDate(t *testing.T) { + queries := db.New(closurePersistenceStub{}) + data := model.EntryData{Closures: []model.Closure{{StartDate: "invalid", EndDate: "2026-12-26"}}} + + err := replaceEntryChildren(context.Background(), queries, uuid.New(), data) + + var parseErr *time.ParseError + require.ErrorAs(t, err, &parseErr) + require.ErrorContains(t, err, "parse closure 1 startDate") + require.NotErrorIs(t, err, errClosurePersisted) +} + +func TestReplaceEntryChildrenReturnsInvalidClosureEndDate(t *testing.T) { + queries := db.New(closurePersistenceStub{}) + data := model.EntryData{Closures: []model.Closure{{StartDate: "2026-12-24", EndDate: "invalid"}}} + + err := replaceEntryChildren(context.Background(), queries, uuid.New(), data) + + var parseErr *time.ParseError + require.ErrorAs(t, err, &parseErr) + require.ErrorContains(t, err, "parse closure 1 endDate") + require.NotErrorIs(t, err, errClosurePersisted) +} + func TestRunImportEntryAttemptsRetriesTransactionConflicts(t *testing.T) { attempts := 0 want := model.RepoResult{Outcome: model.OutcomeImported} From 7764ee2059088d372612afc9380a1c07f9130ea6 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 10:15:19 +0300 Subject: [PATCH 13/18] ILLDEV-484 Fix copilot comments --- directory/app/import_limit_test.go | 14 ++++++++++++++ directory/app/request_validation.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/directory/app/import_limit_test.go b/directory/app/import_limit_test.go index b8179e692..fbe821857 100644 --- a/directory/app/import_limit_test.go +++ b/directory/app/import_limit_test.go @@ -79,6 +79,20 @@ func TestOpenAPIRequestValidationRetainsImportBodyChecks(t *testing.T) { } } +func TestOpenAPIRequestValidationReportsExpectedImportContentType(t *testing.T) { + spec, err := api.GetSpec() + require.NoError(t, err) + handler := openAPIRequestValidationMiddleware(spec)(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + request := httptest.NewRequest(http.MethodPost, BasePath+"/import", strings.NewReader("record")) + request.Header.Set("Content-Type", "application/json") + response := httptest.NewRecorder() + + handler.ServeHTTP(response, request) + + require.Equal(t, http.StatusBadRequest, response.Code) + require.Equal(t, "invalid Content-Type: expected application/x-ndjson\n", response.Body.String()) +} + func TestOpenAPIRequestValidationStillValidatesOtherRequestBodies(t *testing.T) { spec, err := api.GetSpec() require.NoError(t, err) diff --git a/directory/app/request_validation.go b/directory/app/request_validation.go index 26ce74490..4c4da95c2 100644 --- a/directory/app/request_validation.go +++ b/directory/app/request_validation.go @@ -31,7 +31,7 @@ func validateImportRequest(next http.Handler) http.Handler { contentType, _, err := mime.ParseMediaType(request.Header.Get("Content-Type")) if err != nil || contentType != importContentType { - http.Error(writer, "invalid Content-Type", http.StatusBadRequest) + http.Error(writer, "invalid Content-Type: expected "+importContentType, http.StatusBadRequest) return } if request.Body == nil || request.Body == http.NoBody { From b385ca250deb11dd640fd9b402de2eb2c1f620f8 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 10:53:53 +0300 Subject: [PATCH 14/18] ILLDEV-484 Fix copilot comments --- directory/import/db/tier_network.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index fe27106e8..fd3626f35 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -12,6 +12,11 @@ import ( "github.com/jackc/pgx/v5/pgconn" ) +const ( + tierBusinessKeyUniqueConstraint = "tiers_consortium_name_unique" + networkBusinessKeyUniqueConstraint = "networks_consortium_name_unique" +) + func (r *PgImportRepo) ImportTier(ctx context.Context, aggregate model.TierAggregate, policy model.ConflictPolicy) (model.RepoResult, error) { if err := aggregate.NormalizeAndValidate(); err != nil { return model.RepoResult{}, err @@ -222,7 +227,12 @@ func retryableAssignmentImportError(err error) bool { if !errors.As(err, &pgErr) { return false } - return pgErr.Code == "40P01" || pgErr.Code == "40001" || pgErr.Code == "55P03" + return pgErr.Code == "40P01" || + pgErr.Code == "40001" || + pgErr.Code == "55P03" || + (pgErr.Code == "23505" && + (pgErr.ConstraintName == tierBusinessKeyUniqueConstraint || + pgErr.ConstraintName == networkBusinessKeyUniqueConstraint)) } func lockAssignmentEntryRows(ctx context.Context, queries *db.Queries, ids ...uuid.UUID) (map[uuid.UUID]db.Entry, error) { From fe85228a4924eef47460eb20f0a6d476df806250 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 11:36:05 +0300 Subject: [PATCH 15/18] ILLDEV-484 Fix copilot comments --- .../008_import_business_keys.up.sql | 55 +++++++++++++++++-- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/directory/migrations/008_import_business_keys.up.sql b/directory/migrations/008_import_business_keys.up.sql index b59a786f3..c95c35d0f 100644 --- a/directory/migrations/008_import_business_keys.up.sql +++ b/directory/migrations/008_import_business_keys.up.sql @@ -1,11 +1,54 @@ +DO $$ +DECLARE + legacy_row record; + base_name text; + candidate_name text; + suffix integer; +BEGIN + FOR legacy_row IN + SELECT id, consortium FROM tiers + WHERE name IS NULL OR name !~ '[^[:space:]]' + ORDER BY consortium, id + LOOP + base_name := 'Legacy tier ' || legacy_row.id::text; + candidate_name := base_name; + suffix := 0; + WHILE EXISTS ( + SELECT 1 FROM tiers + WHERE consortium = legacy_row.consortium + AND id <> legacy_row.id + AND name = candidate_name + ) LOOP + suffix := suffix + 1; + candidate_name := base_name || ' (' || suffix::text || ')'; + END LOOP; + UPDATE tiers SET name = candidate_name WHERE id = legacy_row.id; + END LOOP; + + FOR legacy_row IN + SELECT id, consortium FROM networks + WHERE name IS NULL OR name !~ '[^[:space:]]' + ORDER BY consortium, id + LOOP + base_name := 'Legacy network ' || legacy_row.id::text; + candidate_name := base_name; + suffix := 0; + WHILE EXISTS ( + SELECT 1 FROM networks + WHERE consortium = legacy_row.consortium + AND id <> legacy_row.id + AND name = candidate_name + ) LOOP + suffix := suffix + 1; + candidate_name := base_name || ' (' || suffix::text || ')'; + END LOOP; + UPDATE networks SET name = candidate_name WHERE id = legacy_row.id; + END LOOP; +END +$$; + DO $$ BEGIN - IF EXISTS (SELECT 1 FROM tiers WHERE name IS NULL OR name !~ '[^[:space:]]') THEN - RAISE EXCEPTION 'cannot add tier business key: tiers contain null or blank names'; - END IF; - IF EXISTS (SELECT 1 FROM networks WHERE name IS NULL OR name !~ '[^[:space:]]') THEN - RAISE EXCEPTION 'cannot add network business key: networks contain null or blank names'; - END IF; IF EXISTS ( SELECT 1 FROM tiers GROUP BY consortium, name HAVING count(*) > 1 ) THEN From bd3ace0c35373c00a29983c3095ec1b98d92f3d9 Mon Sep 17 00:00:00 2001 From: Janis Saldabols Date: Tue, 15 Sep 2026 11:42:45 +0300 Subject: [PATCH 16/18] ILLDEV-484 Fix copilot comments --- directory/import/db/tier_network_test.go | 29 +++++++ .../import_business_key_migration_test.go | 80 +++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 directory/import/db/tier_network_test.go create mode 100644 directory/test/import_business_key_migration_test.go diff --git a/directory/import/db/tier_network_test.go b/directory/import/db/tier_network_test.go new file mode 100644 index 000000000..e64303d1b --- /dev/null +++ b/directory/import/db/tier_network_test.go @@ -0,0 +1,29 @@ +package importdb + +import ( + "fmt" + "testing" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/stretchr/testify/require" +) + +func TestRetryableAssignmentImportErrorHandlesBusinessKeyRaces(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + {name: "tier business key", err: &pgconn.PgError{Code: "23505", ConstraintName: "tiers_consortium_name_unique"}, want: true}, + {name: "wrapped tier business key", err: fmt.Errorf("persist tier: %w", &pgconn.PgError{Code: "23505", ConstraintName: "tiers_consortium_name_unique"}), want: true}, + {name: "network business key", err: &pgconn.PgError{Code: "23505", ConstraintName: "networks_consortium_name_unique"}, want: true}, + {name: "wrapped network business key", err: fmt.Errorf("persist network: %w", &pgconn.PgError{Code: "23505", ConstraintName: "networks_consortium_name_unique"}), want: true}, + {name: "unrelated unique constraint", err: &pgconn.PgError{Code: "23505", ConstraintName: "entry_tiers_pkey"}, want: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + require.Equal(t, test.want, retryableAssignmentImportError(test.err)) + }) + } +} diff --git a/directory/test/import_business_key_migration_test.go b/directory/test/import_business_key_migration_test.go new file mode 100644 index 000000000..0c71322f3 --- /dev/null +++ b/directory/test/import_business_key_migration_test.go @@ -0,0 +1,80 @@ +package test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestImportBusinessKeyMigrationBackfillsLegacyNames(t *testing.T) { + ctx := context.Background() + tx, err := dbpool.Begin(ctx) + require.NoError(t, err) + defer func() { _ = tx.Rollback(ctx) }() + + _, err = tx.Exec(ctx, "CREATE SCHEMA import_business_key_migration_test; SET LOCAL search_path TO import_business_key_migration_test") + require.NoError(t, err) + apply := func(pattern string) { + t.Helper() + paths, globErr := filepath.Glob("../migrations/" + pattern) + require.NoError(t, globErr) + require.NotEmpty(t, paths) + for _, path := range paths { + data, readErr := os.ReadFile(path) + require.NoError(t, readErr) + _, execErr := tx.Exec(ctx, string(data)) + require.NoError(t, execErr, "apply %s", path) + } + } + for version := 1; version <= 7; version++ { + apply(fmt.Sprintf("%03d_*.up.sql", version)) + } + + _, err = tx.Exec(ctx, ` + INSERT INTO entries (id, name, type) VALUES + ('00000000-0000-0000-0000-000000000001', 'Consortium', 'Consortium'), + ('00000000-0000-0000-0000-000000000002', 'Member', 'Institution'); + INSERT INTO tiers (id, consortium, name) VALUES + ('10000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001', NULL), + ('10000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', E' \t'), + ('10000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'Legacy tier 10000000-0000-0000-0000-000000000001'); + INSERT INTO networks (id, consortium, name) VALUES + ('20000000-0000-0000-0000-000000000001', '00000000-0000-0000-0000-000000000001', NULL), + ('20000000-0000-0000-0000-000000000002', '00000000-0000-0000-0000-000000000001', E'\n '), + ('20000000-0000-0000-0000-000000000003', '00000000-0000-0000-0000-000000000001', 'Legacy network 20000000-0000-0000-0000-000000000001'); + INSERT INTO entry_tiers (entry, tier) VALUES + ('00000000-0000-0000-0000-000000000002', '10000000-0000-0000-0000-000000000001'), + ('00000000-0000-0000-0000-000000000002', '10000000-0000-0000-0000-000000000002'), + ('00000000-0000-0000-0000-000000000002', '10000000-0000-0000-0000-000000000003'); + INSERT INTO entry_networks (entry, network, priority) VALUES + ('00000000-0000-0000-0000-000000000002', '20000000-0000-0000-0000-000000000001', 1), + ('00000000-0000-0000-0000-000000000002', '20000000-0000-0000-0000-000000000002', 2), + ('00000000-0000-0000-0000-000000000002', '20000000-0000-0000-0000-000000000003', 3);`) + require.NoError(t, err) + + apply("008_*.up.sql") + + assertLegacyName := func(table, id, want string) { + t.Helper() + var got string + err := tx.QueryRow(ctx, "SELECT name FROM "+table+" WHERE id=$1", id).Scan(&got) //nolint:gosec // fixed test table names + require.NoError(t, err) + require.Equal(t, want, got) + } + assertLegacyName("tiers", "10000000-0000-0000-0000-000000000001", "Legacy tier 10000000-0000-0000-0000-000000000001 (1)") + assertLegacyName("tiers", "10000000-0000-0000-0000-000000000002", "Legacy tier 10000000-0000-0000-0000-000000000002") + assertLegacyName("tiers", "10000000-0000-0000-0000-000000000003", "Legacy tier 10000000-0000-0000-0000-000000000001") + assertLegacyName("networks", "20000000-0000-0000-0000-000000000001", "Legacy network 20000000-0000-0000-0000-000000000001 (1)") + assertLegacyName("networks", "20000000-0000-0000-0000-000000000002", "Legacy network 20000000-0000-0000-0000-000000000002") + assertLegacyName("networks", "20000000-0000-0000-0000-000000000003", "Legacy network 20000000-0000-0000-0000-000000000001") + + var tierAssignments, networkAssignments int + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM entry_tiers").Scan(&tierAssignments)) + require.NoError(t, tx.QueryRow(ctx, "SELECT count(*) FROM entry_networks").Scan(&networkAssignments)) + require.Equal(t, 3, tierAssignments) + require.Equal(t, 3, networkAssignments) +} From ba0ead0f6f0512d8f0940d8907e5e61eb4fde04f Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Tue, 15 Sep 2026 14:47:31 +0200 Subject: [PATCH 17/18] Detect duplicate networks if entry has multiple symbols --- directory/import/db/repo_test.go | 69 +++++++++++++++++++++++++++++ directory/import/db/tier_network.go | 5 +++ 2 files changed, 74 insertions(+) diff --git a/directory/import/db/repo_test.go b/directory/import/db/repo_test.go index 847a6fa2f..b16da5d01 100644 --- a/directory/import/db/repo_test.go +++ b/directory/import/db/repo_test.go @@ -655,6 +655,42 @@ func TestImportTierConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) { require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) } +func TestImportTierRejectsDuplicateEntryAliases(t *testing.T) { + repo, consortium, first, second := importRepoFixture(t) + alias := addImportSymbolAlias(t, first) + ctx := context.Background() + aggregate := model.TierAggregate{ + Key: model.TierKey{Consortium: consortium, Name: "Aliases"}, + Data: model.TierData{Level: "standard", Type: "loan", Cost: 1.5, Entries: []model.SymbolRef{first, alias}}, + } + _, err := repo.ImportTier(ctx, aggregate, model.ConflictPolicyFail) + require.ErrorContains(t, err, "duplicate assignment") + require.ErrorContains(t, err, first.String()) + require.ErrorContains(t, err, alias.String()) + assertTierDoesNotExist(t, consortium, aggregate.Key.Name) + + aggregate.Data.Entries = []model.SymbolRef{second} + _, err = repo.ImportTier(ctx, aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + id := tierIDByKey(t, consortium, aggregate.Key.Name) + aggregate.Data.Entries = []model.SymbolRef{first, alias} + aggregate.Data.Cost = 99 + _, err = repo.ImportTier(ctx, aggregate, model.ConflictPolicyUpdate) + require.ErrorContains(t, err, "duplicate assignment") + require.Equal(t, []model.SymbolRef{second}, tierAssignments(t, id)) + var cost float64 + require.NoError(t, testPool.QueryRow(ctx, `SELECT cost FROM tiers WHERE id=$1`, id).Scan(&cost)) + require.Equal(t, 1.5, cost) +} + +func addImportSymbolAlias(t *testing.T, ref model.SymbolRef) model.SymbolRef { + t.Helper() + alias := model.SymbolRef{Authority: "ALIAS", Symbol: strings.ToUpper(uuid.NewString())} + _, err := testPool.Exec(context.Background(), `INSERT INTO symbols(owner, authority, symbol) VALUES($1, $2, $3)`, entryIDBySymbol(t, ref), alias.Authority, alias.Symbol) + require.NoError(t, err) + return alias +} + func TestExistingTierConflictPolicyPrecedesMissingAssignmentValidation(t *testing.T) { for _, policy := range []model.ConflictPolicy{model.ConflictPolicySkip, model.ConflictPolicyFail} { t.Run(string(policy), func(t *testing.T) { @@ -801,6 +837,39 @@ func TestImportNetworkConflictPoliciesAndUpdateReplacesAssignments(t *testing.T) require.Equal(t, []model.NetworkAssignment{{SymbolRef: second, Priority: 2}}, networkAssignments(t, id)) } +func TestImportNetworkRejectsDuplicateEntryAliases(t *testing.T) { + repo, consortium, first, second := importRepoFixture(t) + alias := addImportSymbolAlias(t, first) + ctx := context.Background() + aggregate := model.NetworkAggregate{ + Key: model.NetworkKey{Consortium: consortium, Name: "Aliases"}, + Data: model.NetworkData{Entries: []model.NetworkAssignment{ + {SymbolRef: first, Priority: 1}, {SymbolRef: alias, Priority: 99}, + }}, + } + _, err := repo.ImportNetwork(ctx, aggregate, model.ConflictPolicyFail) + require.ErrorContains(t, err, "duplicate assignment") + require.ErrorContains(t, err, first.String()) + require.ErrorContains(t, err, alias.String()) + var count int + require.NoError(t, testPool.QueryRow(ctx, `SELECT count(*) FROM networks WHERE consortium=$1 AND name=$2`, entryIDBySymbol(t, consortium), aggregate.Key.Name).Scan(&count)) + require.Zero(t, count) + + aggregate.Data.Entries = []model.NetworkAssignment{{SymbolRef: second, Priority: 5}} + _, err = repo.ImportNetwork(ctx, aggregate, model.ConflictPolicyFail) + require.NoError(t, err) + id := networkIDByKey(t, consortium, aggregate.Key.Name) + aggregate.Data.Entries = []model.NetworkAssignment{{SymbolRef: first, Priority: 1}, {SymbolRef: alias, Priority: 99}} + reciprocal := true + aggregate.Data.Reciprocal = &reciprocal + _, err = repo.ImportNetwork(ctx, aggregate, model.ConflictPolicyUpdate) + require.ErrorContains(t, err, "duplicate assignment") + require.Equal(t, []model.NetworkAssignment{{SymbolRef: second, Priority: 5}}, networkAssignments(t, id)) + var storedReciprocal *bool + require.NoError(t, testPool.QueryRow(ctx, `SELECT reciprocal FROM networks WHERE id=$1`, id).Scan(&storedReciprocal)) + require.Nil(t, storedReciprocal) +} + func TestExistingNetworkConflictPolicyPrecedesMissingAssignmentValidation(t *testing.T) { for _, policy := range []model.ConflictPolicy{model.ConflictPolicySkip, model.ConflictPolicyFail} { t.Run(string(policy), func(t *testing.T) { diff --git a/directory/import/db/tier_network.go b/directory/import/db/tier_network.go index fd3626f35..646b41f32 100644 --- a/directory/import/db/tier_network.go +++ b/directory/import/db/tier_network.go @@ -259,10 +259,15 @@ func lockAssignmentEntryRows(ctx context.Context, queries *db.Queries, ids ...uu func requireAssignmentEntries(assignments []resolvedAssignment) ([]db.Entry, error) { entries := make([]db.Entry, 0, len(assignments)) + seen := make(map[uuid.UUID]model.SymbolRef, len(assignments)) for _, assignment := range assignments { if assignment.entry == nil { return nil, fmt.Errorf("entry %s does not exist", assignment.ref.String()) } + if previous, exists := seen[assignment.entry.ID]; exists { + return nil, fmt.Errorf("duplicate assignment: symbols %s and %s identify the same entry", previous.String(), assignment.ref.String()) + } + seen[assignment.entry.ID] = assignment.ref entries = append(entries, *assignment.entry) } return entries, nil From 93476ee48fcc8970b67e2c79a59bc39d06e1c56a Mon Sep 17 00:00:00 2001 From: Jakub Skoczen Date: Tue, 15 Sep 2026 14:57:15 +0200 Subject: [PATCH 18/18] README --- directory/README.md | 132 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 120 insertions(+), 12 deletions(-) diff --git a/directory/README.md b/directory/README.md index d3e63a0a3..9d262db1e 100644 --- a/directory/README.md +++ b/directory/README.md @@ -19,6 +19,126 @@ applied automatically during startup: DATABASE_URL=postgresql://postgres:directory@localhost:54322/directory make run ``` +## Import + +`POST /directory/import` loads complete directory entries, tiers, and networks +from a newline-delimited JSON (NDJSON) stream. It requires the +`directory.consortium.all` permission. Like the [broker import API](../broker/import/README.md), +it processes each record in its own transaction and supports `fail`, `skip`, +and `update` conflict policies. + +### Request format and sample data + +Each record has `type`, `key`, and `data` fields. Put one complete JSON object +on each physical line, without an enclosing array. Blank lines are ignored. + +| Type | Key | Data | +| --- | --- | --- | +| `entry` | Symbol `authority` and `symbol` | Complete entry fields, owned collections, and configurations | +| `tier` | Consortium symbol and tier `name` | `level`, `type`, `cost`, and entry symbols | +| `network` | Consortium symbol and network `name` | `reciprocal` and entry symbols with individual `priority` values | + +Symbol authorities and values are trimmed and uppercased. An entry's key must +appear exactly once in its `symbols` array. Referenced entries must already +exist or have been imported by an earlier successful record: put consortiums +before institutions, institutions before branches, and members before tiers +and networks. The same applies to references in `illConfig.lendersOfLastResort`. +Different symbols resolving to the same entry cannot appear twice in a tier or +network's membership list. + +Save the following as `directory.ndjson`. This example assumes an empty +directory; only one consortium entry is allowed. For an existing directory, +use its consortium symbol and choose the appropriate conflict policy. + +```ndjson +{"type":"entry","key":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"data":{"name":"Example consortium","type":"Consortium","parent":null,"description":null,"organizationId":null,"contactName":null,"email":null,"fromEmail":null,"tenant":null,"vendor":null,"phoneNumber":null,"lmsLocationCode":null,"hrid":null,"timeZone":null,"symbols":[{"authority":"ISIL","symbol":"EXAMPLE-CON"}],"endpoints":[],"addresses":[],"closures":[],"lmsConfig":null,"catalogConfig":null,"illConfig":null,"holdingsPolicy":null}} +{"type":"entry","key":{"authority":"ISIL","symbol":"EXAMPLE-LIB"},"data":{"name":"Example library","type":"Institution","parent":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"description":null,"organizationId":null,"contactName":null,"email":null,"fromEmail":null,"tenant":null,"vendor":null,"phoneNumber":null,"lmsLocationCode":null,"hrid":null,"timeZone":null,"symbols":[{"authority":"ISIL","symbol":"EXAMPLE-LIB"}],"endpoints":[],"addresses":[],"closures":[],"lmsConfig":null,"catalogConfig":null,"illConfig":null,"holdingsPolicy":null}} +{"type":"tier","key":{"consortium":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"name":"Standard loan"},"data":{"level":"standard","type":"loan","cost":0,"entries":[{"authority":"ISIL","symbol":"EXAMPLE-LIB"}]}} +{"type":"network","key":{"consortium":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"name":"Main network"},"data":{"reciprocal":true,"entries":[{"authority":"ISIL","symbol":"EXAMPLE-LIB","priority":1}]}} +``` + +Entry imports require every field shown, including explicit `null` values for +absent optional values or configurations and empty arrays for empty collections. +Non-null configuration objects also have required fields. Unknown fields are +rejected; see the `ImportEntryRecord`, `ImportTierRecord`, and +`ImportNetworkRecord` schemas in [api.yaml](api.yaml) for the complete contract. +Database IDs are generated by the service. + +### Import with curl + +For the local service started above: + +```sh +curl --fail-with-body \ + -X POST \ + -H 'Content-Type: application/x-ndjson' \ + -H 'X-Okapi-Permissions: ["directory.consortium.all"]' \ + --data-binary @directory.ndjson \ + 'http://localhost:8086/directory/import?conflictPolicy=fail' +``` + +Use `--data-binary` to preserve line boundaries. The permissions header above +is for direct local access. When accessing the service through a gateway, use +that deployment's authentication and tenant headers with an account granted +`directory.consortium.all`. + +### Conflict policies + +Set `conflictPolicy` in the URL to `fail`, `skip`, or `update`. All three policies +create a resource when its key does not exist. For an existing key: + +| Policy | Behavior | +| --- | --- | +| `fail` (default) | Leaves the resource unchanged, increments `failed`, and adds an error detail. Continues with later records. | +| `skip` | Leaves the resource unchanged, increments `skipped`, and adds a diagnostic to `errors`. | +| `update` | Replaces the resource's data, preserves its root database ID, and increments `imported`. | + +An entry matches by its key symbol; a tier or network matches by the resolved +consortium entry and exact name. For example, reimporting the sample with `skip` +leaves all four resources unchanged. With `update`, changing the network's +priority to `5` replaces that library's priority with `5`. + +Updates synchronize complete aggregates, rather than patching individual fields. +For entries, submitted symbols, endpoints, addresses, closures, and configurations +replace the existing values; owned child IDs can change. An empty collection +removes its existing contents, and a `null` configuration removes that +configuration. Entry imports do not change tier or network memberships; import +the corresponding tier or network to replace its full membership list. +For example, updating a network with `"entries":[]` removes all its memberships. +Validation still applies under every policy, and a failed record leaves that +record's existing data unchanged. + +### Response and limits + +A completed stream returns HTTP 200 even if some records failed or were skipped. +The initial sample import returns: + +```json +{ + "entries": {"imported": 2, "failed": 0, "skipped": 0}, + "tiers": {"imported": 1, "failed": 0, "skipped": 0}, + "networks": {"imported": 1, "failed": 0, "skipped": 0}, + "errors": [], + "errorsOmitted": 0 +} +``` + +Always inspect the counters and `errors`, even when curl succeeds. Error details +include `line`, `error`, and, when available, `type` and `key`. Here, `line` is the +one-based **nonblank record number**, not the physical line number. At most 1,000 +error or skip details are retained; `errorsOmitted` counts additional details. + +| Status | Meaning | +| --- | --- | +| `400` | Missing body, unsupported conflict policy, or incorrect content type | +| `401` | Missing consortial-admin permission | +| `413` | Request exceeds 1 GiB or a record exceeds 1 MiB | +| `500` | Import stream could not be read | + +Record-level errors do not stop later records. Fatal stream errors stop the +import, but records already committed remain committed, including when the +response is HTTP 413 or 500. + ## Build and test The SQLC and OpenAPI generator versions are pinned as Go tools in `go.mod`. @@ -40,18 +160,6 @@ make run Run `make generate` before invoking `go build` or `go test` directly. -## Some examples of repositories using SQLC or API generation - -### Contrived -- https://github.com/SeaRoll/api-sqlc-goose/tree/main -- https://github.com/danicc097/openapi-go-gin-postgres-sqlc -- https://github.com/kwryoh/oapi-sample -- https://github.com/aliml92/realworld-gin-sqlc/tree/master - -### Real -- https://github.com/leg100/otf -- https://github.com/helpwave/services/tree/main/services/tasks-svc - ## Environment variables | Name | Description | Default value |