ILLDEV-484 Add directory import endpoint

broker/adapter/api_directory.go

@@ -456,12 +456,10 @@ func getPeerNetworks(peerData dirapi.Entry) map[string]Network {
 	networks := map[string]Network{}
 	if peerData.Networks != nil {
 		for _, n := range *peerData.Networks {
-			if n.Name != nil {
-				networks[*n.Name] = Network{
-					Name:       *n.Name,
-					Priority:   int(n.Priority),
-					Reciprocal: n.Reciprocal,
-				}
+			networks[n.Name] = Network{
+				Name:       n.Name,
+				Priority:   int(n.Priority),
+				Reciprocal: n.Reciprocal,
 			}
 		}
 	}
@@ -472,12 +470,8 @@ func getPeerTiers(peerData dirapi.Entry) []Tier {
 	tiers := []Tier{}
 	if peerData.Tiers != nil {
 		for _, t := range *peerData.Tiers {
-			name := ""
-			if t.Name != nil {
-				name = *t.Name
-			}
 			tiers = append(tiers, Tier{
-				Name:  name,
+				Name:  t.Name,
 				Level: string(t.Level),
 				Type:  string(t.Type),
 				Cost:  t.Cost,

broker/test/adapter/api_directory_test.go

@@ -41,10 +41,6 @@ func boolPtr(v bool) *bool {
 	return &v
 }
 
-func stringPtr(v string) *string {
-	return &v
-}
-
 func withNetworkReciprocal(entry dirapi.Entry, reciprocal *bool) dirapi.Entry {
 	if entry.Networks == nil {
 		return entry
@@ -774,20 +770,20 @@ func TestFilterAndSortUsesCompatibleNetworkPriority(t *testing.T) {
 	appCtx := createLookupCtx()
 	ad := createDirectoryAdapter("")
 	requesterNetworks := []dirapi.EntryNetworkDetails{
-		{Name: stringPtr("Reciprocal"), Priority: 1, Reciprocal: boolPtr(true)},
-		{Name: stringPtr("Paid Low"), Priority: 5, Reciprocal: boolPtr(false)},
-		{Name: stringPtr("Paid High"), Priority: 3, Reciprocal: boolPtr(false)},
+		{Name: "Reciprocal", Priority: 1, Reciprocal: boolPtr(true)},
+		{Name: "Paid Low", Priority: 5, Reciprocal: boolPtr(false)},
+		{Name: "Paid High", Priority: 3, Reciprocal: boolPtr(false)},
 	}
 	paidTier := []dirapi.Tier{
-		{Name: stringPtr("Paid Core Loan"), Level: dirapi.Standard, Type: dirapi.Loan, Cost: 34.4},
+		{Name: "Paid Core Loan", Level: dirapi.Standard, Type: dirapi.Loan, Cost: 34.4},
 	}
 	requesterData := dirapi.Entry{Name: "Requester", Networks: &requesterNetworks}
 	supplierANetworks := []dirapi.EntryNetworkDetails{
-		{Name: stringPtr("Reciprocal"), Priority: 1, Reciprocal: boolPtr(true)},
-		{Name: stringPtr("Paid Low"), Priority: -10, Reciprocal: boolPtr(false)},
+		{Name: "Reciprocal", Priority: 1, Reciprocal: boolPtr(true)},
+		{Name: "Paid Low", Priority: -10, Reciprocal: boolPtr(false)},
 	}
 	supplierBNetworks := []dirapi.EntryNetworkDetails{
-		{Name: stringPtr("Paid High"), Priority: 99, Reciprocal: boolPtr(false)},
+		{Name: "Paid High", Priority: 99, Reciprocal: boolPtr(false)},
 	}
 	entries := []adapter.Supplier{
 		{PeerId: "A", Symbol: "A", CustomData: dirapi.Entry{Name: "Supplier A", Networks: &supplierANetworks, Tiers: &paidTier}},
@@ -916,14 +912,11 @@ func TestCompareSuppliers(t *testing.T) {
 	assert.False(t, suppliers[1].Local)
 }
 
-func strPtr(i string) *string {
-	return &i
-}
 func TestFilterAndSortAppliesHoldingsPolicy(t *testing.T) {
 	appCtx := createLookupCtx()
 	ad := createDirectoryAdapter("")
-	networks := []dirapi.EntryNetworkDetails{{Name: strPtr("Reciprocal"), Priority: 1}}
-	tiers := []dirapi.Tier{{Name: strPtr("Core Loan"), Level: "Core", Type: "Loan", Cost: 0}}
+	networks := []dirapi.EntryNetworkDetails{{Name: "Reciprocal", Priority: 1}}
+	tiers := []dirapi.Tier{{Name: "Core Loan", Level: "Core", Type: "Loan", Cost: 0}}
 	customData := dirapi.Entry{
 		Name: "Supplier", Networks: &networks, Tiers: &tiers,
 		HoldingsPolicy: &dirapi.HoldingsPolicy{
@@ -979,15 +972,15 @@ func TestFilterAndSortAppliesHoldingsPolicy(t *testing.T) {
 }
 
 func TestFilterAndSortResolvesHoldingsPolicyForNonMatchingSuppliers(t *testing.T) {
-	networks := []dirapi.EntryNetworkDetails{{Name: strPtr("Reciprocal"), Priority: 1}}
-	tiers := []dirapi.Tier{{Name: strPtr("Core Loan"), Level: "Core", Type: "Loan", Cost: 0}}
+	networks := []dirapi.EntryNetworkDetails{{Name: "Reciprocal", Priority: 1}}
+	tiers := []dirapi.Tier{{Name: "Core Loan", Level: "Core", Type: "Loan", Cost: 0}}
 	requester := dirapi.Entry{Networks: &networks, Tiers: &tiers}
 	for _, tc := range []struct {
 		name     string
 		networks []dirapi.EntryNetworkDetails
 		tiers    []dirapi.Tier
 	}{
-		{name: "no shared network", networks: []dirapi.EntryNetworkDetails{{Name: strPtr("Other"), Priority: 1}}, tiers: tiers},
+		{name: "no shared network", networks: []dirapi.EntryNetworkDetails{{Name: "Other", Priority: 1}}, tiers: tiers},
 		{name: "no matching tier", networks: networks, tiers: []dirapi.Tier{{Type: "Copy", Cost: 0}}},
 		{name: "no matching cost", networks: networks, tiers: []dirapi.Tier{{Type: "Loan", Cost: 10}}},
 	} {
@@ -1016,3 +1009,7 @@ func TestFilterAndSortResolvesHoldingsPolicyForNonMatchingSuppliers(t *testing.T
 		})
 	}
 }
+
+func strPtr(s string) *string {
+	return &s
+}

directory/README.md

@@ -19,6 +19,126 @@ applied automatically during startup:
 DATABASE_URL=postgresql://postgres:directory@localhost:54322/directory make run
 ```
 
+## Import
+
+`POST /directory/import` loads complete directory entries, tiers, and networks
+from a newline-delimited JSON (NDJSON) stream. It requires the
+`directory.consortium.all` permission. Like the [broker import API](../broker/import/README.md),
+it processes each record in its own transaction and supports `fail`, `skip`,
+and `update` conflict policies.
+
+### Request format and sample data
+
+Each record has `type`, `key`, and `data` fields. Put one complete JSON object
+on each physical line, without an enclosing array. Blank lines are ignored.
+
+| Type | Key | Data |
+| --- | --- | --- |
+| `entry` | Symbol `authority` and `symbol` | Complete entry fields, owned collections, and configurations |
+| `tier` | Consortium symbol and tier `name` | `level`, `type`, `cost`, and entry symbols |
+| `network` | Consortium symbol and network `name` | `reciprocal` and entry symbols with individual `priority` values |
+
+Symbol authorities and values are trimmed and uppercased. An entry's key must
+appear exactly once in its `symbols` array. Referenced entries must already
+exist or have been imported by an earlier successful record: put consortiums
+before institutions, institutions before branches, and members before tiers
+and networks. The same applies to references in `illConfig.lendersOfLastResort`.
+Different symbols resolving to the same entry cannot appear twice in a tier or
+network's membership list.
+
+Save the following as `directory.ndjson`. This example assumes an empty
+directory; only one consortium entry is allowed. For an existing directory,
+use its consortium symbol and choose the appropriate conflict policy.
+
+```ndjson
+{"type":"entry","key":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"data":{"name":"Example consortium","type":"Consortium","parent":null,"description":null,"organizationId":null,"contactName":null,"email":null,"fromEmail":null,"tenant":null,"vendor":null,"phoneNumber":null,"lmsLocationCode":null,"hrid":null,"timeZone":null,"symbols":[{"authority":"ISIL","symbol":"EXAMPLE-CON"}],"endpoints":[],"addresses":[],"closures":[],"lmsConfig":null,"catalogConfig":null,"illConfig":null,"holdingsPolicy":null}}
+{"type":"entry","key":{"authority":"ISIL","symbol":"EXAMPLE-LIB"},"data":{"name":"Example library","type":"Institution","parent":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"description":null,"organizationId":null,"contactName":null,"email":null,"fromEmail":null,"tenant":null,"vendor":null,"phoneNumber":null,"lmsLocationCode":null,"hrid":null,"timeZone":null,"symbols":[{"authority":"ISIL","symbol":"EXAMPLE-LIB"}],"endpoints":[],"addresses":[],"closures":[],"lmsConfig":null,"catalogConfig":null,"illConfig":null,"holdingsPolicy":null}}
+{"type":"tier","key":{"consortium":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"name":"Standard loan"},"data":{"level":"standard","type":"loan","cost":0,"entries":[{"authority":"ISIL","symbol":"EXAMPLE-LIB"}]}}
+{"type":"network","key":{"consortium":{"authority":"ISIL","symbol":"EXAMPLE-CON"},"name":"Main network"},"data":{"reciprocal":true,"entries":[{"authority":"ISIL","symbol":"EXAMPLE-LIB","priority":1}]}}
+```
+
+Entry imports require every field shown, including explicit `null` values for
+absent optional values or configurations and empty arrays for empty collections.
+Non-null configuration objects also have required fields. Unknown fields are
+rejected; see the `ImportEntryRecord`, `ImportTierRecord`, and
+`ImportNetworkRecord` schemas in [api.yaml](api.yaml) for the complete contract.
+Database IDs are generated by the service.
+
+### Import with curl
+
+For the local service started above:
+
+```sh
+curl --fail-with-body \
+  -X POST \
+  -H 'Content-Type: application/x-ndjson' \
+  -H 'X-Okapi-Permissions: ["directory.consortium.all"]' \
+  --data-binary @directory.ndjson \
+  'http://localhost:8086/directory/import?conflictPolicy=fail'
+```
+
+Use `--data-binary` to preserve line boundaries. The permissions header above
+is for direct local access. When accessing the service through a gateway, use
+that deployment's authentication and tenant headers with an account granted
+`directory.consortium.all`.
+
+### Conflict policies
+
+Set `conflictPolicy` in the URL to `fail`, `skip`, or `update`. All three policies
+create a resource when its key does not exist. For an existing key:
+
+| Policy | Behavior |
+| --- | --- |
+| `fail` (default) | Leaves the resource unchanged, increments `failed`, and adds an error detail. Continues with later records. |
+| `skip` | Leaves the resource unchanged, increments `skipped`, and adds a diagnostic to `errors`. |
+| `update` | Replaces the resource's data, preserves its root database ID, and increments `imported`. |
+
+An entry matches by its key symbol; a tier or network matches by the resolved
+consortium entry and exact name. For example, reimporting the sample with `skip`
+leaves all four resources unchanged. With `update`, changing the network's
+priority to `5` replaces that library's priority with `5`.
+
+Updates synchronize complete aggregates, rather than patching individual fields.
+For entries, submitted symbols, endpoints, addresses, closures, and configurations
+replace the existing values; owned child IDs can change. An empty collection
+removes its existing contents, and a `null` configuration removes that
+configuration. Entry imports do not change tier or network memberships; import
+the corresponding tier or network to replace its full membership list.
+For example, updating a network with `"entries":[]` removes all its memberships.
+Validation still applies under every policy, and a failed record leaves that
+record's existing data unchanged.
+
+### Response and limits
+
+A completed stream returns HTTP 200 even if some records failed or were skipped.
+The initial sample import returns:
+
+```json
+{
+  "entries": {"imported": 2, "failed": 0, "skipped": 0},
+  "tiers": {"imported": 1, "failed": 0, "skipped": 0},
+  "networks": {"imported": 1, "failed": 0, "skipped": 0},
+  "errors": [],
+  "errorsOmitted": 0
+}
+```
+
+Always inspect the counters and `errors`, even when curl succeeds. Error details
+include `line`, `error`, and, when available, `type` and `key`. Here, `line` is the
+one-based **nonblank record number**, not the physical line number. At most 1,000
+error or skip details are retained; `errorsOmitted` counts additional details.
+
+| Status | Meaning |
+| --- | --- |
+| `400` | Missing body, unsupported conflict policy, or incorrect content type |
+| `401` | Missing consortial-admin permission |
+| `413` | Request exceeds 1 GiB or a record exceeds 1 MiB |
+| `500` | Import stream could not be read |
+
+Record-level errors do not stop later records. Fatal stream errors stop the
+import, but records already committed remain committed, including when the
+response is HTTP 413 or 500.
+
 ## Build and test
 
 The SQLC and OpenAPI generator versions are pinned as Go tools in `go.mod`.
@@ -40,18 +160,6 @@ make run
 
 Run `make generate` before invoking `go build` or `go test` directly.
 
-## Some examples of repositories using SQLC or API generation
-
-### Contrived
-- https://github.com/SeaRoll/api-sqlc-goose/tree/main
-- https://github.com/danicc097/openapi-go-gin-postgres-sqlc
-- https://github.com/kwryoh/oapi-sample
-- https://github.com/aliml92/realworld-gin-sqlc/tree/master
-
-### Real
-- https://github.com/leg100/otf
-- https://github.com/helpwave/services/tree/main/services/tasks-svc
-
 ## Environment variables
 
 | Name                      | Description                                                                    | Default value                                               |