perf: reduce Vamana memory through shared vector storage

collection.go

@@ -584,9 +584,22 @@ func openCollectionDenseArtifact(
 		return core.OpenScalarQuantizedIVFIndex(ctx, path, kind, reformer)
 	case IndexTypeVamana:
 		if spec.quantize == QuantizeTypeUndefined {
-			return core.OpenVamanaIndex(ctx, path)
+			return core.OpenVamanaIndexWithMmap(ctx, path, useMmap)
 		}
-		return core.OpenScalarQuantizedVamanaIndex(ctx, path, kind, reformer)
+		if field.DataType == DataTypeVectorFP32 {
+			reader, keys, err := collectionEncodedDenseReader(ctx, field, documents)
+			if err != nil {
+				return nil, err
+			}
+			if reader != nil {
+				originals := make(map[uint64][]byte, len(keys))
+				for position, key := range keys {
+					originals[key] = reader.rows[position]
+				}
+				return core.OpenScalarQuantizedVamanaIndexWithEncodedVectors(ctx, path, kind, reformer, originals, useMmap)
+			}
+		}
+		return core.OpenScalarQuantizedVamanaIndexWithMmap(ctx, path, kind, reformer, useMmap)
 	case IndexTypeDiskANN:
 		if spec.quantize == QuantizeTypeUndefined {
 			candidateCount, candidateErr := collectionDenseCandidateCount(ctx, field, documents)
@@ -664,7 +677,7 @@ func (c *Collection) segmentDocumentsLocked(ctx context.Context) ([]collectionSe
 			if err != nil {
 				return nil, err
 			}
-			if spec.indexType == IndexTypeHNSW || (spec.indexType == IndexTypeFlat && spec.quantize != QuantizeTypeUndefined) {
+			if spec.indexType == IndexTypeHNSW || spec.indexType == IndexTypeVamana || (spec.indexType == IndexTypeFlat && spec.quantize != QuantizeTypeUndefined) {
 				borrowedFields[field.Name] = struct{}{}
 			}
 		}
@@ -1027,7 +1040,7 @@ func buildCollectionIndexes(
 			}
 			if field.DataType.IsDenseVector() {
 				var exact collectionDenseIndex
-				useLazyExact := spec.indexType == IndexTypeHNSW ||
+				useLazyExact := spec.indexType == IndexTypeHNSW || spec.indexType == IndexTypeVamana ||
 					(spec.indexType == IndexTypeFlat && spec.quantize != QuantizeTypeUndefined) ||
 					(spec.indexType == IndexTypeDiskANN && spec.quantize == QuantizeTypeUndefined)
 				if field.DataType == DataTypeVectorFP32 && useLazyExact {
@@ -1054,8 +1067,8 @@ func buildCollectionIndexes(
 				var flat collectionDenseIndex
 				if spec.quantize == QuantizeTypeUndefined || spec.indexType == IndexTypeHNSWRaBitQ || spec.indexType == IndexTypeIVFRaBitQ {
 					flat = exact
-				} else if spec.indexType != IndexTypeHNSW {
-					// Quantized HNSW supplies a shared Flat view after opening the graph.
+				} else if spec.indexType != IndexTypeHNSW && spec.indexType != IndexTypeVamana {
+					// Quantized graphs supply a shared Flat view after opening.
 					flat, err = buildCollectionDenseFlat(ctx, schema.Name, field, documents, spec)
 					if err != nil {
 						return fail(err)
@@ -1081,9 +1094,11 @@ func buildCollectionIndexes(
 				}
 				indexes.denseNative[field.Name] = native
 				if flat == nil {
-					quantized, ok := native.(*core.ScalarQuantizedHNSWIndex)
+					quantized, ok := native.(interface {
+						FlatIndex() *core.ScalarQuantizedFlatIndex
+					})
 					if !ok {
-						return fail(fmt.Errorf("quantized HNSW field %q has an incompatible native index", field.Name))
+						return fail(fmt.Errorf("quantized graph field %q has an incompatible native index", field.Name))
 					}
 					indexes.denseFlat[field.Name] = quantized.FlatIndex()
 				}
@@ -2529,7 +2544,7 @@ func buildCollectionDenseVamana(
 	spec collectionVectorIndex,
 	workers int,
 ) (collectionVamanaIndex, error) {
-	candidates, err := collectionDenseCandidates(ctx, field, documents)
+	count, err := collectionDenseCandidateCount(ctx, field, documents)
 	if err != nil {
 		return nil, err
 	}
@@ -2542,6 +2557,24 @@ func buildCollectionDenseVamana(
 		options.MaxOcclusionSize = core.DefaultVamanaMaxOcclusionSize
 	}
 	options.SaturateGraph = spec.vamana.SaturateGraph
+	if field.DataType == DataTypeVectorFP32 {
+		candidates, err := collectionDenseBorrowedCandidates(ctx, field, documents)
+		if err != nil {
+			return nil, err
+		}
+		if spec.quantize == QuantizeTypeUndefined {
+			return core.BuildVamanaWithBorrowedVectors(ctx, int(field.Dimension), options, candidates, workers)
+		}
+		kind, err := toCoreQuantization(spec.quantize)
+		if err != nil {
+			return nil, err
+		}
+		reformer, err := collectionReformer(schemaName, field, spec)
+		if err != nil {
+			return nil, err
+		}
+		return core.BuildScalarQuantizedVamanaWithBorrowedVectors(ctx, int(field.Dimension), options, candidates, workers, kind, reformer)
+	}
 	var builder *core.VamanaBuilder
 	if field.DataType == DataTypeVectorFP16 && spec.quantize == QuantizeTypeUndefined {
 		builder, err = core.NewVamanaBuilderFP16(int(field.Dimension), options)
@@ -2551,17 +2584,27 @@ func buildCollectionDenseVamana(
 	if err != nil {
 		return nil, err
 	}
-	for _, candidate := range candidates {
-		if err := builder.Add(ctx, candidate.Key, candidate.Vector); err != nil {
+	if err := builder.Reserve(count); err != nil {
+		return nil, err
+	}
+	for _, document := range documents {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		value, found := document.Fields[field.Name]
+		if !found || value == nil {
+			continue
+		}
+		vector, err := denseValueToFloat32Borrowed(value)
+		if err != nil {
+			return nil, fmt.Errorf("document %d field %q: %w", document.DocID, field.Name, err)
+		}
+		if err := builder.Add(ctx, document.DocID, vector); err != nil {
 			return nil, err
 		}
-	}
-	base, err := builder.BuildInterleavedWithWorkers(ctx, workers)
-	if err != nil {
-		return nil, err
 	}
 	if spec.quantize == QuantizeTypeUndefined {
-		return base, nil
+		return builder.BuildInterleavedWithWorkers(ctx, workers)
 	}
 	kind, err := toCoreQuantization(spec.quantize)
 	if err != nil {
@@ -2571,7 +2614,7 @@ func buildCollectionDenseVamana(
 	if err != nil {
 		return nil, err
 	}
-	return core.NewScalarQuantizedVamanaIndex(ctx, base, kind, reformer)
+	return builder.BuildScalarQuantizedInterleavedWithWorkers(ctx, workers, kind, reformer)
 }
 
 func buildCollectionDenseDiskANN(

collection_memory_test.go

@@ -26,12 +26,30 @@ import (
 )
 
 func TestHNSWRuntimeSharesFlatAndDefersExact(t *testing.T) {
+	testGraphRuntimeSharesFlatAndDefersExact(t, IndexTypeHNSW)
+}
+
+func TestVamanaRuntimeSharesFlatAndDefersExact(t *testing.T) {
+	testGraphRuntimeSharesFlatAndDefersExact(t, IndexTypeVamana)
+}
+
+func testGraphRuntimeSharesFlatAndDefersExact(t *testing.T, indexType IndexType) {
 	ctx := context.Background()
 	for _, quantize := range []QuantizeType{QuantizeTypeUndefined, QuantizeTypeFP16, QuantizeTypeInt8, QuantizeTypeInt4} {
 		t.Run(fmt.Sprint(quantize), func(t *testing.T) {
-			params := NewHNSWIndexParams(MetricTypeL2)
-			params.M, params.EFConstruction, params.Quantize = 4, 16, quantize
-			params.Quantizer.EnableRotate = quantize == QuantizeTypeInt4 || quantize == QuantizeTypeInt8
+			var params IndexParams
+			rotate := quantize == QuantizeTypeInt4 || quantize == QuantizeTypeInt8
+			if indexType == IndexTypeVamana {
+				value := NewVamanaIndexParams(MetricTypeL2)
+				value.MaxDegree, value.SearchListSize, value.Quantize = 4, 16, quantize
+				value.Quantizer.EnableRotate = rotate
+				params = value
+			} else {
+				value := NewHNSWIndexParams(MetricTypeL2)
+				value.M, value.EFConstruction, value.Quantize = 4, 16, quantize
+				value.Quantizer.EnableRotate = rotate
+				params = value
+			}
 			field := FieldSchema{Name: "embedding", DataType: DataTypeVectorFP32, Dimension: 4, Nullable: true, Index: params}
 			schema := NewCollectionSchema("shared_hnsw", field)
 			documents := annDenseDocuments(48)
@@ -100,14 +118,24 @@ func TestHNSWRuntimeSharesFlatAndDefersExact(t *testing.T) {
 					require.NoError(t, indexes.denseNative[field.Name].(interface {
 						Save(context.Context, string) error
 					}).Save(ctx, path))
-					artifacts = map[string]string{collectionIndexArtifactKey(field.Name, collectionVectorArtifactKind(IndexTypeHNSW)): path}
+					artifacts = map[string]string{collectionIndexArtifactKey(field.Name, collectionVectorArtifactKind(indexType)): path}
 				}
 				require.NoError(t, indexes.Close())
 			}
 		})
 	}
 }
 
+func TestImmutableVamanaDocumentsSharedAcrossQuerySnapshots(t *testing.T) {
+	for _, kind := range []QuantizeType{QuantizeTypeUndefined, QuantizeTypeInt4} {
+		t.Run(fmt.Sprint(kind), func(t *testing.T) {
+			params := NewVamanaIndexParams(MetricTypeL2)
+			params.MaxDegree, params.SearchListSize, params.Quantize = 4, 16, kind
+			testImmutableDocumentsSharedAcrossQuerySnapshots(t, params)
+		})
+	}
+}
+
 func TestImmutableHNSWDocumentsSharedAcrossQuerySnapshots(t *testing.T) {
 	params := NewHNSWIndexParams(MetricTypeL2)
 	params.M, params.EFConstruction, params.Quantize = 4, 16, QuantizeTypeInt4

docs/README.md

@@ -34,9 +34,14 @@ two-pass construction, contiguous-memory mode, ID maps, and refinement are
 disabled. INT4/INT8 enable rotation, verified through the native parameter getter.
 The Vamana-specific CSV columns record these settings and participate in grouping.
 
-These runs use xvec commit `6a8b120d4284bf16853b2b4ad18465c599e72d82` (merged
-PR #91), zvec-go `v0.7.0+rotate`, and the unchanged native zvec library built from
-`8321c1314a559fd5f909e92498f43e5194bf9b99`. They use Go 1.27.1 with
+The xvec rows were rerun on 2026-09-28 using commit
+`5d9b8f5ff98f13ba0a1d5ee65f9d53dd57456997` plus the Vamana shared-vector memory patch
+`586bbd53868e` (the suffix in `backend_version` identifies this uncommitted patch).
+The zvec rows retain the previous measurements using zvec-go `v0.7.0+rotate`
+and the native library built from `8321c1314a559fd5f909e92498f43e5194bf9b99`.
+[Raw reports and provenance](benchmark-runs/vamana-borrowed-20260928/) retain
+commands, binary/dataset/source checksums, process resource measurements, and
+the previous CSV. All rows use Go 1.27.1 with
 `CGO_ENABLED=0`, `GOMAXPROCS=8`, `GOMEMLIMIT=24GiB`, and CPU affinity 0–7
 on e2-standard-8. Each run uses a fresh collection, all 100,000 vectors,
 1,000 serial queries, K=100, batch size 100, optimize concurrency 8,

docs/benchmark-runs/vamana-borrowed-20260928/README.md

@@ -0,0 +1,28 @@
+# Vamana shared-vector memory rerun, 2026-09-28
+
+These artifacts support the four updated xvec rows in
+[`../../benchmark-vamana.csv`](../../benchmark-vamana.csv).
+The zvec rows are unchanged historical measurements. `previous.csv` contains
+the runtime-memory rerun; the original historical CSV is in
+`../vamana-memory-20260928/previous.csv`.
+
+- `xvec-*.json`: original benchmark reports and separate `*.resources.json`
+  files from `wait4` (KiB RSS, seconds for wall/user/system time).
+- `xvec-*.log`: benchmark output.
+- `metadata.json`: build version, command lines, environment settings, binary,
+  source-patch and dataset hashes, and successful exit statuses.
+- `source.patch`: the exact production-code patch applied to the recorded base,
+  including the new borrowed-vector implementation.
+- `previous.csv`: measurements before this rerun.
+- `run.py`: the sequential runner and resource measurement implementation.
+- `update_csv.py`: validation and mapping of raw metrics into CSV fields.
+
+The scripts record this machine's paths; adjust `root` and `repo` when
+reproducing elsewhere. Use the source patch with the recorded base commit,
+build with `CGO_ENABLED=0`, and download the three files from
+`https://assets.zilliz.com/benchmark/cohere_small_100k/` into `dataset/` before
+running. Use fresh collection paths. Downloads and compilation occur outside
+the measured child processes. `run.py` includes untracked production files
+in the patch checksum as well as the tracked-file diff.
+
+See [the analysis](../../benchmark-vamana-memory.md) for results and limitations.