mirror of
https://github.com/FerretDB/documentdb.git
synced 2026-11-07 11:32:47 -05:00
Zinan Ren
f4896e9c81
### Does this PR have any customer impact? No ### Type (Feature, Refactoring, Bugfix, DevOps, Testing, Perf, etc) Testing ### Does it involve schema level changes? (Table, Column, Index, UDF, etc level changes) No ### Are you introducing any new config? If yes, do you have tests with and without them being set? No ### ChangeLog (Refer [Template](../oss/CHANGELOG.md)) ### Description 1. Check the max dimensions when creating the vector index, previously checked by pgvector 2. Add the following release validation tests: - HNSW index (M40 and above) - DiskANN index (M30 and above) - Filtering search (GA) - Exact search (GA) - Compression half (GA) - Compression PQ (GA) ---- #### AI description (iteration 1) #### PR Classification This pull request implements new vector search release validation tests and updates the backend’s vector index dimension checks and error messaging. #### PR Summary The changes add comprehensive release validation tests for vector search covering various compression types (PQ, half), index kinds (ivf, hnsw, diskann), filtering, and exact match scenarios. - `outerLoop/releaseValidation/testSpecs/` now includes multiple new JSON spec files (e.g. ValidateVectorSearchCompressionPQ.json, ValidateVectorSearchCompressionHalf.json, ValidateVectorSearchDiskANN.json, ValidateVectorSearchHNSW.json, ValidateVectorSearchFiltering.json, and ValidateVectorSearchExact.json) with updated expected errors. - Test scripts in `ReleaseValidation.Tests.NodeJs/test/test_signoffTestsNodeJS.js` and C# signoff test file `SignoffTestsCSharp.cs` have been modified to enable and incorporate these new vector search tests. - In `pg_documentdb/src/commands/create_indexes.c`, additional checks enforce maximum dimensions based on compression type (non-compressed: 2000, half: 4000, PQ: as defined) and update error messages accordingly. - Several Postgres gateway and pgmongo files have been updated to remove obsolete regexes and return revised error codes (e.g. error code 67) and messages for exceeding vector dimension limits. <!-- GitOpsUserAgent=GitOps.Apps.Server.pullrequestcopilot -->
94 lines
2.8 KiB
C
94 lines
2.8 KiB
C
/*-------------------------------------------------------------------------
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
*
|
|
* include/vector/vector_common.h
|
|
*
|
|
* Common definitions for vector indexes.
|
|
*
|
|
*-------------------------------------------------------------------------
|
|
*/
|
|
#ifndef VECTOR_COMMON__H
|
|
#define VECTOR_COMMON__H
|
|
|
|
/* pgvector VECTOR_MAX_DIM: 16000 */
|
|
/* dimensions for type vector cannot exceed 16000 */
|
|
#define VECTOR_MAX_DIMENSIONS 16000
|
|
#define VECTOR_MAX_DIMENSIONS_NON_COMPRESSED 2000
|
|
#define VECTOR_MAX_DIMENSIONS_HALF_COMPRESSED 4000
|
|
#define VECTOR_MAX_DIMENSIONS_PQ_COMPRESSED VECTOR_MAX_DIMENSIONS
|
|
|
|
/* ivfflat parameters */
|
|
#define IVFFLAT_DEFAULT_NPROBES 1
|
|
#define IVFFLAT_MIN_NPROBES 1
|
|
#define IVFFLAT_MAX_NPROBES 32768
|
|
|
|
#define IVFFLAT_DEFAULT_LISTS 100
|
|
#define IVFFLAT_MIN_LISTS 1
|
|
#define IVFFLAT_MAX_LISTS 32768
|
|
|
|
#define VECTOR_PARAMETER_NAME_IVF_NPROBES "nProbes"
|
|
#define VECTOR_PARAMETER_NAME_IVF_NPROBES_STR_LEN 7
|
|
#define VECTOR_PARAMETER_NAME_IVF_NLISTS "numLists"
|
|
|
|
/* hnsw parameters */
|
|
#define HNSW_DEFAULT_M 16
|
|
#define HNSW_MIN_M 2
|
|
#define HNSW_MAX_M 100
|
|
|
|
#define HNSW_DEFAULT_EF_CONSTRUCTION 64
|
|
#define HNSW_MIN_EF_CONSTRUCTION 4
|
|
#define HNSW_MAX_EF_CONSTRUCTION 1000
|
|
|
|
#define HNSW_DEFAULT_EF_SEARCH 40
|
|
#define HNSW_MIN_EF_SEARCH 1
|
|
#define HNSW_MAX_EF_SEARCH 1000
|
|
|
|
#define VECTOR_PARAMETER_NAME_HNSW_M "m"
|
|
#define VECTOR_PARAMETER_NAME_HNSW_EF_CONSTRUCTION "efConstruction"
|
|
#define VECTOR_PARAMETER_NAME_HNSW_EF_SEARCH "efSearch"
|
|
#define VECTOR_PARAMETER_NAME_HNSW_EF_SEARCH_STR_LEN 8
|
|
|
|
/* Search parameter name for iterative scan mode */
|
|
#define VECTOR_PARAMETER_NAME_ITERATIVE_SCAN "iterativeScan"
|
|
#define VECTOR_PARAMETER_NAME_ITERATIVE_SCAN_STR_LEN 13
|
|
|
|
/* dynamic calculation of nprobes or efSearch depending on collection size */
|
|
#define VECTOR_SEARCH_SMALL_COLLECTION_ROWS 10000
|
|
|
|
/* metadata field names */
|
|
#define VECTOR_METADATA_FIELD_NAME "__cosmos_meta__"
|
|
#define VECTOR_METADATA_FIELD_NAME_STR_LEN 15
|
|
#define VECTOR_METADATA_SCORE_FIELD_NAME "score"
|
|
#define VECTOR_METADATA_SCORE_FIELD_NAME_STR_LEN 5
|
|
|
|
/*
|
|
* ApiGucPrefix.enableVectorHNSWIndex GUC determines vector indexes
|
|
* and queries are enabled in documentdb_api or not.
|
|
*/
|
|
extern bool EnableVectorHNSWIndex;
|
|
|
|
/*
|
|
* GUC to enable vector pre-filtering feature for vector search.
|
|
* This is disabled by default.
|
|
*/
|
|
extern bool EnableVectorPreFilter;
|
|
extern bool EnableVectorPreFilterV2;
|
|
|
|
/*
|
|
* GUC to set the iterative scan mode for pre-filtering
|
|
*/
|
|
extern int VectorPreFilterIterativeScanMode;
|
|
|
|
/*
|
|
* GUC to enable vector compression feature for vector search.
|
|
*/
|
|
extern bool EnableVectorCompressionHalf;
|
|
extern bool EnableVectorCompressionPQ;
|
|
|
|
/*
|
|
* GUC to enable vector search default search parameter calculation.
|
|
*/
|
|
extern bool EnableVectorCalculateDefaultSearchParameter;
|
|
|
|
#endif
|