Skip to content

Commit 87ce0cf

Browse files
himanshusinghshimanshusinghs
authored
chore: extend accuracy tests to support custom user and cluster config (#655)
1 parent a8c6425 commit 87ce0cf

File tree

5 files changed

+224
-154
lines changed

5 files changed

+224
-154
lines changed

‎tests/accuracy/createIndex.test.ts‎

Lines changed: 135 additions & 130 deletions
Original file line numberDiff line numberDiff line change
@@ -1,140 +1,145 @@
11
import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
22
import { Matcher } from "./sdk/matcher.js";
33

4-
// TODO: supply this with a proper config API once we refactor describeAccuracyTests to support it
5-
process.env.MDB_VOYAGE_API_KEY = "valid-key";
6-
7-
describeAccuracyTests([
8-
{
9-
prompt: "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
10-
expectedToolCalls: [
11-
{
12-
toolName: "create-index",
13-
parameters: {
14-
database: "mflix",
15-
collection: "movies",
16-
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
17-
definition: [
18-
{
19-
type: "classic",
20-
keys: {
21-
release_year: 1,
4+
describeAccuracyTests(
5+
[
6+
{
7+
prompt: "Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
8+
expectedToolCalls: [
9+
{
10+
toolName: "create-index",
11+
parameters: {
12+
database: "mflix",
13+
collection: "movies",
14+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
15+
definition: [
16+
{
17+
type: "classic",
18+
keys: {
19+
release_year: 1,
20+
},
2221
},
23-
},
24-
],
22+
],
23+
},
2524
},
26-
},
27-
],
28-
},
29-
{
30-
prompt: "Create a text index on title field in 'mflix.movies' namespace",
31-
expectedToolCalls: [
32-
{
33-
toolName: "create-index",
34-
parameters: {
35-
database: "mflix",
36-
collection: "movies",
37-
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
38-
definition: [
39-
{
40-
type: "classic",
41-
keys: {
42-
title: "text",
25+
],
26+
},
27+
{
28+
prompt: "Create a text index on title field in 'mflix.movies' namespace",
29+
expectedToolCalls: [
30+
{
31+
toolName: "create-index",
32+
parameters: {
33+
database: "mflix",
34+
collection: "movies",
35+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
36+
definition: [
37+
{
38+
type: "classic",
39+
keys: {
40+
title: "text",
41+
},
4342
},
44-
},
45-
],
43+
],
44+
},
4645
},
47-
},
48-
],
49-
},
50-
{
51-
prompt: "Create a vector search index on 'mydb.movies' namespace on the 'plotSummary' field. The index should use 1024 dimensions.",
52-
expectedToolCalls: [
53-
{
54-
toolName: "create-index",
55-
parameters: {
56-
database: "mydb",
57-
collection: "movies",
58-
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
59-
definition: [
60-
{
61-
type: "vectorSearch",
62-
fields: [
63-
{
64-
type: "vector",
65-
path: "plotSummary",
66-
numDimensions: 1024,
67-
},
68-
],
69-
},
70-
],
46+
],
47+
},
48+
{
49+
prompt: "Create a vector search index on 'mflix.movies' namespace on the 'plotSummary' field. The index should use 1024 dimensions.",
50+
expectedToolCalls: [
51+
{
52+
toolName: "create-index",
53+
parameters: {
54+
database: "mflix",
55+
collection: "movies",
56+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
57+
definition: [
58+
{
59+
type: "vectorSearch",
60+
fields: [
61+
{
62+
type: "vector",
63+
path: "plotSummary",
64+
numDimensions: 1024,
65+
},
66+
],
67+
},
68+
],
69+
},
7170
},
72-
},
73-
],
74-
},
75-
{
76-
prompt: "Create a vector search index on 'mydb.movies' namespace with on the 'plotSummary' field and 'genre' field, both of which contain vector embeddings. Pick a sensible number of dimensions for a voyage 3.5 model.",
77-
expectedToolCalls: [
78-
{
79-
toolName: "create-index",
80-
parameters: {
81-
database: "mydb",
82-
collection: "movies",
83-
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
84-
definition: [
85-
{
86-
type: "vectorSearch",
87-
fields: [
88-
{
89-
type: "vector",
90-
path: "plotSummary",
91-
numDimensions: Matcher.number(
92-
(value) => value % 2 === 0 && value >= 256 && value <= 8192
93-
),
94-
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
95-
},
96-
{
97-
type: "vector",
98-
path: "genre",
99-
numDimensions: Matcher.number(
100-
(value) => value % 2 === 0 && value >= 256 && value <= 8192
101-
),
102-
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
103-
},
104-
],
105-
},
106-
],
71+
],
72+
},
73+
{
74+
prompt: "Create a vector search index on 'mflix.movies' namespace with on the 'plotSummary' field and 'genre' field, both of which contain vector embeddings. Pick a sensible number of dimensions for a voyage 3.5 model.",
75+
expectedToolCalls: [
76+
{
77+
toolName: "create-index",
78+
parameters: {
79+
database: "mflix",
80+
collection: "movies",
81+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
82+
definition: [
83+
{
84+
type: "vectorSearch",
85+
fields: [
86+
{
87+
type: "vector",
88+
path: "plotSummary",
89+
numDimensions: Matcher.number(
90+
(value) => value % 2 === 0 && value >= 256 && value <= 8192
91+
),
92+
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
93+
},
94+
{
95+
type: "vector",
96+
path: "genre",
97+
numDimensions: Matcher.number(
98+
(value) => value % 2 === 0 && value >= 256 && value <= 8192
99+
),
100+
similarity: Matcher.anyOf(Matcher.undefined, Matcher.string()),
101+
},
102+
],
103+
},
104+
],
105+
},
107106
},
108-
},
109-
],
110-
},
111-
{
112-
prompt: "Create a vector search index on 'mydb.movies' namespace where the 'plotSummary' field is indexed as a 1024-dimensional vector and the 'releaseDate' field is indexed as a regular field.",
113-
expectedToolCalls: [
114-
{
115-
toolName: "create-index",
116-
parameters: {
117-
database: "mydb",
118-
collection: "movies",
119-
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
120-
definition: [
121-
{
122-
type: "vectorSearch",
123-
fields: [
124-
{
125-
type: "vector",
126-
path: "plotSummary",
127-
numDimensions: 1024,
128-
},
129-
{
130-
type: "filter",
131-
path: "releaseDate",
132-
},
133-
],
134-
},
135-
],
107+
],
108+
},
109+
{
110+
prompt: "Create a vector search index on 'mflix.movies' namespace where the 'plotSummary' field is indexed as a 1024-dimensional vector and the 'releaseDate' field is indexed as a regular field.",
111+
expectedToolCalls: [
112+
{
113+
toolName: "create-index",
114+
parameters: {
115+
database: "mflix",
116+
collection: "movies",
117+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
118+
definition: [
119+
{
120+
type: "vectorSearch",
121+
fields: [
122+
{
123+
type: "vector",
124+
path: "plotSummary",
125+
numDimensions: 1024,
126+
},
127+
{
128+
type: "filter",
129+
path: "releaseDate",
130+
},
131+
],
132+
},
133+
],
134+
},
136135
},
137-
},
138-
],
139-
},
140-
]);
136+
],
137+
},
138+
],
139+
{
140+
userConfig: { voyageApiKey: "valid-key" },
141+
clusterConfig: {
142+
search: true,
143+
},
144+
}
145+
);

‎tests/accuracy/createIndex.vectorSearchDisabled.test.ts‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/**
2+
* Accuracy tests for when the vector search feature flag is disabled.
3+
*
4+
* TODO: Remove this file once we permanently enable the vector search feature.
5+
*/
6+
import { describeAccuracyTests } from "./sdk/describeAccuracyTests.js";
7+
import { Matcher } from "./sdk/matcher.js";
8+
9+
describeAccuracyTests(
10+
[
11+
{
12+
prompt: "(vectorSearchDisabled) Create an index that covers the following query on 'mflix.movies' namespace - { \"release_year\": 1992 }",
13+
expectedToolCalls: [
14+
{
15+
toolName: "create-index",
16+
parameters: {
17+
database: "mflix",
18+
collection: "movies",
19+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
20+
definition: [
21+
{
22+
type: Matcher.anyOf(Matcher.undefined, Matcher.value("classic")),
23+
keys: {
24+
release_year: 1,
25+
},
26+
},
27+
],
28+
},
29+
},
30+
],
31+
},
32+
{
33+
prompt: "(vectorSearchDisabled) Create a text index on title field in 'mflix.movies' namespace",
34+
expectedToolCalls: [
35+
{
36+
toolName: "create-index",
37+
parameters: {
38+
database: "mflix",
39+
collection: "movies",
40+
name: Matcher.anyOf(Matcher.undefined, Matcher.string()),
41+
definition: [
42+
{
43+
type: Matcher.anyOf(Matcher.undefined, Matcher.value("classic")),
44+
keys: {
45+
title: "text",
46+
},
47+
},
48+
],
49+
},
50+
},
51+
],
52+
},
53+
],
54+
{
55+
userConfig: { voyageApiKey: "" },
56+
}
57+
);

‎tests/accuracy/sdk/accuracyTestingClient.ts‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js"
66
import { MCP_SERVER_CLI_SCRIPT } from "./constants.js";
77
import type { LLMToolCall } from "./accuracyResultStorage/resultStorage.js";
88
import type { VercelMCPClient, VercelMCPClientTools } from "./agent.js";
9+
import type { UserConfig } from "../../../src/lib.js";
910

1011
type ToolResultGeneratorFn = (parameters: Record<string, unknown>) => CallToolResult | Promise<CallToolResult>;
1112
export type MockedTools = Record<string, ToolResultGeneratorFn>;
@@ -81,18 +82,13 @@ export class AccuracyTestingClient {
8182

8283
static async initializeClient(
8384
mdbConnectionString: string,
84-
atlasApiClientId?: string,
85-
atlasApiClientSecret?: string,
86-
voyageApiKey?: string
85+
userConfig: Partial<{ [k in keyof UserConfig]: string }> = {}
8786
): Promise<AccuracyTestingClient> {
88-
const args = [
89-
MCP_SERVER_CLI_SCRIPT,
90-
"--connectionString",
91-
mdbConnectionString,
92-
...(atlasApiClientId ? ["--apiClientId", atlasApiClientId] : []),
93-
...(atlasApiClientSecret ? ["--apiClientSecret", atlasApiClientSecret] : []),
94-
...(voyageApiKey ? ["--voyageApiKey", voyageApiKey] : []),
95-
];
87+
const additionalArgs = Object.entries(userConfig).flatMap(([key, value]) => {
88+
return [`--${key}`, value];
89+
});
90+
91+
const args = [MCP_SERVER_CLI_SCRIPT, "--connectionString", mdbConnectionString, ...additionalArgs];
9692

9793
const clientTransport = new StdioClientTransport({
9894
command: process.execPath,

0 commit comments

Comments
 (0)