From 03876488da9150776f964447bcca9140bec71aa2 Mon Sep 17 00:00:00 2001 From: Hristo Temelski Date: Wed, 29 Oct 2025 09:43:00 +0200 Subject: [PATCH] Added hybrid search command --- packages/search/lib/commands/HYBRID.spec.ts | 379 ++++++++++++++++++++ packages/search/lib/commands/HYBRID.ts | 377 +++++++++++++++++++ packages/search/lib/commands/index.ts | 3 + 3 files changed, 759 insertions(+) create mode 100644 packages/search/lib/commands/HYBRID.spec.ts create mode 100644 packages/search/lib/commands/HYBRID.ts diff --git a/packages/search/lib/commands/HYBRID.spec.ts b/packages/search/lib/commands/HYBRID.spec.ts new file mode 100644 index 0000000000..624c1b9c91 --- /dev/null +++ b/packages/search/lib/commands/HYBRID.spec.ts @@ -0,0 +1,379 @@ +import { strict as assert } from 'node:assert'; +import testUtils, { GLOBAL } from '../test-utils'; +import HYBRID from './HYBRID'; +import { BasicCommandParser } from '@redis/client/lib/client/parser'; + +describe('FT.HYBRID', () => { + describe('parseCommand', () => { + it('minimal command', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index'); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'DIALECT', '2'] + ); + }); + + it('with count expressions', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + countExpressions: 3 + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '3', 'DIALECT', '2'] + ); + }); + + it('with SEARCH expression', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + SEARCH: { + query: '@description: bikes' + } + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'SEARCH', '@description: bikes', 'DIALECT', '2'] + ); + }); + + it('with SEARCH expression and SCORER', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + SEARCH: { + query: '@description: bikes', + SCORER: { + algorithm: 'TFIDF.DOCNORM', + params: ['param1', 'param2'] + }, + YIELD_SCORE_AS: 'search_score' + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'SEARCH', '@description: bikes', + 'SCORER', 'TFIDF.DOCNORM', 'param1', 'param2', + 'YIELD_SCORE_AS', 'search_score', 'DIALECT', '2' + ] + ); + }); + + it('with VSIM expression and KNN method', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + VSIM: { + field: '@vector_field', + vectorData: 'BLOB_DATA', + method: { + KNN: { + K: 10, + EF_RUNTIME: 50, + YIELD_DISTANCE_AS: 'vector_dist' + } + } + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'VSIM', '@vector_field', 'BLOB_DATA', + 'KNN', '1', 'K', '10', 'EF_RUNTIME', '50', 'YIELD_DISTANCE_AS', 'vector_dist', + 'DIALECT', '2' + ] + ); + }); + + it('with VSIM expression and RANGE method', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + VSIM: { + field: '@vector_field', + vectorData: 'BLOB_DATA', + method: { + RANGE: { + RADIUS: 0.5, + EPSILON: 0.01, + YIELD_DISTANCE_AS: 'vector_dist' + } + } + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'VSIM', '@vector_field', 'BLOB_DATA', + 'RANGE', '1', 'RADIUS', '0.5', 'EPSILON', '0.01', 'YIELD_DISTANCE_AS', 'vector_dist', + 'DIALECT', '2' + ] + ); + }); + + it('with VSIM expression and FILTER', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + VSIM: { + field: '@vector_field', + vectorData: 'BLOB_DATA', + FILTER: { + expression: '@category:{bikes}', + POLICY: 'BATCHES', + BATCHES: { + BATCH_SIZE: 100 + } + }, + YIELD_SCORE_AS: 'vsim_score' + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'VSIM', '@vector_field', 'BLOB_DATA', + 'FILTER', '@category:{bikes}', 'POLICY', 'BATCHES', 'BATCHES', 'BATCH_SIZE', '100', + 'YIELD_SCORE_AS', 'vsim_score', 'DIALECT', '2' + ] + ); + }); + + it('with RRF COMBINE method', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + COMBINE: { + method: { + RRF: { + count: 2, + WINDOW: 10, + CONSTANT: 60 + } + }, + YIELD_SCORE_AS: 'combined_score' + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'COMBINE', 'RRF', '2', 'WINDOW', '10', 'CONSTANT', '60', + 'YIELD_SCORE_AS', 'combined_score', 'DIALECT', '2' + ] + ); + }); + + it('with LINEAR COMBINE method', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + COMBINE: { + method: { + LINEAR: { + count: 2, + ALPHA: 0.7, + BETA: 0.3 + } + } + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'COMBINE', 'LINEAR', '2', 'ALPHA', '0.7', 'BETA', '0.3', + 'DIALECT', '2' + ] + ); + }); + + it('with LOAD, SORTBY, and LIMIT', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + LOAD: ['field1', 'field2'], + SORTBY: { + count: 1, + fields: [ + { field: 'score', direction: 'DESC' } + ] + }, + LIMIT: { + offset: 0, + num: 10 + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'LOAD', '2', 'field1', 'field2', + 'SORTBY', '1', 'score', 'DESC', 'LIMIT', '0', '10', 'DIALECT', '2' + ] + ); + }); + + it('with GROUPBY and REDUCE', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + GROUPBY: { + fields: ['@category'], + REDUCE: { + function: 'COUNT', + count: 0, + args: [] + } + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'GROUPBY', '1', '@category', 'REDUCE', 'COUNT', '0', + 'DIALECT', '2' + ] + ); + }); + + it('with APPLY', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + APPLY: { + expression: '@score * 2', + AS: 'double_score' + } + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'APPLY', '@score * 2', 'AS', 'double_score', 'DIALECT', '2'] + ); + }); + + it('with FILTER and post-processing', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + FILTER: '@price:[100 500]' + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'FILTER', '@price:[100 500]', 'DIALECT', '2'] + ); + }); + + it('with PARAMS', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + PARAMS: { + query_vector: 'BLOB_DATA', + min_price: 100 + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'PARAMS', '4', 'query_vector', 'BLOB_DATA', 'min_price', '100', + 'DIALECT', '2' + ] + ); + }); + + it('with EXPLAINSCORE and TIMEOUT', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + EXPLAINSCORE: true, + TIMEOUT: 5000 + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'EXPLAINSCORE', 'TIMEOUT', '5000', 'DIALECT', '2'] + ); + }); + + it('with WITHCURSOR', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + WITHCURSOR: { + COUNT: 100, + MAXIDLE: 300000 + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', 'WITHCURSOR', 'COUNT', '100', 'MAXIDLE', '300000', + 'DIALECT', '2' + ] + ); + }); + + it('complete example with all options', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + countExpressions: 2, + SEARCH: { + query: '@description: bikes', + SCORER: { + algorithm: 'TFIDF.DOCNORM' + }, + YIELD_SCORE_AS: 'text_score' + }, + VSIM: { + field: '@vector_field', + vectorData: '$query_vector', + method: { + KNN: { + K: 5 + } + }, + YIELD_SCORE_AS: 'vector_score' + }, + COMBINE: { + method: { + RRF: { + count: 2, + CONSTANT: 60 + } + }, + YIELD_SCORE_AS: 'final_score' + }, + LOAD: ['description', 'price'], + SORTBY: { + count: 1, + fields: [{ field: 'final_score', direction: 'DESC' }] + }, + LIMIT: { + offset: 0, + num: 10 + }, + PARAMS: { + query_vector: 'BLOB_DATA' + } + }); + assert.deepEqual( + parser.redisArgs, + [ + 'FT.HYBRID', 'index', '2', + 'SEARCH', '@description: bikes', 'SCORER', 'TFIDF.DOCNORM', 'YIELD_SCORE_AS', 'text_score', + 'VSIM', '@vector_field', '$query_vector', 'KNN', '1', 'K', '5', 'YIELD_SCORE_AS', 'vector_score', + 'COMBINE', 'RRF', '2', 'CONSTANT', '60', 'YIELD_SCORE_AS', 'final_score', + 'LOAD', '2', 'description', 'price', + 'SORTBY', '1', 'final_score', 'DESC', + 'LIMIT', '0', '10', + 'PARAMS', '2', 'query_vector', 'BLOB_DATA', + 'DIALECT', '2' + ] + ); + }); + + it('with custom DIALECT', () => { + const parser = new BasicCommandParser(); + HYBRID.parseCommand(parser, 'index', { + DIALECT: 3 + }); + assert.deepEqual( + parser.redisArgs, + ['FT.HYBRID', 'index', '2', 'DIALECT', '3'] + ); + }); + }); + + // Integration tests would need to be added when RediSearch supports FT.HYBRID + // For now, we'll skip them as this is a new command that may not be available yet + describe.skip('client.ft.hybrid', () => { + testUtils.testWithClient('basic hybrid search', async client => { + // This would require a test index and data setup + // similar to how other FT commands are tested + }, GLOBAL.SERVERS.OPEN); + }); +}); \ No newline at end of file diff --git a/packages/search/lib/commands/HYBRID.ts b/packages/search/lib/commands/HYBRID.ts new file mode 100644 index 0000000000..b94e7196da --- /dev/null +++ b/packages/search/lib/commands/HYBRID.ts @@ -0,0 +1,377 @@ +import { CommandParser } from '@redis/client/dist/lib/client/parser'; +import { RedisArgument, Command, ReplyUnion } from '@redis/client/dist/lib/RESP/types'; +import { RedisVariadicArgument, parseOptionalVariadicArgument } from '@redis/client/dist/lib/commands/generic-transformers'; +import { DEFAULT_DIALECT } from '../dialect/default'; +import { FtSearchParams, parseParamsArgument } from './SEARCH'; + +export interface FtHybridSearchExpression { + query: RedisArgument; + SCORER?: { + algorithm: RedisArgument; + params?: Array; + }; + YIELD_SCORE_AS?: RedisArgument; +} + +export interface FtHybridVectorMethod { + KNN?: { + K: number; + EF_RUNTIME?: number; + YIELD_DISTANCE_AS?: RedisArgument; + }; + RANGE?: { + RADIUS: number; + EPSILON?: number; + YIELD_DISTANCE_AS?: RedisArgument; + }; +} + +export interface FtHybridVectorExpression { + field: RedisArgument; + vectorData: RedisArgument; + method?: FtHybridVectorMethod; + FILTER?: { + expression: RedisArgument; + POLICY?: 'ADHOC' | 'BATCHES' | 'ACORN'; + BATCHES?: { + BATCH_SIZE: number; + }; + }; + YIELD_SCORE_AS?: RedisArgument; +} + +export interface FtHybridCombineMethod { + RRF?: { + count: number; + WINDOW?: number; + CONSTANT?: number; + }; + LINEAR?: { + count: number; + ALPHA?: number; + BETA?: number; + }; + FUNCTION?: RedisArgument; +} + +export interface FtHybridOptions { + countExpressions?: number; + SEARCH?: FtHybridSearchExpression; + VSIM?: FtHybridVectorExpression; + COMBINE?: { + method: FtHybridCombineMethod; + YIELD_SCORE_AS?: RedisArgument; + }; + LOAD?: RedisVariadicArgument; + GROUPBY?: { + fields: RedisVariadicArgument; + REDUCE?: { + function: RedisArgument; + count: number; + args: Array; + }; + }; + APPLY?: { + expression: RedisArgument; + AS: RedisArgument; + }; + SORTBY?: { + count: number; + fields: Array<{ + field: RedisArgument; + direction?: 'ASC' | 'DESC'; + }>; + }; + FILTER?: RedisArgument; + LIMIT?: { + offset: number | RedisArgument; + num: number | RedisArgument; + }; + PARAMS?: FtSearchParams; + EXPLAINSCORE?: boolean; + TIMEOUT?: number; + WITHCURSOR?: { + COUNT?: number; + MAXIDLE?: number; + }; + DIALECT?: number; +} + +function parseSearchExpression(parser: CommandParser, search: FtHybridSearchExpression) { + parser.push('SEARCH', search.query); + + if (search.SCORER) { + parser.push('SCORER', search.SCORER.algorithm); + if (search.SCORER.params) { + parser.push(...search.SCORER.params); + } + } + + if (search.YIELD_SCORE_AS) { + parser.push('YIELD_SCORE_AS', search.YIELD_SCORE_AS); + } +} + +function parseVectorExpression(parser: CommandParser, vsim: FtHybridVectorExpression) { + parser.push('VSIM', vsim.field, vsim.vectorData); + + if (vsim.method) { + if (vsim.method.KNN) { + const knn = vsim.method.KNN; + parser.push('KNN', '1', 'K', knn.K.toString()); + + if (knn.EF_RUNTIME !== undefined) { + parser.push('EF_RUNTIME', knn.EF_RUNTIME.toString()); + } + + if (knn.YIELD_DISTANCE_AS) { + parser.push('YIELD_DISTANCE_AS', knn.YIELD_DISTANCE_AS); + } + } + + if (vsim.method.RANGE) { + const range = vsim.method.RANGE; + parser.push('RANGE', '1', 'RADIUS', range.RADIUS.toString()); + + if (range.EPSILON !== undefined) { + parser.push('EPSILON', range.EPSILON.toString()); + } + + if (range.YIELD_DISTANCE_AS) { + parser.push('YIELD_DISTANCE_AS', range.YIELD_DISTANCE_AS); + } + } + } + + if (vsim.FILTER) { + parser.push('FILTER', vsim.FILTER.expression); + + if (vsim.FILTER.POLICY) { + parser.push('POLICY', vsim.FILTER.POLICY); + + if (vsim.FILTER.POLICY === 'BATCHES' && vsim.FILTER.BATCHES) { + parser.push('BATCHES', 'BATCH_SIZE', vsim.FILTER.BATCHES.BATCH_SIZE.toString()); + } + } + } + + if (vsim.YIELD_SCORE_AS) { + parser.push('YIELD_SCORE_AS', vsim.YIELD_SCORE_AS); + } +} + +function parseCombineMethod(parser: CommandParser, combine: FtHybridOptions['COMBINE']) { + if (!combine) return; + + parser.push('COMBINE'); + + if (combine.method.RRF) { + const rrf = combine.method.RRF; + parser.push('RRF', rrf.count.toString()); + + if (rrf.WINDOW !== undefined) { + parser.push('WINDOW', rrf.WINDOW.toString()); + } + + if (rrf.CONSTANT !== undefined) { + parser.push('CONSTANT', rrf.CONSTANT.toString()); + } + } + + if (combine.method.LINEAR) { + const linear = combine.method.LINEAR; + parser.push('LINEAR', linear.count.toString()); + + if (linear.ALPHA !== undefined) { + parser.push('ALPHA', linear.ALPHA.toString()); + } + + if (linear.BETA !== undefined) { + parser.push('BETA', linear.BETA.toString()); + } + } + + if (combine.method.FUNCTION) { + parser.push('FUNCTION', combine.method.FUNCTION); + } + + if (combine.YIELD_SCORE_AS) { + parser.push('YIELD_SCORE_AS', combine.YIELD_SCORE_AS); + } +} + +function parseHybridOptions(parser: CommandParser, options?: FtHybridOptions) { + if (!options) return; + + if (options.SEARCH) { + parseSearchExpression(parser, options.SEARCH); + } + + if (options.VSIM) { + parseVectorExpression(parser, options.VSIM); + } + + if (options.COMBINE) { + parseCombineMethod(parser, options.COMBINE); + } + + parseOptionalVariadicArgument(parser, 'LOAD', options.LOAD); + + if (options.GROUPBY) { + parseOptionalVariadicArgument(parser, 'GROUPBY', options.GROUPBY.fields); + + if (options.GROUPBY.REDUCE) { + parser.push('REDUCE', options.GROUPBY.REDUCE.function, options.GROUPBY.REDUCE.count.toString()); + parser.push(...options.GROUPBY.REDUCE.args); + } + } + + if (options.APPLY) { + parser.push('APPLY', options.APPLY.expression, 'AS', options.APPLY.AS); + } + + if (options.SORTBY) { + parser.push('SORTBY', options.SORTBY.count.toString()); + for (const sortField of options.SORTBY.fields) { + parser.push(sortField.field); + if (sortField.direction) { + parser.push(sortField.direction); + } + } + } + + if (options.FILTER) { + parser.push('FILTER', options.FILTER); + } + + if (options.LIMIT) { + parser.push('LIMIT', options.LIMIT.offset.toString(), options.LIMIT.num.toString()); + } + + parseParamsArgument(parser, options.PARAMS); + + if (options.EXPLAINSCORE) { + parser.push('EXPLAINSCORE'); + } + + if (options.TIMEOUT !== undefined) { + parser.push('TIMEOUT', options.TIMEOUT.toString()); + } + + if (options.WITHCURSOR) { + parser.push('WITHCURSOR'); + + if (options.WITHCURSOR.COUNT !== undefined) { + parser.push('COUNT', options.WITHCURSOR.COUNT.toString()); + } + + if (options.WITHCURSOR.MAXIDLE !== undefined) { + parser.push('MAXIDLE', options.WITHCURSOR.MAXIDLE.toString()); + } + } + + if (options?.DIALECT) { + parser.push('DIALECT', options.DIALECT.toString()); + } +} + +export default { + NOT_KEYED_COMMAND: true, + IS_READ_ONLY: true, + /** + * Performs a hybrid search combining multiple search expressions. + * Supports multiple SEARCH and VECTOR expressions with various fusion methods. + * + * @param parser - The command parser + * @param index - The index name to search + * @param options - Hybrid search options including: + * - countExpressions: Number of expressions (default 2) + * - SEARCH: Text search expression with optional scoring + * - VSIM: Vector similarity expression with KNN/RANGE methods + * - COMBINE: Fusion method (RRF, LINEAR, FUNCTION) + * - Post-processing operations: LOAD, GROUPBY, APPLY, SORTBY, FILTER + * - Tunable options: LIMIT, PARAMS, EXPLAINSCORE, TIMEOUT, WITHCURSOR + */ + parseCommand(parser: CommandParser, index: RedisArgument, options?: FtHybridOptions) { + parser.push('FT.HYBRID', index); + + if (options?.countExpressions !== undefined) { + parser.push(options.countExpressions.toString()); + } else { + parser.push('2'); // Default to 2 expressions + } + + parseHybridOptions(parser, options); + + // Always add DIALECT at the end if not already added + if (!options?.DIALECT) { + parser.push('DIALECT', DEFAULT_DIALECT); + } + }, + transformReply: { + 2: (reply: any): any => { + // Check if this is a cursor reply: [[results...], cursorId] + if (Array.isArray(reply) && reply.length === 2 && typeof reply[1] === 'number') { + // This is a cursor reply + const [searchResults, cursor] = reply; + const transformedResults = transformHybridSearchResults(searchResults); + + return { + ...transformedResults, + cursor + }; + } else { + // Normal reply without cursor + return transformHybridSearchResults(reply); + } + }, + 3: undefined as unknown as () => ReplyUnion + }, + unstableResp3: true +} as const satisfies Command; + +function transformHybridSearchResults(reply: any) { + // Similar structure to FT.SEARCH reply transformation + const withoutDocuments = reply.length > 2 && !Array.isArray(reply[2]); + + const documents = []; + let i = 1; + while (i < reply.length) { + documents.push({ + id: reply[i++], + value: withoutDocuments ? Object.create(null) : documentValue(reply[i++]) + }); + } + + return { + total: reply[0], + documents + }; +} + +function documentValue(tuples: any) { + const message = Object.create(null); + + if (!tuples) { + return message; + } + + let i = 0; + while (i < tuples.length) { + const key = tuples[i++]; + const value = tuples[i++]; + + if (key === '$') { // might be a JSON reply + try { + Object.assign(message, JSON.parse(value)); + continue; + } catch { + // set as a regular property if not a valid JSON + } + } + + message[key] = value; + } + + return message; +} diff --git a/packages/search/lib/commands/index.ts b/packages/search/lib/commands/index.ts index 7aa3f061bf..53030be1ef 100644 --- a/packages/search/lib/commands/index.ts +++ b/packages/search/lib/commands/index.ts @@ -16,6 +16,7 @@ import DICTDUMP from './DICTDUMP'; import DROPINDEX from './DROPINDEX'; import EXPLAIN from './EXPLAIN'; import EXPLAINCLI from './EXPLAINCLI'; +import HYBRID from './HYBRID'; import INFO from './INFO'; import PROFILESEARCH from './PROFILE_SEARCH'; import PROFILEAGGREGATE from './PROFILE_AGGREGATE'; @@ -82,6 +83,8 @@ export default { explain: EXPLAIN, EXPLAINCLI, explainCli: EXPLAINCLI, + HYBRID, + hybrid: HYBRID, INFO, info: INFO, PROFILESEARCH,