diff --git a/CMakeLists.txt b/CMakeLists.txt index 26d8f4b5..e86e5ce8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,6 +132,10 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_c_example_from_env examples/line_sender_c_example_from_env.c) + compile_example( + line_sender_c_example_decimal_binary + examples/concat.c + examples/line_sender_c_example_decimal_binary.c) compile_example( line_sender_cpp_example examples/line_sender_cpp_example.cpp) @@ -165,6 +169,12 @@ if (QUESTDB_TESTS_AND_EXAMPLES) compile_example( line_sender_cpp_example_from_env examples/line_sender_cpp_example_from_env.cpp) + compile_example( + line_sender_cpp_example_decimal_custom + examples/line_sender_cpp_example_decimal_custom.cpp) + compile_example( + line_sender_cpp_example_decimal_binary + examples/line_sender_cpp_example_decimal_binary.cpp) # Include Rust tests as part of the tests run add_test( diff --git a/ci/run_all_tests.py b/ci/run_all_tests.py index e753ed17..19266fbe 100644 --- a/ci/run_all_tests.py +++ b/ci/run_all_tests.py @@ -66,7 +66,7 @@ def main(): run_cmd(str(test_line_sender_path)) run_cmd(str(test_line_sender_path_CXX20)) run_cmd('python3', str(system_test_path), 'run', '--versions', qdb_v, '-v') - #run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb', '-v') + run_cmd('python3', str(system_test_path), 'run', '--repo', './questdb', '-v', '--force-max-version') if __name__ == '__main__': diff --git a/ci/run_tests_pipeline.yaml b/ci/run_tests_pipeline.yaml index 09508219..c5826118 100644 --- a/ci/run_tests_pipeline.yaml +++ b/ci/run_tests_pipeline.yaml @@ -56,7 +56,7 @@ stages: displayName: "Build Rust examples" ############################# temp for test begin ##################### - script: | - git clone --depth 1 https://github.com/questdb/questdb.git ./questdb + git clone --depth 1 --branch rd_decimal_integration https://github.com/questdb/questdb.git ./questdb displayName: git clone questdb - task: Maven@3 displayName: "Compile QuestDB" @@ -64,7 +64,7 @@ stages: mavenPOMFile: "questdb/pom.xml" jdkVersionOption: "1.17" options: "-DskipTests -Pbuild-web-console" - ############################# temp for test end ##################### + ############################# temp for test end ##################### - script: python3 ci/run_all_tests.py env: JAVA_HOME: $(JAVA_HOME_11_X64) @@ -76,7 +76,7 @@ stages: - job: FormatAndLinting displayName: "cargo fmt and clippy" pool: - vmImage: 'ubuntu-latest' + vmImage: "ubuntu-latest" timeoutInMinutes: 10 steps: - checkout: self @@ -117,7 +117,7 @@ stages: - job: TestVsQuestDBMaster displayName: "Vs QuestDB 'master'" pool: - vmImage: 'ubuntu-latest' + vmImage: "ubuntu-latest" timeoutInMinutes: 60 steps: - checkout: self @@ -131,8 +131,8 @@ stages: - task: Maven@3 displayName: "Compile QuestDB" inputs: - mavenPOMFile: 'questdb/pom.xml' - jdkVersionOption: '1.17' + mavenPOMFile: "questdb/pom.xml" + jdkVersionOption: "1.17" options: "-DskipTests -Pbuild-web-console" - script: | python3 system_test/test.py run --repo ./questdb -v diff --git a/cpp_test/test_line_sender.cpp b/cpp_test/test_line_sender.cpp index 441692fd..3027ecd4 100644 --- a/cpp_test/test_line_sender.cpp +++ b/cpp_test/test_line_sender.cpp @@ -196,36 +196,39 @@ TEST_CASE("line_sender c api basics") 2.7, 48121.5, 4.3}; - CHECK(::line_sender_buffer_column_f64_arr_byte_strides( - buffer, - arr_name, - rank, - shape, - strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_byte_strides( + buffer, + arr_name, + rank, + shape, + strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name2 = QDB_COLUMN_NAME_LITERAL("a2"); intptr_t elem_strides[] = {6, 2, 1}; - CHECK(::line_sender_buffer_column_f64_arr_elem_strides( - buffer, - arr_name2, - rank, - shape, - elem_strides, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_elem_strides( + buffer, + arr_name2, + rank, + shape, + elem_strides, + arr_data.data(), + arr_data.size(), + &err)); line_sender_column_name arr_name3 = QDB_COLUMN_NAME_LITERAL("a3"); - CHECK(::line_sender_buffer_column_f64_arr_c_major( - buffer, - arr_name3, - rank, - shape, - arr_data.data(), - arr_data.size(), - &err)); + CHECK( + ::line_sender_buffer_column_f64_arr_c_major( + buffer, + arr_name3, + rank, + shape, + arr_data.data(), + arr_data.size(), + &err)); CHECK(::line_sender_buffer_at_nanos(buffer, 10000000, &err)); CHECK(server.recv() == 0); CHECK(::line_sender_buffer_size(buffer) == 383); @@ -282,7 +285,7 @@ TEST_CASE("line_sender c++ api basics") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -352,7 +355,7 @@ TEST_CASE("line_sender array vector API") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -396,7 +399,7 @@ TEST_CASE("line_sender array span API") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); @@ -443,7 +446,7 @@ TEST_CASE("test multiple lines") questdb::ingress::test::mock_server server; std::string conf_str = "tcp::addr=127.0.0.1:" + std::to_string(server.port()) + - ";protocol_version=2;"; + ";protocol_version=3;"; questdb::ingress::line_sender sender = questdb::ingress::line_sender::from_conf(conf_str); CHECK_FALSE(sender.must_close()); @@ -1061,21 +1064,21 @@ TEST_CASE("Moved View") TEST_CASE("Empty Buffer") { questdb::ingress::line_sender_buffer b1{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; CHECK(b1.size() == 0); questdb::ingress::line_sender_buffer b2{std::move(b1)}; CHECK(b1.size() == 0); CHECK(b2.size() == 0); questdb::ingress::line_sender_buffer b3{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b3 = std::move(b2); CHECK(b2.size() == 0); CHECK(b3.size() == 0); questdb::ingress::line_sender_buffer b4{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b4.table("test").symbol("a", "b").at_now(); questdb::ingress::line_sender_buffer b5{ - questdb::ingress::protocol_version::v2}; + questdb::ingress::protocol_version::v3}; b5 = std::move(b4); CHECK(b4.size() == 0); CHECK(b5.size() == 9); @@ -1111,19 +1114,19 @@ TEST_CASE("HTTP basics") questdb::ingress::protocol::http, "127.0.0.1", 1}; questdb::ingress::opts opts1conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;username=user;password=pass;request_timeout=" - "5000;retry_timeout=5;protocol_version=2;"); + "5000;retry_timeout=5;protocol_version=3;"); questdb::ingress::opts opts2{ questdb::ingress::protocol::https, "localhost", "1"}; questdb::ingress::opts opts2conf = questdb::ingress::opts::from_conf( "http::addr=127.0.0.1:1;token=token;request_min_throughput=1000;retry_" - "timeout=0;protocol_version=2;"); - opts1.protocol_version(questdb::ingress::protocol_version::v2) + "timeout=0;protocol_version=3;"); + opts1.protocol_version(questdb::ingress::protocol_version::v3) .username("user") .password("pass") .max_buf_size(1000000) .request_timeout(5000) .retry_timeout(5); - opts2.protocol_version(questdb::ingress::protocol_version::v2) + opts2.protocol_version(questdb::ingress::protocol_version::v3) .token("token") .request_min_throughput(1000) .retry_timeout(0); @@ -1180,7 +1183,7 @@ TEST_CASE("line sender protocol version v2") questdb::ingress::protocol::tcp, std::string("127.0.0.1"), std::to_string(server.port())}; - opts.protocol_version(questdb::ingress::protocol_version::v2); + opts.protocol_version(questdb::ingress::protocol_version::v3); questdb::ingress::line_sender sender{opts}; CHECK_FALSE(sender.must_close()); server.accept(); diff --git a/doc/C.md b/doc/C.md index 7956c96c..123c8346 100644 --- a/doc/C.md +++ b/doc/C.md @@ -29,6 +29,9 @@ - [Array with element strides](../examples/line_sender_c_example_array_elem_strides.c) - [Array in C-major order](../examples/line_sender_c_example_array_c_major.c) +**Decimal** +- [Decimal in binary format](../examples/line_sender_c_example_decimal_binary.c) + ## API Overview ### Header @@ -90,7 +93,9 @@ line_sender_utf8 symbol_value = QDB_UTF8_LITERAL("ETH-USD"); if (!line_sender_buffer_symbol(buffer, symbol_name, symbol_value, &err)) goto on_error; -if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) +line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); +if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) @@ -101,6 +106,7 @@ if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) if (!line_sender_flush(sender, buffer, &err)) goto on_error; +line_sender_buffer_free(buffer); line_sender_close(sender); ``` diff --git a/doc/CPP.md b/doc/CPP.md index 9f2b1bf8..d1a8f494 100644 --- a/doc/CPP.md +++ b/doc/CPP.md @@ -30,6 +30,10 @@ - [Array in C-major order](../examples/line_sender_cpp_example_array_c_major.cpp) - [Custom array type integration](../examples/line_sender_cpp_example_array_custom.cpp) +**Decimal** +- [Decimal in binary format](../examples/line_sender_cpp_example_decimal_binary.cpp) +- [Custom decimal type integration](../examples/line_sender_cpp_example_decimal_custom.cpp) + ## API Overview ### Header @@ -76,7 +80,7 @@ questdb::ingress::line_sender_buffer buffer; buffer .table("trades") .symbol("symbol", "ETH-USD") - .column("price", 2615.54) + .column("price", "2615.54"_decimal) .at(timestamp_nanos::now()); // To insert more records, call `buffer.table(..)...` again. diff --git a/examples/line_sender_c_example.c b/examples/line_sender_c_example.c index 1dfbe77f..6f711ded 100644 --- a/examples/line_sender_c_example.c +++ b/examples/line_sender_c_example.c @@ -10,7 +10,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -51,7 +51,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -72,6 +74,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_byte_strides.c b/examples/line_sender_c_example_array_byte_strides.c index efe65f33..4f44db75 100644 --- a/examples/line_sender_c_example_array_byte_strides.c +++ b/examples/line_sender_c_example_array_byte_strides.c @@ -82,6 +82,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_c_major.c b/examples/line_sender_c_example_array_c_major.c index 798b13ba..7c9764f8 100644 --- a/examples/line_sender_c_example_array_c_major.c +++ b/examples/line_sender_c_example_array_c_major.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -79,6 +79,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_array_elem_strides.c b/examples/line_sender_c_example_array_elem_strides.c index f2a4272e..f5ef14f4 100644 --- a/examples/line_sender_c_example_array_elem_strides.c +++ b/examples/line_sender_c_example_array_elem_strides.c @@ -13,7 +13,7 @@ static bool example(const char* host, const char* port) line_sender* sender = NULL; line_sender_buffer* buffer = NULL; char* conf_str = - concat("tcp::addr=", host, ":", port, ";protocol_version=2;"); + concat("tcp::addr=", host, ":", port, ";protocol_version=3;"); if (!conf_str) { fprintf(stderr, "Could not concatenate configuration string.\n"); @@ -82,6 +82,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_auth.c b/examples/line_sender_c_example_auth.c index 0be793df..e13e0715 100644 --- a/examples/line_sender_c_example_auth.c +++ b/examples/line_sender_c_example_auth.c @@ -15,7 +15,7 @@ static bool example(const char* host, const char* port) ":", port, ";" - "protocol_version=2;" + "protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -60,7 +60,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -81,6 +83,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_auth_tls.c b/examples/line_sender_c_example_auth_tls.c index d7613d29..4b67117f 100644 --- a/examples/line_sender_c_example_auth_tls.c +++ b/examples/line_sender_c_example_auth_tls.c @@ -15,7 +15,7 @@ static bool example(const char* host, const char* port) ":", port, ";" - "protocol_version=2;" + "protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -61,7 +61,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -82,6 +84,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_decimal_binary.c b/examples/line_sender_c_example_decimal_binary.c new file mode 100644 index 00000000..a4f325c4 --- /dev/null +++ b/examples/line_sender_c_example_decimal_binary.c @@ -0,0 +1,129 @@ +#include +#include +#include +#include +#include "concat.h" + +static bool example(const char* host, const char* port) +{ + line_sender_error* err = NULL; + line_sender* sender = NULL; + line_sender_buffer* buffer = NULL; + char* conf_str = concat("http::addr=", host, ":", port, ";"); + if (!conf_str) + { + fprintf(stderr, "Could not concatenate configuration string.\n"); + return false; + } + line_sender_utf8 conf_str_utf8 = {0, NULL}; + if (!line_sender_utf8_init( + &conf_str_utf8, strlen(conf_str), conf_str, &err)) + goto on_error; + + sender = line_sender_from_conf(conf_str_utf8, &err); + if (!sender) + goto on_error; + + free(conf_str); + conf_str = NULL; + + buffer = line_sender_buffer_new_for_sender(sender); + line_sender_buffer_reserve(buffer, 64 * 1024); // 64KB buffer initial size. + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + line_sender_table_name table_name = + QDB_TABLE_NAME_LITERAL("c_trades_decimal"); + line_sender_column_name symbol_name = QDB_COLUMN_NAME_LITERAL("symbol"); + line_sender_column_name side_name = QDB_COLUMN_NAME_LITERAL("side"); + line_sender_column_name price_name = QDB_COLUMN_NAME_LITERAL("price"); + line_sender_column_name amount_name = QDB_COLUMN_NAME_LITERAL("amount"); + + if (!line_sender_buffer_table(buffer, table_name, &err)) + goto on_error; + + line_sender_utf8 symbol_value = QDB_UTF8_LITERAL("ETH-USD"); + if (!line_sender_buffer_symbol(buffer, symbol_name, symbol_value, &err)) + goto on_error; + + line_sender_utf8 side_value = QDB_UTF8_LITERAL("sell"); + if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) + goto on_error; + + // 123 with a scale of 1 gives a decimal of 12.3 + const uint8_t price_unscaled_value[] = {123}; + if (!line_sender_buffer_column_dec( + buffer, price_name, 1, price_unscaled_value, 1, &err)) + goto on_error; + + if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) + goto on_error; + + // 1997-07-04 04:56:55 UTC + int64_t designated_timestamp = 867992215000000000; + if (!line_sender_buffer_at_nanos(buffer, designated_timestamp, &err)) + goto on_error; + + //// If you want to get the current system timestamp as nanos, call: + // if (!line_sender_buffer_at_nanos(buffer, line_sender_now_nanos(), &err)) + // goto on_error; + + // To insert more records, call `line_sender_buffer_table(..)...` again. + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + if (!line_sender_flush(sender, buffer, &err)) + goto on_error; + + line_sender_buffer_free(buffer); + line_sender_close(sender); + + return true; + +on_error:; + size_t err_len = 0; + const char* err_msg = line_sender_error_msg(err, &err_len); + fprintf(stderr, "Error running example: %.*s\n", (int)err_len, err_msg); + free(conf_str); + line_sender_error_free(err); + line_sender_buffer_free(buffer); + line_sender_close(sender); + return false; +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const char* arg = argv[index]; + if ((strncmp(arg, "-h", 2) == 0) || (strncmp(arg, "--help", 6) == 0)) + { + fprintf(stderr, "Usage:\n"); + fprintf( + stderr, + "line_sender_c_example_decimal_binary: [HOST [PORT]]\n"); + fprintf( + stderr, + " HOST: ILP/HTTP host (defaults to \"localhost\").\n"); + fprintf( + stderr, " PORT: ILP/HTTP port (defaults to \"9000\").\n"); + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + const char* host = "localhost"; + if (argc >= 2) + host = argv[1]; + const char* port = "9000"; + if (argc >= 3) + port = argv[2]; + + return !example(host, port); +} diff --git a/examples/line_sender_c_example_from_conf.c b/examples/line_sender_c_example_from_conf.c index 757071d0..04a46b31 100644 --- a/examples/line_sender_c_example_from_conf.c +++ b/examples/line_sender_c_example_from_conf.c @@ -9,7 +9,7 @@ int main(int argc, const char* argv[]) line_sender_buffer* buffer = NULL; line_sender_utf8 conf = - QDB_UTF8_LITERAL("tcp::addr=localhost:9009;protocol_version=2;"); + QDB_UTF8_LITERAL("tcp::addr=localhost:9009;protocol_version=3;"); line_sender* sender = line_sender_from_conf(conf, &err); if (!sender) goto on_error; @@ -38,7 +38,9 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -59,6 +61,7 @@ int main(int argc, const char* argv[]) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return 0; diff --git a/examples/line_sender_c_example_from_env.c b/examples/line_sender_c_example_from_env.c index 823c5928..d927bac7 100644 --- a/examples/line_sender_c_example_from_env.c +++ b/examples/line_sender_c_example_from_env.c @@ -37,7 +37,9 @@ int main(int argc, const char* argv[]) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -58,6 +60,7 @@ int main(int argc, const char* argv[]) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return 0; diff --git a/examples/line_sender_c_example_http.c b/examples/line_sender_c_example_http.c index 5b9a82cb..349f5e9d 100644 --- a/examples/line_sender_c_example_http.c +++ b/examples/line_sender_c_example_http.c @@ -49,7 +49,9 @@ static bool example(const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -70,6 +72,7 @@ static bool example(const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_c_example_tls_ca.c b/examples/line_sender_c_example_tls_ca.c index a5657ede..a8591ca2 100644 --- a/examples/line_sender_c_example_tls_ca.c +++ b/examples/line_sender_c_example_tls_ca.c @@ -15,7 +15,7 @@ static bool example(const char* ca_path, const char* host, const char* port) ":", port, ";", - "protocol_version=2;" + "protocol_version=3;" "tls_roots=", ca_path, ";", @@ -64,7 +64,9 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_buffer_symbol(buffer, side_name, side_value, &err)) goto on_error; - if (!line_sender_buffer_column_f64(buffer, price_name, 2615.54, &err)) + line_sender_utf8 price_value = QDB_UTF8_LITERAL("2615.54"); + if (!line_sender_buffer_column_dec_str( + buffer, price_name, price_value, &err)) goto on_error; if (!line_sender_buffer_column_f64(buffer, amount_name, 0.00044, &err)) @@ -85,6 +87,7 @@ static bool example(const char* ca_path, const char* host, const char* port) if (!line_sender_flush(sender, buffer, &err)) goto on_error; + line_sender_buffer_free(buffer); line_sender_close(sender); return true; diff --git a/examples/line_sender_cpp_example.cpp b/examples/line_sender_cpp_example.cpp index 97943e74..a1e2f1a2 100644 --- a/examples/line_sender_cpp_example.cpp +++ b/examples/line_sender_cpp_example.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -10,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -25,7 +26,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_array_c_major.cpp b/examples/line_sender_cpp_example_array_c_major.cpp index 01bb869a..122c437f 100644 --- a/examples/line_sender_cpp_example_array_c_major.cpp +++ b/examples/line_sender_cpp_example_array_c_major.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); const auto table_name = "cpp_market_orders_c_major"_tn; const auto symbol_col = "symbol"_cn; diff --git a/examples/line_sender_cpp_example_array_elem_strides.cpp b/examples/line_sender_cpp_example_array_elem_strides.cpp index 7598de48..8864f2fb 100644 --- a/examples/line_sender_cpp_example_array_elem_strides.cpp +++ b/examples/line_sender_cpp_example_array_elem_strides.cpp @@ -14,7 +14,7 @@ static bool array_example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;"); + ";protocol_version=3;"); const auto table_name = "cpp_market_orders_elem_strides"_tn; const auto symbol_col = "symbol"_cn; diff --git a/examples/line_sender_cpp_example_auth.cpp b/examples/line_sender_cpp_example_auth.cpp index 9ec2f5e7..07cb7c6c 100644 --- a/examples/line_sender_cpp_example_auth.cpp +++ b/examples/line_sender_cpp_example_auth.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -10,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcp::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -29,7 +30,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_auth_tls.cpp b/examples/line_sender_cpp_example_auth_tls.cpp index 3f08d8c7..dc1ba25f 100644 --- a/examples/line_sender_cpp_example_auth_tls.cpp +++ b/examples/line_sender_cpp_example_auth_tls.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -10,7 +11,7 @@ static bool example(std::string_view host, std::string_view port) { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -29,7 +30,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_decimal_binary.cpp b/examples/line_sender_cpp_example_decimal_binary.cpp new file mode 100644 index 00000000..7115265d --- /dev/null +++ b/examples/line_sender_cpp_example_decimal_binary.cpp @@ -0,0 +1,84 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; + +static bool example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_trades_decimal"_tn; + const auto symbol_name = "symbol"_cn; + const auto side_name = "side"_cn; + const auto price_name = "price"_cn; + const auto amount_name = "amount"_cn; + const uint8_t price_unscaled_value[] = {123}; + // 123 with a scale of 1 gives a decimal of 12.3 + const auto price_value = + questdb::ingress::decimal::decimal_view(1, price_unscaled_value); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_name, "ETH-USD"_utf8) + .symbol(side_name, "sell"_utf8) + .column(price_name, price_value) + .column(amount_name, 0.00044) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "Error running example: " << err.what() << std::endl; + + return false; + } +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const std::string_view arg{argv[index]}; + if ((arg == "-h"sv) || (arg == "--help"sv)) + { + std::cerr + << "Usage:\n" + << " " << argv[0] << ": [HOST [PORT]]\n" + << " HOST: ILP/HTTP host (defaults to \"localhost\").\n" + << " PORT: ILP/HTTP port (defaults to \"9000\")." + << std::endl; + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9000"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_decimal_custom.cpp b/examples/line_sender_cpp_example_decimal_custom.cpp new file mode 100644 index 00000000..b71931ea --- /dev/null +++ b/examples/line_sender_cpp_example_decimal_custom.cpp @@ -0,0 +1,137 @@ +#include +#include + +using namespace std::literals::string_view_literals; +using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; + +struct ViewHolder +{ + std::array data; + uint32_t scale; + questdb::ingress::decimal::decimal_view view() const + { + return {scale, data}; + } +}; + +namespace custom_decimal +{ +class Decimal32 +{ +public: + Decimal32(uint32_t scale, int32_t unscaled_value) + : _scale(scale) + , _unscaled_value(unscaled_value) + { + } + + int32_t unscaled_value() const + { + return _unscaled_value; + } + + uint32_t scale() const + { + return _scale; + } + +private: + uint32_t _scale; + int32_t _unscaled_value; +}; + +// Customization point for QuestDB decimal API (discovered via König lookup) +// If you need to support a 3rd party type, put this function in the namespace +// of the type in question or in the `questdb::ingress::decimal` namespace +inline auto to_decimal_view_state_impl(const Decimal32& d) +{ + int32_t unscaled_value = d.unscaled_value(); + return ViewHolder{ + { + // Big-Endian bytes + static_cast(unscaled_value >> 24), + static_cast(unscaled_value >> 16), + static_cast(unscaled_value >> 8), + static_cast(unscaled_value >> 0), + }, + d.scale()}; +} +} + +static bool example(std::string_view host, std::string_view port) +{ + try + { + auto sender = questdb::ingress::line_sender::from_conf( + "http::addr=" + std::string{host} + ":" + std::string{port} + ";"); + + // We prepare all our table names and column names in advance. + // If we're inserting multiple rows, this allows us to avoid + // re-validating the same strings over and over again. + const auto table_name = "cpp_trades_decimal"_tn; + const auto symbol_name = "symbol"_cn; + const auto side_name = "side"_cn; + const auto price_name = "price"_cn; + const auto amount_name = "amount"_cn; + + // 123 with a scale of 1 gives a decimal of 12.3 + const auto price_value = custom_decimal::Decimal32(1, 123); + + questdb::ingress::line_sender_buffer buffer = sender.new_buffer(); + buffer.table(table_name) + .symbol(symbol_name, "ETH-USD"_utf8) + .symbol(side_name, "sell"_utf8) + .column(price_name, price_value) + .column(amount_name, 0.00044) + .at(questdb::ingress::timestamp_nanos::now()); + + // To insert more records, call `buffer.table(..)...` again. + + sender.flush(buffer); + + // It's recommended to keep a timer and/or maximum buffer size to flush + // the buffer periodically with any accumulated records. + + return true; + } + catch (const questdb::ingress::line_sender_error& err) + { + std::cerr << "Error running example: " << err.what() << std::endl; + + return false; + } +} + +static bool displayed_help(int argc, const char* argv[]) +{ + for (int index = 1; index < argc; ++index) + { + const std::string_view arg{argv[index]}; + if ((arg == "-h"sv) || (arg == "--help"sv)) + { + std::cerr << "Usage:\n" + << " " << argv[0] << ": [HOST [PORT]]\n" + << " HOST: ILP host (defaults to \"localhost\").\n" + << " PORT: ILP port (defaults to \"9000\")." + << std::endl; + return true; + } + } + return false; +} + +int main(int argc, const char* argv[]) +{ + if (displayed_help(argc, argv)) + return 0; + + auto host = "localhost"sv; + if (argc >= 2) + host = std::string_view{argv[1]}; + auto port = "9000"sv; + if (argc >= 3) + port = std::string_view{argv[2]}; + + return !example(host, port); +} diff --git a/examples/line_sender_cpp_example_from_conf.cpp b/examples/line_sender_cpp_example_from_conf.cpp index 697850e0..142efb27 100644 --- a/examples/line_sender_cpp_example_from_conf.cpp +++ b/examples/line_sender_cpp_example_from_conf.cpp @@ -3,13 +3,14 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; int main(int argc, const char* argv[]) { try { auto sender = questdb::ingress::line_sender::from_conf( - "tcp::addr=localhost:9009;protocol_version=2;"); + "tcp::addr=localhost:9009;protocol_version=3;"); // We prepare all our table names and column names in advance. // If we're inserting multiple rows, this allows us to avoid @@ -24,7 +25,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_from_env.cpp b/examples/line_sender_cpp_example_from_env.cpp index 3bf1c02a..515a68a2 100644 --- a/examples/line_sender_cpp_example_from_env.cpp +++ b/examples/line_sender_cpp_example_from_env.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; int main(int argc, const char* argv[]) { @@ -23,7 +24,7 @@ int main(int argc, const char* argv[]) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/examples/line_sender_cpp_example_http.cpp b/examples/line_sender_cpp_example_http.cpp index 1e675935..80969151 100644 --- a/examples/line_sender_cpp_example_http.cpp +++ b/examples/line_sender_cpp_example_http.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example(std::string_view host, std::string_view port) { @@ -24,7 +25,7 @@ static bool example(std::string_view host, std::string_view port) buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); @@ -52,11 +53,12 @@ static bool displayed_help(int argc, const char* argv[]) const std::string_view arg{argv[index]}; if ((arg == "-h"sv) || (arg == "--help"sv)) { - std::cerr << "Usage:\n" - << "line_sender_c_example: [HOST [PORT]]\n" - << " HOST: ILP host (defaults to \"localhost\").\n" - << " PORT: ILP port (defaults to \"9009\")." - << std::endl; + std::cerr + << "Usage:\n" + << " " << argv[0] << ": [HOST [PORT]]\n" + << " HOST: ILP/HTTP host (defaults to \"localhost\").\n" + << " PORT: ILP/HTTP port (defaults to \"9000\")." + << std::endl; return true; } } @@ -71,7 +73,7 @@ int main(int argc, const char* argv[]) auto host = "localhost"sv; if (argc >= 2) host = std::string_view{argv[1]}; - auto port = "9009"sv; + auto port = "9000"sv; if (argc >= 3) port = std::string_view{argv[2]}; diff --git a/examples/line_sender_cpp_example_tls_ca.cpp b/examples/line_sender_cpp_example_tls_ca.cpp index 4e3d0f10..c7a847ee 100644 --- a/examples/line_sender_cpp_example_tls_ca.cpp +++ b/examples/line_sender_cpp_example_tls_ca.cpp @@ -3,6 +3,7 @@ using namespace std::literals::string_view_literals; using namespace questdb::ingress::literals; +using namespace questdb::ingress::decimal; static bool example( std::string_view ca_path, std::string_view host, std::string_view port) @@ -11,7 +12,7 @@ static bool example( { auto sender = questdb::ingress::line_sender::from_conf( "tcps::addr=" + std::string{host} + ":" + std::string{port} + - ";protocol_version=2;" + ";protocol_version=3;" "username=admin;" "token=5UjEMuA0Pj5pjK8a-fa24dyIf-Es5mYny3oE_Wmus48;" "token_x=fLKYEaoEb9lrn3nkwLDA-M_xnuFOdSt9y0Z7_vWSHLU;" @@ -32,7 +33,7 @@ static bool example( buffer.table(table_name) .symbol(symbol_name, "ETH-USD"_utf8) .symbol(side_name, "sell"_utf8) - .column(price_name, 2615.54) + .column(price_name, "2615.54"_decimal) .column(amount_name, 0.00044) .at(questdb::ingress::timestamp_nanos::now()); diff --git a/include/questdb/ingress/line_sender.h b/include/questdb/ingress/line_sender.h index b250d31c..10b9e332 100644 --- a/include/questdb/ingress/line_sender.h +++ b/include/questdb/ingress/line_sender.h @@ -83,6 +83,9 @@ typedef enum line_sender_error_code /** Line sender protocol version error. */ line_sender_error_protocol_version_error, + + /** The supplied decimal is invalid. */ + line_sender_error_invalid_decimal, } line_sender_error_code; /** The protocol used to connect with. */ @@ -119,6 +122,15 @@ typedef enum line_sender_protocol_version * `line_sender_protocol_version_2` support. */ line_sender_protocol_version_2 = 2, + + /** + * Version 3 of InfluxDB Line Protocol. + * Supports the decimal data type in text and binary formats. + * This version is specific to QuestDB and not compatible with InfluxDB. + * QuestDB server version 9.2.0 or later is required for + * `line_sender_protocol_version_3` support. + */ + line_sender_protocol_version_3 = 3, } line_sender_protocol_version; /** Possible sources of the root certificates used to validate the server's @@ -496,6 +508,42 @@ bool line_sender_buffer_column_str( line_sender_utf8 value, line_sender_error** err_out); +/** + * Record a decimal string value for the given column. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] value Column value. + * @param[out] err_out Set on error. + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_dec_str( + line_sender_buffer* buffer, + line_sender_column_name name, + line_sender_utf8 value, + line_sender_error** err_out); + +/** + * Record a decimal value for the given column. + * + * @param[in] buffer Line buffer object. + * @param[in] name Column name. + * @param[in] scale Number of digits after the decimal point + * @param[in] data Unscaled value in two's complement format, big-endian + * @param[in] data_len Length of the unscaled value array + * @param[out] err_out Set on error. + * @return true on success, false on error. + */ +LINESENDER_API +bool line_sender_buffer_column_dec( + line_sender_buffer* buffer, + line_sender_column_name name, + const unsigned int scale, + const uint8_t* data, + size_t data_len, + line_sender_error** err_out); + /** * Record a multidimensional array of `double` values in C-major order. * @@ -842,6 +890,9 @@ bool line_sender_opts_token_y( * * QuestDB server version 9.0.0 or later is required for * `line_sender_protocol_version_2` support. + * + * QuestDB server version 9.2.0 or later is required for + * `line_sender_protocol_version_3` support. */ LINESENDER_API bool line_sender_opts_protocol_version( diff --git a/include/questdb/ingress/line_sender.hpp b/include/questdb/ingress/line_sender.hpp index 32b89bbc..7a2fc58e 100644 --- a/include/questdb/ingress/line_sender.hpp +++ b/include/questdb/ingress/line_sender.hpp @@ -24,610 +24,11 @@ #pragma once -#include "line_sender.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#if __cplusplus >= 202002L -# include -#endif +#include "line_sender_array.hpp" +#include "line_sender_decimal.hpp" namespace questdb::ingress { -constexpr const char* inaddr_any = "0.0.0.0"; - -class line_sender; -class line_sender_buffer; -class opts; - -/** Category of error. */ -enum class line_sender_error_code -{ - /** The host, port, or interface was incorrect. */ - could_not_resolve_addr, - - /** Called methods in the wrong order. E.g. `symbol` after `column`. */ - invalid_api_call, - - /** A network error connecting or flushing data out. */ - socket_error, - - /** The string or symbol field is not encoded in valid UTF-8. */ - invalid_utf8, - - /** The table name or column name contains bad characters. */ - invalid_name, - - /** The supplied timestamp is invalid. */ - invalid_timestamp, - - /** Error during the authentication process. */ - auth_error, - - /** Error during TLS handshake. */ - tls_error, - - /** The server does not support ILP over HTTP. */ - http_not_supported, - - /** Error sent back from the server during flush. */ - server_flush_error, - - /** Bad configuration. */ - config_error, -}; - -/** The protocol used to connect with. */ -enum class protocol -{ - /** InfluxDB Line Protocol over TCP. */ - tcp, - - /** InfluxDB Line Protocol over TCP with TLS. */ - tcps, - - /** InfluxDB Line Protocol over HTTP. */ - http, - - /** InfluxDB Line Protocol over HTTP with TLS. */ - https, -}; - -enum class protocol_version -{ - /** InfluxDB Line Protocol v1. */ - v1 = 1, - - /** - * InfluxDB Line Protocol v2. - * QuestDB server version 9.0.0 or later is required for - * `v2` support. - */ - v2 = 2, -}; - -/* Possible sources of the root certificates used to validate the server's TLS - * certificate. */ -enum class ca -{ - /** Use the set of root certificates provided by the `webpki` crate. */ - webpki_roots, - - /** Use the set of root certificates provided by the operating system. */ - os_roots, - - /** Combine the set of root certificates provided by the `webpki` crate and - * the operating system. */ - webpki_and_os_roots, - - /** Use the root certificates provided in a PEM-encoded file. */ - pem_file, -}; - -/** - * An error that occurred when using the line sender. - * - * Call `.what()` to obtain the ASCII-encoded error message. - */ -class line_sender_error : public std::runtime_error -{ -public: - line_sender_error(line_sender_error_code code, const std::string& what) - : std::runtime_error{what} - , _code{code} - { - } - - /** Error code categorizing the error. */ - line_sender_error_code code() const noexcept - { - return _code; - } - -private: - inline static line_sender_error from_c(::line_sender_error* c_err) - { - line_sender_error_code code = static_cast( - static_cast(::line_sender_error_get_code(c_err))); - size_t c_len{0}; - const char* c_msg{::line_sender_error_msg(c_err, &c_len)}; - std::string msg{c_msg, c_len}; - line_sender_error err{code, msg}; - ::line_sender_error_free(c_err); - return err; - } - - template - inline static auto wrapped_call(F&& f, Args&&... args) - { - ::line_sender_error* c_err{nullptr}; - auto obj = f(std::forward(args)..., &c_err); - if (obj) - return obj; - else - throw from_c(c_err); - } - - friend class line_sender; - friend class line_sender_buffer; - friend class opts; - - template < - typename T, - bool (*F)(T*, size_t, const char*, ::line_sender_error**)> - friend class basic_view; - - line_sender_error_code _code; -}; - -/** - * Non-owning validated string. - * - * See `table_name_view`, `column_name_view` and `utf8_view` along with the - * `_utf8`, `_tn` and `_cn` literal suffixes in the `literals` namespace. - */ -template -class basic_view -{ -public: - basic_view(const char* buf, size_t len) - : _impl{0, nullptr} - { - line_sender_error::wrapped_call(F, &_impl, len, buf); - } - - template - basic_view(const char (&buf)[N]) - : basic_view{buf, N - 1} - { - } - - basic_view(std::string_view s_view) - : basic_view{s_view.data(), s_view.size()} - { - } - - basic_view(const std::string& s) - : basic_view{s.data(), s.size()} - { - } - - size_t size() const noexcept - { - return _impl.len; - } - - const char* data() const noexcept - { - return _impl.buf; - } - - std::string_view to_string_view() const noexcept - { - return std::string_view{_impl.buf, _impl.len}; - } - -private: - T _impl; - - friend class line_sender; - friend class line_sender_buffer; - friend class opts; -}; - -using utf8_view = basic_view<::line_sender_utf8, ::line_sender_utf8_init>; - -using table_name_view = - basic_view<::line_sender_table_name, ::line_sender_table_name_init>; - -using column_name_view = - basic_view<::line_sender_column_name, ::line_sender_column_name_init>; - -namespace literals -{ -/** - * Utility to construct `utf8_view` objects from string literals. - * @code {.cpp} - * auto validated = "A UTF-8 encoded string"_utf8; - * @endcode - */ -inline utf8_view operator"" _utf8(const char* buf, size_t len) -{ - return utf8_view{buf, len}; -} - -/** - * Utility to construct `table_name_view` objects from string literals. - * @code {.cpp} - * auto table_name = "events"_tn; - * @endcode - */ -inline table_name_view operator"" _tn(const char* buf, size_t len) -{ - return table_name_view{buf, len}; -} - -/** - * Utility to construct `column_name_view` objects from string literals. - * @code {.cpp} - * auto column_name = "events"_cn; - * @endcode - */ -inline column_name_view operator"" _cn(const char* buf, size_t len) -{ - return column_name_view{buf, len}; -} -} // namespace literals - -class timestamp_micros -{ -public: - template - explicit timestamp_micros(std::chrono::time_point tp) - : _ts{std::chrono::duration_cast( - tp.time_since_epoch()) - .count()} - { - } - - explicit timestamp_micros(int64_t ts) noexcept - : _ts{ts} - { - } - - int64_t as_micros() const noexcept - { - return _ts; - } - - static inline timestamp_micros now() noexcept - { - return timestamp_micros{::line_sender_now_micros()}; - } - -private: - int64_t _ts; -}; - -class timestamp_nanos -{ -public: - template - explicit timestamp_nanos(std::chrono::time_point tp) - : _ts{std::chrono::duration_cast( - tp.time_since_epoch()) - .count()} - { - } - - explicit timestamp_nanos(int64_t ts) noexcept - : _ts{ts} - { - } - - int64_t as_nanos() const noexcept - { - return _ts; - } - - static inline timestamp_nanos now() noexcept - { - return timestamp_nanos{::line_sender_now_nanos()}; - } - -private: - int64_t _ts; -}; - -#if __cplusplus < 202002L -class buffer_view final -{ -public: - /** - * Default constructor. Creates an empty buffer view. - */ - buffer_view() noexcept = default; - - /** - * Construct a buffer view from raw byte data. - * @param data Pointer to the underlying byte array (may be nullptr if - * length=0). - * @param length Number of bytes in the array. - */ - constexpr buffer_view(const std::byte* data, size_t length) noexcept - : buf(data) - , len(length) - { - } - - /** - * Obtain a pointer to the underlying byte array. - * - * @return Const pointer to the data (may be nullptr if empty()). - */ - constexpr const std::byte* data() const noexcept - { - return buf; - } - - /** - * Obtain the number of bytes in the view. - * - * @return Size of the view in bytes. - */ - constexpr size_t size() const noexcept - { - return len; - } - - /** - * Check if the buffer view is empty. - * @return true if the view has no bytes (size() == 0). - */ - constexpr bool empty() const noexcept - { - return len == 0; - } - - /** - * Check byte-wise if two buffer views are equal. - * @return true if both views have the same size and - * the same byte content. - */ - friend bool operator==( - const buffer_view& lhs, const buffer_view& rhs) noexcept - { - return lhs.size() == rhs.size() && - std::equal(lhs.buf, lhs.buf + lhs.len, rhs.buf); - } - -private: - const std::byte* buf{nullptr}; - size_t len{0}; -}; -#endif - -namespace array -{ -enum class strides_mode -{ - /** Strides are provided in bytes */ - bytes, - - /** Strides are provided in elements */ - elements, -}; - -/** - * A view over a multi-dimensional array with custom strides. - * - * The strides can be expressed as bytes offsets or as element counts. - * The `rank` is the number of dimensions in the array, and the `shape` - * describes the size of each dimension. - * - * If the data is stored in a row-major order, it may be more convenient and - * efficient to use the `row_major_view` instead of `strided_view`. - * - * The `data` pointer must point to a contiguous block of memory that contains - * the array data. - */ -template -class strided_view -{ -public: - using element_type = T; - static constexpr strides_mode stride_size_mode = M; - - strided_view( - size_t rank, - const uintptr_t* shape, - const intptr_t* strides, - const T* data, - size_t data_size) - : _rank{rank} - , _shape{shape} - , _strides{strides} - , _data{data} - , _data_size{data_size} - { - } - - size_t rank() const - { - return _rank; - } - - const uintptr_t* shape() const - { - return _shape; - } - - const intptr_t* strides() const - { - return _strides; - } - - const T* data() const - { - return _data; - } - - size_t data_size() const - { - return _data_size; - } - - const strided_view& view() const - { - return *this; - } - -private: - size_t _rank; - const uintptr_t* _shape; - const intptr_t* _strides; - const T* _data; - size_t _data_size; -}; - -/** - * A view over a multi-dimensional array in row-major order. - * - * The `rank` is the number of dimensions in the array, and the `shape` - * describes the size of each dimension. - * - * The `data` pointer must point to a contiguous block of memory that contains - * the array data. - * - * If the source array is not stored in a row-major order, you may express - * the strides explicitly using the `strided_view` class. - * - * This class provides a simpler and more efficient interface for row-major - * arrays. - */ -template -class row_major_view -{ -public: - using element_type = T; - - row_major_view( - size_t rank, const uintptr_t* shape, const T* data, size_t data_size) - : _rank{rank} - , _shape{shape} - , _data{data} - , _data_size{data_size} - { - } - - size_t rank() const - { - return _rank; - } - const uintptr_t* shape() const - { - return _shape; - } - const T* data() const - { - return _data; - } - - size_t data_size() const - { - return _data_size; - } - - const row_major_view& view() const - { - return *this; - } - -private: - size_t _rank; - const uintptr_t* _shape; - const T* _data; - size_t _data_size; -}; - -template -struct row_major_1d_holder -{ - uintptr_t shape[1]; - const T* data; - size_t size; - - row_major_1d_holder(const T* d, size_t s) - : data(d) - , size(s) - { - shape[0] = static_cast(s); - } - - array::row_major_view view() const - { - return {1, shape, data, size}; - } -}; - -template -inline auto to_array_view_state_impl(const std::vector& vec) -{ - return row_major_1d_holder::type>( - vec.data(), vec.size()); -} - -#if __cplusplus >= 202002L -template -inline auto to_array_view_state_impl(const std::span& span) -{ - return row_major_1d_holder::type>( - span.data(), span.size()); -} -#endif - -template -inline auto to_array_view_state_impl(const std::array& arr) -{ - return row_major_1d_holder::type>(arr.data(), N); -} - -/** - * Customization point to enable serialization of additonal types as arrays. - * - * Forwards to a namespace or ADL (König) lookup function. - * The customized `to_array_view_state_impl` for your custom type can be placed - * in either: - * * The namespace of the type in question. - * * In the `questdb::ingress::array` namespace. -/// - * The function can either return a view object directly (either - * `row_major_view` or `strided_view`), or, if you need to place some fields on - * the stack, an object with a `.view()` method which returns a `const&` to one - * "materialize" shape or strides information into contiguous memory. - * of the two view types. Returning an object may be useful if you need to - */ -struct to_array_view_state_fn -{ - template - auto operator()(const T& array) const - { - // Implement your own `to_array_view_state_impl` as needed. - return to_array_view_state_impl(array); - } -}; - -inline constexpr to_array_view_state_fn to_array_view_state{}; - -} // namespace array - class line_sender_buffer { public: @@ -988,7 +389,11 @@ class line_sender_buffer * @param name Column name. * @param array Multi-dimensional array. */ - template + template < + typename ToArrayViewT, + std::enable_if_t< + questdb::ingress::array::has_array_view_state_v, + int> = 0> line_sender_buffer& column(column_name_view name, ToArrayViewT array) { may_init(); @@ -1026,6 +431,102 @@ class line_sender_buffer return column(name, utf8_view{value}); } + /** + * Record an arbitrary-precision decimal value from a text representation. + * + * This sends the decimal as a string (e.g., "123.456") to be parsed by + * the QuestDB server. + * + * For better performance and precision control, consider using the binary + * format via `decimal::decimal_view` instead. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * @param name Column name. + * @param value Decimal value as a validated UTF-8 string. + */ + line_sender_buffer& column( + column_name_view name, decimal::decimal_str_view value) + { + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_dec_str, + _impl, + name._impl, + value.view()._impl); + return *this; + } + + /** + * Record an arbitrary-precision decimal value in binary format. + * + * The decimal is represented as an unscaled integer (mantissa) and a scale. + * This provides precise control over the decimal representation and is more + * efficient than text-based serialization. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * # Constraints + * + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes (protocol limitation) + * + * @param name Column name. + * @param decimal Binary decimal view with scale and mantissa bytes. + */ + line_sender_buffer& column( + column_name_view name, const decimal::decimal_view& decimal) + { + may_init(); + line_sender_error::wrapped_call( + ::line_sender_buffer_column_dec, + _impl, + name._impl, + decimal.scale(), + decimal.data(), + decimal.data_size()); + return *this; + } + + /** + * Record a decimal value using a custom type via a customization point. + * + * This overload allows you to serialize custom decimal types by + * implementing a `to_decimal_view_state_impl` function for your type. + * + * QuestDB server version 9.2.0 or later is required for decimal support. + * + * # Customization + * + * To support your custom decimal type, implement + * `to_decimal_view_state_impl` in either: + * - The namespace of your type (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function should return either: + * - A `decimal::decimal_view` directly, or + * - An object with a `.view()` method returning `const + * decimal::decimal_view&` + * + * Include your customization point before including `line_sender.hpp`. + * + * @tparam ToDecimalViewT Type convertible to decimal::decimal_view. + * @param name Column name. + * @param decimal Custom decimal value. + */ + template < + typename ToDecimalViewT, + std::enable_if_t< + questdb::ingress::decimal::has_decimal_view_state_v, + int> = 0> + line_sender_buffer& column(column_name_view name, ToDecimalViewT decimal) + { + may_init(); + const auto decimal_view_state = + questdb::ingress::decimal::to_decimal_view_state(decimal); + return column(name, decimal_view_state.view()); + } + /** Record a nanosecond timestamp value for the given column. */ template line_sender_buffer& column( diff --git a/include/questdb/ingress/line_sender_array.hpp b/include/questdb/ingress/line_sender_array.hpp new file mode 100644 index 00000000..d4042b25 --- /dev/null +++ b/include/questdb/ingress/line_sender_array.hpp @@ -0,0 +1,266 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender.h" + +#include +#include +#include +#include +#include +#include +#if __cplusplus >= 202002L +# include +#endif + +namespace questdb::ingress::array +{ +enum class strides_mode +{ + /** Strides are provided in bytes */ + bytes, + + /** Strides are provided in elements */ + elements, +}; + +/** + * A view over a multi-dimensional array with custom strides. + * + * The strides can be expressed as bytes offsets or as element counts. + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * If the data is stored in a row-major order, it may be more convenient and + * efficient to use the `row_major_view` instead of `strided_view`. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + */ +template +class strided_view +{ +public: + using element_type = T; + static constexpr strides_mode stride_size_mode = M; + + strided_view( + size_t rank, + const uintptr_t* shape, + const intptr_t* strides, + const T* data, + size_t data_size) + : _rank{rank} + , _shape{shape} + , _strides{strides} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + + const uintptr_t* shape() const + { + return _shape; + } + + const intptr_t* strides() const + { + return _strides; + } + + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + + const strided_view& view() const + { + return *this; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const intptr_t* _strides; + const T* _data; + size_t _data_size; +}; + +/** + * A view over a multi-dimensional array in row-major order. + * + * The `rank` is the number of dimensions in the array, and the `shape` + * describes the size of each dimension. + * + * The `data` pointer must point to a contiguous block of memory that contains + * the array data. + * + * If the source array is not stored in a row-major order, you may express + * the strides explicitly using the `strided_view` class. + * + * This class provides a simpler and more efficient interface for row-major + * arrays. + */ +template +class row_major_view +{ +public: + using element_type = T; + + row_major_view( + size_t rank, const uintptr_t* shape, const T* data, size_t data_size) + : _rank{rank} + , _shape{shape} + , _data{data} + , _data_size{data_size} + { + } + + size_t rank() const + { + return _rank; + } + const uintptr_t* shape() const + { + return _shape; + } + const T* data() const + { + return _data; + } + + size_t data_size() const + { + return _data_size; + } + + const row_major_view& view() const + { + return *this; + } + +private: + size_t _rank; + const uintptr_t* _shape; + const T* _data; + size_t _data_size; +}; + +template +struct row_major_1d_holder +{ + uintptr_t shape[1]; + const T* data; + size_t size; + + row_major_1d_holder(const T* d, size_t s) + : data(d) + , size(s) + { + shape[0] = static_cast(s); + } + + array::row_major_view view() const + { + return {1, shape, data, size}; + } +}; + +template +inline auto to_array_view_state_impl(const std::vector& vec) +{ + return row_major_1d_holder::type>( + vec.data(), vec.size()); +} + +#if __cplusplus >= 202002L +template +inline auto to_array_view_state_impl(const std::span& span) +{ + return row_major_1d_holder::type>( + span.data(), span.size()); +} +#endif + +template +inline auto to_array_view_state_impl(const std::array& arr) +{ + return row_major_1d_holder::type>(arr.data(), N); +} + +/** + * Customization point to enable serialization of additional types as arrays. + * + * Forwards to a namespace or ADL (König) lookup function. + * The customized `to_array_view_state_impl` for your custom type can be placed + * in either: + * * The namespace of the type in question. + * * In the `questdb::ingress::array` namespace. +/// + * The function can either return a view object directly (either + * `row_major_view` or `strided_view`), or, if you need to place some fields on + * the stack, an object with a `.view()` method which returns a `const&` to one + * "materialize" shape or strides information into contiguous memory. + * of the two view types. Returning an object may be useful if you need to + */ +struct to_array_view_state_fn +{ + template + auto operator()(const T& array) const + { + // Implement your own `to_array_view_state_impl` as needed. + return to_array_view_state_impl(array); + } +}; + +inline constexpr to_array_view_state_fn to_array_view_state{}; + +template +struct has_array_view_state : std::false_type +{ +}; + +template +struct has_array_view_state< + T, + std::void_t()))>> + : std::true_type +{ +}; + +template +inline constexpr bool has_array_view_state_v = has_array_view_state::value; +} // namespace questdb::ingress::array diff --git a/include/questdb/ingress/line_sender_core.hpp b/include/questdb/ingress/line_sender_core.hpp new file mode 100644 index 00000000..8fd3685b --- /dev/null +++ b/include/questdb/ingress/line_sender_core.hpp @@ -0,0 +1,432 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#if __cplusplus >= 202002L +# include +#endif + +namespace questdb::ingress +{ +constexpr const char* inaddr_any = "0.0.0.0"; + +class line_sender; +class line_sender_buffer; +class opts; + +/** Category of error. */ +enum class line_sender_error_code +{ + /** The host, port, or interface was incorrect. */ + could_not_resolve_addr, + + /** Called methods in the wrong order. E.g. `symbol` after `column`. */ + invalid_api_call, + + /** A network error connecting or flushing data out. */ + socket_error, + + /** The string or symbol field is not encoded in valid UTF-8. */ + invalid_utf8, + + /** The table name or column name contains bad characters. */ + invalid_name, + + /** The supplied timestamp is invalid. */ + invalid_timestamp, + + /** Error during the authentication process. */ + auth_error, + + /** Error during TLS handshake. */ + tls_error, + + /** The server does not support ILP over HTTP. */ + http_not_supported, + + /** Error sent back from the server during flush. */ + server_flush_error, + + /** Bad configuration. */ + config_error, + + /** There was an error serializing an array. */ + array_error, + + /** Line sender protocol version error. */ + protocol_version_error, + + /** The supplied decimal is invalid. */ + invalid_decimal, +}; + +/** The protocol used to connect with. */ +enum class protocol +{ + /** InfluxDB Line Protocol over TCP. */ + tcp, + + /** InfluxDB Line Protocol over TCP with TLS. */ + tcps, + + /** InfluxDB Line Protocol over HTTP. */ + http, + + /** InfluxDB Line Protocol over HTTP with TLS. */ + https, +}; + +enum class protocol_version +{ + /** InfluxDB Line Protocol v1. */ + v1 = 1, + + /** + * InfluxDB Line Protocol v2. + * QuestDB server version 9.0.0 or later is required for + * `v2` support. + */ + v2 = 2, + + /** + * InfluxDB Line Protocol v3. + * QuestDB server version 9.2.0 or later is required for + * `v3` support. + */ + v3 = 3, +}; + +/* Possible sources of the root certificates used to validate the server's TLS + * certificate. */ +enum class ca +{ + /** Use the set of root certificates provided by the `webpki` crate. */ + webpki_roots, + + /** Use the set of root certificates provided by the operating system. */ + os_roots, + + /** Combine the set of root certificates provided by the `webpki` crate and + * the operating system. */ + webpki_and_os_roots, + + /** Use the root certificates provided in a PEM-encoded file. */ + pem_file, +}; + +/** + * An error that occurred when using the line sender. + * + * Call `.what()` to obtain the ASCII-encoded error message. + */ +class line_sender_error : public std::runtime_error +{ +public: + line_sender_error(line_sender_error_code code, const std::string& what) + : std::runtime_error{what} + , _code{code} + { + } + + /** Error code categorizing the error. */ + line_sender_error_code code() const noexcept + { + return _code; + } + +private: + inline static line_sender_error from_c(::line_sender_error* c_err) + { + line_sender_error_code code = static_cast( + static_cast(::line_sender_error_get_code(c_err))); + size_t c_len{0}; + const char* c_msg{::line_sender_error_msg(c_err, &c_len)}; + std::string msg{c_msg, c_len}; + line_sender_error err{code, msg}; + ::line_sender_error_free(c_err); + return err; + } + + template + inline static auto wrapped_call(F&& f, Args&&... args) + { + ::line_sender_error* c_err{nullptr}; + auto obj = f(std::forward(args)..., &c_err); + if (obj) + return obj; + else + throw from_c(c_err); + } + + friend class line_sender; + friend class line_sender_buffer; + friend class opts; + + template < + typename T, + bool (*F)(T*, size_t, const char*, ::line_sender_error**)> + friend class basic_view; + + line_sender_error_code _code; +}; + +/** + * Non-owning validated string. + * + * See `table_name_view`, `column_name_view` and `utf8_view` along with the + * `_utf8`, `_tn` and `_cn` literal suffixes in the `literals` namespace. + */ +template +class basic_view +{ +public: + basic_view(const char* buf, size_t len) + : _impl{0, nullptr} + { + line_sender_error::wrapped_call(F, &_impl, len, buf); + } + + template + basic_view(const char (&buf)[N]) + : basic_view{buf, N - 1} + { + } + + basic_view(std::string_view s_view) + : basic_view{s_view.data(), s_view.size()} + { + } + + basic_view(const std::string& s) + : basic_view{s.data(), s.size()} + { + } + + size_t size() const noexcept + { + return _impl.len; + } + + const char* data() const noexcept + { + return _impl.buf; + } + + std::string_view to_string_view() const noexcept + { + return std::string_view{_impl.buf, _impl.len}; + } + +private: + T _impl; + + friend class line_sender; + friend class line_sender_buffer; + friend class opts; +}; + +using utf8_view = basic_view<::line_sender_utf8, ::line_sender_utf8_init>; + +using table_name_view = + basic_view<::line_sender_table_name, ::line_sender_table_name_init>; + +using column_name_view = + basic_view<::line_sender_column_name, ::line_sender_column_name_init>; + +namespace literals +{ +/** + * Utility to construct `utf8_view` objects from string literals. + * @code {.cpp} + * auto validated = "A UTF-8 encoded string"_utf8; + * @endcode + */ +inline utf8_view operator"" _utf8(const char* buf, size_t len) +{ + return utf8_view{buf, len}; +} + +/** + * Utility to construct `table_name_view` objects from string literals. + * @code {.cpp} + * auto table_name = "events"_tn; + * @endcode + */ +inline table_name_view operator"" _tn(const char* buf, size_t len) +{ + return table_name_view{buf, len}; +} + +/** + * Utility to construct `column_name_view` objects from string literals. + * @code {.cpp} + * auto column_name = "events"_cn; + * @endcode + */ +inline column_name_view operator"" _cn(const char* buf, size_t len) +{ + return column_name_view{buf, len}; +} +} // namespace literals + +class timestamp_micros +{ +public: + template + explicit timestamp_micros(std::chrono::time_point tp) + : _ts{std::chrono::duration_cast( + tp.time_since_epoch()) + .count()} + { + } + + explicit timestamp_micros(int64_t ts) noexcept + : _ts{ts} + { + } + + int64_t as_micros() const noexcept + { + return _ts; + } + + static inline timestamp_micros now() noexcept + { + return timestamp_micros{::line_sender_now_micros()}; + } + +private: + int64_t _ts; +}; + +class timestamp_nanos +{ +public: + template + explicit timestamp_nanos(std::chrono::time_point tp) + : _ts{std::chrono::duration_cast( + tp.time_since_epoch()) + .count()} + { + } + + explicit timestamp_nanos(int64_t ts) noexcept + : _ts{ts} + { + } + + int64_t as_nanos() const noexcept + { + return _ts; + } + + static inline timestamp_nanos now() noexcept + { + return timestamp_nanos{::line_sender_now_nanos()}; + } + +private: + int64_t _ts; +}; + +#if __cplusplus < 202002L +class buffer_view final +{ +public: + /** + * Default constructor. Creates an empty buffer view. + */ + buffer_view() noexcept = default; + + /** + * Construct a buffer view from raw byte data. + * @param data Pointer to the underlying byte array (may be nullptr if + * length=0). + * @param length Number of bytes in the array. + */ + constexpr buffer_view(const std::byte* data, size_t length) noexcept + : buf(data) + , len(length) + { + } + + /** + * Obtain a pointer to the underlying byte array. + * + * @return Const pointer to the data (may be nullptr if empty()). + */ + constexpr const std::byte* data() const noexcept + { + return buf; + } + + /** + * Obtain the number of bytes in the view. + * + * @return Size of the view in bytes. + */ + constexpr size_t size() const noexcept + { + return len; + } + + /** + * Check if the buffer view is empty. + * @return true if the view has no bytes (size() == 0). + */ + constexpr bool empty() const noexcept + { + return len == 0; + } + + /** + * Check byte-wise if two buffer views are equal. + * @return true if both views have the same size and + * the same byte content. + */ + friend bool operator==( + const buffer_view& lhs, const buffer_view& rhs) noexcept + { + return lhs.size() == rhs.size() && + std::equal(lhs.buf, lhs.buf + lhs.len, rhs.buf); + } + +private: + const std::byte* buf{nullptr}; + size_t len{0}; +}; +#endif + +} // namespace questdb::ingress \ No newline at end of file diff --git a/include/questdb/ingress/line_sender_decimal.hpp b/include/questdb/ingress/line_sender_decimal.hpp new file mode 100644 index 00000000..c6501e24 --- /dev/null +++ b/include/questdb/ingress/line_sender_decimal.hpp @@ -0,0 +1,282 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +#pragma once + +#include "line_sender_core.hpp" + +/** + * Types and utilities for working with arbitrary-precision decimal numbers. + * + * Decimals are represented as an unscaled integer value (mantissa) and a scale. + * For example, the decimal "123.45" with scale 2 is represented as: + * - Unscaled value: 12345 + * - Scale: 2 (meaning divide by 10^2 = 100) + * + * QuestDB supports decimal values with: + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes in binary format + * + * QuestDB server version 9.2.0 or later is required for decimal support. + */ +namespace questdb::ingress::decimal +{ + +/** + * A validated UTF-8 string view for text-based decimal representation. + * + * This is a wrapper around utf8_view that allows the compiler to distinguish + * between regular strings and decimal strings. + * + * Use this to send decimal values as strings (e.g., "123.456"). + * The string will be parsed by the QuestDB server as a decimal column type. + */ +class decimal_str_view +{ +public: + decimal_str_view(const char* buf, size_t len) + : _view{buf, len} + { + } + + template + decimal_str_view(const char (&buf)[N]) + : _view{buf} + { + } + + decimal_str_view(std::string_view s_view) + : _view{s_view} + { + } + + decimal_str_view(const std::string& s) + : _view{s} + { + } + + const utf8_view& view() const noexcept + { + return _view; + } + +private: + utf8_view _view; + + friend class line_sender_buffer; +}; + +/** + * Literal suffix to construct `decimal_str_view` objects from string literals. + * + * @code {.cpp} + * using namespace questdb::ingress::decimal; + * buffer.column("price"_cn, "123.456"_decimal); + * @endcode + */ +inline decimal_str_view operator"" _decimal(const char* buf, size_t len) +{ + return decimal_str_view{buf, len}; +} + +/** + * A view over a decimal number in binary format. + * + * The decimal is represented as: + * - A scale (number of decimal places) + * - An unscaled value (mantissa) encoded as bytes in two's complement + * big-endian format + * + * # Example + * + * To represent the decimal "123.45": + * - Scale: 2 + * - Unscaled value: 12345 = 0x3039 in big-endian format + * + * ```c++ + * // Represent 123.45 with scale 2 (unscaled value is 12345) + * uint8_t mantissa[] = {0x30, 0x39}; // 12345 in two's complement big-endian + * auto decimal = questdb::ingress::decimal::decimal_view(2, mantissa, + * sizeof(mantissa)); buffer.column("price"_cn, decimal); + * ``` + * + * # Constraints + * + * - Maximum scale: 76 (QuestDB server limitation) + * - Maximum mantissa size: 127 bytes (protocol limitation) + */ +class decimal_view +{ +public: + /** + * Construct a binary decimal view from raw bytes. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Pointer to unscaled value in two's complement big-endian + * format + * @param data_size Number of bytes in the mantissa (must be ≤ 127) + */ + decimal_view(uint32_t scale, const uint8_t* data, size_t data_size) + : _scale{scale} + , _data{data} + , _data_size{data_size} + { + } + + /** + * Construct a binary decimal view from a fixed-size array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data Fixed-size array containing the unscaled value + */ + template + decimal_view(uint32_t scale, const uint8_t (&data)[N]) + : _scale{scale} + , _data{data} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::array. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param data std::array containing the unscaled value + */ + template + decimal_view(uint32_t scale, const std::array& data) + : _scale{scale} + , _data{data.data()} + , _data_size{N} + { + } + + /** + * Construct a binary decimal view from a std::vector. + * + * @param scale Number of decimal places (must be ≤ 76) + * @param vec Vector containing the unscaled value + */ + decimal_view(uint32_t scale, const std::vector& vec) + : _scale{scale} + , _data{vec.data()} + , _data_size{vec.size()} + { + } + +#if __cplusplus >= 202002L + /** + * Construct a binary decimal view from a std::span (C++20). + * + * @param scale Number of decimal places (must be ≤ 76) + * @param span Span containing the unscaled value + */ + decimal_view(uint32_t scale, const std::span& span) + : _scale{scale} + , _data{span.data()} + , _data_size{span.size()} + { + } +#endif + + /** Get the scale (number of decimal places). */ + uint32_t scale() const + { + return _scale; + } + + /** Get a pointer to the unscaled value bytes. */ + const uint8_t* data() const + { + return _data; + } + + /** Get the size of the unscaled value in bytes. */ + size_t data_size() const + { + return _data_size; + } + + /** Get a const reference to this view (for customization point + * compatibility). */ + const decimal_view& view() const + { + return *this; + } + +private: + uint32_t _scale; + const uint8_t* _data; + size_t _data_size; +}; +/** + * Customization point to enable serialization of additional types as decimals. + * + * This allows you to support custom decimal types by implementing a conversion + * function. The customized `to_decimal_view_state_impl` for your type can be + * placed in either: + * - The namespace of the type in question (ADL/Koenig lookup) + * - The `questdb::ingress::decimal` namespace + * + * The function can either: + * - Return a `decimal_view` object directly, or + * - Return an object with a `.view()` method that returns `const decimal_view&` + * (useful if you need to store temporary data like shape/strides on the +stack) + */ +struct to_decimal_view_state_fn +{ + template + auto operator()(const T& decimal) const + { + // Implement your own `to_decimal_view_state_impl` as needed. + // ADL lookup for user-defined to_decimal_view_state_impl + return to_decimal_view_state_impl(decimal); + } +}; + +inline constexpr to_decimal_view_state_fn to_decimal_view_state{}; + +template +struct has_decimal_view_state : std::false_type +{ +}; + +template +struct has_decimal_view_state< + T, + std::void_t()))>> + : std::true_type +{ +}; + +template +inline constexpr bool has_decimal_view_state_v = + has_decimal_view_state::value; +} // namespace questdb::ingress::decimal + +namespace questdb::ingress +{ +using decimal::decimal_view; +} // namespace questdb::ingress \ No newline at end of file diff --git a/questdb-rs-ffi/src/lib.rs b/questdb-rs-ffi/src/lib.rs index ab3fe28b..d7b76c98 100644 --- a/questdb-rs-ffi/src/lib.rs +++ b/questdb-rs-ffi/src/lib.rs @@ -25,6 +25,7 @@ #![allow(non_camel_case_types, clippy::missing_safety_doc)] use libc::{c_char, size_t}; +use questdb::ingress::DecimalView; use std::ascii; use std::boxed::Box; use std::convert::{From, Into}; @@ -223,6 +224,9 @@ pub enum line_sender_error_code { /// Line sender protocol version error. line_sender_error_protocol_version_error, + + /// The supplied decimal is invalid. + line_sender_error_invalid_decimal, } impl From for line_sender_error_code { @@ -251,6 +255,7 @@ impl From for line_sender_error_code { ErrorCode::ProtocolVersionError => { line_sender_error_code::line_sender_error_protocol_version_error } + ErrorCode::InvalidDecimal => line_sender_error_code::line_sender_error_invalid_decimal, } } } @@ -306,8 +311,14 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.0.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` support. V2 = 2, + + /// Version 3 of InfluxDB Line Protocol. + /// Supports the decimal data type in text and binary formats. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 9.2.0 or later is required for `V3` support. + V3 = 3, } impl From for ingress::ProtocolVersion { @@ -315,6 +326,7 @@ impl From for ingress::ProtocolVersion { match version { ProtocolVersion::V1 => ingress::ProtocolVersion::V1, ProtocolVersion::V2 => ingress::ProtocolVersion::V2, + ProtocolVersion::V3 => ingress::ProtocolVersion::V3, } } } @@ -324,6 +336,7 @@ impl From for ProtocolVersion { match version { ingress::ProtocolVersion::V1 => ProtocolVersion::V1, ingress::ProtocolVersion::V2 => ProtocolVersion::V2, + ingress::ProtocolVersion::V3 => ProtocolVersion::V3, } } } @@ -575,7 +588,7 @@ pub struct line_sender_column_name { } impl line_sender_column_name { - unsafe fn as_name<'a>(&self) -> ColumnName<'a> { + fn as_name<'a>(&self) -> ColumnName<'a> { unsafe { let str_name = str::from_utf8_unchecked(slice::from_raw_parts(self.buf as *const u8, self.len)); @@ -989,6 +1002,61 @@ pub unsafe extern "C" fn line_sender_buffer_column_str( } } +/// Record a decimal string value for the given column. +/// +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] value Column value. +/// @param[out] err_out Set on error. +/// @return true on success, false on error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn line_sender_buffer_column_dec_str( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + value: line_sender_utf8, + err_out: *mut *mut line_sender_error, +) -> bool { + let buffer = unsafe { unwrap_buffer_mut(buffer) }; + let name = name.as_name(); + let value = value.as_str(); + unsafe { + bubble_err_to_c!(err_out, buffer.column_dec(name, value)); + } + true +} + +/// Record a decimal value for the given column. +/// +/// @param[in] buffer Line buffer object. +/// @param[in] name Column name. +/// @param[in] scale Number of digits after the decimal point +/// @param[in] data Unscaled value in two's complement format, big-endian +/// @param[in] data_len Length of the unscaled value array +/// @param[out] err_out Set on error. +/// @return true on success, false on error. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn line_sender_buffer_column_dec( + buffer: *mut line_sender_buffer, + name: line_sender_column_name, + scale: u32, + data: *const u8, + data_len: size_t, + err_out: *mut *mut line_sender_error, +) -> bool { + unsafe { + let data = if data.is_null() { + &[] + } else { + slice::from_raw_parts(data, data_len) + }; + let buffer = unwrap_buffer_mut(buffer); + let name = name.as_name(); + let decimal = bubble_err_to_c!(err_out, DecimalView::try_new_scaled(scale, data)); + bubble_err_to_c!(err_out, buffer.column_dec(name, decimal)); + } + true +} + /// Records a float64 multidimensional array with **C-MAJOR memory layout**. /// /// @param[in] buffer Line buffer object. diff --git a/questdb-rs/Cargo.toml b/questdb-rs/Cargo.toml index b5339c18..520e076b 100644 --- a/questdb-rs/Cargo.toml +++ b/questdb-rs/Cargo.toml @@ -39,6 +39,8 @@ serde_json = { version = "1", optional = true } questdb-confstr = "0.1.1" rand = { version = "0.9.0", optional = true } ndarray = { version = "0.16", optional = true } +rust_decimal = { version = "1.38.0", optional = true } +bigdecimal = { version = "0.4.8", optional = true } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3.9", features = ["ws2def"] } @@ -67,7 +69,13 @@ sync-sender = ["sync-sender-tcp", "sync-sender-http"] sync-sender-tcp = ["_sync-sender", "_sender-tcp", "dep:socket2"] ## Sync ILP/HTTP -sync-sender-http = ["_sync-sender", "_sender-http", "dep:ureq", "dep:serde_json", "dep:rand"] +sync-sender-http = [ + "_sync-sender", + "_sender-http", + "dep:ureq", + "dep:serde_json", + "dep:rand", +] ## Allow use OS-provided root TLS certificates tls-native-certs = ["dep:rustls-native-certs"] @@ -85,11 +93,17 @@ ring-crypto = ["dep:ring", "rustls/ring"] insecure-skip-verify = [] ## Enable code-generation in `build.rs` for additional tests. -json_tests = [] +json_tests = ["bigdecimal"] ## Enable methods to create timestamp objects from chrono::DateTime objects. chrono_timestamp = ["chrono"] +## Enable serialization of rust_decimal::Decimal in ILP +rust_decimal = ["dep:rust_decimal"] + +## Enable serialization of bigdecimal::BigDecimal in ILP +bigdecimal = ["dep:bigdecimal"] + # Hidden derived features, used in code to enable-disable code sections. Don't use directly. _sender-tcp = [] _sender-http = [] @@ -108,7 +122,9 @@ almost-all-features = [ "insecure-skip-verify", "json_tests", "chrono_timestamp", - "ndarray" + "ndarray", + "rust_decimal", + "bigdecimal", ] [[example]] @@ -125,8 +141,8 @@ required-features = ["chrono_timestamp"] [[example]] name = "http" -required-features = ["sync-sender-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray", "rust_decimal"] [[example]] name = "protocol_version" -required-features = ["sync-sender-http", "ndarray"] +required-features = ["sync-sender-http", "ndarray", "bigdecimal"] diff --git a/questdb-rs/build.rs b/questdb-rs/build.rs index 20aa6529..69b01f17 100644 --- a/questdb-rs/build.rs +++ b/questdb-rs/build.rs @@ -62,6 +62,12 @@ pub mod json_tests { value: bool, } + #[derive(Debug, Serialize, Deserialize)] + struct DecimalColumn { + name: String, + value: String, + } + #[derive(Debug, Serialize, Deserialize)] #[serde(tag = "type", rename_all = "UPPERCASE")] enum Column { @@ -69,6 +75,7 @@ pub mod json_tests { Long(LongColumn), Double(DoubleColumn), Boolean(BooleanColumn), + Decimal(DecimalColumn), } #[derive(Debug, Serialize, Deserialize)] @@ -88,10 +95,19 @@ pub mod json_tests { Error, } + fn default_minimum_protocol_version() -> u32 { + 1 + } + #[derive(Debug, Serialize, Deserialize)] struct TestSpec { #[serde(rename = "testName")] test_name: String, + #[serde( + rename = "minimumProtocolVersion", + default = "default_minimum_protocol_version" + )] + minimum_protocol_version: u32, table: String, symbols: Vec, columns: Vec, @@ -126,6 +142,8 @@ pub mod json_tests { use base64ct::Base64; use base64ct::Encoding; use rstest::rstest; + use bigdecimal::BigDecimal; + use std::str::FromStr; fn matches_any_line(line: &[u8], expected: &[&str]) -> bool { for &exp in expected { @@ -133,6 +151,7 @@ pub mod json_tests { return true; } } + let line = String::from_utf8_lossy(line); eprintln!("Could not match:\n {line:?}\nTo any of: {expected:#?}"); false } @@ -140,18 +159,24 @@ pub mod json_tests { )?; for (index, spec) in specs.iter().enumerate() { - writeln!(output, "/// {}", spec.test_name)?; - // for line in serde_json::to_string_pretty(&spec).unwrap().split("\n") { - // writeln!(output, "/// {}", line)?; - // } - writeln!(output, "#[rstest]")?; - writeln!( + write!( output, - "fn test_{:03}_{}(\n #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion,\n) -> TestResult {{", + indoc! {r#" + /// {} + #[rstest] + fn test_{:03}_{}( + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] version: ProtocolVersion + ) -> TestResult {{ + if (version as u8) < {} {{ + return Ok(()); + }} + let mut buffer = Buffer::new(version); + "#}, + spec.test_name, index, - slugify!(&spec.test_name, separator = "_") + slugify!(&spec.test_name, separator = "_"), + spec.minimum_protocol_version )?; - writeln!(output, " let mut buffer = Buffer::new(version);")?; let (expected, indent) = match &spec.result { Outcome::Success(line) => (Some(line), ""), @@ -191,6 +216,11 @@ pub mod json_tests { "{} .column_bool({:?}, {:?})?", indent, column.name, column.value )?, + Column::Decimal(column) => writeln!( + output, + "{} .column_dec({:?}, &BigDecimal::from_str({:?}).unwrap())?", + indent, column.name, column.value + )?, } } writeln!(output, "{indent} .at_now()?;")?; diff --git a/questdb-rs/examples/basic.rs b/questdb-rs/examples/basic.rs index 22155c39..a9674187 100644 --- a/questdb-rs/examples/basic.rs +++ b/questdb-rs/examples/basic.rs @@ -7,8 +7,8 @@ use questdb::{ fn main() -> Result<()> { let host: String = std::env::args().nth(1).unwrap_or("localhost".to_string()); - let port: &str = &std::env::args().nth(2).unwrap_or("9009".to_string()); - let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};"))?; + let port: String = std::env::args().nth(2).unwrap_or("9009".to_string()); + let mut sender = Sender::from_conf(format!("tcp::addr={host}:{port};protocol_version=3;"))?; let mut buffer = sender.new_buffer(); let designated_timestamp = TimestampNanos::from_datetime(Utc.with_ymd_and_hms(1997, 7, 4, 4, 56, 55).unwrap())?; @@ -16,7 +16,7 @@ fn main() -> Result<()> { .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_dec("price", "2615.54")? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/http.rs b/questdb-rs/examples/http.rs index 9895be49..44acb94b 100644 --- a/questdb-rs/examples/http.rs +++ b/questdb-rs/examples/http.rs @@ -1,17 +1,23 @@ +use std::str::FromStr; + use ndarray::arr1; use questdb::{ Result, ingress::{Sender, TimestampNanos}, }; +use rust_decimal::Decimal; fn main() -> Result<()> { - let mut sender = Sender::from_conf("https::addr=localhost:9000;username=foo;password=bar;")?; + let mut sender = Sender::from_conf( + "https::addr=localhost:9000;username=foo;password=bar;protocol_version=3;", + )?; let mut buffer = sender.new_buffer(); + let price = Decimal::from_str("2615.54").unwrap(); buffer .table("trades")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_dec("price", &price)? .column_f64("amount", 0.00044)? // QuestDB server version 9.0.0 or later is required for array support. .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? diff --git a/questdb-rs/examples/protocol_version.rs b/questdb-rs/examples/protocol_version.rs index 840b4e2e..2749aeb0 100644 --- a/questdb-rs/examples/protocol_version.rs +++ b/questdb-rs/examples/protocol_version.rs @@ -1,3 +1,6 @@ +use std::str::FromStr; + +use bigdecimal::BigDecimal; use ndarray::arr1; use questdb::{ Result, @@ -6,7 +9,7 @@ use questdb::{ fn main() -> Result<()> { let mut sender = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", + "http::addr=localhost:9000;username=foo;password=bar;protocol_version=1;", )?; let mut buffer = sender.new_buffer(); buffer @@ -18,16 +21,17 @@ fn main() -> Result<()> { .at(TimestampNanos::now())?; sender.flush(&mut buffer)?; - // QuestDB server version 9.0.0 or later is required for `protocol_version=2` support. + // QuestDB server version 9.2.0 or later is required for `protocol_version=3` support. let mut sender2 = Sender::from_conf( - "https::addr=localhost:9000;username=foo;password=bar;protocol_version=2;", + "http::addr=localhost:9000;username=foo;password=bar;protocol_version=3;", )?; - let mut buffer2 = sender.new_buffer(); + let price = BigDecimal::from_str("2615.54").unwrap(); + let mut buffer2 = sender2.new_buffer(); buffer2 - .table("trades_ilp_v2")? + .table("trades_ilp_v3")? .symbol("symbol", "ETH-USD")? .symbol("side", "sell")? - .column_f64("price", 2615.54)? + .column_dec("price", &price)? .column_f64("amount", 0.00044)? .column_arr("location", &arr1(&[100.0, 100.1, 100.2]).view())? .at(TimestampNanos::now())?; diff --git a/questdb-rs/src/error.rs b/questdb-rs/src/error.rs index a28344b3..8b0c1c0e 100644 --- a/questdb-rs/src/error.rs +++ b/questdb-rs/src/error.rs @@ -78,6 +78,9 @@ pub enum ErrorCode { /// Validate protocol version error. ProtocolVersionError, + + /// The supplied decimal is invalid. + InvalidDecimal, } /// An error that occurred when using QuestDB client library. diff --git a/questdb-rs/src/ingress/buffer.rs b/questdb-rs/src/ingress/buffer.rs index 4ed554b8..5cee2497 100644 --- a/questdb-rs/src/ingress/buffer.rs +++ b/questdb-rs/src/ingress/buffer.rs @@ -21,6 +21,8 @@ * limitations under the License. * ******************************************************************************/ + +use crate::ingress::decimal::DecimalView; use crate::ingress::ndarr::{ArrayElementSealed, check_and_get_array_bytes_size}; use crate::ingress::{ ARRAY_BINARY_FORMAT_TYPE, ArrayElement, DOUBLE_BINARY_FORMAT_TYPE, DebugBytes, MAX_ARRAY_DIMS, @@ -975,6 +977,99 @@ impl Buffer { Ok(self) } + /// Record a decimal value for the given column. + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// buffer.column_dec("col_name", "123.45")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// or + /// + /// ```no_run + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use questdb::ingress::ColumnName; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let col_name = ColumnName::new("col_name")?; + /// buffer.column_dec(col_name, "123.45")?; + /// # Ok(()) + /// # } + /// ``` + /// + /// With `rust_decimal` feature enabled: + /// + /// ```no_run + /// # #[cfg(feature = "rust_decimal")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use rust_decimal::Decimal; + /// use std::str::FromStr; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value = Decimal::from_str("123.45").unwrap(); + /// buffer.column_dec("col_name", &value)?; + /// # Ok(()) + /// # } + /// # } + /// ``` + /// + /// With `bigdecimal` feature enabled: + /// + /// ```no_run + /// # #[cfg(feature = "bigdecimal")] + /// # { + /// # use questdb::Result; + /// # use questdb::ingress::{Buffer, SenderBuilder}; + /// use bigdecimal::BigDecimal; + /// use std::str::FromStr; + /// + /// # fn main() -> Result<()> { + /// # let mut sender = SenderBuilder::from_conf("https::addr=localhost:9000;")?.build()?; + /// # let mut buffer = sender.new_buffer(); + /// # buffer.table("x")?; + /// let value = BigDecimal::from_str("0.123456789012345678901234567890").unwrap(); + /// buffer.column_dec("col_name", &value)?; + /// # Ok(()) + /// # } + /// # } + /// ``` + pub fn column_dec<'a, N, S>(&mut self, name: N, value: S) -> crate::Result<&mut Self> + where + N: TryInto>, + S: TryInto>, + Error: From, + Error: From, + { + if self.protocol_version < ProtocolVersion::V3 { + return Err(error::fmt!( + ProtocolVersionError, + "Protocol version {} does not support the decimal datatype", + self.protocol_version + )); + } + + let value: DecimalView = value.try_into()?; + self.write_column_key(name)?; + value.serialize(&mut self.output); + Ok(self) + } + /// Record a multidimensional array value for the given column. /// /// Supports arrays with up to [`MAX_ARRAY_DIMS`] dimensions. The array elements must @@ -1030,10 +1125,11 @@ impl Buffer { D: ArrayElement + ArrayElementSealed, Error: From, { - if self.protocol_version == ProtocolVersion::V1 { + if self.protocol_version < ProtocolVersion::V2 { return Err(error::fmt!( ProtocolVersionError, - "Protocol version v1 does not support array datatype", + "Protocol version {} does not support array datatype", + self.protocol_version )); } let ndim = view.ndim(); diff --git a/questdb-rs/src/ingress/decimal.rs b/questdb-rs/src/ingress/decimal.rs new file mode 100644 index 00000000..74159da3 --- /dev/null +++ b/questdb-rs/src/ingress/decimal.rs @@ -0,0 +1,275 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::{Result, error}; +use std::borrow::Cow; + +/// A decimal value backed by either a string representation or a scaled mantissa. +/// +/// Decimal values can be serialized in two formats: +/// +/// ### Text Format +/// The decimal is written as a string representation followed by a `'d'` suffix. +/// +/// Example: `"123.45d"` or `"1.5e-3d"` +/// +/// ### Binary Format +/// A more compact binary encoding consisting of: +/// +/// 1. Binary format marker: `'='` (0x3D) +/// 2. Type identifier: [`DECIMAL_BINARY_FORMAT_TYPE`](crate::ingress::DECIMAL_BINARY_FORMAT_TYPE) byte +/// 3. Scale: 1 byte (0-76 inclusive) - number of decimal places +/// 4. Length: 1 byte - number of bytes in the unscaled value +/// 5. Unscaled value: variable-length byte array in two's complement format, big-endian +/// +/// Example: For decimal `123.45` with scale 2: +/// ```text +/// Unscaled value: 12345 +/// Binary representation: +/// = [23] [2] [2] [0x30] [0x39] +/// │ │ │ │ └───────────┘ +/// │ │ │ │ └─ Mantissa bytes (12345 in big-endian) +/// │ │ │ └─ Length: 2 bytes +/// │ │ └─ Scale: 2 +/// │ └─ Type: DECIMAL_BINARY_FORMAT_TYPE (23) +/// └─ Binary marker: '=' +/// ``` +/// +/// #### Binary Format Notes +/// - The unscaled value must be encoded in two's complement big-endian format +/// - Maximum scale is 76 +/// - Length byte indicates how many bytes follow for the unscaled value +#[derive(Debug)] +pub enum DecimalView<'a> { + String { value: &'a str }, + Scaled { scale: u8, value: Cow<'a, [u8]> }, +} + +impl<'a> DecimalView<'a> { + /// Creates a [`DecimalView::Scaled`] from a mantissa buffer and scale. + /// + /// Validates that: + /// - `scale` does not exceed the QuestDB maximum of 76 decimal places. + /// - The mantissa fits into at most 127 bytes (ILP binary limit). + /// + /// Returns an [`error::ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) + /// error if either constraint is violated. + pub fn try_new_scaled(scale: u32, value: T) -> Result + where + T: Into>, + { + if scale > 76 { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support decimal scale greater than 76, got {}", + scale + )); + } + let value: Cow<'a, [u8]> = value.into(); + if value.len() > i8::MAX as usize { + return Err(error::fmt!( + InvalidDecimal, + "QuestDB ILP does not support decimal longer than {} bytes, got {}", + i8::MAX, + value.len() + )); + } + Ok(DecimalView::Scaled { + scale: scale as u8, + value, + }) + } + + /// Creates a [`DecimalView::String`] from a textual decimal representation. + /// + /// Thousand separators (commas) are not allowed and the decimal point must be a dot (`.`). + /// + /// Performs lightweight validation and rejects values containing ILP-reserved characters. + /// Accepts plain decimals, optional `+/-` prefixes, `NaN`, `Infinity`, and scientific + /// notation (`e`/`E`). + /// + /// Returns [`error::ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) + /// if disallowed characters are encountered. + pub fn try_new_string(value: &'a str) -> Result { + // Basic validation: ensure only numerical characters are present (accepts NaN, Inf[inity], and e-notation) + for b in value.chars() { + match b { + '0'..='9' + | '.' + | '-' + | '+' + | 'e' + | 'E' + | 'N' + | 'a' + | 'I' + | 'n' + | 'f' + | 'i' + | 't' + | 'y' => {} + _ => { + return Err(error::fmt!( + InvalidDecimal, + "Decimal string contains ILP reserved character {:?}", + b + )); + } + } + } + Ok(DecimalView::String { value }) + } + + /// Serializes the decimal view into the provided output buffer using the ILP encoding. + /// + /// Delegates to [`serialize_string`] for textual representations and [`serialize_scaled`] for + /// the compact binary format. + pub(crate) fn serialize(&self, out: &mut Vec) { + match self { + DecimalView::String { value } => Self::serialize_string(value, out), + DecimalView::Scaled { scale, value } => { + Self::serialize_scaled(*scale, value.as_ref(), out) + } + } + } + + /// Serializes a textual decimal by copying the string and appending the `d` suffix. + fn serialize_string(value: &str, out: &mut Vec) { + // Pre-allocate space for the string content plus the 'd' suffix + out.reserve(value.len() + 1); + + out.extend_from_slice(value.as_bytes()); + + // Append the 'd' suffix to mark this as a decimal value + out.push(b'd'); + } + + /// Serializes a scaled decimal into the binary ILP format, writing the marker, type tag, + /// scale, mantissa length, and mantissa bytes. + fn serialize_scaled(scale: u8, value: &[u8], out: &mut Vec) { + // Write binary format: '=' marker + type + scale + length + mantissa bytes + out.push(b'='); + out.push(crate::ingress::DECIMAL_BINARY_FORMAT_TYPE); + out.push(scale); + out.push(value.len() as u8); + out.extend_from_slice(value); + } +} + +/// Implementation for string slices containing decimal representations. +/// +/// This implementation uses the text format. +/// +/// # Format +/// The string is validated and written as-is, followed by the 'd' suffix. Thousand separators +/// (commas) are not allowed and the decimal point must be a dot (`.`). +/// +/// # Validation +/// The implementation performs **partial validation only**: +/// - Rejects non-numerical characters (not -/+, 0-9, ., Infinity, NaN, e/E) +/// - Does NOT validate the actual decimal syntax (e.g., "e2e" would pass) +/// +/// This is intentional: full parsing would add overhead. The QuestDB server performs complete +/// validation and will reject malformed decimals. +/// +/// # Examples +/// - `"123.45"` → `"123.45d"` +/// - `"1.5e-3"` → `"1.5e-3d"` +/// - `"-0.001"` → `"-0.001d"` +/// +/// # Errors +/// Returns [`Error`] with [`ErrorCode::InvalidDecimal`](crate::error::ErrorCode::InvalidDecimal) +/// if the string contains non-numerical characters. +impl<'a> TryInto> for &'a str { + type Error = crate::Error; + + fn try_into(self) -> Result> { + DecimalView::try_new_string(self) + } +} + +#[cfg(feature = "rust_decimal")] +impl<'a> TryInto> for &'a rust_decimal::Decimal { + type Error = crate::Error; + + fn try_into(self) -> Result> { + let raw = self.mantissa().to_be_bytes(); + let bytes = trim_leading_sign_bytes(&raw); + DecimalView::try_new_scaled(self.scale(), bytes) + } +} + +#[cfg(feature = "bigdecimal")] +impl<'a> TryInto> for &'a bigdecimal::BigDecimal { + type Error = crate::Error; + + fn try_into(self) -> Result> { + let (unscaled, mut scale) = self.as_bigint_and_scale(); + + // QuestDB binary ILP doesn't support negative scale, we need to upscale the + // unscaled value to be compliant + let bytes = if scale < 0 { + use bigdecimal::num_bigint; + let unscaled = + unscaled.into_owned() * num_bigint::BigInt::from(10).pow((-scale) as u32); + scale = 0; + unscaled.to_signed_bytes_be() + } else { + unscaled.to_signed_bytes_be() + }; + + let bytes = trim_leading_sign_bytes(&bytes); + + DecimalView::try_new_scaled(scale as u32, bytes) + } +} + +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +fn trim_leading_sign_bytes(bytes: &[u8]) -> Vec { + if bytes.is_empty() { + return vec![0]; + } + + let negative = bytes[0] & 0x80 != 0; + let mut keep_from = 0usize; + + while keep_from < bytes.len() - 1 { + let current = bytes[keep_from]; + let next = bytes[keep_from + 1]; + + let should_trim = if negative { + current == 0xFF && (next & 0x80) == 0x80 + } else { + current == 0x00 && (next & 0x80) == 0x00 + }; + + if should_trim { + keep_from += 1; + } else { + break; + } + } + + bytes[keep_from..].to_vec() +} diff --git a/questdb-rs/src/ingress/mod.md b/questdb-rs/src/ingress/mod.md index 97fa5ee6..16806795 100644 --- a/questdb-rs/src/ingress/mod.md +++ b/questdb-rs/src/ingress/mod.md @@ -263,7 +263,7 @@ arrays using several convenient types: - native Rust vectors (up to 3-dimensional) - arrays from the [`ndarray`](https://docs.rs/ndarray) crate -You must use protocol version 2 to ingest arrays. HTTP transport will +You must use protocol version 2 to ingest arrays. The HTTP transport will automatically enable it as long as you're connecting to an up-to-date QuestDB server (version 9.0.0 or later), but with TCP you must explicitly specify it in the configuration string: `protocol_version=2;`. @@ -326,6 +326,21 @@ buffer.table(table_name)?.column_f64(price_name, 39269.98)?.at(TimestampNanos::n # } ``` +## Decimal Datatype + +The [`Buffer::column_dec`](Buffer::column_dec) method supports efficient ingestion of decimal values using several convenient types: + +- native Rust String slices +- decimals from the [`rust_decimal`](https://docs.rs/rust_decimal) crate +- decimals from the [`bigdecimal`](https://docs.rs/bigdecimal) crate + +You must use protocol version 3 to ingest decimals. The HTTP transport will +automatically enable it as long as you're connecting to an up-to-date QuestDB +server (version 9.2.0 or later), but with TCP you must explicitly specify it in +the configuration string: `protocol_version=3;`. + +**Note**: QuestDB server version 9.2.0 or later is required for decimal support. + ## Check out the CONSIDERATIONS Document The [Library diff --git a/questdb-rs/src/ingress/mod.rs b/questdb-rs/src/ingress/mod.rs index ad4938a9..d8b73b89 100644 --- a/questdb-rs/src/ingress/mod.rs +++ b/questdb-rs/src/ingress/mod.rs @@ -62,6 +62,9 @@ pub use buffer::*; mod sender; pub use sender::*; +mod decimal; +pub use decimal::DecimalView; + const MAX_NAME_LEN_DEFAULT: usize = 127; /// The maximum allowed dimensions for arrays. @@ -71,9 +74,11 @@ pub const MAX_ARRAY_DIM_LEN: usize = 0x0FFF_FFFF; // 1 << 28 - 1 pub(crate) const ARRAY_BINARY_FORMAT_TYPE: u8 = 14; pub(crate) const DOUBLE_BINARY_FORMAT_TYPE: u8 = 16; +#[allow(dead_code)] +pub const DECIMAL_BINARY_FORMAT_TYPE: u8 = 23; /// The version of InfluxDB Line Protocol used to communicate with the server. -#[derive(Debug, Copy, Clone, PartialEq)] +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd)] pub enum ProtocolVersion { /// Version 1 of Line Protocol. /// Full-text protocol. @@ -83,15 +88,29 @@ pub enum ProtocolVersion { /// Version 2 of InfluxDB Line Protocol. /// Uses binary format serialization for f64, and supports the array data type. /// This version is specific to QuestDB and is not compatible with InfluxDB. - /// QuestDB server version 9.0.0 or later is required for `V2` supported. + /// QuestDB server version 9.0.0 or later is required for `V2` support. V2 = 2, + + /// Version 3 of InfluxDB Line Protocol. + /// Supports the decimal data type in text and binary formats. + /// This version is specific to QuestDB and is not compatible with InfluxDB. + /// QuestDB server version 9.2.0 or later is required for `V3` support. + V3 = 3, } +/// List of supported protocol versions, in order of preference (highest to lowest). +const SUPPORTED_PROTOCOL_VERSIONS: [ProtocolVersion; 3] = [ + ProtocolVersion::V3, + ProtocolVersion::V2, + ProtocolVersion::V1, +]; + impl Display for ProtocolVersion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { ProtocolVersion::V1 => write!(f, "v1"), ProtocolVersion::V2 => write!(f, "v2"), + ProtocolVersion::V3 => write!(f, "v3"), } } } @@ -417,11 +436,12 @@ impl SenderBuilder { "protocol_version" => match val { "1" => builder.protocol_version(ProtocolVersion::V1)?, "2" => builder.protocol_version(ProtocolVersion::V2)?, + "3" => builder.protocol_version(ProtocolVersion::V3)?, "auto" => builder, invalid => { return Err(error::fmt!( ConfigError, - "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2]]]\"]" + "invalid \"protocol_version\" [value={invalid}, allowed-values=[auto, 1, 2, 3]]" )); } }, @@ -1146,16 +1166,17 @@ impl SenderBuilder { let (protocol_versions, server_max_name_len) = read_server_settings(http_state, settings_url, max_name_len)?; max_name_len = server_max_name_len; - if protocol_versions.contains(&ProtocolVersion::V2) { - ProtocolVersion::V2 - } else if protocol_versions.contains(&ProtocolVersion::V1) { - ProtocolVersion::V1 - } else { - return Err(fmt!( - ProtocolVersionError, - "Server does not support current client" - )); - } + SUPPORTED_PROTOCOL_VERSIONS + .iter() + .find(|version| protocol_versions.contains(version)) + .copied() + .ok_or_else(|| { + fmt!( + ProtocolVersionError, + "Server does not support any of the client protocol versions: {:?}", + SUPPORTED_PROTOCOL_VERSIONS + ) + })? } else { unreachable!("HTTP handler should be used for HTTP protocol"); } diff --git a/questdb-rs/src/ingress/sender/http.rs b/questdb-rs/src/ingress/sender/http.rs index fa34cc74..7332acf4 100644 --- a/questdb-rs/src/ingress/sender/http.rs +++ b/questdb-rs/src/ingress/sender/http.rs @@ -466,6 +466,7 @@ pub(crate) fn read_server_settings( match v { 1 => support_versions.push(ProtocolVersion::V1), 2 => support_versions.push(ProtocolVersion::V2), + 3 => support_versions.push(ProtocolVersion::V3), _ => {} } } diff --git a/questdb-rs/src/tests/decimal.rs b/questdb-rs/src/tests/decimal.rs new file mode 100644 index 00000000..4356a45d --- /dev/null +++ b/questdb-rs/src/tests/decimal.rs @@ -0,0 +1,633 @@ +/******************************************************************************* + * ___ _ ____ ____ + * / _ \ _ _ ___ ___| |_| _ \| __ ) + * | | | | | | |/ _ \/ __| __| | | | _ \ + * | |_| | |_| | __/\__ \ |_| |_| | |_) | + * \__\_\\__,_|\___||___/\__|____/|____/ + * + * Copyright (c) 2014-2019 Appsicle + * Copyright (c) 2019-2025 QuestDB + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + ******************************************************************************/ + +use crate::ErrorCode; +use crate::ingress::{Buffer, DecimalView, ProtocolVersion}; +use crate::tests::{TestResult, assert_err_contains}; +use rstest::rstest; + +fn serialize_decimal(decimal: DecimalView) -> Vec { + let mut out = Vec::new(); + decimal.serialize(&mut out); + out +} + +// ============================================================================ +// Tests for &str implementation +// ============================================================================ + +#[test] +fn test_str_positive_decimal() -> TestResult { + let decimal = DecimalView::try_new_string("123.45")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"123.45d"); + Ok(()) +} + +#[test] +fn test_str_negative_decimal() -> TestResult { + let decimal = DecimalView::try_new_string("-123.45")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"-123.45d"); + Ok(()) +} + +#[test] +fn test_str_zero() -> TestResult { + let decimal = DecimalView::try_new_string("0")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"0d"); + Ok(()) +} + +#[test] +fn test_str_nan() -> TestResult { + let decimal = DecimalView::try_new_string("NaN")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"NaNd"); + Ok(()) +} + +#[test] +fn test_str_inf() -> TestResult { + let decimal = DecimalView::try_new_string("Infinity")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"Infinityd"); + Ok(()) +} + +#[test] +fn test_str_negative_infinity() -> TestResult { + let decimal = DecimalView::try_new_string("-Infinity")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"-Infinityd"); + Ok(()) +} + +#[test] +fn test_str_scientific_notation() -> TestResult { + let decimal = DecimalView::try_new_string("1.5e-3")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"1.5e-3d"); + Ok(()) +} + +#[test] +fn test_str_large_decimal() -> TestResult { + let decimal = DecimalView::try_new_string("999999999999999999.123456789")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"999999999999999999.123456789d"); + Ok(()) +} + +#[test] +fn test_str_with_leading_zero() -> TestResult { + let decimal = DecimalView::try_new_string("0.001")?; + let result = serialize_decimal(decimal); + assert_eq!(result, b"0.001d"); + Ok(()) +} + +#[test] +fn test_str_rejects_space() -> TestResult { + let result = DecimalView::try_new_string("12 3.45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); + Ok(()) +} + +#[test] +fn test_str_rejects_comma() -> TestResult { + let result = DecimalView::try_new_string("1,234.56"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); + Ok(()) +} + +#[test] +fn test_str_rejects_equals() -> TestResult { + let result = DecimalView::try_new_string("123=45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); + Ok(()) +} + +#[test] +fn test_str_rejects_newline() -> TestResult { + let result = DecimalView::try_new_string("123\n45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); + Ok(()) +} + +#[test] +fn test_str_rejects_backslash() -> TestResult { + let result = DecimalView::try_new_string("123\\45"); + assert_err_contains(result, ErrorCode::InvalidDecimal, "reserved character"); + Ok(()) +} + +/// Validates the binary format structure and extracts the components. +#[cfg(any(feature = "rust_decimal", feature = "bigdecimal"))] +fn parse_binary_decimal(bytes: &[u8]) -> (u8, i128) { + // Validate format markers + + use crate::ingress::DECIMAL_BINARY_FORMAT_TYPE; + assert_eq!(bytes[0], b'=', "Missing binary format marker"); + assert_eq!( + bytes[1], DECIMAL_BINARY_FORMAT_TYPE, + "Invalid decimal type byte" + ); + + let scale = bytes[2]; + let length = bytes[3] as usize; + + assert!(scale <= 76, "Scale {} exceeds maximum of 76", scale); + assert_eq!( + bytes.len(), + 4 + length, + "Binary data length mismatch: expected {} bytes, got {}", + 4 + length, + bytes.len() + ); + + // Parse mantissa bytes as big-endian two's complement + let mantissa_bytes = &bytes[4..]; + + // Convert from big-endian bytes to i128 + // We need to sign-extend if the value is negative (high bit set) + let mut i128_bytes = [0u8; 16]; + let offset = 16 - length; + + // Copy mantissa bytes to the lower part of i128_bytes + i128_bytes[offset..].copy_from_slice(mantissa_bytes); + + // Sign extend if negative (check if high bit of mantissa is set) + if mantissa_bytes[0] & 0x80 != 0 { + // Fill upper bytes with 0xFF for negative numbers + i128_bytes[..offset].fill(0xFF); + } + + let unscaled = i128::from_be_bytes(i128_bytes); + + (scale, unscaled) +} + +// ============================================================================ +// Tests for rust_decimal::Decimal implementation +// ============================================================================ + +#[cfg(feature = "rust_decimal")] +mod rust_decimal_tests { + use super::*; + use crate::ingress::DecimalView; + use rust_decimal::Decimal; + use std::convert::TryInto; + use std::str::FromStr; + + #[test] + fn test_decimal_binary_format_zero() -> TestResult { + let dec = Decimal::ZERO; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Zero should have scale 0"); + assert_eq!(unscaled, 0, "Zero should have unscaled value 0"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_positive() -> TestResult { + let dec = Decimal::from_str("123.45")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_negative() -> TestResult { + let dec = Decimal::from_str("-123.45")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-123.45 should have scale 2"); + assert_eq!( + unscaled, -12345, + "-123.45 should have unscaled value -12345" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_one() -> TestResult { + let dec = Decimal::ONE; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "One should have scale 0"); + assert_eq!(unscaled, 1, "One should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_max_scale() -> TestResult { + // Create a decimal with maximum scale (28 for rust_decimal) + let dec = Decimal::from_str("0.0000000000000000000000000001")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 28, "Should have maximum scale of 28"); + assert_eq!(unscaled, 1, "Should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_large_value() -> TestResult { + let dec = Decimal::MAX; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, 79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_large_value2() -> TestResult { + let dec = Decimal::MIN; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, -79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_small_negative() -> TestResult { + let dec = Decimal::from_str("-0.01")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-0.01 should have scale 2"); + assert_eq!(unscaled, -1, "-0.01 should have unscaled value -1"); + Ok(()) + } + + #[test] + fn test_decimal_binary_format_trailing_zeros() -> TestResult { + let dec = Decimal::from_str("1.00")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + // rust_decimal normalizes trailing zeros + assert_eq!(scale, 2, "1.00 should have scale 2"); + assert_eq!(unscaled, 100, "1.00 should have unscaled value 100"); + Ok(()) + } +} + +// ============================================================================ +// Tests for bigdecimal::BigDecimal implementation +// ============================================================================ + +#[cfg(feature = "bigdecimal")] +mod bigdecimal_tests { + use super::*; + use crate::ingress::DecimalView; + use bigdecimal::BigDecimal; + use std::convert::TryInto; + use std::str::FromStr; + + #[test] + fn test_bigdecimal_binary_format_zero() -> TestResult { + let dec = BigDecimal::from_str("0")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Zero should have scale 0"); + assert_eq!(unscaled, 0, "Zero should have unscaled value 0"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_positive() -> TestResult { + let dec = BigDecimal::from_str("123.45")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_negative() -> TestResult { + let dec = BigDecimal::from_str("-123.45")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-123.45 should have scale 2"); + assert_eq!( + unscaled, -12345, + "-123.45 should have unscaled value -12345" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_one() -> TestResult { + let dec = BigDecimal::from_str("1")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "One should have scale 0"); + assert_eq!(unscaled, 1, "One should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_high_precision() -> TestResult { + // BigDecimal can handle arbitrary precision, test a value with many decimal places + let dec = BigDecimal::from_str("0.123456789012345678901234567890")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 30, "Should preserve high precision scale"); + assert_eq!( + unscaled, 123456789012345678901234567890i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_large_value() -> TestResult { + // Test a very large value that BigDecimal can represent + let dec = BigDecimal::from_str("79228162514264337593543950335")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large integer should have scale 0"); + assert_eq!( + unscaled, 79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_large_negative() -> TestResult { + let dec = BigDecimal::from_str("-79228162514264337593543950335")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 0, "Large negative integer should have scale 0"); + assert_eq!( + unscaled, -79228162514264337593543950335i128, + "Should have correct unscaled value" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_small_negative() -> TestResult { + let dec = BigDecimal::from_str("-0.01")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 2, "-0.01 should have scale 2"); + assert_eq!(unscaled, -1, "-0.01 should have unscaled value -1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_trailing_zeros() -> TestResult { + let dec = BigDecimal::from_str("1.00")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + // BigDecimal may normalize trailing zeros differently than rust_decimal + assert_eq!(scale, 2, "1.00 should have scale 2"); + assert_eq!(unscaled, 100, "1.00 should have unscaled value 100"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_max_scale() -> TestResult { + // Test with scale at QuestDB's limit of 76 + let dec = BigDecimal::from_str( + "0.0000000000000000000000000000000000000000000000000000000000000000000000000001", + )?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + assert_eq!(scale, 76, "Should have maximum scale of 76"); + assert_eq!(unscaled, 1, "Should have unscaled value 1"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_format_exceeds_max_scale() -> TestResult { + // Test that exceeding scale 76 returns an error + let dec = BigDecimal::from_str( + "0.00000000000000000000000000000000000000000000000000000000000000000000000000001", + )?; + let result: crate::Result = (&dec).try_into(); + assert_err_contains(result, ErrorCode::InvalidDecimal, "scale greater than 76"); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_negative_scale() -> TestResult { + // Test with a negative scale + let dec = BigDecimal::from_str("1.23e12")?; + let ilp_dec: DecimalView = (&dec).try_into()?; + let result = serialize_decimal(ilp_dec); + + let (scale, unscaled) = parse_binary_decimal(&result); + // QuestDB does not support negative scale, instead the value should be + // scaled properly + assert_eq!(scale, 0, "Should have scale of 0"); + assert_eq!( + unscaled, 1230000000000, + "Should have unscaled value 1230000000000" + ); + Ok(()) + } + + #[test] + fn test_bigdecimal_binary_value_too_large() -> TestResult { + // QuestDB cannot accept arrays that are larger than what an i8 can fit + let dec = BigDecimal::from_str("1e1000")?; + let result: crate::Result = (&dec).try_into(); + assert_err_contains(result, ErrorCode::InvalidDecimal, "decimal longer than"); + Ok(()) + } +} + +// ============================================================================ +// Buffer integration tests +// ============================================================================ + +#[rstest] +fn test_buffer_column_decimal_str( + #[values(ProtocolVersion::V3)] version: ProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new(version); + buffer + .table("test")? + .symbol("sym", "val")? + .column_dec("dec", "123.45")? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.starts_with("test,sym=val dec=123.45d")); + Ok(()) +} + +#[rstest] +fn test_buffer_column_decimal_str_unsupported( + #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, +) -> TestResult { + let mut buffer = Buffer::new(version); + let result = buffer + .table("test")? + .symbol("sym", "val")? + .column_dec("dec", "123.45"); + assert_err_contains( + result, + ErrorCode::ProtocolVersionError, + "does not support the decimal datatype", + ); + Ok(()) +} + +#[cfg(feature = "rust_decimal")] +#[test] +fn test_buffer_column_decimal_rust_decimal() -> TestResult { + use rust_decimal::Decimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V3); + let dec = Decimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_dec("dec", &dec)? + .at_now()?; + + let bytes = buffer.as_bytes(); + // Should start with table name and symbol + assert!(bytes.starts_with(b"test,sym=val dec=")); + assert!(bytes.ends_with(b"\n")); + + // Skip the prefix and \n suffix + let dec_binary = &bytes[17..bytes.len() - 1]; + let (scale, unscaled) = parse_binary_decimal(dec_binary); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) +} + +#[test] +fn test_buffer_multiple_decimals() -> TestResult { + let mut buffer = Buffer::new(ProtocolVersion::V3); + buffer + .table("test")? + .column_dec("dec1", "123.45")? + .column_dec("dec2", "-67.89")? + .column_dec("dec3", "0.001")? + .at_now()?; + + let output = std::str::from_utf8(buffer.as_bytes())?; + assert!(output.contains("dec1=123.45d")); + assert!(output.contains("dec2=-67.89d")); + assert!(output.contains("dec3=0.001d")); + Ok(()) +} + +#[test] +fn test_decimal_column_name_too_long() -> TestResult { + let mut buffer = Buffer::with_max_name_len(ProtocolVersion::V3, 4); + let name = "a name too long"; + let err = buffer.table("tbl")?.column_dec(name, "123.45").unwrap_err(); + assert_eq!(err.code(), ErrorCode::InvalidName); + assert_eq!( + err.msg(), + r#"Bad name: "a name too long": Too long (max 4 characters)"# + ); + Ok(()) +} + +#[cfg(feature = "bigdecimal")] +#[test] +fn test_buffer_column_decimal_bigdecimal() -> TestResult { + use bigdecimal::BigDecimal; + use std::str::FromStr; + + let mut buffer = Buffer::new(ProtocolVersion::V3); + let dec = BigDecimal::from_str("123.45")?; + buffer + .table("test")? + .symbol("sym", "val")? + .column_dec("dec", &dec)? + .at_now()?; + + let bytes = buffer.as_bytes(); + // Should start with table name and symbol + assert!(bytes.starts_with(b"test,sym=val dec=")); + assert!(bytes.ends_with(b"\n")); + + // Skip the prefix and \n suffix + let dec_binary = &bytes[17..bytes.len() - 1]; + let (scale, unscaled) = parse_binary_decimal(dec_binary); + assert_eq!(scale, 2, "123.45 should have scale 2"); + assert_eq!(unscaled, 12345, "123.45 should have unscaled value 12345"); + Ok(()) +} diff --git a/questdb-rs/src/tests/http.rs b/questdb-rs/src/tests/http.rs index 7055fd0c..c5feb330 100644 --- a/questdb-rs/src/tests/http.rs +++ b/questdb-rs/src/tests/http.rs @@ -785,7 +785,7 @@ fn test_sender_auto_protocol_version_only_v2() -> TestResult { #[test] fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { - let mut server = MockServer::new()?.configure_settings_response(&[3, 4], 127); + let mut server = MockServer::new()?.configure_settings_response(&[4, 5], 127); let sender_builder = server.lsb_http(); let server_thread = std::thread::spawn(move || -> io::Result { server.accept()?; @@ -795,7 +795,7 @@ fn test_sender_auto_protocol_version_unsupported_client() -> TestResult { assert_err_contains( sender_builder.build(), ErrorCode::ProtocolVersionError, - "Server does not support current client", + "Server does not support any of the client protocol versions", ); // We keep the server around til the end of the test to ensure that the response is fully received. diff --git a/questdb-rs/src/tests/interop/ilp-client-interop-test.json b/questdb-rs/src/tests/interop/ilp-client-interop-test.json index 0acedad7..2b3bdd62 100644 --- a/questdb-rs/src/tests/interop/ilp-client-interop-test.json +++ b/questdb-rs/src/tests/interop/ilp-client-interop-test.json @@ -1635,5 +1635,48 @@ "result": { "status": "ERROR" } + }, + { + "testName": "decimal", + "minimumProtocolVersion": 3, + "table": "decimals", + "symbols": [], + "columns": [ + { + "type": "DECIMAL", + "name": "zero", + "value": "0.0" + }, + { + "type": "DECIMAL", + "name": "neg_zero", + "value": "-0.0" + }, + { + "type": "DECIMAL", + "name": "one", + "value": "1.0" + }, + { + "type": "DECIMAL", + "name": "large", + "value": "99999999999999.999" + }, + { + "type": "DECIMAL", + "name": "small", + "value": "0.001" + }, + { + "type": "DECIMAL", + "name": "neg_small", + "value": "-0.001" + } + ], + "result": { + "status": "SUCCESS", + "line": "decimals zero=0d,neg_zero=0d,one=1.0d,large=99999999999999.999d,small=0.001d,neg_small=-0.001d", + "binaryBase64": "ZGVjaW1hbHMgemVybz09FwEBACxuZWdfemVybz09FwEBACxvbmU9PRcBAQosbGFyZ2U9PRcDCAFjRXhdif//LHNtYWxsPT0XAwEBLG5lZ19zbWFsbD09FwMB/wo=" + } } -] +] \ No newline at end of file diff --git a/questdb-rs/src/tests/mod.rs b/questdb-rs/src/tests/mod.rs index 5611c74f..3abf3ab7 100644 --- a/questdb-rs/src/tests/mod.rs +++ b/questdb-rs/src/tests/mod.rs @@ -30,6 +30,7 @@ mod http; mod mock; mod sender; +mod decimal; mod ndarr; #[cfg(feature = "json_tests")] diff --git a/questdb-rs/src/tests/sender.rs b/questdb-rs/src/tests/sender.rs index 6ff41acd..b6b71200 100644 --- a/questdb-rs/src/tests/sender.rs +++ b/questdb-rs/src/tests/sender.rs @@ -241,7 +241,8 @@ fn test_array_f64_for_ndarray() -> TestResult { #[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_max_buf_size( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let max = 1024; let mut server = MockServer::new()?; @@ -271,7 +272,7 @@ fn test_max_buf_size( "Could not flush buffer: Buffer size of 1026 exceeds maximum configured allowed size of 1024 bytes." ); } - ProtocolVersion::V2 => { + ProtocolVersion::V2 | ProtocolVersion::V3 => { assert_eq!( err.msg(), "Could not flush buffer: Buffer size of 1025 exceeds maximum configured allowed size of 1024 bytes." @@ -662,7 +663,8 @@ fn test_arr_column_name_too_long() -> TestResult { #[cfg(feature = "sync-sender-tcp")] #[rstest] fn test_tls_with_file_ca( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let mut ca_path = certs_dir(); ca_path.push("server_rootCA.pem"); @@ -777,7 +779,8 @@ fn test_plain_to_tls_server() -> TestResult { #[cfg(feature = "insecure-skip-verify")] #[rstest] fn test_tls_insecure_skip_verify( - #[values(ProtocolVersion::V1, ProtocolVersion::V2)] version: ProtocolVersion, + #[values(ProtocolVersion::V1, ProtocolVersion::V2, ProtocolVersion::V3)] + version: ProtocolVersion, ) -> TestResult { let server = MockServer::new()?; let lsb = server @@ -863,7 +866,7 @@ pub(crate) fn f64_to_bytes(name: &str, value: f64, version: ProtocolVersion) -> let mut ser = F64Serializer::new(value); buf.extend_from_slice(ser.as_str().as_bytes()); } - ProtocolVersion::V2 => { + ProtocolVersion::V2 | ProtocolVersion::V3 => { buf.push(b'='); buf.push(DOUBLE_BINARY_FORMAT_TYPE); buf.extend_from_slice(&value.to_le_bytes()); diff --git a/system_test/questdb_line_sender.py b/system_test/questdb_line_sender.py index 53d25de3..fec8becc 100644 --- a/system_test/questdb_line_sender.py +++ b/system_test/questdb_line_sender.py @@ -49,6 +49,7 @@ from datetime import datetime from functools import total_ordering from enum import Enum +from decimal import Decimal from ctypes import ( c_bool, @@ -101,6 +102,7 @@ class CertificateAuthority(Enum): class ProtocolVersion(Enum): V1 = (c_protocol_version(1), '1') V2 = (c_protocol_version(2), '2') + V3 = (c_protocol_version(3), '3') @classmethod def from_int(cls, value: c_protocol_version): @@ -290,6 +292,13 @@ def set_sig(fn, restype, *argtypes): c_line_sender_column_name, c_line_sender_utf8, c_line_sender_error_p_p) + set_sig( + dll.line_sender_buffer_column_dec_str, + c_bool, + c_line_sender_buffer_p, + c_line_sender_column_name, + c_line_sender_utf8, + c_line_sender_error_p_p) set_sig( dll.line_sender_buffer_column_f64_arr_byte_strides, c_bool, @@ -704,6 +713,12 @@ def column( self._impl, _column_name(name), _utf8(value)) + elif isinstance(value, Decimal): + _error_wrapped_call( + _DLL.line_sender_buffer_column_dec_str, + self._impl, + _column_name(name), + _utf8(str(value))) elif isinstance(value, TimestampMicros): _error_wrapped_call( _DLL.line_sender_buffer_column_ts_micros, @@ -727,7 +742,7 @@ def column( fqn = _fully_qual_name(value) raise ValueError( f'Bad field value of type {fqn}: Expected one of ' - '`bool`, `int`, `float` or `str`.') + '`bool`, `int`, `float`, `str`, `Decimal`, `TimestampMicros`, or `datetime`.') return self def column_f64_arr(self, name: str, @@ -915,7 +930,7 @@ def symbol(self, name: str, value: str): def column( self, name: str, - value: Union[bool, int, float, str, TimestampMicros, TimestampNanos, datetime]): + value: Union[bool, int, float, str, Decimal, TimestampMicros, TimestampNanos, datetime]): self._buffer.column(name, value) return self diff --git a/system_test/test.py b/system_test/test.py index f5d263ca..1b6147d8 100755 --- a/system_test/test.py +++ b/system_test/test.py @@ -49,7 +49,7 @@ list_questdb_releases, AUTH) import subprocess -from collections import namedtuple +from decimal import Decimal QDB_FIXTURE: QuestDbFixtureBase = None TLS_PROXY_FIXTURE: TlsProxyFixture = None @@ -57,6 +57,7 @@ # The first QuestDB version that supports array types. FIRST_ARRAYS_RELEASE = (8, 3, 3) +DECIMAL_RELEASE = (9, 2, 0) def retry_check_table(*args, **kwargs): @@ -121,18 +122,16 @@ def _ns_to_qdb_date(self, at_ts_ns, exp_nanos: bool): @property def client_driven_nanos_supported(self) -> bool: - return False - ### Re-enable once https://github.com/questdb/questdb/pull/6220 is merged. # """True if the QuestDB server supports nanos and also respects the client's precision for the designated timestamp.""" - # if QDB_FIXTURE.version <= (9, 1, 0): - # return False + if QDB_FIXTURE.version <= (9, 1, 0): + return False - # if QDB_FIXTURE.http: - # return QDB_FIXTURE.protocol_version != qls.ProtocolVersion.V1 - # elif QDB_FIXTURE.protocol_version is None: - # return False # TCP defaults to ProtocolVersion.V1 - # else: - # return QDB_FIXTURE.protocol_version >= qls.ProtocolVersion.V2 + if QDB_FIXTURE.http: + return QDB_FIXTURE.protocol_version != qls.ProtocolVersion.V1 + elif QDB_FIXTURE.protocol_version is None: + return False # TCP defaults to ProtocolVersion.V1 + else: + return QDB_FIXTURE.protocol_version >= qls.ProtocolVersion.V2 @property def expected_protocol_version(self) -> qls.ProtocolVersion: @@ -144,6 +143,9 @@ def expected_protocol_version(self) -> qls.ProtocolVersion: if QDB_FIXTURE.version >= FIRST_ARRAYS_RELEASE: return qls.ProtocolVersion.V2 + if QDB_FIXTURE.version >= DECIMAL_RELEASE: + return qls.ProtocolVersion.V3 + return qls.ProtocolVersion.V1 return QDB_FIXTURE.protocol_version @@ -564,6 +566,42 @@ def test_timestamp_column(self): scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) + def test_decimal_column(self): + if QDB_FIXTURE.version < DECIMAL_RELEASE: + self.skipTest('No decimal support in this version of QuestDB.') + if self.expected_protocol_version < qls.ProtocolVersion.V3: + self.skipTest('communicating over old protocol which does not support decimals') + + table_name = uuid.uuid4().hex + pending = None + decimals = [ + Decimal("12.99"), + Decimal("-12.34"), + Decimal("0.001"), + Decimal("10000000.0"), + Decimal("NaN"), + Decimal("Infinity"), + Decimal("0"), + Decimal("-0"), + Decimal("1e3") + ] + with self._mk_linesender() as sender: + for dec in decimals: + sender.table(table_name) + sender.column('dec', dec) + sender.at_now() + pending = sender.buffer.peek() + + resp = retry_check_table(table_name, min_rows=len(decimals), log_ctx=pending) + exp_columns = [ + {'name': 'dec', 'type': 'DECIMAL(18,3)'}, + {'name': 'timestamp', 'type': 'TIMESTAMP'}] + self.assertEqual(resp['columns'], exp_columns) + # By default, the decimal created as a scale of 3 + exp_dataset = [['12.990'], ['-12.340'], ['0.001'], ['10000000.000'], [None], [None], ['0.000'], ['0.000'], ['1000.000']] + scrubbed_dataset = [row[:-1] for row in resp['dataset']] + self.assertEqual(scrubbed_dataset, exp_dataset) + def test_f64_arr_column(self): if self.expected_protocol_version < qls.ProtocolVersion.V2: self.skipTest('communicating over old protocol which does not support arrays') @@ -801,20 +839,23 @@ def _test_example(self, bin_name, table_name, tls=False): exp_columns = [ {'name': 'symbol', 'type': 'SYMBOL'}, {'name': 'side', 'type': 'SYMBOL'}, - {'name': 'price', 'type': 'DOUBLE'}, + {'name': 'price', 'type': 'DECIMAL(18,3)'}, {'name': 'amount', 'type': 'DOUBLE'}, {'name': 'timestamp', 'type': exp_ts_type}] self.assertEqual(resp['columns'], exp_columns) exp_dataset = [['ETH-USD', 'sell', - 2615.54, + '2615.540', 0.00044]] # Comparison excludes timestamp column. scrubbed_dataset = [row[:-1] for row in resp['dataset']] self.assertEqual(scrubbed_dataset, exp_dataset) def test_c_example(self): + if QDB_FIXTURE.version < DECIMAL_RELEASE: + self.skipTest('No decimal support in this version of QuestDB.') + suffix = '_auth' if QDB_FIXTURE.auth else '' suffix += '_http' if QDB_FIXTURE.http else '' self._test_example( @@ -822,6 +863,9 @@ def test_c_example(self): f'c_trades{suffix}') def test_cpp_example(self): + if QDB_FIXTURE.version < DECIMAL_RELEASE: + self.skipTest('No decimal support in this version of QuestDB.') + suffix = '_auth' if QDB_FIXTURE.auth else '' suffix += '_http' if QDB_FIXTURE.http else '' self._test_example( @@ -829,12 +873,18 @@ def test_cpp_example(self): f'cpp_trades{suffix}') def test_c_tls_example(self): + if QDB_FIXTURE.version < DECIMAL_RELEASE: + self.skipTest('No decimal support in this version of QuestDB.') + self._test_example( 'line_sender_c_example_tls_ca', 'c_trades_tls_ca', tls=True) def test_cpp_tls_example(self): + if QDB_FIXTURE.version < DECIMAL_RELEASE: + self.skipTest('No decimal support in this version of QuestDB.') + self._test_example( 'line_sender_cpp_example_tls_ca', 'cpp_trades_tls_ca', @@ -1137,6 +1187,11 @@ def parse_args(): parser = argparse.ArgumentParser('Run system tests.') sub_p = parser.add_subparsers(dest='command') run_p = sub_p.add_parser('run', help='Run tests') + run_p.add_argument( + '--force-max-version', + action='store_true', + help='Force the client to assume the max version (999,999,999)' + ) run_p.add_argument( '--unittest-help', action='store_true', @@ -1190,7 +1245,7 @@ def run_with_existing(args): (999, 999, 999), True, False, - qls.ProtocolVersion.V2 + qls.ProtocolVersion.V3 ) unittest.main() @@ -1243,6 +1298,8 @@ def run_with_fixtures(args): try: sys.stderr.write(f'>>>> STARTING {questdb_dir} [auth={auth}] <<<<\n') QDB_FIXTURE.start() + if getattr(args, 'force_max_version', False): + QDB_FIXTURE.version = (999, 999, 999) for http, protocol_version, build_mode in itertools.product( (False, True), # http [None] + list(qls.ProtocolVersion), # None is for `auto`