From d359d6292bace24c0b006253c1ff07009b59bf93 Mon Sep 17 00:00:00 2001 From: lmangani Date: Sat, 4 Jan 2025 13:13:07 +0000 Subject: [PATCH 01/10] VITE_SELFSERVICE option --- README.md | 16 ++++++++++++++++ src/components/common/AppInit.tsx | 11 ++++++++++- src/pages/Home.tsx | 8 ++------ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index e35118d..8574fbe 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,22 @@ For production npm run preview ``` +#### Configuration +The application will defaults to the connection setup on first access. + + +To preset connection details use the following ENV variables: +``` +VITE_CLICKHOUSE_URL +VITE_CLICKHOUSE_USER +VITE_CLICKHOUSE_PASS +``` + +For self-serving setups such as quackpipe +``` +VITE_SELFSERVICE +``` + ## 📄 License [ch-ui](https://github.com/caioricciuti/ch-ui) fork licensed under the MIT License - see the [LICENSE](LICENSE) file for details. diff --git a/src/components/common/AppInit.tsx b/src/components/common/AppInit.tsx index 7fa3da0..66418e6 100644 --- a/src/components/common/AppInit.tsx +++ b/src/components/common/AppInit.tsx @@ -49,7 +49,16 @@ const AppInitializer = ({ children }: { children: ReactNode }) => { const envUseAdvanced = window.env?.VITE_CLICKHOUSE_USE_ADVANCED; const envCustomPath = window.env?.VITE_CLICKHOUSE_CUSTOM_PATH; - if (envUrl && envUser) { + if (import.meta.env?.VITE_SELFSERVICE || window.env?.VITE_SELFSERVICE) { + setCredential({ + url: window.location.origin, + username: 'default', + password: envPass || "", + useAdvanced: false, + customPath: "", + }); + setCredentialSource("auto"); + } else if (envUrl) { setCredential({ url: envUrl, username: envUser, diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index d474f94..97953bf 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -17,18 +17,14 @@ function HomePage() { return (
-{/* -*/} - -{/* + - -*/} + Date: Sat, 4 Jan 2025 14:47:20 +0100 Subject: [PATCH 02/10] Hide Database browser Pending chsql_system integration, hide the database browser column and tools --- src/pages/Home.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 97953bf..6a61dab 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -17,13 +17,19 @@ function HomePage() { return (
+{/* +*/} + +{/* + +*/} Date: Sat, 4 Jan 2025 15:52:14 +0100 Subject: [PATCH 03/10] Update AppInit.tsx --- src/components/common/AppInit.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/components/common/AppInit.tsx b/src/components/common/AppInit.tsx index 66418e6..e4da575 100644 --- a/src/components/common/AppInit.tsx +++ b/src/components/common/AppInit.tsx @@ -9,6 +9,7 @@ declare global { VITE_CLICKHOUSE_PASS?: string; VITE_CLICKHOUSE_USE_ADVANCED?: boolean; VITE_CLICKHOUSE_CUSTOM_PATH?: string; + VITE_SELFSERVICE?: string; }; } } @@ -46,18 +47,18 @@ const AppInitializer = ({ children }: { children: ReactNode }) => { const envUrl = ( import.meta.env?.VITE_CLICKHOUSE_URL || window.env?.VITE_CLICKHOUSE_URL ); const envUser = ( import.meta.env?.VITE_CLICKHOUSE_USER || window.env?.VITE_CLICKHOUSE_USER ); const envPass = ( import.meta.env?.VITE_CLICKHOUSE_PASS || window.env?.VITE_CLICKHOUSE_PASS ); - const envUseAdvanced = window.env?.VITE_CLICKHOUSE_USE_ADVANCED; - const envCustomPath = window.env?.VITE_CLICKHOUSE_CUSTOM_PATH; + const envUseAdvanced = ( import.meta.env?.VITE_CLICKHOUSE_USE_ADVANCED || window.env?.VITE_CLICKHOUSE_USE_ADVANCED ); + const envCustomPath = ( import.meta.env?.VITE_CLICKHOUSE_CUSTOM_PATH || window.env?.VITE_CLICKHOUSE_CUSTOM_PATH ); if (import.meta.env?.VITE_SELFSERVICE || window.env?.VITE_SELFSERVICE) { setCredential({ url: window.location.origin, username: 'default', - password: envPass || "", + password: '', useAdvanced: false, customPath: "", }); - setCredentialSource("auto"); + setCredentialSource("env"); } else if (envUrl) { setCredential({ url: envUrl, From 06bf0f1aa17cc2f139885c50c06618f725823d23 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Mon, 6 Jan 2025 11:36:27 +0100 Subject: [PATCH 04/10] Enable DB/Table browser --- src/pages/Home.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 6a61dab..0dbf53a 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -23,13 +23,13 @@ function HomePage() { */} -{/* + -*/} + Date: Mon, 6 Jan 2025 11:43:56 +0100 Subject: [PATCH 05/10] Test Stats --- src/features/metrics/config/metricsConfig.ts | 167 ++++++++++++++----- 1 file changed, 127 insertions(+), 40 deletions(-) diff --git a/src/features/metrics/config/metricsConfig.ts b/src/features/metrics/config/metricsConfig.ts index c6e9e38..c0e496c 100644 --- a/src/features/metrics/config/metricsConfig.ts +++ b/src/features/metrics/config/metricsConfig.ts @@ -1,6 +1,3 @@ -//ignore TS check -// @ts-nocheck - import { ChartConfig } from "@/components/ui/chart"; import { HomeIcon, @@ -13,6 +10,7 @@ import { CpuIcon, AlertTriangleIcon, } from "lucide-react"; +import { ReactNode, ComponentType } from "react"; export interface Metrics { title: string; @@ -28,10 +26,31 @@ export interface MetricItem { type: "card" | "table" | "chart"; chartType?: "bar" | "line" | "area" | "pie" | "radar" | "radial"; description: string; - chartConfig?: ChartConfig; + chartConfig?: CustomChartConfig; tiles?: number; } + +export type ChartTheme = { + light: string; + dark: string; +} + + +export type ChartDataConfig = { + label?: ReactNode; + icon?: ComponentType<{}>; +} & ({ color?: string; theme?: never } | { color?: never; theme: ChartTheme }); + + + +export type CustomChartConfig = { + indexBy: string; + [key: string]: ChartDataConfig | string | undefined; +} + + + export const metrics: Metrics[] = [ { title: "Overview", @@ -41,7 +60,12 @@ export const metrics: Metrics[] = [ items: [ { title: "Server Uptime (days)", - query: `SELECT 1`, + query: ` + -- set max decimal places to 2 + + SELECT + ROUND(uptime() / 86400, 2) AS uptime_days + `, type: "card", description: "Total time the server has been running in seconds, minutes, hours, and days.", @@ -50,7 +74,9 @@ export const metrics: Metrics[] = [ { title: "Total Databases", query: ` - SELECT count(database) as total_databases FROM (SHOW ALL TABLES) + SELECT COUNT(*) AS total_databases + FROM system.databases + WHERE name NOT IN ('system', 'information_schema') `, type: "card", description: "Total number of databases excluding system databases.", @@ -59,7 +85,9 @@ export const metrics: Metrics[] = [ { title: "Total Tables", query: ` - SELECT count(name) as total_tables FROM (SHOW ALL TABLES); + SELECT COUNT(*) AS total_tables + FROM system.tables + WHERE database NOT IN ('system', 'information_schema') `, type: "card", description: "Total number of user tables excluding temporary tables.", @@ -73,6 +101,30 @@ export const metrics: Metrics[] = [ "Version of the ClickHouse server running on the current instance.", tiles: 1, }, + { + title: "Running Queries", + query: `SELECT 1`, + type: "table", + description: "Currently running queries excluding system queries.", + tiles: 4, + }, + { + title: "Daily Query Count", + description: "Number of queries per day for the last 30 days.", + type: "chart", + chartType: "bar", + query: ` + SELECT 1 + `, + chartConfig: { + indexBy: "day", + query_count: { + label: "Query Count", + color: "hsl(var(--chart-1))", + }, + }, + tiles: 4, + }, ], }, { @@ -83,37 +135,58 @@ export const metrics: Metrics[] = [ items: [ { title: "Total Tables", - query: `SELECT count(name) as total_tables FROM (SHOW ALL TABLES);`, + query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE lower(database) NOT IN ('system', 'information_schema')`, type: "card", description: "Total number of user-defined tables.", tiles: 1, }, { title: "Total System Tables", - query: `SELECT count(name) as total_tables FROM (SHOW ALL TABLES);`, + query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE lower(database) IN ('system', 'information_schema')`, type: "card", description: "Total number of system tables.", tiles: 1, }, - ], - }, - { - title: "Queries", - scope: "queries", - description: "Comprehensive metrics related to queries in the system.", - icon: TerminalSquareIcon, - items: [ { - title: "Queries Per Second (QPS)", - query: `SELECT 0`, + title: "Total Temporary Tables", + query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE is_temporary = 1`, + type: "card", + description: "Total number of temporary tables.", + tiles: 1, + }, + // card + { + title: "Biggest Table", + query: `SELECT name AS table FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_bytes DESC LIMIT 1`, + type: "card", + description: "Largest table in the system.", + tiles: 1, + }, + { + title: "Table Cardinality", + query: `SELECT database, name AS table, total_rows FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_rows DESC LIMIT 10`, + type: "table", + description: "Number of rows in the top 10 tables.", + tiles: 2, + }, + { + title: "Table Row Counts", + query: `SELECT database, name AS table, total_rows FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_rows DESC LIMIT 10`, + type: "table", + description: "Number of rows in the top 10 tables.", + tiles: 2, + }, + { + title: "Table Sizes (MB)", + query: `SELECT name AS name, total_bytes / 1024 / 1024 AS total_mb FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_mb DESC LIMIT 20`, type: "chart", - chartType: "area", - description: "Rate of queries per second over the last hour.", + chartType: "bar", + description: "Size distribution of the top 30 largest tables.", chartConfig: { - indexBy: "minute", - qps: { - label: "QPS", - color: "hsl(var(--chart-3))", + indexBy: "name", + total_mb: { + label: "Size (MB)", + color: "hsl(var(--chart-2))", }, }, tiles: 2, @@ -121,27 +194,41 @@ export const metrics: Metrics[] = [ ], }, { - title: "Performance", - scope: "performance", - description: "Performance-related metrics.", - icon: CpuIcon, + title: "Storage", + scope: "storage", + description: "Storage-related metrics.", + icon: HardDriveIcon, items: [ { - title: "CPU Usage", - query: ` - SELECT 0 - `, + title: "Disk Usage", + query: `SELECT + name, + round(total_space / 1024 / 1024 / 1024, 2) AS total_gb, + round(free_space / 1024 / 1024 / 1024, 2) AS free_gb, + round((1 - free_space / total_space) * 100, 2) AS used_percent + FROM system.disks`, + type: "table", + description: "Detailed disk usage information.", + }, + { + title: "Database Sizes", + query: `SELECT + database, + round(sum(total_bytes) / 1024 / 1024 / 1024, 2) AS size_gb + FROM system.tables + GROUP BY database + ORDER BY size_gb DESC`, type: "chart", - chartType: "line", - description: "CPU usage over the last hour.", + chartType: "bar", + description: "Size distribution of databases.", chartConfig: { - indexBy: "minute", - cpu_usage: { - label: "CPU Usage", - color: "hsl(var(--chart-5))", + indexBy: "database", + size_gb: { + label: "Size (GB)", + color: "hsl(var(--chart-2))", }, }, - tiles: 2, + tiles: 4, }, ], }, From a3790ad968ba371b63c3d571e7ec597a10ea0028 Mon Sep 17 00:00:00 2001 From: Lorenzo Mangani Date: Mon, 6 Jan 2025 12:35:36 +0100 Subject: [PATCH 06/10] Update Home.tsx --- src/pages/Home.tsx | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/pages/Home.tsx b/src/pages/Home.tsx index 0dbf53a..18eae81 100644 --- a/src/pages/Home.tsx +++ b/src/pages/Home.tsx @@ -16,21 +16,15 @@ function HomePage() { }, []); return ( -
-{/* +
-*/} - - - - Date: Mon, 6 Jan 2025 16:59:56 +0100 Subject: [PATCH 07/10] Update metricsConfig.ts --- src/features/metrics/config/metricsConfig.ts | 44 +------------------- 1 file changed, 1 insertion(+), 43 deletions(-) diff --git a/src/features/metrics/config/metricsConfig.ts b/src/features/metrics/config/metricsConfig.ts index c0e496c..5939432 100644 --- a/src/features/metrics/config/metricsConfig.ts +++ b/src/features/metrics/config/metricsConfig.ts @@ -61,10 +61,7 @@ export const metrics: Metrics[] = [ { title: "Server Uptime (days)", query: ` - -- set max decimal places to 2 - - SELECT - ROUND(uptime() / 86400, 2) AS uptime_days + SELECT ROUND(uptime() / 86400, 2) AS uptime_days `, type: "card", description: @@ -193,43 +190,4 @@ export const metrics: Metrics[] = [ }, ], }, - { - title: "Storage", - scope: "storage", - description: "Storage-related metrics.", - icon: HardDriveIcon, - items: [ - { - title: "Disk Usage", - query: `SELECT - name, - round(total_space / 1024 / 1024 / 1024, 2) AS total_gb, - round(free_space / 1024 / 1024 / 1024, 2) AS free_gb, - round((1 - free_space / total_space) * 100, 2) AS used_percent - FROM system.disks`, - type: "table", - description: "Detailed disk usage information.", - }, - { - title: "Database Sizes", - query: `SELECT - database, - round(sum(total_bytes) / 1024 / 1024 / 1024, 2) AS size_gb - FROM system.tables - GROUP BY database - ORDER BY size_gb DESC`, - type: "chart", - chartType: "bar", - description: "Size distribution of databases.", - chartConfig: { - indexBy: "database", - size_gb: { - label: "Size (GB)", - color: "hsl(var(--chart-2))", - }, - }, - tiles: 4, - }, - ], - }, ]; From dbb973a71ea20fa478eb0c5da0e85926f20fb67b Mon Sep 17 00:00:00 2001 From: lmangani Date: Wed, 8 Jan 2025 12:57:28 +0000 Subject: [PATCH 08/10] test query conversion --- src/features/metrics/config/metricsConfig.ts | 411 ++++++++++++++++-- src/features/workspace/components/HomeTab.tsx | 24 +- src/features/workspace/editor/appQueries.ts | 30 +- src/store/index.ts | 62 +-- 4 files changed, 440 insertions(+), 87 deletions(-) diff --git a/src/features/metrics/config/metricsConfig.ts b/src/features/metrics/config/metricsConfig.ts index 5939432..5a53a4b 100644 --- a/src/features/metrics/config/metricsConfig.ts +++ b/src/features/metrics/config/metricsConfig.ts @@ -30,50 +30,47 @@ export interface MetricItem { tiles?: number; } - export type ChartTheme = { light: string; dark: string; } - export type ChartDataConfig = { label?: ReactNode; icon?: ComponentType<{}>; } & ({ color?: string; theme?: never } | { color?: never; theme: ChartTheme }); - - export type CustomChartConfig = { - indexBy: string; - [key: string]: ChartDataConfig | string | undefined; + indexBy: string; + [key: string]: ChartDataConfig | string | undefined; } - - export const metrics: Metrics[] = [ { title: "Overview", scope: "overview", - description: "Overview of ClickHouse metrics.", + description: "Overview of DuckDB metrics.", icon: HomeIcon, items: [ { title: "Server Uptime (days)", query: ` - SELECT ROUND(uptime() / 86400, 2) AS uptime_days + SELECT + ROUND(SUM(julianday('now') - julianday(start_time)), 2) AS uptime_days + FROM pragma_database_list + WHERE name = 'main' `, type: "card", description: - "Total time the server has been running in seconds, minutes, hours, and days.", + "Total time the server has been running in days.", tiles: 1, }, { title: "Total Databases", query: ` SELECT COUNT(*) AS total_databases - FROM system.databases - WHERE name NOT IN ('system', 'information_schema') + FROM pragma_database_list + WHERE name NOT IN ('main', 'temp') `, type: "card", description: "Total number of databases excluding system databases.", @@ -83,8 +80,8 @@ export const metrics: Metrics[] = [ title: "Total Tables", query: ` SELECT COUNT(*) AS total_tables - FROM system.tables - WHERE database NOT IN ('system', 'information_schema') + FROM information_schema.tables + WHERE table_schema NOT IN ('main', 'temp') `, type: "card", description: "Total number of user tables excluding temporary tables.", @@ -92,17 +89,17 @@ export const metrics: Metrics[] = [ }, { title: "Version", - query: `SELECT version() AS version`, + query: `SELECT sqlite_version() AS version`, type: "card", description: - "Version of the ClickHouse server running on the current instance.", + "Version of the DuckDB server running on the current instance.", tiles: 1, }, { title: "Running Queries", - query: `SELECT 1`, + query: `SELECT * FROM pragma_busy_list`, type: "table", - description: "Currently running queries excluding system queries.", + description: "Currently running queries.", tiles: 4, }, { @@ -111,9 +108,13 @@ export const metrics: Metrics[] = [ type: "chart", chartType: "bar", query: ` - SELECT 1 + SELECT COUNT(*) AS query_count, DATE(event_time) AS day + FROM pragma_sql_log + WHERE event_time > DATE('now', '-30 days') + GROUP BY day + ORDER BY day `, - chartConfig: { + chartConfig: { indexBy: "day", query_count: { label: "Query Count", @@ -132,50 +133,88 @@ export const metrics: Metrics[] = [ items: [ { title: "Total Tables", - query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE lower(database) NOT IN ('system', 'information_schema')`, + query: ` + SELECT COUNT(*) AS total_tables + FROM information_schema.tables + WHERE table_schema NOT IN ('main', 'temp') + `, type: "card", description: "Total number of user-defined tables.", tiles: 1, }, { title: "Total System Tables", - query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE lower(database) IN ('system', 'information_schema')`, + query: ` + SELECT COUNT(*) AS total_tables + FROM information_schema.tables + WHERE table_schema IN ('main', 'temp') + `, type: "card", description: "Total number of system tables.", tiles: 1, }, { title: "Total Temporary Tables", - query: `SELECT COUNT(*) AS total_tables FROM system.tables WHERE is_temporary = 1`, + query: ` + SELECT COUNT(*) AS total_tables + FROM information_schema.tables + WHERE table_schema LIKE 'temp%' + `, type: "card", description: "Total number of temporary tables.", tiles: 1, }, - // card { title: "Biggest Table", - query: `SELECT name AS table FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_bytes DESC LIMIT 1`, + query: ` + SELECT table_name AS table + FROM information_schema.tables + WHERE table_schema NOT IN ('main', 'temp') + ORDER BY table_name DESC + LIMIT 1 + `, type: "card", description: "Largest table in the system.", tiles: 1, }, { title: "Table Cardinality", - query: `SELECT database, name AS table, total_rows FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_rows DESC LIMIT 10`, + query: ` + SELECT table_schema, table_name AS table, COUNT(*) AS total_rows + FROM information_schema.tables + WHERE table_schema NOT IN ('main', 'temp') + GROUP BY table_schema, table_name + ORDER BY total_rows DESC + LIMIT 10 + `, type: "table", description: "Number of rows in the top 10 tables.", tiles: 2, }, { title: "Table Row Counts", - query: `SELECT database, name AS table, total_rows FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_rows DESC LIMIT 10`, + query: ` + SELECT table_schema, table_name AS table, COUNT(*) AS total_rows + FROM information_schema.tables + WHERE table_schema NOT IN ('main', 'temp') + GROUP BY table_schema, table_name + ORDER BY total_rows DESC + LIMIT 10 + `, type: "table", description: "Number of rows in the top 10 tables.", tiles: 2, }, { title: "Table Sizes (MB)", - query: `SELECT name AS name, total_bytes / 1024 / 1024 AS total_mb FROM system.tables WHERE database NOT IN ('system', 'information_schema') ORDER BY total_mb DESC LIMIT 20`, + query: ` + SELECT table_name AS name, SUM(pgsize) / 1024 / 1024 AS total_mb + FROM pragma_table_info + WHERE table_schema NOT IN ('main', 'temp') + GROUP BY table_name + ORDER BY total_mb DESC + LIMIT 20 + `, type: "chart", chartType: "bar", description: "Size distribution of the top 30 largest tables.", @@ -188,6 +227,320 @@ export const metrics: Metrics[] = [ }, tiles: 2, }, + { + title: "Most Used Tables", + query: ` + SELECT table_name AS tables, COUNT(*) AS query_count + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + GROUP BY table_name + ORDER BY query_count DESC + LIMIT 10 + `, + type: "chart", + chartType: "bar", + description: "Top 10 most queried tables in the last 24 hours.", + chartConfig: { + indexBy: "tables", + query_count: { + label: "Query Count", + color: "hsl(var(--chart-2))", + }, + }, + tiles: 2, + }, + ], + }, + { + title: "Queries", + scope: "queries", + description: "Comprehensive metrics related to queries in the system.", + icon: TerminalSquareIcon, + items: [ + { + title: "Running Queries Count", + query: ` + SELECT COUNT(*) AS running_queries + FROM pragma_busy_list + `, + type: "card", + description: "Current number of active queries in the system.", + tiles: 1, + }, + { + title: "Query Error Rate", + query: ` + SELECT + ROUND(100 * SUM(CASE WHEN type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') THEN 1 ELSE 0 END) / COUNT(*), 2) AS error_rate + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + `, + type: "card", + description: "Percentage of failed queries over the last 24 hours.", + tiles: 1, + }, + { + title: "Average Query Duration", + query: ` + SELECT + ROUND(avg(CAST(query_duration_ms AS DOUBLE)), 2) AS avg_duration_ms + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + `, + type: "card", + description: + "Average duration of queries executed in the last 24 hours.", + tiles: 1, + }, + { + title: "Total Queries (Last 24h)", + query: ` + SELECT COUNT(*) AS total_queries + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + `, + type: "card", + description: "Total number of queries executed in the last 24 hours.", + tiles: 1, + }, + { + title: "Query Duration Distribution", + query: ` + SELECT + CASE + WHEN query_duration_ms < 10 THEN '<10ms' + WHEN query_duration_ms < 20 THEN '10ms-20ms' + WHEN query_duration_ms < 50 THEN '20ms-50ms' + WHEN query_duration_ms < 100 THEN '50ms-100ms' + WHEN query_duration_ms < 200 THEN '100ms-200ms' + WHEN query_duration_ms < 500 THEN '200ms-500ms' + WHEN query_duration_ms < 1000 THEN '500ms-1s' + WHEN query_duration_ms < 5000 THEN '1s-5s' + WHEN query_duration_ms < 30000 THEN '5s-30s' + ELSE '>30s' + END AS duration_bucket, + COUNT(*) AS query_count + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + GROUP BY duration_bucket + ORDER BY duration_bucket + `, + type: "chart", + chartType: "bar", + description: + "Granular distribution of query durations over the last 24 hours.", + chartConfig: { + indexBy: "duration_bucket", + query_count: { + label: "Query Count", + color: "hsl(var(--chart-1))", + }, + }, + tiles: 2, + }, + { + title: "Queries Per Second (QPS)", + query: ` + SELECT + strftime('%Y-%m-%d %H:%M', event_time) AS minute, + COUNT(*) AS qps + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 hour') + GROUP BY minute + ORDER BY minute + `, + type: "chart", + chartType: "area", + description: "Rate of queries per second over the last hour.", + chartConfig: { + indexBy: "minute", + qps: { + label: "QPS", + color: "hsl(var(--chart-3))", + }, + }, + tiles: 2, + }, + { + title: "Queries Per User", + query: ` + SELECT user, COUNT(*) AS query_count + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + GROUP BY user + ORDER BY query_count DESC + LIMIT 10 + `, + type: "table", + description: + "Top 10 users by the number of queries executed in the last 24 hours.", + tiles: 4, + }, + { + title: "Top Slow Queries", + query: ` + SELECT query, query_duration_ms + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + ORDER BY query_duration_ms DESC + LIMIT 10 + `, + type: "table", + description: "Top 10 slowest queries executed in the last 24 hours.", + tiles: 4, + }, + ], + }, + { + title: "Storage", + scope: "storage", + description: "Storage-related metrics.", + icon: HardDriveIcon, + items: [ + { + title: "Database Sizes", + query: ` + SELECT + name AS database, + round(total_bytes / 1024 / 1024 / 1024, 2) AS size_gb + FROM pragma_database_list + WHERE name NOT IN ('main', 'temp') + ORDER BY size_gb DESC + `, + type: "chart", + chartType: "bar", + description: "Size distribution of databases.", + chartConfig: { + indexBy: "database", + size_gb: { + label: "Size (GB)", + color: "hsl(var(--chart-2))", + }, + }, + tiles: 4, + }, + ], + }, + { + title: "Settings & Config", + scope: "settings", + description: "Settings and configuration.", + icon: Settings2, + items: [ + { + title: "Current Settings", + query: `SELECT * FROM pragma_settings`, + type: "table", + description: "Current DuckDB settings.", + tiles: 4, + }, + ], + }, + { + title: "Query Exceptions", + scope: "exceptions", + description: "Overview of query exceptions in the system.", + icon: AlertTriangleIcon, + items: [ + { + title: "Exceptions (Last 24h)", + query: ` + SELECT COUNT(*) AS total_exceptions + FROM pragma_sql_log + WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') + AND event_time > DATE('now', '-1 day') + `, + type: "card", + description: + "Total number of exceptions recorded in the last 24 hours.", + tiles: 1, + }, + { + title: "Exception Rate (Last 24h)", + query: ` + SELECT + ROUND(100 * SUM(CASE WHEN type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') THEN 1 ELSE 0 END) / COUNT(*), 2) AS exception_rate + FROM pragma_sql_log + WHERE event_time > DATE('now', '-1 day') + `, + type: "card", + description: + "Percentage of queries that resulted in exceptions over the last 24 hours.", + tiles: 1, + }, + { + title: "Recent Exceptions", + query: ` + SELECT + event_time, + user, + query, + exception + FROM pragma_sql_log + WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') + AND event_time > DATE('now', '-1 hour') + ORDER BY event_time DESC + LIMIT 10 + `, + type: "table", + description: "Last 10 exceptions recorded in the last hour.", + tiles: 4, + }, + { + title: "Exceptions by User", + query: ` + SELECT user, COUNT(*) AS exception_count + FROM pragma_sql_log + WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') + AND event_time > DATE('now', '-1 day') + GROUP BY user + ORDER BY exception_count DESC + LIMIT 10 + `, + type: "table", + description: + "Top 10 users with the most exceptions in the last 24 hours.", + tiles: 4, + }, + { + title: "Exceptions Over Time", + query: ` + SELECT + strftime('%Y-%m-%d %H:00', event_time) AS hourERROR, + COUNT(*) AS exception_count + FROM pragma_sql_log + WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') + AND event_time > DATE('now', '-1 day') + GROUP BY hourERROR + ORDER BY hourERROR + `, + type: "chart", + chartType: "line", + description: "Count of exceptions recorded over the last 24 hours.", + chartConfig: { + indexBy: "hourERROR", + exception_count: { + label: "Exception Count", + color: "hsl(var(--chart-2))", + }, + }, + tiles: 2, + }, + { + title: "Most Common Exceptions", + query: ` + SELECT exception, COUNT(*) AS count + FROM pragma_sql_log + WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') + AND event_time > DATE('now', '-1 day') + GROUP BY exception + ORDER BY count DESC + LIMIT 10 + `, + type: "table", + description: "Top 10 most common exceptions in the last 24 hours.", + tiles: 2, + }, ], }, ]; diff --git a/src/features/workspace/components/HomeTab.tsx b/src/features/workspace/components/HomeTab.tsx index a5aa823..2518521 100644 --- a/src/features/workspace/components/HomeTab.tsx +++ b/src/features/workspace/components/HomeTab.tsx @@ -93,27 +93,26 @@ const HomeTab = () => { setLoading(true); setError(null); try { - const recentQueries = await runQuery(` - SELECT DISTINCT - replaceAll(query, 'FORMAT JSON', '') AS cleaned_query, - max(event_time) AS latest_event_time, + const recentQueries = await runQuery(` + SELECT DISTINCT + REPLACE(query, 'FORMAT JSON', '') AS cleaned_query, + MAX(event_time) AS latest_event_time, query_kind, - length(replaceAll(query, 'FORMAT JSON', '')) AS query_length + LENGTH(REPLACE(query, 'FORMAT JSON', '')) AS query_length FROM - system.query_log + pragma_sql_log WHERE user = '${credential.username}' - AND event_time >= (current_timestamp() - INTERVAL 2 DAY) - AND arrayExists(db -> db NOT LIKE '%system%', databases) + AND event_time >= (CURRENT_TIMESTAMP - INTERVAL '2' DAY) AND query NOT LIKE 'SELECT DISTINCT%' GROUP BY cleaned_query, query_kind ORDER BY latest_event_time DESC - LIMIT - 6; - `); - setRecentItems(recentQueries.data); + LIMIT + 6; + `); + setRecentItems(recentQueries.data); } catch (err) { setError("Failed to load recent queries"); console.error(err); @@ -122,6 +121,7 @@ const HomeTab = () => { } }; + const truncateQuery = (query: string, length: number = 50) => { return query.length > length ? `${query.slice(0, length)}...` : query; }; diff --git a/src/features/workspace/editor/appQueries.ts b/src/features/workspace/editor/appQueries.ts index 3465836..5f8e873 100644 --- a/src/features/workspace/editor/appQueries.ts +++ b/src/features/workspace/editor/appQueries.ts @@ -6,30 +6,30 @@ export const appQueries: Record = { getIntellisense: { query: ` SELECT - database, - table, - name AS column_name, - type AS column_type - FROM system.columns - ORDER BY database, table, column_name; + table_catalog AS database, + table_schema AS schema, + table_name AS table, + column_name, + data_type AS column_type + FROM information_schema.columns + ORDER BY table_catalog, table_schema, table_name, column_name; `, }, getDatabasesTables: { query: ` SELECT - databases.name AS database_name, - tables.name AS table_name, - tables.engine AS table_type - FROM system.databases AS databases - LEFT JOIN system.tables AS tables - ON databases.name = tables.database - ORDER BY database_name, table_name; + table_catalog AS database_name, + table_schema AS schema_name, + table_name AS table_name, + table_type + FROM information_schema.tables + ORDER BY table_catalog, table_schema, table_name; `, }, getClickHouseFunctions: { - query: `SELECT name from system.functions`, + query: `SELECT function_name AS name FROM duckdb_functions`, }, getKeywords: { - query: `SELECT keyword FROM system.keywords`, + query: `SELECT keyword FROM duckdb_keywords`, }, }; diff --git a/src/store/index.ts b/src/store/index.ts index 9ae993c..f4f9057 100644 --- a/src/store/index.ts +++ b/src/store/index.ts @@ -459,10 +459,7 @@ const useAppStore = create()( try { const result = await clickHouseClient.query({ query: ` - SELECT if(grant_option = 1, true, false) AS is_admin - FROM system.grants - WHERE user_name = currentUser() - LIMIT 1 + SELECT false AS is_admin `, }); @@ -495,12 +492,16 @@ const useAppStore = create()( })); try { - const result = await runQuery(` - SELECT COUNT(*) as exists - FROM system.tables - WHERE database = 'CH_UI' - AND name = 'saved_queries' - `); + + + await runQuery("ATTACH '/tmp/chui.db' AS CH_UI"); + const result = await runQuery(` + SELECT COUNT(*) as exists + FROM information_schema.tables + WHERE table_catalog = 'CH_UI' + AND table_name = 'saved_queries' + `); + const response = result as SavedQueriesCheckResponse; const isActive = response.data[0]?.exists > 0; @@ -545,26 +546,24 @@ const useAppStore = create()( try { // Run queries in sequence with proper error handling - await runQuery("CREATE DATABASE IF NOT EXISTS CH_UI").then( - async () => { - await runQuery(` - CREATE TABLE IF NOT EXISTS CH_UI.saved_queries ( - id String, - name String, - query String, - created_at DateTime64(3), - updated_at DateTime64(3), - owner String, - is_public Boolean DEFAULT false, - tags Array(String) DEFAULT [], - description String DEFAULT '', - PRIMARY KEY (id) - ) ENGINE = MergeTree() - ORDER BY (id, created_at) - SETTINGS index_granularity = 8192 - `); - } - ); + await runQuery("ATTACH '/tmp/chui.db' AS CH_UI").then( + async () => { + await runQuery(` + CREATE TABLE IF NOT EXISTS CH_UI.saved_queries ( + id STRING, + name STRING, + query STRING, + created_at TIMESTAMP, + updated_at TIMESTAMP, + owner STRING, + is_public BOOLEAN DEFAULT false, + tags STRING[] DEFAULT [], + description STRING DEFAULT '', + PRIMARY KEY (id) + ) + `); + } + ); // Verify the table was created successfully const isActive = await get().checkSavedQueriesStatus(); @@ -605,7 +604,8 @@ const useAppStore = create()( })); try { - await runQuery("DROP TABLE IF EXISTS CH_UI.saved_queries"); + await runQuery("ATTACH '/tmp/chui.db' AS CH_UI"); + await runQuery("DROP TABLE IF EXISTS CH_UI.saved_queries"); // Verify the table was dropped successfully const isActive = await get().checkSavedQueriesStatus(); From 333a8e2b78164a520c4e8b8626b7bafd5006b30b Mon Sep 17 00:00:00 2001 From: lmangani Date: Wed, 8 Jan 2025 13:26:25 +0000 Subject: [PATCH 09/10] remove engine, on cluster --- .../explorer/components/CreateDatabase.tsx | 14 -------------- src/features/explorer/components/CreateTable.tsx | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/features/explorer/components/CreateDatabase.tsx b/src/features/explorer/components/CreateDatabase.tsx index 94b0dad..7c8eede 100644 --- a/src/features/explorer/components/CreateDatabase.tsx +++ b/src/features/explorer/components/CreateDatabase.tsx @@ -163,20 +163,6 @@ const CreateDatabase = () => { } sqlStatement += `${databaseName} `; - if (onCluster && clusterName) { - sqlStatement += `ON CLUSTER ${clusterName} `; - } - - if (engine === "Lazy") { - sqlStatement += `ENGINE = Lazy(${expirationTimeInSeconds}) `; - } else if (engine) { - sqlStatement += `ENGINE = ${engine} `; - } - - if (comment) { - sqlStatement += `COMMENT '${comment}'`; - } - setSql(sqlStatement.trim()); setErrors({}); return sqlStatement.trim(); diff --git a/src/features/explorer/components/CreateTable.tsx b/src/features/explorer/components/CreateTable.tsx index 1ce9094..377bbf1 100644 --- a/src/features/explorer/components/CreateTable.tsx +++ b/src/features/explorer/components/CreateTable.tsx @@ -171,7 +171,7 @@ const CreateTable = () => { return `${field.name} ${typeStr} ${nullableStr}${commentStr}`; }).join(",\n "); - let sql = `CREATE TABLE ${database}.${tableName}\n(\n ${fieldDefinitions}\n) ENGINE = ${engine}`; + let sql = `CREATE TABLE ${database}.${tableName}\n(\n ${fieldDefinitions}\n)`; if (primaryKeyFields.length > 0) { sql += `\nPRIMARY KEY (${primaryKeyFields.join(", ")})`; @@ -350,4 +350,4 @@ const CreateTable = () => { ); }; -export default CreateTable; \ No newline at end of file +export default CreateTable; From 4d533e68484bc8438d24a34c2a3db1467e724002 Mon Sep 17 00:00:00 2001 From: lmangani Date: Wed, 8 Jan 2025 13:39:43 +0000 Subject: [PATCH 10/10] resync --- src/features/metrics/config/metricsConfig.ts | 351 +------------------ 1 file changed, 1 insertion(+), 350 deletions(-) diff --git a/src/features/metrics/config/metricsConfig.ts b/src/features/metrics/config/metricsConfig.ts index 5a53a4b..d9176ee 100644 --- a/src/features/metrics/config/metricsConfig.ts +++ b/src/features/metrics/config/metricsConfig.ts @@ -89,40 +89,12 @@ export const metrics: Metrics[] = [ }, { title: "Version", - query: `SELECT sqlite_version() AS version`, + query: `SELECT version() AS version`, type: "card", description: "Version of the DuckDB server running on the current instance.", tiles: 1, }, - { - title: "Running Queries", - query: `SELECT * FROM pragma_busy_list`, - type: "table", - description: "Currently running queries.", - tiles: 4, - }, - { - title: "Daily Query Count", - description: "Number of queries per day for the last 30 days.", - type: "chart", - chartType: "bar", - query: ` - SELECT COUNT(*) AS query_count, DATE(event_time) AS day - FROM pragma_sql_log - WHERE event_time > DATE('now', '-30 days') - GROUP BY day - ORDER BY day - `, - chartConfig: { - indexBy: "day", - query_count: { - label: "Query Count", - color: "hsl(var(--chart-1))", - }, - }, - tiles: 4, - }, ], }, { @@ -205,220 +177,6 @@ export const metrics: Metrics[] = [ description: "Number of rows in the top 10 tables.", tiles: 2, }, - { - title: "Table Sizes (MB)", - query: ` - SELECT table_name AS name, SUM(pgsize) / 1024 / 1024 AS total_mb - FROM pragma_table_info - WHERE table_schema NOT IN ('main', 'temp') - GROUP BY table_name - ORDER BY total_mb DESC - LIMIT 20 - `, - type: "chart", - chartType: "bar", - description: "Size distribution of the top 30 largest tables.", - chartConfig: { - indexBy: "name", - total_mb: { - label: "Size (MB)", - color: "hsl(var(--chart-2))", - }, - }, - tiles: 2, - }, - { - title: "Most Used Tables", - query: ` - SELECT table_name AS tables, COUNT(*) AS query_count - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - GROUP BY table_name - ORDER BY query_count DESC - LIMIT 10 - `, - type: "chart", - chartType: "bar", - description: "Top 10 most queried tables in the last 24 hours.", - chartConfig: { - indexBy: "tables", - query_count: { - label: "Query Count", - color: "hsl(var(--chart-2))", - }, - }, - tiles: 2, - }, - ], - }, - { - title: "Queries", - scope: "queries", - description: "Comprehensive metrics related to queries in the system.", - icon: TerminalSquareIcon, - items: [ - { - title: "Running Queries Count", - query: ` - SELECT COUNT(*) AS running_queries - FROM pragma_busy_list - `, - type: "card", - description: "Current number of active queries in the system.", - tiles: 1, - }, - { - title: "Query Error Rate", - query: ` - SELECT - ROUND(100 * SUM(CASE WHEN type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') THEN 1 ELSE 0 END) / COUNT(*), 2) AS error_rate - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - `, - type: "card", - description: "Percentage of failed queries over the last 24 hours.", - tiles: 1, - }, - { - title: "Average Query Duration", - query: ` - SELECT - ROUND(avg(CAST(query_duration_ms AS DOUBLE)), 2) AS avg_duration_ms - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - `, - type: "card", - description: - "Average duration of queries executed in the last 24 hours.", - tiles: 1, - }, - { - title: "Total Queries (Last 24h)", - query: ` - SELECT COUNT(*) AS total_queries - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - `, - type: "card", - description: "Total number of queries executed in the last 24 hours.", - tiles: 1, - }, - { - title: "Query Duration Distribution", - query: ` - SELECT - CASE - WHEN query_duration_ms < 10 THEN '<10ms' - WHEN query_duration_ms < 20 THEN '10ms-20ms' - WHEN query_duration_ms < 50 THEN '20ms-50ms' - WHEN query_duration_ms < 100 THEN '50ms-100ms' - WHEN query_duration_ms < 200 THEN '100ms-200ms' - WHEN query_duration_ms < 500 THEN '200ms-500ms' - WHEN query_duration_ms < 1000 THEN '500ms-1s' - WHEN query_duration_ms < 5000 THEN '1s-5s' - WHEN query_duration_ms < 30000 THEN '5s-30s' - ELSE '>30s' - END AS duration_bucket, - COUNT(*) AS query_count - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - GROUP BY duration_bucket - ORDER BY duration_bucket - `, - type: "chart", - chartType: "bar", - description: - "Granular distribution of query durations over the last 24 hours.", - chartConfig: { - indexBy: "duration_bucket", - query_count: { - label: "Query Count", - color: "hsl(var(--chart-1))", - }, - }, - tiles: 2, - }, - { - title: "Queries Per Second (QPS)", - query: ` - SELECT - strftime('%Y-%m-%d %H:%M', event_time) AS minute, - COUNT(*) AS qps - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 hour') - GROUP BY minute - ORDER BY minute - `, - type: "chart", - chartType: "area", - description: "Rate of queries per second over the last hour.", - chartConfig: { - indexBy: "minute", - qps: { - label: "QPS", - color: "hsl(var(--chart-3))", - }, - }, - tiles: 2, - }, - { - title: "Queries Per User", - query: ` - SELECT user, COUNT(*) AS query_count - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - GROUP BY user - ORDER BY query_count DESC - LIMIT 10 - `, - type: "table", - description: - "Top 10 users by the number of queries executed in the last 24 hours.", - tiles: 4, - }, - { - title: "Top Slow Queries", - query: ` - SELECT query, query_duration_ms - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - ORDER BY query_duration_ms DESC - LIMIT 10 - `, - type: "table", - description: "Top 10 slowest queries executed in the last 24 hours.", - tiles: 4, - }, - ], - }, - { - title: "Storage", - scope: "storage", - description: "Storage-related metrics.", - icon: HardDriveIcon, - items: [ - { - title: "Database Sizes", - query: ` - SELECT - name AS database, - round(total_bytes / 1024 / 1024 / 1024, 2) AS size_gb - FROM pragma_database_list - WHERE name NOT IN ('main', 'temp') - ORDER BY size_gb DESC - `, - type: "chart", - chartType: "bar", - description: "Size distribution of databases.", - chartConfig: { - indexBy: "database", - size_gb: { - label: "Size (GB)", - color: "hsl(var(--chart-2))", - }, - }, - tiles: 4, - }, ], }, { @@ -436,111 +194,4 @@ export const metrics: Metrics[] = [ }, ], }, - { - title: "Query Exceptions", - scope: "exceptions", - description: "Overview of query exceptions in the system.", - icon: AlertTriangleIcon, - items: [ - { - title: "Exceptions (Last 24h)", - query: ` - SELECT COUNT(*) AS total_exceptions - FROM pragma_sql_log - WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') - AND event_time > DATE('now', '-1 day') - `, - type: "card", - description: - "Total number of exceptions recorded in the last 24 hours.", - tiles: 1, - }, - { - title: "Exception Rate (Last 24h)", - query: ` - SELECT - ROUND(100 * SUM(CASE WHEN type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') THEN 1 ELSE 0 END) / COUNT(*), 2) AS exception_rate - FROM pragma_sql_log - WHERE event_time > DATE('now', '-1 day') - `, - type: "card", - description: - "Percentage of queries that resulted in exceptions over the last 24 hours.", - tiles: 1, - }, - { - title: "Recent Exceptions", - query: ` - SELECT - event_time, - user, - query, - exception - FROM pragma_sql_log - WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') - AND event_time > DATE('now', '-1 hour') - ORDER BY event_time DESC - LIMIT 10 - `, - type: "table", - description: "Last 10 exceptions recorded in the last hour.", - tiles: 4, - }, - { - title: "Exceptions by User", - query: ` - SELECT user, COUNT(*) AS exception_count - FROM pragma_sql_log - WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') - AND event_time > DATE('now', '-1 day') - GROUP BY user - ORDER BY exception_count DESC - LIMIT 10 - `, - type: "table", - description: - "Top 10 users with the most exceptions in the last 24 hours.", - tiles: 4, - }, - { - title: "Exceptions Over Time", - query: ` - SELECT - strftime('%Y-%m-%d %H:00', event_time) AS hourERROR, - COUNT(*) AS exception_count - FROM pragma_sql_log - WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') - AND event_time > DATE('now', '-1 day') - GROUP BY hourERROR - ORDER BY hourERROR - `, - type: "chart", - chartType: "line", - description: "Count of exceptions recorded over the last 24 hours.", - chartConfig: { - indexBy: "hourERROR", - exception_count: { - label: "Exception Count", - color: "hsl(var(--chart-2))", - }, - }, - tiles: 2, - }, - { - title: "Most Common Exceptions", - query: ` - SELECT exception, COUNT(*) AS count - FROM pragma_sql_log - WHERE type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') - AND event_time > DATE('now', '-1 day') - GROUP BY exception - ORDER BY count DESC - LIMIT 10 - `, - type: "table", - description: "Top 10 most common exceptions in the last 24 hours.", - tiles: 2, - }, - ], - }, ];