Update TPC-DS tests.

Fix using virtual environment, avoid TPC-DS tests failing when fast query status could not be obtained.
Travis CI: enable TPC-DS tests, allow tests to fail on unsupported versions.

Author: sokolcati

.travis.yml

@@ -22,24 +22,22 @@ notifications:
         on_failure: always
 
 env:
-    - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=0
+    - PG_VERSION=16 LEVEL=hardcore USE_TPCDS=1
     - PG_VERSION=16
-    - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=0
+    - PG_VERSION=15 LEVEL=hardcore USE_TPCDS=1
     - PG_VERSION=15
-    - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=0
+    - PG_VERSION=14 LEVEL=hardcore USE_TPCDS=1
     - PG_VERSION=14
-    - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=0
+    - PG_VERSION=13 LEVEL=hardcore USE_TPCDS=1
     - PG_VERSION=13
-    - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=0
+    - PG_VERSION=12 LEVEL=hardcore USE_TPCDS=1
     - PG_VERSION=12
-    - PG_VERSION=11 LEVEL=hardcore USE_TPCDS=0
     - PG_VERSION=11
-    - PG_VERSION=10 LEVEL=hardcore USE_TPCDS=0
     - PG_VERSION=10
-    - PG_VERSION=9.6 LEVEL=hardcore
     - PG_VERSION=9.6
 
 matrix:
     allow_failures:
-        - env: PG_VERSION=10 LEVEL=nightmare
-        - env: PG_VERSION=9.6 LEVEL=nightmare
+        - env: PG_VERSION=11
+        - env: PG_VERSION=10
+        - env: PG_VERSION=9.6

run_tests.sh

@@ -13,6 +13,9 @@
 set -ux
 status=0
 
+venv_path=tmp/env
+rm -rf "$venv_path"
+
 # global exports
 export PGPORT=55435
 export VIRTUAL_ENV_DISABLE_PROMPT=1
@@ -148,13 +151,14 @@ if [ -f regression.diffs ]; then cat regression.diffs; fi
 
 # run python tests
 set +x -e
-python3 -m venv /tmp/env && source /tmp/env/bin/activate &&
-pip install -r ./tests/requirements.txt
+python3 -m venv "$venv_path" && source "$venv_path/bin/activate"
+pip3 install --upgrade -t "$venv_path" -r ./tests/requirements.txt
+#pip3 install -e "./$venv_path"
 set -e #exit virtualenv with error code
-python tests/pg_qs_test_runner.py --port $PGPORT
+python3 tests/pg_qs_test_runner.py --port $PGPORT
 if [[ "$USE_TPCDS" == "1" ]]; then
-	python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-setup
-	python tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-run
+	python3 tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-setup
+	python3 tests/pg_qs_test_runner.py --port $PGPORT --tpc-ds-run
 fi
 deactivate
 set -x
@@ -179,4 +183,5 @@ gcov $CUSTOM_PG_SRC/contrib/pg_query_state/*.c $CUSTOM_PG_SRC/contrib/pg_query_s
 set +ux
 
 # send coverage stats to Codecov
+export CODECOV_TOKEN=55ab7421-9277-45af-a329-d8b40db96b2a
 bash <(curl -s https://codecov.io/bash)

tests/pg_qs_test_runner.py

@@ -8,9 +8,11 @@
 import os
 import sys
 
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(os.path.abspath('tmp/env'))
+
 import psycopg2
 
-sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 from test_cases import *
 import tpcds
 

tests/tpcds.py

@@ -8,6 +8,10 @@
 import time
 
 import progressbar
+# This actually imports progressbar2 but `import progressbar2' itself doesn't work.
+# In case of problems with the progressbar/progressbar2, check that you have the
+# progressbar2 installed and the path to it or venv is specified.
+
 import psycopg2.extensions
 
 import common
@@ -22,7 +26,10 @@ def setup_tpcds(config):
 	try:
 		conn = psycopg2.connect(**config)
 		cur = conn.cursor()
+	except Exception as e:
+		raise DataLoadException('Load failed: %s' % e)
 
+	try:
 		# Create pg_query_state extension
 		cur.execute('CREATE EXTENSION IF NOT EXISTS pg_query_state')
 
@@ -55,13 +62,13 @@ def run_tpcds(config):
 	TPC_DS_STATEMENT_TIMEOUT = 20000	# statement_timeout in ms
 
 	print('Preparing TPC-DS queries...')
+	err_count = 0
 	queries = []
 	for query_file in sorted(os.listdir('tmp_stress/tpcds-result-reproduction/query_qualification/')):
 		with open('tmp_stress/tpcds-result-reproduction/query_qualification/%s' % query_file, 'r') as f:
 			queries.append(f.read())
 
 	acon, = common.n_async_connect(config)
-	pid = acon.get_backend_pid()
 
 	print('Starting TPC-DS queries...')
 	timeout_list = []
@@ -84,8 +91,25 @@ def run_tpcds(config):
 			PG_QS_DELAY, BEFORE_GETTING_QS_DELAY = 0.1, 0.1
 			BEFORE_GETTING_QS, GETTING_QS = range(2)
 			state, n_first_getting_qs_retries = BEFORE_GETTING_QS, 0
+
+			pg_qs_args = {
+				'config': config,
+				'pid': acon.get_backend_pid()
+			}
+
 			while True:
-				result, notices = common.pg_query_state(config, pid)
+				try:
+					result, notices = common.pg_query_state(**pg_qs_args)
+				except Exception as e:
+					# do not consider the test failed if the "error in message
+					# queue data transmitting" is received, this may happen with
+					# some small probability, but if it happens too often it is
+					# a problem, we will handle this case after the loop
+					if "error in message queue data transmitting" in e.pgerror:
+						err_count += 1
+					else:
+						raise e
+
 				# run state machine to determine the first getting of query state
 				# and query finishing
 				if state == BEFORE_GETTING_QS:
@@ -109,6 +133,12 @@ def run_tpcds(config):
 		except psycopg2.extensions.QueryCanceledError:
 			timeout_list.append(i + 1)
 
+	if err_count > 2:
+		print("\nERROR: error in message queue data transmitting")
+		raise Exception('error was received %d times'%err_count)
+	elif err_count > 0:
+		print(err_count, " times there was error in message queue data transmitting")
+
 	common.n_close((acon,))
 
 	if len(timeout_list) > 0: