Transfer tests to python3

Author: ololobus

Dockerfile.tmpl

@@ -7,7 +7,7 @@ RUN apk add --no-cache \
 	make musl-dev gcc bison flex coreutils \
 	zlib-dev libedit-dev \
 	clang clang-analyzer linux-headers \
-	python2 python2-dev py2-virtualenv;
+	python3 python3-dev py3-virtualenv;
 
 
 # Install fresh valgrind
@@ -29,6 +29,7 @@ RUN	chown postgres:postgres ${PGDATA} && \
 	chmod a+rwx /usr/local/share/postgresql/extension
 
 COPY run_tests.sh /run.sh
+COPY tests/requirements.txt /requirements.txt
 RUN chmod 755 /run.sh
 
 ADD . /pg/testdir

Makefile

@@ -9,7 +9,7 @@ DATA_built = $(EXTENSION)--$(EXTVERSION).sql
 PGFILEDESC = "pg_query_state - facility to track progress of plan execution"
 
 EXTRA_CLEAN = ./isolation_output $(EXTENSION)--$(EXTVERSION).sql \
-	Dockerfile ./tests/*.pyc
+	Dockerfile ./tests/*.pyc ./tmp_stress
 
 ifdef USE_PGXS
 PG_CONFIG ?= pg_config

mk_dockerfile.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env sh
+
 if [ -z ${PG_VERSION+x} ]; then
 	echo PG_VERSION is not set!
 	exit 1

run_tests.sh

@@ -143,8 +143,8 @@ if [ -f regression.diffs ]; then cat regression.diffs; fi
 
 # run python tests
 set +x -e
-virtualenv /tmp/env && source /tmp/env/bin/activate &&
-pip install PyYAML && pip install psycopg2
+python3 -m venv /tmp/env && source /tmp/env/bin/activate &&
+pip install -r /requirements.txt
 set -e #exit virtualenv with error code
 python tests/pg_qs_test_runner.py --port $PGPORT
 deactivate

tests/pg_qs_test_runner.py

@@ -3,10 +3,15 @@
 				Tests extract query state from running backend (including concurrent extracts)
 Copyright (c) 2016-2016, Postgres Professional
 '''
+from __future__ import print_function, division, absolute_import
 
+import os
+import sys
 import argparse
+import getpass
 import psycopg2
-import sys
+
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 from test_cases import *
 
 class PasswordPromptAction(argparse.Action):
@@ -34,50 +39,51 @@ class TeardownException(Exception): pass
 	'drop table foo cascade',
 	'drop table bar cascade',
 	'drop extension pg_query_state cascade',
-	]
+]
 
 tests = [
-        test_deadlock,
-        test_simple_query,
-        test_concurrent_access,
-        test_nested_call,
-        test_trigger,
-        test_costs,
-        test_buffers,
-        test_timing,
-        test_formats,
-        test_timing_buffers_conflicts,
-        test_insert_on_conflict,
-        ]
+	test_deadlock,
+	test_simple_query,
+	test_concurrent_access,
+	test_nested_call,
+	test_trigger,
+	test_costs,
+	test_buffers,
+	test_timing,
+	test_formats,
+	test_timing_buffers_conflicts,
+	test_insert_on_conflict,
+]
 
 def setup(con):
 	''' Creates pg_query_state extension, creates tables for tests, fills it with data '''
-	print 'setting up...'
+	print('setting up...')
 	try:
 		cur = con.cursor()
 		for cmd in setup_cmd:
 			cur.execute(cmd)
 		con.commit()
 		cur.close()
-	except Exception, e:
+	except Exception as e:
 		raise SetupException('Setup failed: %s' % e)
-	print 'done!'
+	print('done!')
 
 def teardown(con):
 	''' Drops table and extension '''
-	print 'tearing down...'
+	print('tearing down...')
 	try:
 		cur = con.cursor()
 		for cmd in teardown_cmd:
 			cur.execute(cmd)
 		con.commit()
 		cur.close()
-	except Exception, e:
+	except Exception as e:
 		raise TeardownException('Teardown failed: %s' % e)
-	print 'done!'
+	print('done!')
 
 def main(config):
 	''' Main test function '''
+
 	con = psycopg2.connect(**config)
 	setup(con)
 
@@ -86,19 +92,27 @@ def main(config):
 			descr = test.__doc__
 		else:
 			descr = 'test case %d' % (i+1)
-		print ("%s..." % descr),; sys.stdout.flush()
+		print(("%s..." % descr))
+		sys.stdout.flush()
 		test(config)
-		print 'ok!'
+		print('ok!')
+
+	if os.environ['LEVEL'] == 'stress':
+		print('Starting stress test')
+		stress_test(config)
+		print('Stress finished successfully')
 
 	teardown(con)
 	con.close()
 
 if __name__ == '__main__':
 	parser = argparse.ArgumentParser(description='Query state of running backends tests')
+
 	parser.add_argument('--host', default='localhost', help='postgres server host')
 	parser.add_argument('--port', type=int, default=5432, help='postgres server port')
 	parser.add_argument('--user', dest='user', default='postgres', help='user name')
 	parser.add_argument('--database', dest='database', default='postgres', help='database name')
 	parser.add_argument('--password', dest='password', nargs=0, action=PasswordPromptAction, default='')
+
 	args = parser.parse_args()
 	main(args.__dict__)

tests/prepare_stress.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env sh
+mkdir -p tmp_stress
+cd tmp_stress
+rm -rf ./*
+
+git clone https://github.com/gregrahn/tpcds-kit.git
+cd tpcds-kit/tools
+make -s
+
+#Generate data
+./dsdgen -FORCE -VERBOSE
+
+#Prepare data
+mkdir tables -p
+for i in `ls *.dat`; do
+  echo "Preparing file " $i
+  sed 's/|$//' $i > tables/$i
+done
+
+#Generate queries
+./dsqgen -DIRECTORY ../query_templates \
+         -INPUT ../query_templates/templates.lst \
+         -VERBOSE Y \
+         -QUALIFY Y \
+         -DIALECT netezza

tests/requirements.txt

@@ -0,0 +1,3 @@
+PyYAML
+psycopg2
+progressbar2