Support building C extensions in benchmarks

Add ctypes and ctypes_argtypes benchmarks

Author: mdboom

.gitignore

@@ -10,7 +10,7 @@
 # Created by setup.py sdist
 build/
 dist/
-pyperformance.egg-info/
+*.egg-info/
 
 # Created by the pyperformance script
 venv/

MANIFEST.in

@@ -15,5 +15,6 @@ include pyperformance/data-files/benchmarks/MANIFEST
 include pyperformance/data-files/benchmarks/bm_*/*.toml
 include pyperformance/data-files/benchmarks/bm_*/*.py
 include pyperformance/data-files/benchmarks/bm_*/requirements.txt
+include pyperformance/data-files/benchmarks/bm_*/*.c
 recursive-include pyperformance/data-files/benchmarks/bm_*/data *
 recursive-exclude pyperformance/tests *

doc/benchmarks.rst

@@ -130,6 +130,16 @@ deepcopy
 Benchmark the Python `copy.deepcopy` method. The `deepcopy` method is
 performed on a nested dictionary and a dataclass.
 
+ctypes
+------
+
+Benchmark to measure the function call overhead of calling C functions using ctypes.
+
+The ``ctypes`` benchmark lets ``ctypes`` infer the argument types from the passed in
+values. The ``ctypes_argtypes`` benchmark `explicitly specifies the argument types
+<https://docs.python.org/3.10/library/ctypes.html?highlight=ctypes#specifying-the-required-argument-types-function-prototypes>`_,
+which is slower than inferred argument types.
+
 deltablue
 ---------
 

doc/custom_benchmarks.rst

@@ -324,16 +324,17 @@ All other PEP 621 fields are optional (e.g. ``requires-python = ">=3.8"``,
 The ``[tool.pyperformance]`` Section
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-=============== ===== === === ===
-field           type  R   B   F 
-=============== ===== === === ===
-tool.name       str   X       X
-tool.tags       [str]     X    
-tool.extra_opts [str]     X    
-tool.inherits   file           
-tool.runscript  file      X    
-tool.datadir    file      X    
-=============== ===== === === ===
+================== ===== === === ===
+field              type  R   B   F
+================== ===== === === ===
+tool.name          str   X       X
+tool.tags          [str]     X
+tool.extra_opts    [str]     X
+tool.inherits      file
+tool.runscript     file      X
+tool.datadir       file      X
+tool.install_setup bool
+================== ===== === === ===
 
 "R": required
 "B": inferred from the inherited metadata
@@ -342,3 +343,6 @@ tool.datadir    file      X
 * tags: optional list of names to group benchmarks
 * extra_opts: optional list of args to pass to ``tool.runscript``
 * runscript: the benchmark script to use instead of run_benchmark.py.
+* install_setup: when ``true``, run ``pip install -e .`` in the
+  benchmark directory to install it in the virtual environment. This has the
+  effect of running a ``setup.py`` file, if present.

pyperformance/_benchmark.py

@@ -164,6 +164,13 @@ def runscript(self):
     def extra_opts(self):
         return self._get_metadata_value('extra_opts', ())
 
+    @property
+    def setup_py(self):
+        if not self._get_metadata_value('install_setup', False):
+            return None
+        filename = os.path.join(os.path.dirname(self.metafile), 'setup.py')
+        return filename if os.path.exists(filename) else None
+
     # Other metadata keys:
     # * base
     # * python

pyperformance/_benchmark_metadata.py

@@ -32,6 +32,7 @@
     'datadir': None,
     'runscript': None,
     'extra_opts': None,
+    'install_setup': None,
 }
 
 
@@ -228,6 +229,9 @@ def _resolve_value(field, value, rootdir):
         for opt in value:
             if not opt or not isinstance(opt, str):
                 raise TypeError(f'extra_opts should be a list of strings, got {value!r}')
+    elif field == 'install_setup':
+        if not isinstance(value, bool):
+            raise TypeError(f'install_setup should be a bool, got {value!r}')
     else:
         raise NotImplementedError(field)
     return value

pyperformance/_pip.py

@@ -150,7 +150,7 @@ def install_requirements(reqs, *extra,
     if upgrade:
         args.append('-U')  # --upgrade
     for reqs in [reqs, *extra]:
-        if os.path.exists(reqs):
+        if os.path.isfile(reqs):
             args.append('-r')  # --requirement
         args.append(reqs)
     return run_pip('install', *args, **kwargs)

pyperformance/data-files/benchmarks/MANIFEST

@@ -11,6 +11,8 @@ generators	<local>
 chameleon	<local>
 chaos	<local>
 crypto_pyaes	<local>
+ctypes	<local>
+ctypes_argtypes	<local:ctypes>
 deepcopy	<local>
 deltablue	<local>
 django_template	<local>
@@ -69,6 +71,7 @@ xml_etree	<local>
 #apps
 #math
 #template
+#extension
 
 
 [group default]

pyperformance/data-files/benchmarks/bm_ctypes/bm_ctypes_argtypes.toml

@@ -0,0 +1,11 @@
+[project]
+name = "pyperformance_bm_ctypes_argtypes"
+requires-python = ">=3.7"
+dependencies = ["pyperf"]
+urls = {repository = "https://github.com/python/pyperformance"}
+dynamic = ["version"]
+
+[tool.pyperformance]
+name = "ctypes_argtypes"
+tags = "extension"
+extra_opts = ["--argtypes"]

pyperformance/data-files/benchmarks/bm_ctypes/cmodule.c

@@ -0,0 +1,54 @@
+#include <Python.h>
+
+#if defined(_WIN32) || defined(__CYGWIN__)
+#define EXPORTED_SYMBOL __declspec(dllexport)
+#else
+#define EXPORTED_SYMBOL
+#endif
+
+
+EXPORTED_SYMBOL
+void void_foo_void(void) {
+
+}
+
+EXPORTED_SYMBOL
+int int_foo_int(int a) {
+    return a + 1;
+}
+
+EXPORTED_SYMBOL
+void void_foo_int(int a) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int(int a, int b) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int_int(int a, int b, int c) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_int_int_int_int(int a, int b, int c, int d) {
+
+}
+
+EXPORTED_SYMBOL
+void void_foo_constchar(const char* str) {
+
+}
+
+PyMODINIT_FUNC
+PyInit_cmodule(void) {
+  // DELIBERATELY EMPTY
+
+  // This isn't actually a Python extension module (it's used via ctypes), so
+  // this entry point function will never be called. However, we are utilizing
+  // setuptools to build it, and on Windows, setuptools explicitly passes the
+  // flag /EXPORT:PyInit_cmodule, so it must be defined.
+  return NULL;
+}