From 79af13b699df9219e06d172efe1b1d9fad9777cd Mon Sep 17 00:00:00 2001 From: Cristian Ferretti <37232625+jcferretti@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:17:53 -0400 Subject: [PATCH] Gradle build target and CI checks for python ticking client (#4688) * First draft. * More. * It works! * Improved the test (followup to Jianfeng's comment.). * Avoid pydeephaven_ticking as an explicit import target (hat tip Corey and Jianfeng) * Followup to comments from Colin (thanks). * Followup to Jianfeng and Corey's comments. * Removing references to pydeephaven_ticking from README.md * Halfway thru followup (Colin). * Following up on Colin's review comments. * Use new image with python package unittest-xml-reporting; rename cpp-clients image. --- .github/workflows/tag-base-images.yml | 2 +- cpp-client/build.gradle | 12 +- .../cpp-clients-fat-base/gradle.properties | 4 - .../build.gradle | 0 .../cpp-clients-multi-base/gradle.properties | 4 + py/client-ticking/README.md | 2 +- py/client-ticking/build.gradle | 114 ++++++++++++++++++ py/client-ticking/demos/demo_simple.py | 1 + py/client-ticking/gradle.properties | 1 + py/client-ticking/tests/__init__.py | 3 + py/client-ticking/tests/suite.py | 14 +++ py/client-ticking/tests/test_ticking_basic.py | 47 ++++++++ settings.gradle | 3 + 13 files changed, 195 insertions(+), 12 deletions(-) delete mode 100644 docker/registry/cpp-clients-fat-base/gradle.properties rename docker/registry/{cpp-clients-fat-base => cpp-clients-multi-base}/build.gradle (100%) create mode 100644 docker/registry/cpp-clients-multi-base/gradle.properties create mode 100644 py/client-ticking/build.gradle create mode 100644 py/client-ticking/gradle.properties create mode 100644 py/client-ticking/tests/__init__.py create mode 100644 py/client-ticking/tests/suite.py create mode 100644 py/client-ticking/tests/test_ticking_basic.py diff --git a/.github/workflows/tag-base-images.yml b/.github/workflows/tag-base-images.yml index 98668797371..3a9dbb91b21 100644 --- a/.github/workflows/tag-base-images.yml +++ b/.github/workflows/tag-base-images.yml @@ -59,7 +59,7 @@ jobs: - name: Tag upstream images if: ${{ startsWith(github.ref, 'refs/heads/release/v') }} run: | - ./docker/registry/cpp-clients-fat-base/build/crane/retag.sh + ./docker/registry/cpp-clients-multi-base/build/crane/retag.sh ./docker/registry/protoc-base/build/crane/retag.sh ./docker/registry/slim-base/build/crane/retag.sh ./docker/registry/server-base/build/crane/retag.sh diff --git a/cpp-client/build.gradle b/cpp-client/build.gradle index 0d7b6bedfd1..0ba12afcf8e 100644 --- a/cpp-client/build.gradle +++ b/cpp-client/build.gradle @@ -7,11 +7,11 @@ plugins { id 'license' } -// We use the cpp-clients-fat-base image instead of the cpp-client-base, +// We use the cpp-clients-multi-base image instead of the cpp-client-base, // so that the test tasks in py/client-ticking and R can in turn use the // image we will generate here as a base. -// See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-fat -evaluationDependsOn Docker.registryProject('cpp-clients-fat-base') +// See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-multi +evaluationDependsOn Docker.registryProject('cpp-clients-multi-base') configurations { cpp {} @@ -79,7 +79,7 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') { dockerfile { // See comment at the beginning of this file for why we use this base image. - from('deephaven/cpp-clients-fat-base:local-build') + from('deephaven/cpp-clients-multi-base:local-build') // // Build and install client. // @@ -112,11 +112,11 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') { rm -fr ${PREFIX}/src/deephaven/build ''') // Note environment variables defined here are inherited by other images - // using this image as a base. + // using this image as a base ("from"). environmentVariable 'DH_PREFIX', prefix environmentVariable 'LD_LIBRARY_PATH', "${prefix}/lib" } - parentContainers = [ Docker.registryTask(project, 'cpp-clients-fat-base') ] + parentContainers = [ Docker.registryTask(project, 'cpp-clients-multi-base') ] } def testCppClient = Docker.registerDockerTask(project, 'testCppClient') { diff --git a/docker/registry/cpp-clients-fat-base/gradle.properties b/docker/registry/cpp-clients-fat-base/gradle.properties deleted file mode 100644 index ae9ddfc242f..00000000000 --- a/docker/registry/cpp-clients-fat-base/gradle.properties +++ /dev/null @@ -1,4 +0,0 @@ -io.deephaven.project.ProjectType=DOCKER_REGISTRY -deephaven.registry.imageName=ghcr.io/deephaven/cpp-clients-fat-base:latest -deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-fat-base@sha256:e71faadaf9a37f163a940dbbef9ee0eecf117a83678530b76054b796e2550c5f -deephaven.registry.platform=linux/amd64 diff --git a/docker/registry/cpp-clients-fat-base/build.gradle b/docker/registry/cpp-clients-multi-base/build.gradle similarity index 100% rename from docker/registry/cpp-clients-fat-base/build.gradle rename to docker/registry/cpp-clients-multi-base/build.gradle diff --git a/docker/registry/cpp-clients-multi-base/gradle.properties b/docker/registry/cpp-clients-multi-base/gradle.properties new file mode 100644 index 00000000000..fdb2928a9dc --- /dev/null +++ b/docker/registry/cpp-clients-multi-base/gradle.properties @@ -0,0 +1,4 @@ +io.deephaven.project.ProjectType=DOCKER_REGISTRY +deephaven.registry.imageName=ghcr.io/deephaven/cpp-clients-multi-base:latest +deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:0ac6473b7c533a504b1761257e90cec2046a69efc9c1b8f428b6b39cc83da0a4 +deephaven.registry.platform=linux/amd64 diff --git a/py/client-ticking/README.md b/py/client-ticking/README.md index d3a8ae429d1..c1391934bb6 100644 --- a/py/client-ticking/README.md +++ b/py/client-ticking/README.md @@ -51,7 +51,7 @@ cd ${DHROOT}/py/client-ticking ``` # Ensure the DHCPP environment variable is set per the instructions above rm -rf build # Ensure we clean the remnants of any pre-existing build. -CFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python setup.py build_ext -i +CFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python3 setup.py build_ext -i ``` ### Install pydeephaven-ticking diff --git a/py/client-ticking/build.gradle b/py/client-ticking/build.gradle new file mode 100644 index 00000000000..11ed082157a --- /dev/null +++ b/py/client-ticking/build.gradle @@ -0,0 +1,114 @@ +plugins { + id 'com.bmuschko.docker-remote-api' + id 'io.deephaven.project.register' + id 'io.deephaven.deephaven-in-docker' +} + +configurations { + pythonWheel +} + +dependencies { + pythonWheel project(':py-client') +} + +evaluationDependsOn(':cpp-client') + +def prefix = '/opt/deephaven' + +// start a grpc-api server +String randomSuffix = UUID.randomUUID().toString(); +deephavenDocker { + envVars.set([ + 'START_OPTS':'-Xmx512m -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler' + ]) + containerName.set "pydeephaven-test-container-${randomSuffix}" + networkName.set "pydeephaven-network-${randomSuffix}" +} + +def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking') { + // Only tested on x86-64, and we only build dependencies for x86-64 + platform = 'linux/amd64' + + copyIn { + from(layout.projectDirectory) { + include 'setup.py' + include 'README.md' + include 'src/**' + } + from(configurations.pythonWheel) { + into 'wheels' + } + } + copyOut { + into layout.buildDirectory.dir('wheel') + } + dockerfile { + from('deephaven/cpp-client:local-build') + runCommand("""mkdir -p \\ + /out \\ + ${prefix}/log \\ + ${prefix}/src/py-client-ticking/src \\ + ${prefix}/src/py-client-ticking/in-wheels + + """) + copyFile('setup.py', "${prefix}/src/py-client-ticking") + copyFile('README.md', "${prefix}/src/py-client-ticking") + copyFile('src/', "${prefix}/src/py-client-ticking/src/") + copyFile('wheels/', "${prefix}/src/py-client-ticking/in-wheels") + runCommand("PREFIX=${prefix}; " + + '''set -eux ; \ + cd "${PREFIX}/src/py-client-ticking"; \ + . "${PREFIX}/env.sh"; \ + MAKEFLAGS="-j${NCPUS}" \ + CFLAGS="-I${DHCPP}/include" \ + LDFLAGS="-L${DHCPP}/lib" \ + python3 setup.py build_ext -i; \ + python3 setup.py bdist_wheel; \ + pip3 install in-wheels/*.whl; \ + pip3 install --force --no-deps dist/*.whl; \ + ln dist/*.whl /out; \ + cd /; \ + rm -fr "${PREFIX}/src/py-client-ticking" + ''') + } + parentContainers = [ project.tasks.getByPath(':cpp-client:cppClient') ] +} + +def testPyClientTicking = Docker.registerDockerTask(project, 'testPyClientTicking') { + // Only tested on x86-64, and we only build dependencies for x86-64 + platform = 'linux/amd64' + copyIn { + from(layout.projectDirectory) { + include 'tests/**' + } + } + dockerfile { + from('deephaven/py-client-ticking:local-build') + runCommand("PREFIX=${prefix}; " + + '''set -eux ; \ + rm -fr /out; \ + mkdir -p \ + /out/report \ + /project/tests + ''') + copyFile('tests/', "/project/tests/") + workingDir('/project') + // + // Setup for test run. + // + environmentVariable 'DH_HOST', deephavenDocker.containerName.get() + environmentVariable 'DH_PORT', '10000' + } + containerDependencies.dependsOn = [deephavenDocker.healthyTask] + containerDependencies.finalizedBy = deephavenDocker.endTask + network = deephavenDocker.networkName.get() + parentContainers = [ project.tasks.getByName('pyClientTicking') ] + entrypoint = ['python3', '-m', 'xmlrunner', 'discover', 'tests', '-v', '-o', '/out/report'] + copyOut { + into layout.buildDirectory.dir('test-results') + } +} + +tasks.getByName('check').dependsOn(testPyClientTicking) +deephavenDocker.shouldLogIfTaskFails testPyClientTicking diff --git a/py/client-ticking/demos/demo_simple.py b/py/client-ticking/demos/demo_simple.py index 83ee2b30c16..5d69af63f55 100644 --- a/py/client-ticking/demos/demo_simple.py +++ b/py/client-ticking/demos/demo_simple.py @@ -48,3 +48,4 @@ def _show_deltas(self, what: str, dict: Dict[str, pa.Array]): print("Waking up and stopping the listener") listener_handle.stop() +session.close() diff --git a/py/client-ticking/gradle.properties b/py/client-ticking/gradle.properties new file mode 100644 index 00000000000..8f42cc0940f --- /dev/null +++ b/py/client-ticking/gradle.properties @@ -0,0 +1 @@ +io.deephaven.project.ProjectType=BASIC diff --git a/py/client-ticking/tests/__init__.py b/py/client-ticking/tests/__init__.py new file mode 100644 index 00000000000..2519e05c6e2 --- /dev/null +++ b/py/client-ticking/tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending +# diff --git a/py/client-ticking/tests/suite.py b/py/client-ticking/tests/suite.py new file mode 100644 index 00000000000..17a3c87893d --- /dev/null +++ b/py/client-ticking/tests/suite.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending +# + +import unittest + +from tests.test_ticking_basic import TickingBasicTestCase + +if __name__ == '__main__': + suite = unittest.TestSuite() + suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TickingBasicTestCase)) + + runner = unittest.TextTestRunner(verbosity=2) + runner.run(suite) diff --git a/py/client-ticking/tests/test_ticking_basic.py b/py/client-ticking/tests/test_ticking_basic.py new file mode 100644 index 00000000000..1269abfd1f0 --- /dev/null +++ b/py/client-ticking/tests/test_ticking_basic.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending +# + +import unittest + +import pydeephaven as dh +import time +import sys + +class TickingBasicTestCase(unittest.TestCase): + def test_ticking_basic_time_table(self): + session = dh.Session() + half_second_in_nanos = 200 * 1000 * 1000 + table = session.time_table(period=half_second_in_nanos).update(formulas=["Col1 = i"]) + session.bind_table(name="my_ticking_table", table=table) + table_added_last_col1_seen = -1 + table_added_update_count = 0 + def update_table_added(added): + nonlocal table_added_update_count + nonlocal table_added_last_col1_seen + for value in added['Col1'].to_pylist(): + prev = table_added_last_col1_seen + table_added_last_col1_seen = value + if prev != -1: + self.assertTrue(prev + 1 == table_added_last_col1_seen) + table_added_update_count += 1 + listener_handle = dh.listen(table, lambda update : update_table_added(update.added('Col1'))) + listener_handle.start() + seen_rows = 0 + start_seconds = time.time() + # Wait until we see a given number of updates or timeout. Note the callback for the updates + # is already checking they are of the right form. + col1_target = 10 + timeout_seconds = 10 + while True: + time.sleep(1) + if table_added_last_col1_seen >= col1_target: + break + now_seconds = time.time() + self.assertTrue(now_seconds - start_seconds < timeout_seconds) # eventually fail + self.assertTrue(4 >= table_added_update_count) + listener_handle.stop() + session.close() + +if __name__ == '__main__': + unittest.main() diff --git a/settings.gradle b/settings.gradle index 9e3d72b84eb..6cfad7e1b31 100644 --- a/settings.gradle +++ b/settings.gradle @@ -331,6 +331,9 @@ project(':jpy-integration').projectDir = file('py/jpy-integration') include(':py-client') project(':py-client').projectDir = file('py/client') +include(':py-client-ticking') +project(':py-client-ticking').projectDir = file('py/client-ticking') + include(':py-server') project(':py-server').projectDir = file('py/server')