Gradle build target and CI checks for python ticking client (#4688)

* First draft. * More. * It works! * Improved the test (followup to Jianfeng's comment.). * Avoid pydeephaven_ticking as an explicit import target (hat tip Corey and Jianfeng) * Followup to comments from Colin (thanks). * Followup to Jianfeng and Corey's comments. * Removing references to pydeephaven_ticking from README.md * Halfway thru followup (Colin). * Following up on Colin's review comments. * Use new image with python package unittest-xml-reporting; rename cpp-clients image.
deephaven · Oct 24, 2023 · 79af13b · 79af13b
1 parent d0ca524
commit 79af13b
Show file tree

Hide file tree

Showing 13 changed files with 195 additions and 12 deletions.
diff --git a/.github/workflows/tag-base-images.yml b/.github/workflows/tag-base-images.yml
@@ -59,7 +59,7 @@ jobs:
       - name: Tag upstream images
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}
         run: |
-          ./docker/registry/cpp-clients-fat-base/build/crane/retag.sh
+          ./docker/registry/cpp-clients-multi-base/build/crane/retag.sh
           ./docker/registry/protoc-base/build/crane/retag.sh
           ./docker/registry/slim-base/build/crane/retag.sh
           ./docker/registry/server-base/build/crane/retag.sh
diff --git a/cpp-client/build.gradle b/cpp-client/build.gradle
@@ -7,11 +7,11 @@ plugins {
     id 'license'
 }
 
-// We use the cpp-clients-fat-base image instead of the cpp-client-base,
+// We use the cpp-clients-multi-base image instead of the cpp-client-base,
 // so that the test tasks in py/client-ticking and R can in turn use the
 // image we will generate here as a base.
-// See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-fat
-evaluationDependsOn Docker.registryProject('cpp-clients-fat-base')
+// See https://github.com/deephaven/deephaven-base-images/tree/main/cpp-clients-multi
+evaluationDependsOn Docker.registryProject('cpp-clients-multi-base')
 
 configurations {
     cpp {}
@@ -79,7 +79,7 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
 
     dockerfile {
         // See comment at the beginning of this file for why we use this base image.
-        from('deephaven/cpp-clients-fat-base:local-build')
+        from('deephaven/cpp-clients-multi-base:local-build')
         //
         // Build and install client.
         //
@@ -112,11 +112,11 @@ def buildCppClientImage = Docker.registerDockerTask(project, 'cppClient') {
                       rm -fr ${PREFIX}/src/deephaven/build
                    ''')
         // Note environment variables defined here are inherited by other images
-        // using this image as a base.
+        // using this image as a base ("from").
         environmentVariable 'DH_PREFIX', prefix
         environmentVariable 'LD_LIBRARY_PATH', "${prefix}/lib"
     }
-    parentContainers = [ Docker.registryTask(project, 'cpp-clients-fat-base') ]
+    parentContainers = [ Docker.registryTask(project, 'cpp-clients-multi-base') ]
 }
 
 def testCppClient = Docker.registerDockerTask(project, 'testCppClient') {

diff --git a/docker/registry/cpp-clients-fat-base/gradle.properties b/docker/registry/cpp-clients-fat-base/gradle.properties
diff --git a/...egistry/cpp-clients-fat-base/build.gradle → ...istry/cpp-clients-multi-base/build.gradle b/...egistry/cpp-clients-fat-base/build.gradle → ...istry/cpp-clients-multi-base/build.gradle
diff --git a/docker/registry/cpp-clients-multi-base/gradle.properties b/docker/registry/cpp-clients-multi-base/gradle.properties
@@ -0,0 +1,4 @@
+io.deephaven.project.ProjectType=DOCKER_REGISTRY
+deephaven.registry.imageName=ghcr.io/deephaven/cpp-clients-multi-base:latest
+deephaven.registry.imageId=ghcr.io/deephaven/cpp-clients-multi-base@sha256:0ac6473b7c533a504b1761257e90cec2046a69efc9c1b8f428b6b39cc83da0a4
+deephaven.registry.platform=linux/amd64
diff --git a/py/client-ticking/README.md b/py/client-ticking/README.md
@@ -51,7 +51,7 @@ cd ${DHROOT}/py/client-ticking
 ```
 # Ensure the DHCPP environment variable is set per the instructions above
 rm -rf build  # Ensure we clean the remnants of any pre-existing build.
-CFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python setup.py build_ext -i
+CFLAGS="-I${DHCPP}/include" LDFLAGS="-L${DHCPP}/lib" python3 setup.py build_ext -i
 ```
 
 ### Install pydeephaven-ticking

diff --git a/py/client-ticking/build.gradle b/py/client-ticking/build.gradle
@@ -0,0 +1,114 @@
+plugins {
+    id 'com.bmuschko.docker-remote-api'
+    id 'io.deephaven.project.register'
+    id 'io.deephaven.deephaven-in-docker'
+}
+
+configurations {
+    pythonWheel
+}
+
+dependencies {
+    pythonWheel project(':py-client')
+}
+
+evaluationDependsOn(':cpp-client')
+
+def prefix = '/opt/deephaven'
+
+// start a grpc-api server
+String randomSuffix = UUID.randomUUID().toString();
+deephavenDocker {
+    envVars.set([
+        'START_OPTS':'-Xmx512m -DAuthHandlers=io.deephaven.auth.AnonymousAuthenticationHandler'
+    ])
+    containerName.set "pydeephaven-test-container-${randomSuffix}"
+    networkName.set "pydeephaven-network-${randomSuffix}"
+}
+
+def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking') {
+    // Only tested on x86-64, and we only build dependencies for x86-64
+    platform = 'linux/amd64'
+
+    copyIn {
+        from(layout.projectDirectory) {
+            include 'setup.py'
+            include 'README.md'
+            include 'src/**'
+        }
+        from(configurations.pythonWheel) {
+            into 'wheels'
+        }
+    }
+    copyOut {
+        into layout.buildDirectory.dir('wheel')
+    }
+    dockerfile {
+        from('deephaven/cpp-client:local-build')
+        runCommand("""mkdir -p \\
+                        /out \\
+                        ${prefix}/log \\
+                        ${prefix}/src/py-client-ticking/src \\
+                        ${prefix}/src/py-client-ticking/in-wheels
+
+                   """)
+        copyFile('setup.py', "${prefix}/src/py-client-ticking")
+        copyFile('README.md', "${prefix}/src/py-client-ticking")
+        copyFile('src/', "${prefix}/src/py-client-ticking/src/")
+        copyFile('wheels/', "${prefix}/src/py-client-ticking/in-wheels")
+        runCommand("PREFIX=${prefix}; " +
+                   '''set -eux ; \
+                     cd "${PREFIX}/src/py-client-ticking"; \
+                     . "${PREFIX}/env.sh"; \
+                     MAKEFLAGS="-j${NCPUS}" \
+                       CFLAGS="-I${DHCPP}/include" \
+                       LDFLAGS="-L${DHCPP}/lib" \
+                       python3 setup.py build_ext -i; \
+                     python3 setup.py bdist_wheel; \
+                     pip3 install in-wheels/*.whl; \
+                     pip3 install --force --no-deps dist/*.whl; \
+                     ln dist/*.whl /out; \
+                     cd /; \
+                     rm -fr "${PREFIX}/src/py-client-ticking"
+                   ''')
+    }
+    parentContainers = [ project.tasks.getByPath(':cpp-client:cppClient') ]
+}
+
+def testPyClientTicking = Docker.registerDockerTask(project, 'testPyClientTicking') {
+    // Only tested on x86-64, and we only build dependencies for x86-64
+    platform = 'linux/amd64'
+    copyIn {
+        from(layout.projectDirectory) {
+            include 'tests/**'
+        }
+    }
+    dockerfile {
+        from('deephaven/py-client-ticking:local-build')
+        runCommand("PREFIX=${prefix}; " +
+                   '''set -eux ; \
+                      rm -fr /out; \
+                      mkdir -p \
+                        /out/report \
+                        /project/tests                      
+                   ''')
+        copyFile('tests/', "/project/tests/")
+        workingDir('/project')
+        //
+        // Setup for test run.
+        //
+        environmentVariable 'DH_HOST', deephavenDocker.containerName.get()
+        environmentVariable 'DH_PORT', '10000'
+    }
+    containerDependencies.dependsOn = [deephavenDocker.healthyTask]
+    containerDependencies.finalizedBy = deephavenDocker.endTask
+    network = deephavenDocker.networkName.get()
+    parentContainers = [ project.tasks.getByName('pyClientTicking') ]
+    entrypoint = ['python3', '-m', 'xmlrunner', 'discover', 'tests', '-v', '-o', '/out/report']
+    copyOut {
+        into layout.buildDirectory.dir('test-results')
+    }
+}
+
+tasks.getByName('check').dependsOn(testPyClientTicking)
+deephavenDocker.shouldLogIfTaskFails testPyClientTicking
diff --git a/py/client-ticking/demos/demo_simple.py b/py/client-ticking/demos/demo_simple.py
@@ -48,3 +48,4 @@ def _show_deltas(self, what: str, dict: Dict[str, pa.Array]):
 print("Waking up and stopping the listener")
 
 listener_handle.stop()
+session.close()
diff --git a/py/client-ticking/gradle.properties b/py/client-ticking/gradle.properties
@@ -0,0 +1 @@
+io.deephaven.project.ProjectType=BASIC
diff --git a/py/client-ticking/tests/__init__.py b/py/client-ticking/tests/__init__.py
@@ -0,0 +1,3 @@
+#
+#     Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
+#
diff --git a/py/client-ticking/tests/suite.py b/py/client-ticking/tests/suite.py
@@ -0,0 +1,14 @@
+#
+# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
+#
+
+import unittest
+
+from tests.test_ticking_basic import TickingBasicTestCase
+
+if __name__ == '__main__':
+    suite = unittest.TestSuite()
+    suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TickingBasicTestCase))
+
+    runner = unittest.TextTestRunner(verbosity=2)
+    runner.run(suite)
diff --git a/py/client-ticking/tests/test_ticking_basic.py b/py/client-ticking/tests/test_ticking_basic.py
@@ -0,0 +1,47 @@
+#
+# Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
+#
+
+import unittest
+
+import pydeephaven as dh
+import time
+import sys
+
+class TickingBasicTestCase(unittest.TestCase):
+    def test_ticking_basic_time_table(self):
+        session = dh.Session()
+        half_second_in_nanos = 200 * 1000 * 1000
+        table = session.time_table(period=half_second_in_nanos).update(formulas=["Col1 = i"])
+        session.bind_table(name="my_ticking_table", table=table)
+        table_added_last_col1_seen = -1
+        table_added_update_count = 0
+        def update_table_added(added):
+            nonlocal table_added_update_count
+            nonlocal table_added_last_col1_seen
+            for value in added['Col1'].to_pylist():
+                prev = table_added_last_col1_seen
+                table_added_last_col1_seen = value
+                if prev != -1:
+                    self.assertTrue(prev + 1 == table_added_last_col1_seen)
+            table_added_update_count += 1
+        listener_handle = dh.listen(table, lambda update : update_table_added(update.added('Col1')))
+        listener_handle.start()
+        seen_rows = 0
+        start_seconds = time.time()
+        # Wait until we see a given number of updates or timeout.  Note the callback for the updates
+        # is already checking they are of the right form.
+        col1_target = 10
+        timeout_seconds = 10
+        while True:
+            time.sleep(1)
+            if table_added_last_col1_seen >= col1_target:
+                break
+            now_seconds = time.time()
+            self.assertTrue(now_seconds - start_seconds < timeout_seconds)  # eventually fail
+        self.assertTrue(4 >= table_added_update_count)
+        listener_handle.stop()
+        session.close()
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/settings.gradle b/settings.gradle
@@ -331,6 +331,9 @@ project(':jpy-integration').projectDir = file('py/jpy-integration')
 include(':py-client')
 project(':py-client').projectDir = file('py/client')
 
+include(':py-client-ticking')
+project(':py-client-ticking').projectDir = file('py/client-ticking')
+
 include(':py-server')
 project(':py-server').projectDir = file('py/server')
Original file line number	Diff line number	Diff line change
Expand Up		@@ -48,3 +48,4 @@ def _show_deltas(self, what: str, dict: Dict[str, pa.Array]):
		print("Waking up and stopping the listener")

		listener_handle.stop()
		session.close()