Add in-development version support

.github/workflows/publish-ci.yml

@@ -63,6 +63,7 @@ jobs:
           ORG_GRADLE_PROJECT_signingKey: ${{ secrets.CI_AT_DEEPHAVEN_KEY }}
           ORG_GRADLE_PROJECT_signingPassword: ${{ secrets.CI_AT_DEEPHAVEN_PASSWORD }}
           ORG_GRADLE_PROJECT_signingRequired: true
+          ORG_GRADLE_PROJECT_deephavenBaseQualifier: ""
 
       - name: Upload Artifacts
         if: ${{ startsWith(github.ref, 'refs/heads/release/v') }}

RELEASE.md

@@ -247,18 +247,9 @@ mention the version explicitly. These files are listed below:
 
 ```
 #
-# Edit files for version change, updating from 0.31.0 to 0.32.0
+# Edit files for version change
 #
-authorization-codegen/protoc-gen-contextual-auth-wiring
-authorization-codegen/protoc-gen-service-auth-wiring
-buildSrc/src/main/groovy/io.deephaven.common-conventions.gradle
-py/client-ticking/README.md
-py/client-ticking/setup.py
-py/client/README.md
-py/client/pydeephaven/__init__.py
-py/client/setup.py
-py/embedded-server/deephaven_server/__init__.py
-py/server/deephaven/__init__.py
+gradle.properties
 R/rdeephaven/DESCRIPTION
 ```
 

authorization-codegen/protoc-gen-contextual-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-contextual-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.31.0-all.jar io.deephaven.auth.codegen.GenerateContextualAuthWiring
+java -cp "authorization-codegen/build/libs/deephaven-authorization-codegen-${DEEPHAVEN_VERSION}-all.jar" io.deephaven.auth.codegen.GenerateContextualAuthWiring

authorization-codegen/protoc-gen-service-auth-wiring

@@ -1,2 +1,2 @@
 # protoc-gen-service-auth-wiring
-java -cp authorization-codegen/build/libs/deephaven-authorization-codegen-0.31.0-all.jar io.deephaven.auth.codegen.GenerateServiceAuthWiring
+java -cp "authorization-codegen/build/libs/deephaven-authorization-codegen-${DEEPHAVEN_VERSION}-all.jar" io.deephaven.auth.codegen.GenerateServiceAuthWiring

authorization/README.md

@@ -27,11 +27,12 @@ Here is a sample bash script to generate the provided authorizing wiring if you
 ```bash
 ./gradlew :authorization-codegen:shadowJar
 
+DEEPHAVEN_VERSION="$(./gradlew printVersion -q)"
 OUT_DIR=authorization/src/main/java/
 PROTO_DIR=proto/proto-backplane-grpc/src/main/proto/
 ROOT_DIR=$PROTO_DIR/deephaven/proto
 
-PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION} PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
      $ROOT_DIR/application.proto                                        \
      $ROOT_DIR/console.proto                                            \
      $ROOT_DIR/config.proto                                             \
@@ -41,7 +42,7 @@ PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $P
      $ROOT_DIR/storage.proto                                            \
      $ROOT_DIR/ticket.proto
 
-PATH=authorization-codegen:$PATH protoc --contextual-auth-wiring_out=$OUT_DIR -I $PROTO_DIR \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION} PATH=authorization-codegen:$PATH protoc --contextual-auth-wiring_out=$OUT_DIR -I $PROTO_DIR \
      $ROOT_DIR/table.proto                                              \
      $ROOT_DIR/inputtable.proto                                         \
      $ROOT_DIR/partitionedtable.proto                                   \
@@ -51,6 +52,8 @@ OUT_DIR=authorization/src/main/java/
 PROTO_DIR=../grpc/src/proto/grpc/health/v1/
 ROOT_DIR=$PROTO_DIR
 
-PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
+DEEPHAVEN_VERSION=${DEEPHAVEN_VERSION}  PATH=authorization-codegen:$PATH protoc --service-auth-wiring_out=$OUT_DIR -I $PROTO_DIR    \
      $ROOT_DIR/health.proto
+
+./gradlew :authorization:spotlessApply
 ```

buildSrc/src/main/groovy/io.deephaven.common-conventions.gradle

@@ -5,7 +5,11 @@ plugins {
 }
 
 group = 'io.deephaven'
-version = '0.31.0'
+
+def deephavenBaseVersion = project.property('deephavenBaseVersion').toString().trim()
+def deephavenBaseQualifier = project.property('deephavenBaseQualifier').toString().trim()
+def versionSeparator = deephavenBaseQualifier.isEmpty() ? "" : "-"
+version = "${deephavenBaseVersion}${versionSeparator}${deephavenBaseQualifier}"
 
 if (!name.startsWith('deephaven-')) {
     archivesBaseName = "deephaven-${name}"

gradle.properties

@@ -1,3 +1,17 @@
+# This structure gives DHC and re-builders control over the version. When deephavenBaseQualifier is non-empty, the
+# project version will be "${deephavenBaseVersion}-${deephavenBaseQualifier}". When deephavenBaseQualifier is empty, the
+# project version will be deephavenBaseVersion.
+#
+# The default values will represent the in-development version for the next release: "X.Y.Z-SNAPSHOT".
+#
+# During the normal DHC release process, the publishing code will set -PdeephavenBaseQualifier="": "X.Y.Z".
+#
+# Re-builders who want to inherit the base version, but have their own qualifier can set -PdeephavenBaseQualifier="customQualifier": "X.Y.Z-customQualifier".
+#
+# Re-builders who want a fully custom version can set -PdeephavenBaseVersion="customVersion" -PdeephavenBaseQualifier="": "customVersion".
+deephavenBaseVersion=0.31.0
+deephavenBaseQualifier=SNAPSHOT
+
 #org.gradle.debug
 ## Enable to attach debugger to port 5005 when running gradle
 ## Note, you can use -Dorg.gradle.debug=true as well.

py/client-ticking/README.md

@@ -66,7 +66,7 @@ Then install the package.
 Note the actual name of the `.whl` file may be different depending on system details.
 
 ```
-pip3 install --force --no-deps dist/pydeephaven_ticking-0.31.0-cp310-cp310-linux_x86_64.whl
+pip3 install --force --no-deps dist/pydeephaven_ticking-<x.y.z>-cp310-cp310-linux_x86_64.whl
 ```
 
 The reason for the "--force" flag is to overwrite any previously-built version of the package that

py/client-ticking/build.gradle

@@ -56,15 +56,16 @@ def buildPyClientTicking = Docker.registerDockerTask(project, 'pyClientTicking')
         copyFile('README.md', "${prefix}/src/py-client-ticking")
         copyFile('src/', "${prefix}/src/py-client-ticking/src/")
         copyFile('wheels/', "${prefix}/src/py-client-ticking/in-wheels")
-        runCommand("PREFIX=${prefix}; " +
+        runCommand("PREFIX=${prefix}; DEEPHAVEN_VERSION=${project.version};" +
                    '''set -eux ; \
                      cd "${PREFIX}/src/py-client-ticking"; \
                      . "${PREFIX}/env.sh"; \
                      MAKEFLAGS="-j${NCPUS}" \
                        CFLAGS="-I${DHCPP}/include" \
                        LDFLAGS="-L${DHCPP}/lib" \
+                       DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" \
                        python3 setup.py build_ext -i; \
-                     python3 setup.py bdist_wheel; \
+                     DEEPHAVEN_VERSION="${DEEPHAVEN_VERSION}" python3 setup.py bdist_wheel; \
                      pip3 install in-wheels/*.whl; \
                      pip3 install --force --no-deps dist/*.whl; \
                      ln dist/*.whl /out; \

py/client-ticking/setup.py

@@ -1,21 +1,40 @@
 #
 #  Copyright (c) 2016-2023 Deephaven Data Labs and Patent Pending
 #
+import os
 import pathlib
+
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
 from setuptools import find_packages, setup, Extension
 from Cython.Build import cythonize
 
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README.md").read_text(encoding="utf-8")
+
+def _normalize_version(java_version):
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
+
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
-# The text of the README file
-README = (HERE / "README.md").read_text()
+_version = _compute_version()
 
 setup(
     name='pydeephaven-ticking',
-    version='0.30.0.dev0',
+    version=_version,
     description='The Deephaven Python Client for Ticking Tables',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type="text/markdown",
     packages=find_packages(where="src", exclude=("tests",)),
     package_dir={"": "src"},
@@ -44,5 +63,5 @@
                    libraries=["dhcore_static"]
         )]),
     python_requires='>=3.8',
-    install_requires=['pydeephaven==0.31.0']
+    install_requires=[f"pydeephaven=={_version}"]
 )

py/client/README.md

@@ -38,7 +38,7 @@ $ python3 -m examples.demo_asof_join
 Note the actual name of the `.whl` file may be different depending on system details.
 
 ``` shell
-$ pip3 install dist/pydeephaven-0.31.0-py3-none-any.whl
+$ pip3 install dist/pydeephaven-<x.y.z>-py3-none-any.whl
 ```
 ## Quick start
 

py/client/pydeephaven/__init__.py

@@ -23,6 +23,8 @@
     >>> session.close()
 """
 
+import importlib.metadata
+
 from .session import Session
 from .dherror import DHError
 from ._table_interface import SortDirection
@@ -35,4 +37,8 @@
     pass
 
 __all__ = ["Session", "DHError", "SortDirection"]
-__version__ = "0.31.0"
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('pydeephaven')

py/client/setup.py

@@ -1,20 +1,37 @@
 #
 #  Copyright (c) 2016-2021 Deephaven Data Labs and Patent Pending
 #
+import os
 import pathlib
+
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
 from setuptools import find_packages, setup
 
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README.md").read_text(encoding="utf-8")
+
+def _normalize_version(java_version):
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
 
-# The text of the README file
-README = (HERE / "README.md").read_text()
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
 setup(
     name='pydeephaven',
-    version='0.31.0',
+    version=_compute_version(),
     description='The Deephaven Python Client',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type="text/markdown",
     packages=find_packages(exclude=("tests",)),
     url='https://deephaven.io/',

py/embedded-server/deephaven_server/__init__.py

@@ -1,10 +1,16 @@
 #
 # Copyright (c) 2016-2022 Deephaven Data Labs and Patent Pending
 #
-__version__ = "0.31.0"
+
+import importlib.metadata
 
 from .start_jvm import DEFAULT_JVM_PROPERTIES, DEFAULT_JVM_ARGS, start_jvm
 from .server import Server
 
 from deephaven_internal.jvm import check_py_env
 check_py_env()
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('deephaven-server')

py/embedded-server/setup.py

@@ -3,33 +3,37 @@
 #
 import os
 import pathlib
-from setuptools.extern import packaging
-from setuptools import find_namespace_packages, setup
-
-# The directory containing this file
-HERE = pathlib.Path(__file__).parent
 
-# The text of the README file
-README = (HERE / "README_PyPi.md").read_text()
+# Note: pkg_resources is deprecated https://setuptools.pypa.io/en/latest/pkg_resources.html, and it is suggested
+# to use an external library `packaging`. From the context of building a wheel though, we'd prefer to not have to
+# install extra dependencies, at least until we can more properly manage the build environment (pyproject.toml).
+# TODO(deephaven-core#2233): upgrade setup.py to pyproject.toml
+from pkg_resources import parse_version
+from setuptools import find_namespace_packages, setup
 
+def _get_readme() -> str:
+    # The directory containing this file
+    HERE = pathlib.Path(__file__).parent
+    # The text of the README file
+    return (HERE / "README_PyPi.md").read_text(encoding="utf-8")
 
-# Versions should comply with PEP440.  For a discussion on single-sourcing
-# the version across setup.py and the project code, see
-# https://packaging.python.org/en/latest/single_source_version.html
-# todo: does DH versions align w/ PEP440?
-# see https://github.com/pypa/setuptools/blob/v40.8.0/setuptools/dist.py#L470
-def normalize_version(version):
-    return str(packaging.version.Version(version))
+def _normalize_version(java_version) -> str:
+    partitions = java_version.partition("-")
+    regular_version = partitions[0]
+    local_segment = partitions[2]
+    python_version = f"{regular_version}+{local_segment}" if local_segment else regular_version
+    return str(parse_version(python_version))
 
+def _compute_version():
+    return _normalize_version(os.environ['DEEPHAVEN_VERSION'])
 
-__deephaven_version__ = os.environ['DEEPHAVEN_VERSION']
-__normalized_version__ = normalize_version(__deephaven_version__)
+_version = _compute_version()
 
 setup(
     name='deephaven-server',
-    version=__normalized_version__,
+    version=_version,
     description='Deephaven Embedded Server Python Package',
-    long_description=README,
+    long_description=_get_readme(),
     long_description_content_type='text/markdown',
     packages=find_namespace_packages(exclude=("tests")),
     package_data={'deephaven_server': ['jars/*']},
@@ -56,6 +60,6 @@ def normalize_version(version):
     install_requires=[
         'jpy>=0.14.0',
         "java-utilities",
-        f"deephaven-core[autocomplete]=={__normalized_version__}",
+        f"deephaven-core[autocomplete]=={_version}",
     ]
 )

py/server/deephaven/__init__.py

@@ -7,7 +7,7 @@
 
 """
 
-__version__ = "0.31.0"
+import importlib.metadata
 
 from deephaven_internal import jvm
 
@@ -31,3 +31,8 @@
 __all__ = ["read_csv", "write_csv", "kafka_consumer", "kafka_producer", "empty_table", "time_table", "merge",
            "merge_sorted", "new_table", "input_table", "ring_table", "function_generated_table", "DynamicTableWriter",
            "TableReplayer", "garbage_collect", "read_sql", "DHError", "SortDirection"]
+
+# Note: this is the _distribution_ name, not the _package_ name. Until 3.10, there is not an easy way to get the
+# distribution name from the package name.
+# https://docs.python.org/3/library/importlib.metadata.html#package-distributions
+__version__ = importlib.metadata.version('deephaven-core')