Merge pull request #133 from MTG/python3-migration

Python3 migration & Emscripten update

.github/workflows/build-dev.yml

@@ -0,0 +1,43 @@
+name: build-dev
+
+on: 
+  push:
+    branches:
+      - dev
+      - master
+    paths:
+      - 'src/python/requirements.txt'
+      - 'build-libs.sh'
+      - 'Dockerfile'
+      - 'Makefile'
+  pull_request:
+    branches:
+      - dev
+      - master
+    paths:
+      - 'src/python/requirements.txt'
+      - 'build-libs.sh'
+      - 'Dockerfile'
+      - 'Makefile'
+    types:
+      - opened
+      - edited
+      - synchronize
+      - reopened
+
+jobs: 
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout close
+        uses: actions/checkout@v2
+      - name: Build new Docker image
+        working-directory: ${{ github.workspace }}
+        run: docker build -t mtgupf/essentia-emscripten:dev .
+      - name: Build library
+        working-directory: ${{ github.workspace }}
+        run: |
+          docker run --rm -v `pwd`:/essentia/ mtgupf/essentia-emscripten:dev /essentia/build-libs.sh
+        shell: bash
+
+  # unit-test:

.github/workflows/deploy.yml

@@ -10,7 +10,7 @@ on:
       # - 'test/**'
       - 'build-libs.sh'
       - 'Dockerfile'
-      - 'Makefile.essentiajs'
+      - 'Makefile'
       - 'rollup*.js'
       - 'tsconfig.json'
       - '!docs/**'

.github/workflows/integration.yml

@@ -10,7 +10,7 @@ on:
       # - 'test/**'
       - 'build-libs.sh'
       - 'Dockerfile'
-      - 'Makefile.essentiajs'
+      - 'Makefile'
       - 'rollup*.js'
       - 'tsconfig.json'
       - '!docs/**'
@@ -24,7 +24,7 @@ on:
       # - 'test/**'
       - 'build-libs.sh'
       - 'Dockerfile'
-      - 'Makefile.essentiajs'
+      - 'Makefile'
       - 'rollup*.js'
       - 'tsconfig.json'
       - '!docs/**'

Dockerfile

@@ -1,26 +1,26 @@
-ARG EMSCRIPTEN_VERSION=1.39.19
+ARG EMSCRIPTEN_VERSION=3.1.24
 FROM emscripten/emsdk:${EMSCRIPTEN_VERSION}
 
 ENV LANG C.UTF-8
 
 # compile essentia with emscripten and selected third-party dependencies
 RUN apt-get update \
-    && apt-get install -y cmake curl nano python-dev python-numpy-dev libpython2.7 python-pip libeigen3-dev \
+    && apt-get install -y cmake curl nano python3-dev python3-numpy-dev python3-numpy python3-yaml python3-six libeigen3-dev python3-pip \
     && mkdir /essentia && cd /essentia && git clone https://github.com/MTG/essentia.git \
     && cd /essentia/essentia/packaging/debian_3rdparty \
     && bash -C "./build_eigen3.sh" && cd ../../  && chmod +x waf \
-    && emconfigure sh -c './waf configure --prefix=$EMSCRIPTEN/system/local/ --build-static --fft=KISS --emscripten --static-dependencies' \
-    && emmake ./waf && emmake ./waf install \
-    &&  apt-get remove -y python-dev libeigen3-dev \
+    && emconfigure sh -c 'python3 waf configure --prefix=$EMSCRIPTEN/system/local/ --build-static --fft=KISS --emscripten --static-dependencies' \
+    && emmake python3 waf && emmake python3 waf install \
+    &&  apt-get remove -y python3-dev libeigen3-dev \
     && apt-get autoremove -y \
     && apt-get clean -y \
     && rm -rf /var/lib/apt/lists/* \
     && cd / && rm -rf /essentia/essentia
 
 # copy and install python dependencies
 COPY src/python/requirements.txt /tmp/requirements.txt
-RUN pip install --upgrade setuptools \
-    && pip install --no-cache-dir -r /tmp/requirements.txt \
+RUN pip3 install --upgrade setuptools \
+    && pip3 install --no-cache-dir -r /tmp/requirements.txt \
     && rm /tmp/requirements.txt
 
 # add latest eigen3 header files for linking the essentia.js binding builds
@@ -30,6 +30,6 @@ RUN wget  -P /usr/local/include/ "https://gitlab.com/libeigen/eigen/-/archive/3.
     && rm -f eigen-${EIGEN_VERSION}.tar.gz && mv eigen-${EIGEN_VERSION} eigen3
 
 ENV EIGEN_PATH /usr/local/include/eigen3
-ENV EMSCRIPTEN /emsdk/upstream/emscripten
+ENV EMSCRIPTEN /emsdk/upstream/emscripten 
 
 WORKDIR /essentia/

Makefile.essentiajs → Makefile

@@ -1,6 +1,7 @@
 ESSENTIAJS_VERSION=0.1.3
 ## Path to libs for Emscripten
 LIB_DIR_ESSENTIA=$(EMSCRIPTEN)/system/local/lib
+INCLUDE_DIR_ESSENTIA=$(EMSCRIPTEN)/system/local/include
 EIGEN_PATH=/usr/local/include/eigen3
 ## Pass custom build and dist directories using system environment variables
 BUILD_DIR_ES := $(or $(ESSENTIAJS_WASM_BUILDS_DIR),builds)
@@ -22,25 +23,18 @@ POST_JS_UMD_WASM=src/js/wasm.umd.post.js
 
 codegen:
 	@echo "Generating cpp source code from essentia python bindings ..."
-	@cd src/python && python configure_bindings.py
+	@cd src/python && python3 configure_bindings.py
 
 build:
 	@mkdir -p $(BUILD_DIR_ES)
 
-	@echo "Compiling the emscripten embind cpp bindings to bitcode ..."
+	@echo "Compiling + linking emscripten embind cpp bindings directly to js, wasm files ..."
 
-	@emcc -I $(EIGEN_PATH) \
-	   --bind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \
-	   -o $(BUILD_DIR_ES)/essentiajs.bc \
-	   -s EXCEPTION_DEBUG \
-	   -s ASSERTIONS=2 \
-	   -s DISABLE_EXCEPTION_CATCHING=2 || exit 1
-	@echo "Done ..."
-
-	@echo "Linking and compiling the bindings with essentia to js, wasm files ..."
-	@echo "compiling async builds..."
-	@emcc --bind -Oz $(BUILD_DIR_ES)/essentiajs.bc ${LIB_DIR_ESSENTIA}/essentia.a \
-	   -s WASM=1 \
+	@echo "... compiling async builds..."
+	@emcc -lembind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \
+	   -lembind -lessentia -L $(LIB_DIR_ESSENTIA) \
+		 -I $(EIGEN_PATH) -I $(INCLUDE_DIR_ESSENTIA)\
+		 -s WASM=1 \
 	   -o $(ESSENTIA_JS_WEB) \
 	   -s EXCEPTION_DEBUG \
 	   -s ASSERTIONS=2 \
@@ -51,8 +45,10 @@ build:
 	   -s ALLOW_MEMORY_GROWTH=1 || exit 1
 	@echo "Done ..."
 
-	@echo "compiling sync builds..."
-	@emcc --bind -Oz $(BUILD_DIR_ES)/essentiajs.bc ${LIB_DIR_ESSENTIA}/essentia.a \
+	@echo "... compiling sync builds..."
+	@emcc --bind -Oz $(BINDING_ESSENTIAJS) $(INCLUDE_ESSENTIAJS) \
+		 -lembind -lessentia -L $(LIB_DIR_ESSENTIA) \
+		 -I $(EIGEN_PATH) -I $(INCLUDE_DIR_ESSENTIA)\
 	   -s WASM=1 \
 	   -o $(ESSENTIA_JS_MODULE) \
 	   -s BINARYEN_ASYNC_COMPILATION=0 \
@@ -78,9 +74,6 @@ build:
 
 	@echo "Done ..."
 
-	@echo "Removing unnecessary files ..."
-	@rm $(BUILD_DIR_ES)/essentiajs.bc
-
 	@echo "Builds ..."
 	@ls $(BUILD_DIR_ES)
 

build-libs.sh

@@ -4,26 +4,26 @@ set -e -x
 # Custom path to the node installation in the essentia-emscripten docker image
 # provided along with emsdk. This can be removed once the 
 # entrypoint issues in the essentia-emscripten docker image got resolved.
-NPM_PATH=/emsdk/node/12.18.1_64bit/bin/npm
-NODE_PATH=/emsdk/node/12.18.1_64bit/bin/node
+# NPM_PATH=/emsdk/node/12.18.1_64bit/bin/npm
+# NODE_PATH=/emsdk/node/12.18.1_64bit/bin/node
 # set permission
-ln -sf $NODE_PATH /usr/bin/node
+# ln -sf $NODE_PATH /usr/bin/node
 
 # Generate Essentia.js source code from Essentia docs
-$NPM_PATH run gen-code
+npm run gen-code
 
 # Build Essentia WASM backend
-$NPM_PATH run build-wasm
+npm run build-wasm
 
 # Install essentia.js node dependecies
-$NPM_PATH install
+npm install
 
 # Build essentia.js JS API and add-on modules
-$NPM_PATH run build-js-api
+npm run build-js-api
 
 # Minified build of JS API and add-on modules
-$NPM_PATH run build-js-api rollup.config.min.js
+npm run build-js-api rollup.config.min.js
 
 # Run tests
 echo "Running tests ..."
-$NPM_PATH test
+npm test

docs/tutorials/4. Building from source.md

@@ -63,7 +63,7 @@ You need to have a local installation of docker.
     docker run --rm -v `pwd`:/srv/workspace/ \
                     mtgupf/essentia-emscripten:latest \
                     /srv/workspace/build-bindings.sh \
-                    Makefile.essentiajs
+                    Makefile
     ```
 
 OR 
@@ -101,7 +101,7 @@ OR
   Spawn a subshell inside the emscripten `emconfigure` in order to properlly access the emscripten variables.
 
   ```bash
-  emconfigure sh -c './build-bindings.sh Makefile.essentiajs'
+  emconfigure sh -c './build-bindings.sh Makefile'
   ```
 
   > Note: make you added the emscripten env variables to your bash profile.

package.json

@@ -15,8 +15,8 @@
     "CHANGELOG.md"
   ],
   "scripts": {
-    "gen-code": "make -f Makefile.essentiajs codegen",
-    "build-wasm": "make -f Makefile.essentiajs build",
+    "gen-code": "make -f Makefile codegen",
+    "build-wasm": "make -f Makefile build",
     "build-js-api": "rollup --config",
     "build-api-docs": "./build-docs.sh",
     "test": "mocha"

src/python/bindings.cog

@@ -35,7 +35,7 @@ EMSCRIPTEN_BINDINGS(CLASS_EssentiaJS) {
     /*[[[cog
     import logging
     import cog
-    from .code_generator import TO_INCLUDE_ALGOS
+    from code_generator import TO_INCLUDE_ALGOS
     logging.basicConfig(level='INFO')
     logging.info("Generating emscripten bindings for the essentia...")
     for algo_name in TO_INCLUDE_ALGOS:

src/python/configure_bindings.py

@@ -1,4 +1,6 @@
 # -*- coding: utf-8 -*-
+from __future__ import print_function
+from builtins import str
 import os
 import argparse
 import essentia.standard as estd
@@ -30,6 +32,7 @@ def savelist_to_file(path_list, filename):
                         # requires Gaia and Tensorflow dependencies
                         'GaiaTransform', 'MusicExtractorSVM', 
                         'TensorflowPredict', 'TensorflowPredictMusiCNN', 'TensorflowPredictVGGish',
+                        'TensorNormalize', 'TensorTranspose',
                         # these algortihms expect a matrix_real input or output types which are not yet supported for the JS bindings 
                         'BpmHistogram', 'FadeDetection', 'HumDetector', 'Onsets', 'Panning', 'SBic', 'SingleGaussian',
                         # these algorithms expect std::complex** type for wither input, parameters or outputs, which are not yet supported for the JS bindings 

src/python/excluded_algos.md

@@ -29,6 +29,8 @@ MusicExtractorSVM
 TensorflowPredict
 TensorflowPredictMusiCNN
 TensorflowPredictVGGish
+TensorNormalize
+TensorTranspose
 BpmHistogram
 FadeDetection
 HumDetector

src/python/header.cog

@@ -42,7 +42,7 @@ class EssentiaJS {
     // NOTE: The following code snippets are machine generated. Do not edit.    
     /*[[[cog
     import cog
-    from .code_generator import generate_headers, TO_INCLUDE_ALGOS
+    from code_generator import generate_headers, TO_INCLUDE_ALGOS
     cog.outl(" ")
     cog.outl("// class property which stores all the list of essentia algorithm names available in essentia.js")
     c_algo_list = str(TO_INCLUDE_ALGOS)

src/python/included_algos.md

@@ -179,6 +179,7 @@ TempoTapDegara
 TempoTapMaxAgreement
 TempoTapTicks
 TensorflowInputMusiCNN
+TensorflowInputTempoCNN
 TensorflowInputVGGish
 TonalExtractor
 TonicIndianArtMusic

src/python/js_wrapper.cog

@@ -198,7 +198,7 @@ class Essentia {
   // NOTE: The following code snippets are machine generated. Do not edit.
   /*[[[cog
   import cog
-  from .code_generator import generate_typescript_wrapper
+  from code_generator import generate_typescript_wrapper
   algos = generate_typescript_wrapper()
   for algo in algos:
     for ln in algo:

src/python/library.cog

@@ -161,7 +161,7 @@ val EssentiaJS::LoudnessEBUR128(std::vector<float>& left_channel, std::vector<fl
 // NOTE: The following code snippets are machine generated. Do not edit.
 /*[[[cog
 import cog
-from .code_generator import generate_algorithms
+from code_generator import generate_algorithms
 algos = generate_algorithms()
 for algo in algos:
   for ln in algo: