Remove builtin Joern, depend on PyJoern (#2)

* Remove builtin Joern, depend on PyJoern

* Update container for new pyjoern

* update setup

* Fix setup again

* update java version

* fix counting bug

.github/workflows/release.yml

@@ -0,0 +1,40 @@
+name: Release
+
+on:
+  push:
+    tags:
+    - "v**"
+
+jobs:
+
+  release-github:
+    name: Create Github Release
+    permissions: write-all
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          generateReleaseNotes: true
+
+  release-pypi:
+    name: Release pypi package
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout source
+      uses: actions/checkout@v2
+    - name: Setup Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
+    - name: Install build
+      run: pip install build
+    - name: Build dists
+      run: python -m build
+    - name: Release to PyPI
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

.gitignore

@@ -6,7 +6,6 @@ testing/
 /final_results/*
 tests/cfged/angr_sailr*
 tests/cfged/*.dwarf.linemaps
-sailreval/joern/bin/joern-cli/*
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Dockerfile

@@ -27,8 +27,8 @@ RUN apt-get update && apt-get -o APT::Immediate-Configure=0 install -y \
 COPY ./sailreval /SAILR/sailreval
 COPY ./*.py /SAILR/
 COPY ./setup.cfg /SAILR/
-# also installs joern
-RUN pip3 install -e ./SAILR
+# also inits the pyjoern project to avoid later download in init
+RUN pip3 install -e ./SAILR && pyjoern --install
 
 #
 #

sailreval/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.0"
+__version__ = "1.0.0"
 
 # create loggers
 import logging

sailreval/analysis/counting.py

@@ -1,6 +1,7 @@
 import itertools
 import os
 import sys
+from copy import deepcopy
 from pathlib import Path
 from collections import defaultdict
 from typing import List, Dict, Tuple, Set
@@ -12,7 +13,7 @@
 from sailreval.utils import load_tomls_by_bin_name, bcolors
 from sailreval.utils.sailr_target import SAILRTarget
 from sailreval.utils.compile import DEFAULT_OPTIMIZATION_LEVELS, OPTIMIZATION_LEVELS
-from sailreval.joern import JoernServer, JoernClient
+from pyjoern import JoernServer, JoernClient
 
 from tqdm import tqdm
 import toml
@@ -529,7 +530,7 @@ def summarize_sailr_targets(
             toml_dirs, decompilers, metrics, unique_funcs=unique_funcs, min_func_size=min_func_size,
             max_func_size=max_func_size,
         )
-        full_summary_doc += save_data_as_markdown_table(_summary, show_stats=show_stats) 
+        full_summary_doc += save_data_as_markdown_table(deepcopy(_summary), show_stats=show_stats)
         #pkg_name = list(_summary.keys())[0]
         for name_with_opt in _summary:
             for opt_level in opt_levels:

sailreval/analysis/measure.py

@@ -10,10 +10,10 @@
 from typing import List
 
 import toml
+from pyjoern import JoernClient, JoernServer, fast_cfgs_from_source
+from pyjoern.mapping import cfg_root_node, correct_source_cfg_addrs
 
 from sailreval import ALL_DECOMPILERS, ALL_METRICS, SAILR_DECOMPILERS, SAILR_METRICS, JOERNLESS_SERVER_METRICS
-from sailreval.joern import JoernClient, JoernServer
-from sailreval.joern.cfg.utils import cfgs_from_source, correct_source_cfg_addrs
 from sailreval.metrics import get_metric_function, POST_METRICS
 from sailreval.metrics.ged_to_source import has_cfged_required_files, has_cfged_required_src_files
 from sailreval.utils import bcolors, SAILR_DECOMPILATION_RESULTS_DIR, timeout, SAILR_MEASURE_RESULTS_DIR, WorkDirContext
@@ -192,7 +192,7 @@ def measure_files(file_dir: Path, basename: str, decompilers=None, metrics=None,
                 if require_cfgs:
                     if has_cfged_required_src_files(tfile, target_binary):
                         linemaps_path = tfile.with_suffix(".linemaps")
-                        extracted_cfgs = cfgs_from_source(tfile.absolute())
+                        extracted_cfgs = fast_cfgs_from_source(tfile.absolute())
                         if extracted_cfgs:
                             source_cfgs = correct_source_cfg_addrs(
                                 extracted_cfgs,
@@ -227,7 +227,7 @@ def measure_files(file_dir: Path, basename: str, decompilers=None, metrics=None,
             # extract cfgs if needed
             if require_cfgs and has_cfged_required_files(target_file.absolute()):
                 try:
-                    dec_cfgs = cfgs_from_source(target_file.absolute()) if dec_name != "source" else source_cfgs
+                    dec_cfgs = fast_cfgs_from_source(target_file.absolute()) if dec_name != "source" else source_cfgs
                 except Exception:
                     dec_cfgs = {}