Add operator tables (#67)

* Administrivia and bump version
* First cut at creating an operator table
* Comment out fields that don't YAML parse easily (yet)
* Add program to convert operator CSV to YAML
* Spelling corrections
* Add program to create operator table from YAML
* Need to use special branch to test... And we'll have to remove this later when it gets merged to master. Sigh.
* CI woes
* Another attempt at packaging

.github/workflows/mathics.yml

@@ -11,11 +11,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9]
+        python-version: ['3.11']
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install OS dependencies
@@ -31,7 +31,10 @@ jobs:
     - name: Test Mathics3
       run: |
         # Until next Mathics3/mathics-core release is out...
-        python -m pip install -e git+https://github.com/Mathics3/mathics-core#egg=Mathics3[full]
-        (cd src/mathics3 && ./admin-tools/make-op-tables.sh )
+        git clone https://github.com/Mathics3/mathics-core.git
+        cd mathics-core/
+        git checkout table-driven-operator-precedence
+        make PIP_INSTALL_OPTS='[full]'
         # pip install Mathics3[full]
+        cd ..
         MATHICS_CHARACTER_ENCODING="ASCII" make check-mathics

.gitignore

@@ -18,12 +18,15 @@
 /.python-version
 /Mathics_Scanner.egg-info
 ChangeLog
+ChangeLog.orig
+ChangeLog.rej
 Documents/
 Homepage/
 Test/
 _Copies_/
 _Database_/
 build/
 /mathics_scanner/data/characters.json
+/mathics_scanner/data/operators.json
 dist/
 tmp

Makefile

@@ -8,6 +8,7 @@ GIT2CL ?= admin-tools/git2cl
 PYTHON ?= python
 PIP ?= pip3
 RM  ?= rm
+PIP_INSTALL_OPTS ?=
 
 .PHONY: all build \
    check check-full check-mathics clean \
@@ -25,19 +26,23 @@ mathics_scanner/data/characters.json: mathics_scanner/data/named-characters.yml
 	$(PIP) install -r requirements-dev.txt
 	$(PYTHON) mathics_scanner/generate/build_tables.py
 
+mathics_scanner/data/operators.json: mathics_scanner/data/operators.yml
+	$(PIP) install -r requirements-dev.txt
+	$(PYTHON) mathics_scanner/generate/build_operator_tables.py
+
 #: build everything needed to install
 build: mathics_scanner/data/characters.json
 	$(PYTHON) ./setup.py build
 
 #: Set up to run from the source tree
-develop: mathics_scanner/data/characters.json
-	$(PIP) install -e .
+develop: mathics_scanner/data/characters.json mathics_scanner/data/operators.json
+	$(PIP) install -e .$(PIP_INSTALL_OPTS)
 
 #: Build distribution
 dist: admin-tools/make-dist.sh
 	$(SHELL) admin-tools/make-dist.sh
 
-#: Install mathics
+#: Install mathics-scanner
 install: build
 	$(PYTHON) setup.py install
 
@@ -57,7 +62,7 @@ doc:  mathics_scanner/data/characters.json
 #: Remove derived files
 clean:
 	@find . -name *.pyc -type f -delete; \
-	$(RM) -f mathics_scanner/data/characters.json || true
+	$(RM) -f mathics_scanner/data/characters.json mathics_scanner/data/operators.json || true
 
 #: Run py.test tests. Use environment variable "o" for pytest options
 pytest: mathics_scanner/data/characters.json

README.rst

@@ -48,13 +48,14 @@ To regenerate JSON-format tables run:
 
 ::
 
-   $ mathics-generate-json-table
+   $ mathics3-generate-json-table
+   $ mathics3-operator-json-table
 
-Without options ``mathics-generate-json-table`` produces the maximum set of correspondences.
+Without options ``mathics3-generate-json-table`` produces the maximum set of correspondences.
 
 In most applications though you may need just a few of these. The
 ``--field`` option can be used to narrow the list of entries to output in JSON. Run
-``mathics-generate-json-table --help`` for a full list of fields.
+``mathics3-generate-json-table --help`` for a full list of fields.
 
 
 Implementation

admin-tools/check-versions.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 function finish {
-  cd $owd
+  cd $mathics_scanner_owd
 }
 
 # FIXME put some of the below in a common routine
-owd=$(pwd)
+mathics_scanner_owd=$(pwd)
 trap finish EXIT
 
 cd $(dirname ${BASH_SOURCE[0]})
@@ -24,3 +24,4 @@ for version in $PYVERSIONS; do
     fi
     echo === $version ===
 done
+finish

admin-tools/make-dist.sh

@@ -3,11 +3,11 @@ PACKAGE=mathics-scanner
 
 # FIXME put some of the below in a common routine
 function finish {
-  cd $owd
+  cd $mathics_scanner_owd
 }
 
 cd $(dirname ${BASH_SOURCE[0]})
-owd=$(pwd)
+mathics_scanner_owd=$(pwd)
 trap finish EXIT
 
 if ! source ./pyenv-versions ; then
@@ -25,9 +25,9 @@ for pyversion in $PYVERSIONS; do
     # pip bdist_egg create too-general wheels. So
     # we narrow that by moving the generated wheel.
 
-    # Pick out first two number of version, e.g. 3.7.9 -> 37
     rm -fr build
     python setup.py bdist_egg
 done
 python setup.py bdist_wheel
 python ./setup.py sdist
+finish

admin-tools/make-tables.sh

@@ -7,3 +7,4 @@ PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics_scanner/data
 $PYTHON ../generate/build_tables.py -o characters.json
+$PYTHON ../generate/build_operator_tables.py -o operators.json