tokenizer gets no-meaning infix ops from JSON

.github/workflows/mathics.yml

@@ -31,7 +31,9 @@ jobs:
     - name: Test Mathics3
       run: |
         # Until next Mathics3/mathics-core release is out...
-        git clone https://github.com/Mathics3/mathics-core.git
+        # git clone https://github.com/Mathics3/mathics-core.git
+        # Until next operator-info-from-JSON is merges
+        git clone -b operator-info-from-JSON https://github.com/Mathics3/mathics-core.git
         cd mathics-core/
         make PIP_INSTALL_OPTS='[full]'
         # pip install Mathics3[full]

admin-tools/make-tables.sh

@@ -7,5 +7,4 @@ PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics_scanner/data
 $PYTHON ../generate/build_tables.py -o characters.json
-$PYTHON ../generate/operator_csv_to_yml.py
 $PYTHON ../generate/build_operator_tables.py -o operators.json

mathics_scanner/__init__.py

@@ -26,7 +26,7 @@
 )
 
 # TODO: Move is_symbol_name to the characters module
-from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name
+# from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name
 from mathics_scanner.version import __version__
 
 __all__ = [
@@ -37,12 +37,12 @@
     "MultiLineFeeder",
     "ScanError",
     "SingleLineFeeder",
-    "Token",
-    "Tokeniser",
+    # "Token",
+    # "Tokeniser",
     "TranslateError",
     "__version__",
     "aliased_characters",
-    "is_symbol_name",
+    # "is_symbol_name",
     "named_characters",
     "replace_unicode_with_wl",
     "replace_wl_with_plain_text",

mathics_scanner/data/named-characters.yml

@@ -4933,21 +4933,20 @@ FreakedSmiley:
 # the YAML name doesn't have to match since there is just ASCII for that.
 # Since we use the name Function as the key here, the YAML name for "&" has
 # to be different, since YAML keys need to be unique.
-
+# Note \[Function] is not the same as \[RightTeeArrow] the
+# unicode for each has to be different. Here we add
+# "LONG".
 Function:
   amslatex: "\\mathsto"
   ascii: "|->"
   has-unicode-inverse: false
   is-letter-like: false
   operator-name: Function
-  # This needs to be greater than Alternative "|" which is
-  # 160 so that we don't treat |-> as | ->.
-  unicode-equivalent: "\u21A6"
-  unicode-equivalent-name: RIGHTWARDS ARROW FROM BAR
+  unicode-equivalent: "\u27FC"
+  unicode-equivalent-name: LONG RIGHTWARDS ARROW FROM BAR
   unicode-reference: https://www.compart.com/en/unicode/U+21A6
   wl-reference: https://reference.wolfram.com/language/ref/character/Function.html
   wl-unicode: "\uF4A1"
-  wl-unicode-name: RIGHTWARDS ARROW FROM BAR
 
 # Alternative form of Function. We use FunctionAmpersand
 # since Function is already taken. See note above.