Toward getting this working...

Function unicode change so as not to conflict with RightTeeArrow.
Function unicode is a long arrow.

Remove CSV to YML stuff. CSV is beyond hope of keeping in sync.

Remove tokeniser import from __init__.py. Workaround for now.
We need this so we can create operator JSON without needing the
JSON table to be prevously around.

clanup tokeniser.py a little bit.

SPlit typing changing variables like tokens and literal_tokens into
separate variables for each type they can hold.

admin-tools/make-tables.sh

@@ -7,5 +7,4 @@ PYTHON=${PYTHON:-python}
 
 cd $mydir/../mathics_scanner/data
 $PYTHON ../generate/build_tables.py -o characters.json
-$PYTHON ../generate/operator_csv_to_yml.py
 $PYTHON ../generate/build_operator_tables.py -o operators.json

mathics_scanner/__init__.py

@@ -26,7 +26,7 @@
 )
 
 # TODO: Move is_symbol_name to the characters module
-from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name
+# from mathics_scanner.tokeniser import Token, Tokeniser, is_symbol_name
 from mathics_scanner.version import __version__
 
 __all__ = [
@@ -37,12 +37,12 @@
     "MultiLineFeeder",
     "ScanError",
     "SingleLineFeeder",
-    "Token",
-    "Tokeniser",
+    # "Token",
+    # "Tokeniser",
     "TranslateError",
     "__version__",
     "aliased_characters",
-    "is_symbol_name",
+    # "is_symbol_name",
     "named_characters",
     "replace_unicode_with_wl",
     "replace_wl_with_plain_text",

mathics_scanner/data/named-characters.yml

@@ -4933,21 +4933,20 @@ FreakedSmiley:
 # the YAML name doesn't have to match since there is just ASCII for that.
 # Since we use the name Function as the key here, the YAML name for "&" has
 # to be different, since YAML keys need to be unique.
-
+# Note \[Function] is not the same as \[RightTeeArrow] the
+# unicode for each has to be different. Here we add
+# "LONG".
 Function:
   amslatex: "\\mathsto"
   ascii: "|->"
   has-unicode-inverse: false
   is-letter-like: false
   operator-name: Function
-  # This needs to be greater than Alternative "|" which is
-  # 160 so that we don't treat |-> as | ->.
-  unicode-equivalent: "\u21A6"
-  unicode-equivalent-name: RIGHTWARDS ARROW FROM BAR
+  unicode-equivalent: "\u27FC"
+  unicode-equivalent-name: LONG RIGHTWARDS ARROW FROM BAR
   unicode-reference: https://www.compart.com/en/unicode/U+21A6
   wl-reference: https://reference.wolfram.com/language/ref/character/Function.html
   wl-unicode: "\uF4A1"
-  wl-unicode-name: RIGHTWARDS ARROW FROM BAR
 
 # Alternative form of Function. We use FunctionAmpersand
 # since Function is already taken. See note above.