Get box operator for scanner from JSON

Mathics3 · Dec 8, 2024 · 8d4aa62 · 8d4aa62
1 parent 7f8d8ec
commit 8d4aa62
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 43 deletions.
diff --git a/mathics_scanner/data/operators.yml b/mathics_scanner/data/operators.yml
@@ -6062,14 +6062,13 @@ SubscriptBox:
   WolframLanguageData-corrected: 8
   UnicodeCharacters.tr:
   UnicodeCharacters-corrected.tr: 775
-  # N-tokens: {}
-  # L-tokens: {"\_"}
-  # O-tokens: {}
-  # usage: "expr1 \_ expr2"
+  operator: "\\_"
+  usage: "\\(x\\\_y\\)"
   FullForm:
   arity: Binary
   affix: Infix
   associativity: "unknown"
+  box-operator: true
   meaningful: true
   # comments:
 
@@ -6313,6 +6312,22 @@ Superset:
   meaningful: false
   # comments:
 
+SuperscriptBox:
+  precedence: 590
+  WolframLanguageData: 21
+  WolframLanguageData-corrected: 21
+  UnicodeCharacters.tr:
+  UnicodeCharacters-corrected.tr: 660
+  operator: "\\^"
+  usage: "\\(x\\^_y\\)"
+  FullForm:
+  arity: Binary
+  affix: Infix
+  associativity: "unknown"
+  box-operator: true
+  meaningful: true
+  # comments:
+
 SupersetEqual:
   precedence: 250
   WolframLanguageData:
@@ -6330,23 +6345,6 @@ SupersetEqual:
   meaningful: false
   # comments:
 
-SupercriptBox:
-  Precedence-Function: 690
-  precedence: 590
-  WolframLanguageData: 21
-  WolframLanguageData-corrected: 21
-  UnicodeCharacters.tr:
-  UnicodeCharacters-corrected.tr: 660
-  operator: "\\^"
-  usage: "\\(x\\^y\\)"
-  FullForm:
-  arity: Binary
-  affix: Infix
-  associativity: right
-  box-operator: true
-  meaningful: true
-  # comments:
-
 TagSet:
   Precedence-Function: 670
   precedence: 40

diff --git a/mathics_scanner/tokeniser.py b/mathics_scanner/tokeniser.py
@@ -109,18 +109,6 @@ def init_module():
         #
         ("LeftRowBox", r" \\\( "),
         ("RightRowBox", r" \\\) "),
-        # Box Operators which are valid only inside Box delimiters
-        ("InterpretedBox", r" \\\! "),
-        ("SuperscriptBox", r" \\\^ "),
-        ("SubscriptBox", r" \\\_ "),
-        ("OverscriptBox", r" \\\& "),
-        ("UnderscriptBox", r" \\\+ "),
-        ("OtherscriptBox", r" \\\% "),
-        ("FractionBox", r" \\\/ "),
-        ("SqrtBox", r" \\\@ "),
-        ("RadicalBox", r" \\\@ "),
-        ("FormBox", r" \\\` "),
-        #
         # End Box Operators
         #
         ("Information", r"\?\?"),
@@ -222,11 +210,11 @@ def init_module():
         ("VerticalSeparator", r" \uF432 "),
     ]
 
-    for table in ("no-meaning-infix-operators",):
+    for table in ("box-operators", "no-meaning-infix-operators"):
         table_info = OPERATOR_DATA[table]
         for operator_name, unicode in table_info.items():
-            # if any([tup[0] == operator_name for tup in tokens]):
-            #     print(f"Please remove {operator_name}")
+            if any([tup[0] == operator_name for tup in tokens]):
+                print(f"Please remove {operator_name}")
             tokens.append((operator_name, f" {unicode} "))
 
     literal_tokens = {
@@ -273,17 +261,17 @@ def init_module():
         "\\": [
             "LeftRowBox",
             "RightRowBox",
+            "FormBox",
+            "FractionBox",
             "InterpretedBox",
-            "SuperscriptBox",
-            "SubscriptBox",
+            "OverunderscriptBox",
             "OverscriptBox",
-            "UnderscriptBox",
-            "OtherscriptBox",
-            "FractionBox",
-            "SqrtBox",
             "RadicalBox",
-            "FormBox",
             "RawBackslash",
+            "SqrtBox",
+            "SubscriptBox",
+            "SuperscriptBox",
+            "UnderscriptBox",
         ],
         "]": ["RawRightBracket"],
         "^": ["UpSetDelayed", "UpSet", "Power"],