Skip to content

Commit

Permalink
Reverted tokenizer and language regex eslint fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
@kpal81xd
kpal81xd committed Nov 14, 2024
1 parent 7545cb6 commit 004e471
Show file tree
Hide file tree
Showing 4 changed files with 30 additions and 28 deletions.
4 changes: 3 additions & 1 deletion examples/eslint.config.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ export default [
'assets/scripts/utils/area-light-lut-bin-gen.js',
'cache',
'dist',
'src/lib'
'src/lib',
'src/app/monaco/languages',
'src/app/monaco/tokenizer-rules.mjs'
]
}
];
24 changes: 12 additions & 12 deletions examples/src/app/monaco/languages/glsl.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ export const conf = {
{ open: '[', close: ']' },
{ open: '{', close: '}' },
{ open: '(', close: ')' },
{ open: '\'', close: '\'', notIn: ['string', 'comment'] },
{ open: "'", close: "'", notIn: ['string', 'comment'] },
{ open: '"', close: '"', notIn: ['string'] }
],
surroundingPairs: [
{ open: '{', close: '}' },
{ open: '[', close: ']' },
{ open: '(', close: ')' },
{ open: '"', close: '"' },
{ open: '\'', close: '\'' }
{ open: "'", close: "'" }
]
};

Expand Down Expand Up @@ -212,17 +212,17 @@ export const language = {
'>>=',
'>>>='
],
symbols: /[=><!~?:&|+\-*/^%]+/,
symbols: /[=><!~?:&|+\-*\/\^%]+/,
escapes: /\\(?:[abfnrtv\\"']|x[0-9A-Fa-f]{1,4}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8})/,
integersuffix: /([uU](ll|LL|l|L)|(ll|LL|l|L)?[uU]?)/,
floatsuffix: /[fl]?/i,
encoding: /[uUL]|u8/, /* eslint-disable-line regexp/no-dupe-disjunctions */
floatsuffix: /[fFlL]?/,
encoding: /u|u8|U|L/,

tokenizer: {
root: [
// identifiers and keywords
[
/[a-z_]\w*/i,
/[a-zA-Z_]\w*/,
{
cases: {
'@keywords': { token: 'keyword.$0' },
Expand All @@ -238,7 +238,7 @@ export const language = {
{ include: '@whitespace' },

// delimiters and operators
[/[{}()[\]]/, '@brackets'],
[/[{}()\[\]]/, '@brackets'],
[
/@symbols/,
{
Expand All @@ -250,11 +250,11 @@ export const language = {
],

// numbers
[/\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
[/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, 'number.float'],
[/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, 'number.float'],
[/0[xX][0-9a-fA-F']*[0-9a-fA-F](@integersuffix)/, 'number.hex'],
[/0[0-7']*[0-7](@integersuffix)/, 'number.octal'],
[/0[bB][01']*[01](@integersuffix)/, 'number.binary'],
[/0[bB][0-1']*[0-1](@integersuffix)/, 'number.binary'],
[/\d[\d']*\d(@integersuffix)/, 'number'],
[/\d(@integersuffix)/, 'number'],

Expand All @@ -263,10 +263,10 @@ export const language = {
],

comment: [
[/[^/*]+/, 'comment'],
[/[^\/*]+/, 'comment'],
[/\/\*/, 'comment', '@push'],
['\\*/', 'comment', '@pop'],
[/[/*]/, 'comment']
[/[\/*]/, 'comment']
],

// Does it have strings?
Expand All @@ -290,4 +290,4 @@ export const language = {
[/\/\/.*$/, 'comment']
]
}
};
};
26 changes: 13 additions & 13 deletions examples/src/app/monaco/languages/wgsl.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ export const language = {
predeclared_intrinsics,
operators,

symbols: /[!%&*+\-./:;<=>^|_~]+/,
symbols: /[!%&*+\-\.\/:;<=>^|_~]+/,

tokenizer: {
root: [
Expand All @@ -402,7 +402,7 @@ export const language = {

[/;:\./, 'delimiter'],
[/,/, 'delimiter'], // Hack: Should be in previous rule
[/[{}()[\]]/, '@brackets'],
[/[{}()\[\]]/, '@brackets'],
['@', 'annotation', '@attribute'],
[
/@symbols/,
Expand All @@ -424,13 +424,13 @@ export const language = {

blockComment: [
// Soak up uninteresting text: anything except * or /
[/[^/*]+/, 'comment'],
[/[^\/*]+/, 'comment'],
// Recognize the start of a nested block comment.
[/\/\*/, 'comment', '@push'],
// Recognize the end of a nested block comment.
[/\*\//, 'comment', '@pop'],
// Recognize insignificant * and /
[/[/*]/, 'comment']
[/[\/*]/, 'comment']
],

attribute: [
Expand All @@ -457,19 +457,19 @@ export const language = {
// 0, with type-specifying suffix.
[/0[fh]/, 'number.float'],
// Other decimal integer, with type-specifying suffix.
[/[1-9]\d*[fh]/, 'number.float'],
[/[1-9][0-9]*[fh]/, 'number.float'],
// Has decimal point, at least one digit after decimal.
[/\d*\.\d+([eE][+-]?\d+)?[fh]?/, 'number.float'],
[/[0-9]*\.[0-9]+([eE][+-]?[0-9]+)?[fh]?/, 'number.float'],
// Has decimal point, at least one digit before decimal.
[/\d+\.\d*([eE][+-]?\d+)?[fh]?/, 'number.float'],
[/[0-9]+\.[0-9]*([eE][+-]?[0-9]+)?[fh]?/, 'number.float'],
// Has at least one digit, and has an exponent.
[/\d+[eE][+-]?\d+[fh]?/, 'number.float'],
[/[0-9]+[eE][+-]?[0-9]+[fh]?/, 'number.float'],

// Hex float literals
// https://www.w3.org/TR/WGSL/#syntax-hex_float_literal
[/0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+(?:[pP][+-]?\d+[fh]?)?/, 'number.hex'],
[/0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*(?:[pP][+-]?\d+[fh]?)?/, 'number.hex'],
[/0[xX][0-9a-fA-F]+[pP][+-]?\d+[fh]?/, 'number.hex'],
[/0[xX][0-9a-fA-F]*\.[0-9a-fA-F]+(?:[pP][+-]?[0-9]+[fh]?)?/, 'number.hex'],
[/0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*(?:[pP][+-]?[0-9]+[fh]?)?/, 'number.hex'],
[/0[xX][0-9a-fA-F]+[pP][+-]?[0-9]+[fh]?/, 'number.hex'],

// Hexadecimal integer literals
// https://www.w3.org/TR/WGSL/#syntax-hex_int_literal
Expand All @@ -478,8 +478,8 @@ export const language = {
// Decimal integer literals
// https://www.w3.org/TR/WGSL/#syntax-decimal_int_literal
// We need two rules here because 01 is not valid.
[/[1-9]\d*[iu]?/, 'number'],
[/[1-9][0-9]*[iu]?/, 'number'],
[/0[iu]?/, 'number'] // Must match last
]
}
};
};
4 changes: 2 additions & 2 deletions examples/src/app/monaco/tokenizer-rules.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ export const jsRules = {
[/./, 'comment.doc']
],
jsdocBrackets: [
[/(@link)([^}]+)/, ['keyword', 'identifier']],
[/([@]link)(\s*[^\}]+)/, ['keyword', 'identifier']],
[/\{/, 'comment.doc', '@push'],
[/\}/, 'comment.doc', '@pop'],
[/./, 'type.identifier']
]
};
};

0 comments on commit 004e471

Please sign in to comment.