fix: typo checker and fixer #771 (#788)

* build: added typo check and fix

* build: added multiple test cases to cover picking and ffoo

.eslintrc.js

@@ -20,5 +20,6 @@ module.exports = {
     '@typescript-eslint/explicit-function-return-type': 'off',
     '@typescript-eslint/explicit-module-boundary-types': 'off',
     '@typescript-eslint/no-explicit-any': 'off',
+    '@typescript-eslint/no-var-requires': 0,
   },
 };

.github/workflows/build.yml

@@ -13,6 +13,8 @@ jobs:
         run: npm ci
       - name: lint check
         run: npm run lint
+      - name: run tests
+        run: npm run test
 
   build:
     runs-on: ubuntu-latest

README.md

@@ -110,6 +110,12 @@ $ npm run build
 $ npm start
 ```
 
+## Running the tests
+
+```bash
+$ npm test
+```
+
 ## Support
 
 EddieBot is an MIT-licensed open source project. It can grow thanks to the contributors and the community members. If you'd like to join them, feel free to make a pull request and we'll review it.

babel.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+    presets: [
+        ['@babel/preset-env', {targets: {node: 'current'}}],
+        '@babel/preset-typescript',
+    ],
+};

package.json

@@ -20,31 +20,40 @@
     "start": "node -r dotenv/config prod/main.js",
     "lint": "eslint \"{src,apps,libs,test}/**/*.ts\"",
     "lint:fix": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
-    "prepare": "if test \"$HUSKY\" != \"0\" ; then husky install ; fi"
+    "prepare": "if test \"$HUSKY\" != \"0\" ; then husky install ; fi",
+    "test": "jest"
   },
   "dependencies": {
     "@discordjs/rest": "^1.0.1",
     "alex": "^9.1.1",
     "discord.js": "^14.1.2",
     "dotenv": "^16.0.1",
     "express": "^4.18.2",
-    "mongoose": "^6.3.6",
+    "mongoose": "^6.4.6",
     "rimraf": "^3.0.2",
+    "typo-js-ts": "^2.0.4",
     "winston": "^3.7.2"
   },
   "devDependencies": {
+    "@babel/preset-env": "^7.23.2",
+    "@babel/preset-typescript": "^7.23.2",
     "@commitlint/cli": "^17.6.1",
     "@commitlint/config-conventional": "^12.1.1",
+    "@jest/globals": "^29.7.0",
     "@types/alex": "^9.1.0",
-    "@types/express": "^4.17.14",
-    "@types/node": "^14.14.36",
+    "@types/express": "^4.17.20",
+    "@types/jest": "^29.5.6",
+    "@types/node": "^14.18.63",
     "@typescript-eslint/eslint-plugin": "^5.31.0",
     "@typescript-eslint/parser": "^5.31.0",
     "eslint": "^8.20.0",
     "eslint-config-prettier": "^8.1.0",
     "eslint-plugin-prettier": "^3.3.1",
     "husky": "^6.0.0",
+    "jest": "^29.7.0",
     "prettier": "^2.2.1",
-    "typescript": "^4.7.3"
+    "ts-jest": "^29.1.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^4.9.5"
   }
-}
+}

src/alexjs/__tests__/stripSpecialCharacters.test.ts

@@ -0,0 +1,15 @@
+import { describe, expect, test } from '@jest/globals';
+import { stripSpecialCharacters } from '../stripSpecialCharacters';
+describe('Strip special characters module tests', () => {
+  test('Should remove special characters from words.', async () => {
+    // given
+    const expectedOutput = 'K ing';
+    const inputWord = 'K?ing';
+
+    // when
+    const strippedOutput = stripSpecialCharacters(inputWord);
+
+    // then
+    expect(strippedOutput).toBe(expectedOutput);
+  });
+});

src/config/DictionaryOptions.ts

@@ -0,0 +1,5 @@
+const Typo = require('typo-js-ts').Typo;
+
+export const dict = new Typo('en_US', null, null, {
+  dictionaryPath: 'src/typo/dict',
+});

src/events/onMessage.ts

@@ -6,6 +6,7 @@ import { ExtendedClient } from '../interfaces/ExtendedClient';
 import { errorHandler } from '../utils/errorHandler';
 import Warnings from '../database/models/Warnings';
 import Statistics from '../database/models/Statistics';
+import { sentenceTypoFixer } from '../utils/typoFixer';
 
 export const onMessage = async (bot: ExtendedClient, message: Message) => {
   try {
@@ -14,7 +15,9 @@ export const onMessage = async (bot: ExtendedClient, message: Message) => {
     }
 
     const triggeredWarnings: EmbedBuilder[] = [];
-    const cleaned = stripSpecialCharacters(message.content);
+    const cleaned = await sentenceTypoFixer(
+      stripSpecialCharacters(message.content),
+    );
     triggeredWarnings.push(
       ...(await checkContent(bot, cleaned, message.guild.id)),
     );

src/events/onUpdate.ts

@@ -6,6 +6,7 @@ import { ExtendedClient } from '../interfaces/ExtendedClient';
 import { errorHandler } from '../utils/errorHandler';
 import Warnings from '../database/models/Warnings';
 import Statistics from '../database/models/Statistics';
+import { sentenceTypoFixer } from '../utils/typoFixer';
 
 export const onUpdate = async (
   bot: ExtendedClient,
@@ -26,7 +27,9 @@ export const onUpdate = async (
 
   try {
     const triggeredWarnings: EmbedBuilder[] = [];
-    const cleaned = stripSpecialCharacters(newMessage.content);
+    const cleaned = await sentenceTypoFixer(
+      stripSpecialCharacters(newMessage.content),
+    );
     triggeredWarnings.push(
       ...(await checkContent(bot, cleaned, newMessage.guild.id)),
     );

src/typo/dict/en_US/en_US.aff

@@ -0,0 +1,207 @@
+SET UTF-8
+TRY esianrtolcdugmphbyfvkwzESIANRTOLCDUGMPHBYFVKWZ'
+ICONV 1
+ICONV ’ '
+NOSUGGEST !
+
+# ordinal numbers
+COMPOUNDMIN 1
+# only in compounds: 1th, 2th, 3th
+ONLYINCOMPOUND c
+# compound rules:
+# 1. [0-9]*1[0-9]th (10th, 11th, 12th, 56714th, etc.)
+# 2. [0-9]*[02-9](1st|2nd|3rd|[4-9]th) (21st, 22nd, 123rd, 1234th, etc.)
+COMPOUNDRULE 2
+COMPOUNDRULE n*1t
+COMPOUNDRULE n*mp
+
+# Jeroen: removed numbers from WORDCHARS for R
+WORDCHARS ’
+
+PFX A Y 1
+PFX A   0     re         .
+
+PFX I Y 1
+PFX I   0     in         .
+
+PFX U Y 1
+PFX U   0     un         .
+
+PFX C Y 1
+PFX C   0     de          .
+
+PFX E Y 1
+PFX E   0     dis         .
+
+PFX F Y 1
+PFX F   0     con         .
+
+PFX K Y 1
+PFX K   0     pro         .
+
+SFX V N 2
+SFX V   e     ive        e
+SFX V   0     ive        [^e]
+
+SFX N Y 3
+SFX N   e     ion        e
+SFX N   y     ication    y 
+SFX N   0     en         [^ey] 
+
+SFX X Y 3
+SFX X   e     ions       e
+SFX X   y     ications   y
+SFX X   0     ens        [^ey]
+
+SFX H N 2
+SFX H   y     ieth       y
+SFX H   0     th         [^y] 
+
+SFX Y Y 1
+SFX Y   0     ly         .
+
+SFX G Y 2
+SFX G   e     ing        e
+SFX G   0     ing        [^e] 
+
+SFX J Y 2
+SFX J   e     ings       e
+SFX J   0     ings       [^e]
+
+SFX D Y 4
+SFX D   0     d          e
+SFX D   y     ied        [^aeiou]y
+SFX D   0     ed         [^ey]
+SFX D   0     ed         [aeiou]y
+
+SFX T N 4
+SFX T   0     st         e
+SFX T   y     iest       [^aeiou]y
+SFX T   0     est        [aeiou]y
+SFX T   0     est        [^ey]
+
+SFX R Y 4
+SFX R   0     r          e
+SFX R   y     ier        [^aeiou]y
+SFX R   0     er         [aeiou]y
+SFX R   0     er         [^ey]
+
+SFX Z Y 4
+SFX Z   0     rs         e
+SFX Z   y     iers       [^aeiou]y
+SFX Z   0     ers        [aeiou]y
+SFX Z   0     ers        [^ey]
+
+SFX S Y 4
+SFX S   y     ies        [^aeiou]y
+SFX S   0     s          [aeiou]y
+SFX S   0     es         [sxzh]
+SFX S   0     s          [^sxzhy]
+
+SFX P Y 3
+SFX P   y     iness      [^aeiou]y
+SFX P   0     ness       [aeiou]y
+SFX P   0     ness       [^y]
+
+SFX M Y 1
+SFX M   0     's         .
+
+SFX B Y 3
+SFX B   0     able       [^aeiou]
+SFX B   0     able       ee
+SFX B   e     able       [^aeiou]e
+
+SFX L Y 1
+SFX L   0     ment       .
+
+REP 90
+REP a ei
+REP ei a
+REP a ey
+REP ey a
+REP ai ie
+REP ie ai
+REP alot a_lot
+REP are air
+REP are ear
+REP are eir
+REP air are
+REP air ere
+REP ere air
+REP ere ear
+REP ere eir
+REP ear are
+REP ear air
+REP ear ere
+REP eir are
+REP eir ere
+REP ch te
+REP te ch
+REP ch ti
+REP ti ch
+REP ch tu
+REP tu ch
+REP ch s
+REP s ch
+REP ch k
+REP k ch
+REP f ph
+REP ph f
+REP gh f
+REP f gh
+REP i igh
+REP igh i
+REP i uy
+REP uy i
+REP i ee
+REP ee i
+REP j di
+REP di j
+REP j gg
+REP gg j
+REP j ge
+REP ge j
+REP s ti
+REP ti s
+REP s ci
+REP ci s
+REP k cc
+REP cc k
+REP k qu
+REP qu k
+REP kw qu
+REP o eau
+REP eau o
+REP o ew
+REP ew o
+REP oo ew
+REP ew oo
+REP ew ui
+REP ui ew
+REP oo ui
+REP ui oo
+REP ew u
+REP u ew
+REP oo u
+REP u oo
+REP u oe
+REP oe u
+REP u ieu
+REP ieu u
+REP ue ew
+REP ew ue
+REP uff ough
+REP oo ieu
+REP ieu oo
+REP ier ear
+REP ear ier
+REP ear air
+REP air ear
+REP w qu
+REP qu w
+REP z ss
+REP ss z
+REP shun tion
+REP shun sion
+REP shun cion
+REP size cise