From 2c06d2a8f0fcea310e69adae030d0e6dba2eef25 Mon Sep 17 00:00:00 2001 From: Martin Dvorak Date: Mon, 8 Jan 2024 23:09:06 +0100 Subject: [PATCH] Removing Mitie and NER which resolves #1522 --- .gitmodules | 3 - app/app.pro | 32 ---- .../dialogs/ner_choose_tag_types_dialog.cpp | 84 --------- .../qt/dialogs/ner_choose_tag_types_dialog.h | 72 ------- app/src/qt/dialogs/ner_result_dialog.cpp | 118 ------------ app/src/qt/dialogs/ner_result_dialog.h | 68 ------- app/src/qt/dialogs/wingman_dialog.cpp | 1 + app/src/qt/main_menu_presenter.cpp | 6 - app/src/qt/main_menu_view.cpp | 21 --- app/src/qt/main_menu_view.h | 6 - app/src/qt/main_window_presenter.cpp | 170 +---------------- app/src/qt/main_window_presenter.h | 29 --- app/src/qt/ner_leaderboard_model.cpp | 109 ----------- app/src/qt/ner_leaderboard_model.h | 46 ----- app/src/qt/ner_leaderboard_view.cpp | 61 ------ app/src/qt/ner_leaderboard_view.h | 44 ----- app/src/qt/ner_main_window_worker_thread.cpp | 38 ---- app/src/qt/ner_main_window_worker_thread.h | 91 --------- deps/mitie | 1 - lib/lib.pro | 21 --- lib/src/mind/ai/ai.cpp | 9 - lib/src/mind/ai/ai.h | 33 ---- .../mind/ai/nlp/named_entity_recognition.cpp | 178 ------------------ .../mind/ai/nlp/named_entity_recognition.h | 89 --------- lib/src/mind/ai/nlp/ner_named_entity.cpp | 22 --- lib/src/mind/ai/nlp/ner_named_entity.h | 47 ----- lib/src/mind/mind.cpp | 17 -- lib/src/mind/mind.h | 14 -- mindforger.pro | 2 - 29 files changed, 2 insertions(+), 1430 deletions(-) delete mode 100644 app/src/qt/dialogs/ner_choose_tag_types_dialog.cpp delete mode 100644 app/src/qt/dialogs/ner_choose_tag_types_dialog.h delete mode 100644 app/src/qt/dialogs/ner_result_dialog.cpp delete mode 100644 app/src/qt/dialogs/ner_result_dialog.h delete mode 100644 app/src/qt/ner_leaderboard_model.cpp delete mode 100644 app/src/qt/ner_leaderboard_model.h delete mode 100644 app/src/qt/ner_leaderboard_view.cpp delete mode 100644 app/src/qt/ner_leaderboard_view.h delete mode 100644 app/src/qt/ner_main_window_worker_thread.cpp delete mode 100644 app/src/qt/ner_main_window_worker_thread.h delete mode 160000 deps/mitie delete mode 100644 lib/src/mind/ai/nlp/named_entity_recognition.cpp delete mode 100644 lib/src/mind/ai/nlp/named_entity_recognition.h delete mode 100644 lib/src/mind/ai/nlp/ner_named_entity.cpp delete mode 100644 lib/src/mind/ai/nlp/ner_named_entity.h diff --git a/.gitmodules b/.gitmodules index 5402d0a8..ad167075 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "doc"] path = doc url = https://github.com/dvorka/mindforger-repository.git -[submodule "deps/mitie"] - path = deps/mitie - url = https://github.com/dvorka/MITIE.git [submodule "deps/cmark-gfm"] path = deps/cmark-gfm url = https://github.com/dvorka/cmark.git diff --git a/app/app.pro b/app/app.pro index f3df8237..0c5da17c 100644 --- a/app/app.pro +++ b/app/app.pro @@ -58,15 +58,6 @@ mfoldhunspell | equals(OS_DISTRO_VERSION, "Windows") | equals(OS_DISTRO_VERSION, message("Hunspell: configuring use of NEW API on OS: $$OS_DISTRO_VERSION") } -mfllamacpp { - DEFINES += MF_LLAMA_CPP -} - -# Named Entity Recognition -mfner { - DEFINES += MF_NER -} - # webkit is supposed to be OBSOLETED by webengine, but webengine is disabled # on Linux since Qt 5.9 due to its tragic performance -> conditional compilation # seems to be the only way: @@ -127,12 +118,6 @@ win32 { } } -# NER library -mfner { - # MF links MITIE for AI/NLP/DL - LIBS += -L$$OUT_PWD/../deps/mitie/mitielib -lmitie -} - # Zlib win32 { INCLUDEPATH += $$PWD/../deps/zlib-win/include @@ -337,14 +322,6 @@ HEADERS += \ win32|macx|mfwebengine { HEADERS += ./src/qt/web_engine_page_link_navigation_policy.h } -mfner { - HEADERS += \ - src/qt/dialogs/ner_choose_tag_types_dialog.h \ - src/qt/dialogs/ner_result_dialog.h \ - src/qt/ner_leaderboard_model.h \ - src/qt/ner_leaderboard_view.h \ - src/qt/ner_main_window_worker_thread.h -} SOURCES += \ ./src/qt/mindforger.cpp \ @@ -463,15 +440,6 @@ win32|macx|mfwebengine { SOURCES += ./src/qt/web_engine_page_link_navigation_policy.cpp } -mfner { - SOURCES += \ - src/qt/dialogs/ner_choose_tag_types_dialog.cpp \ - src/qt/dialogs/ner_result_dialog.cpp \ - src/qt/ner_leaderboard_model.cpp \ - src/qt/ner_leaderboard_view.cpp \ - src/qt/ner_main_window_worker_thread.cpp -} - win32 { HEADERS += \ ../deps/getopt/getopt.h diff --git a/app/src/qt/dialogs/ner_choose_tag_types_dialog.cpp b/app/src/qt/dialogs/ner_choose_tag_types_dialog.cpp deleted file mode 100644 index fab55896..00000000 --- a/app/src/qt/dialogs/ner_choose_tag_types_dialog.cpp +++ /dev/null @@ -1,84 +0,0 @@ -/* - ner_choose_tag_types_dialog.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_choose_tag_types_dialog.h" - -namespace m8r { - -NerChooseTagTypesDialog::NerChooseTagTypesDialog(QWidget* parent) - : QDialog(parent) -{ - // widgets - label = new QLabel{tr("Choose entity types to be extracted:")}; - - personsCheckBox = new QCheckBox{tr("persons")}; - locationsCheckBox = new QCheckBox{tr("locations")}; - organizationsCheckBox = new QCheckBox{tr("organizations")}; - miscCheckBox = new QCheckBox{tr("other entities")}; - - chooseButton = new QPushButton{tr("&Choose")}; - chooseButton->setDefault(true); - - closeButton = new QPushButton{tr("&Cancel")}; - - // signals - connect(closeButton, SIGNAL(clicked()), this, SLOT(close())); - connect(personsCheckBox, SIGNAL(stateChanged(int)), this, SLOT(slotChooseButtonEnable(int))); - connect(locationsCheckBox, SIGNAL(stateChanged(int)), this, SLOT(slotChooseButtonEnable(int))); - connect(organizationsCheckBox, SIGNAL(stateChanged(int)), this, SLOT(slotChooseButtonEnable(int))); - connect(miscCheckBox, SIGNAL(stateChanged(int)), this, SLOT(slotChooseButtonEnable(int))); - - // assembly - QVBoxLayout* mainLayout = new QVBoxLayout{}; - mainLayout->addWidget(label); - mainLayout->addWidget(personsCheckBox); - mainLayout->addWidget(locationsCheckBox); - mainLayout->addWidget(organizationsCheckBox); - mainLayout->addWidget(miscCheckBox); - - QHBoxLayout* buttonLayout = new QHBoxLayout{}; - buttonLayout->addStretch(1); - buttonLayout->addWidget(closeButton); - buttonLayout->addWidget(chooseButton); - buttonLayout->addStretch(); - - mainLayout->addLayout(buttonLayout); - setLayout(mainLayout); - - // dialog - setWindowTitle(tr("Choose Entity Type")); - resize(fontMetrics().averageCharWidth()*35, height()); - setModal(true); -} - -NerChooseTagTypesDialog::~NerChooseTagTypesDialog() -{ -} - -void NerChooseTagTypesDialog::slotChooseButtonEnable(int state) -{ - UNUSED_ARG(state); - - if(personsCheckBox->isChecked() || locationsCheckBox->isChecked() || organizationsCheckBox->isChecked() || miscCheckBox->isChecked()) { - chooseButton->setEnabled(true); - } else { - chooseButton->setEnabled(false); - } -} - -} // m8r namespace diff --git a/app/src/qt/dialogs/ner_choose_tag_types_dialog.h b/app/src/qt/dialogs/ner_choose_tag_types_dialog.h deleted file mode 100644 index 9c56248e..00000000 --- a/app/src/qt/dialogs/ner_choose_tag_types_dialog.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - ner_choose_tag_types_dialog.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8RUI_NER_CHOOSE_TAG_TYPES_DIALOG_H -#define M8RUI_NER_CHOOSE_TAG_TYPES_DIALOG_H - -#include - -#include "../../lib/src/gear/lang_utils.h" -#include "../../lib/src/debug.h" - -namespace m8r { - -class NerChooseTagTypesDialog : public QDialog -{ - Q_OBJECT - -private: - QLabel* label; - // IMPROVE tag types are hardcoded - allow loading them dynamically for generic NER models - QCheckBox* personsCheckBox; - QCheckBox* locationsCheckBox; - QCheckBox* organizationsCheckBox; - QCheckBox* miscCheckBox; - QPushButton* chooseButton; - QPushButton* closeButton; - -public: - explicit NerChooseTagTypesDialog(QWidget* parent); - NerChooseTagTypesDialog(const NerChooseTagTypesDialog&) = delete; - NerChooseTagTypesDialog(const NerChooseTagTypesDialog&&) = delete; - NerChooseTagTypesDialog &operator=(const NerChooseTagTypesDialog&) = delete; - NerChooseTagTypesDialog &operator=(const NerChooseTagTypesDialog&&) = delete; - ~NerChooseTagTypesDialog(); - - void clearCheckboxes() { - personsCheckBox->setChecked(false); - locationsCheckBox->setChecked(false); - organizationsCheckBox->setChecked(false); - miscCheckBox->setChecked(false); - } - - QCheckBox* getPersonsCheckbox() const { return personsCheckBox; } - QCheckBox* getLocationsCheckbox() const { return locationsCheckBox; } - QCheckBox* getOrganizationsCheckbox() const { return organizationsCheckBox; } - QCheckBox* getMiscCheckbox() const { return miscCheckBox; } - - QPushButton* getChooseButton() const { return chooseButton; } - - void show() { slotChooseButtonEnable(0); QDialog::show(); } - -private slots: - void slotChooseButtonEnable(int); -}; - -} -#endif // M8RUI_NER_CHOOSE_TAG_TYPES_DIALOG_H diff --git a/app/src/qt/dialogs/ner_result_dialog.cpp b/app/src/qt/dialogs/ner_result_dialog.cpp deleted file mode 100644 index c02862f9..00000000 --- a/app/src/qt/dialogs/ner_result_dialog.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* - ner_result_dialog.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_result_dialog.h" - -namespace m8r { - -using namespace std; - -NerResultDialog::NerResultDialog(QWidget* parent) - : QDialog(parent) -{ - // widgets - leaderboardModel = new NerLeaderboardModel(this); - leaderboardView = new NerLeaderboardView(this); - leaderboardView->setModel(leaderboardModel); - - label = new QLabel{tr("Recognized named entities:")}; - - findButton = new QPushButton{tr("&Find Entity in Notes")}; - findButton->setDefault(true); - findButton->setEnabled(false); - - closeButton = new QPushButton{tr("&Cancel")}; - - // signals - QObject::connect(findButton, SIGNAL(clicked()), this, SLOT(handleChoice())); - QObject::connect(closeButton, SIGNAL(clicked()), this, SLOT(close())); - QObject::connect( - leaderboardView->selectionModel(), - SIGNAL(selectionChanged(const QItemSelection&, const QItemSelection&)), - this, - SLOT(slotRowSelected(const QItemSelection&, const QItemSelection&))); - - // assembly - QVBoxLayout *mainLayout = new QVBoxLayout{}; - mainLayout->addWidget(label); - mainLayout->addWidget(leaderboardView); - - QHBoxLayout *buttonLayout = new QHBoxLayout{}; - buttonLayout->addStretch(1); - buttonLayout->addWidget(closeButton); - buttonLayout->addWidget(findButton); - buttonLayout->addStretch(); - - mainLayout->addLayout(buttonLayout); - setLayout(mainLayout); - - // dialog - setWindowTitle(tr("Find Named Entities")); - // height is set to make sure listview gets enough lines - resize(fontMetrics().averageCharWidth()*75, fontMetrics().height()*30); - setModal(true); -} - -NerResultDialog::~NerResultDialog() -{ - delete label; - delete leaderboardView; - delete leaderboardModel; - delete closeButton; -} - -void NerResultDialog::show(std::vector& entities) -{ - choice.clear(); - leaderboardModel->removeAllRows(); - - if(entities.size()) { - for(NerNamedEntity& e:entities) { - leaderboardModel->addRow(e.name, e.type, e.score); - } - } - - QDialog::show(); - - leaderboardView->sortByColumn(2, Qt::SortOrder::DescendingOrder); -} - -void NerResultDialog::handleChoice() -{ - QDialog::close(); - emit choiceFinished(); -} - -void NerResultDialog::slotRowSelected(const QItemSelection& selected, const QItemSelection& deselected) -{ - Q_UNUSED(deselected); - - QModelIndexList indices = selected.indexes(); - if(indices.size()) { - const QModelIndex& index = indices.at(0); - QStandardItem* item = leaderboardModel->itemFromIndex(index); - choice = item->text().toStdString(); - if(choice.size()) { - findButton->setEnabled(true); - } else { - findButton->setEnabled(false); - } - } -} - -} // m8r namespace diff --git a/app/src/qt/dialogs/ner_result_dialog.h b/app/src/qt/dialogs/ner_result_dialog.h deleted file mode 100644 index 5713251d..00000000 --- a/app/src/qt/dialogs/ner_result_dialog.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - ner_result_dialog.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8RUI_NER_RESULT_DIALOG_H -#define M8RUI_NER_RESULT_DIALOG_H - -#include - -#include "../../../../lib/src/mind/ai/nlp/ner_named_entity.h" - -#include "../ner_leaderboard_model.h" -#include "../ner_leaderboard_view.h" - -#include "../../../../lib/src/debug.h" - -namespace m8r { - -class NerResultDialog : public QDialog -{ - Q_OBJECT - -private: - std::string choice; - - QLabel* label; - NerLeaderboardModel* leaderboardModel; - NerLeaderboardView* leaderboardView; - QPushButton* closeButton; - QPushButton* findButton; - -public: - explicit NerResultDialog(QWidget* parent); - NerResultDialog(const NerResultDialog&) = delete; - NerResultDialog(const NerResultDialog&&) = delete; - NerResultDialog &operator=(const NerResultDialog&) = delete; - NerResultDialog &operator=(const NerResultDialog&&) = delete; - ~NerResultDialog(); - - QPushButton* getFindButton() const { return findButton; } - std::string getChoice() const { return choice; } - - void show(std::vector& entities); - -signals: - void choiceFinished(); - -private slots: - void handleChoice(); - void slotRowSelected(const QItemSelection& selected, const QItemSelection& deselected); -}; - -} -#endif // M8RUI_NER_RESULT_DIALOG_H diff --git a/app/src/qt/dialogs/wingman_dialog.cpp b/app/src/qt/dialogs/wingman_dialog.cpp index 8fb2df1a..80de6716 100644 --- a/app/src/qt/dialogs/wingman_dialog.cpp +++ b/app/src/qt/dialogs/wingman_dialog.cpp @@ -44,6 +44,7 @@ const vector WingmanDialog::notePrompts( QString{"Rewrite to be funny."}, QString{"Chat with the content."}, // other UCs: + // - NER UCs // - simplify // - beautify // - translate diff --git a/app/src/qt/main_menu_presenter.cpp b/app/src/qt/main_menu_presenter.cpp index 0e9d5579..608383c8 100644 --- a/app/src/qt/main_menu_presenter.cpp +++ b/app/src/qt/main_menu_presenter.cpp @@ -85,12 +85,6 @@ MainMenuPresenter::MainMenuPresenter(MainWindowPresenter* mwp) QObject::connect(view->actionFindNoteByName, SIGNAL(triggered()), mwp, SLOT(doActionFindNoteByName())); QObject::connect(view->actionFindOutlineByTag, SIGNAL(triggered()), mwp, SLOT(doActionFindOutlineByTag())); QObject::connect(view->actionFindNoteByTag, SIGNAL(triggered()), mwp, SLOT(doActionFindNoteByTag())); -#ifdef MF_NER - QObject::connect(view->actionFindNerPersons, SIGNAL(triggered()), mwp, SLOT(doActionFindNerPersons())); - QObject::connect(view->actionFindNerLocations, SIGNAL(triggered()), mwp, SLOT(doActionFindNerLocations())); - QObject::connect(view->actionFindNerOrganizations, SIGNAL(triggered()), mwp, SLOT(doActionFindNerOrganizations())); - QObject::connect(view->actionFindNerMisc, SIGNAL(triggered()), mwp, SLOT(doActionFindNerMisc())); -#endif // menu: view QObject::connect( diff --git a/app/src/qt/main_menu_view.cpp b/app/src/qt/main_menu_view.cpp index 415a6840..d3fe51b8 100644 --- a/app/src/qt/main_menu_view.cpp +++ b/app/src/qt/main_menu_view.cpp @@ -206,20 +206,6 @@ MainMenuView::MainMenuView(MainWindowView& mainWindowView) actionFindDocByName->setStatusTip(tr("Find Document by name")); #endif -#ifdef MF_NER - actionFindNerPersons = new QAction(QIcon(":/menu-icons/find.svg"), tr("Recall &Persons"), mainWindow); - actionFindNerPersons->setStatusTip(tr("Find persons using Named-entity recognition (NER)")); - - actionFindNerLocations = new QAction(QIcon(":/menu-icons/find.svg"), tr("Recall &Locations"), mainWindow); - actionFindNerLocations->setStatusTip(tr("Find locations using Named-entity recognition (NER)")); - - actionFindNerOrganizations = new QAction(QIcon(":/menu-icons/find.svg"), tr("Recall Organizations"), mainWindow); - actionFindNerOrganizations->setStatusTip(tr("Find organizations using Named-entity recognition (NER)")); - - actionFindNerMisc = new QAction(QIcon(":/menu-icons/find.svg"), tr("Recall Other Entities"), mainWindow); - actionFindNerMisc->setStatusTip(tr("Find miscellaneous entities using Named-entity recognition (NER)")); -#endif - menuFind = qMenuBar->addMenu(tr("&Recall")); menuFind->addAction(actionFindFts); menuFind->addAction(actionFindOutlineByName); @@ -229,13 +215,6 @@ MainMenuView::MainMenuView(MainWindowView& mainWindowView) #ifdef MF_WIP menuFind->addAction(actionFindDocByName); #endif -#ifdef MF_NER - menuFind->addSeparator(); - menuFind->addAction(actionFindNerPersons); - menuFind->addAction(actionFindNerLocations); - menuFind->addAction(actionFindNerOrganizations); - menuFind->addAction(actionFindNerMisc); -#endif // menu: view diff --git a/app/src/qt/main_menu_view.h b/app/src/qt/main_menu_view.h index caf8e8fb..ad29635d 100644 --- a/app/src/qt/main_menu_view.h +++ b/app/src/qt/main_menu_view.h @@ -96,12 +96,6 @@ class MainMenuView : public QObject #ifdef MF_WIP QAction* actionFindDocByName; #endif -#ifdef MF_NER - QAction* actionFindNerPersons; - QAction* actionFindNerLocations; - QAction* actionFindNerOrganizations; - QAction* actionFindNerMisc; -#endif // menu: View QAction* actionViewHome; diff --git a/app/src/qt/main_window_presenter.cpp b/app/src/qt/main_window_presenter.cpp index 65240f8a..9d071a8c 100644 --- a/app/src/qt/main_window_presenter.cpp +++ b/app/src/qt/main_window_presenter.cpp @@ -93,10 +93,7 @@ MainWindowPresenter::MainWindowPresenter(MainWindowView& view) QString::fromStdString(File::EXTENSION_CSV), &view ); -#ifdef MF_NER - nerChooseTagsDialog = new NerChooseTagTypesDialog(&view); - nerResultDialog = new NerResultDialog(&view); -#endif + // show/hide widgets based on configuration handleMindPreferences(); @@ -249,21 +246,11 @@ MainWindowPresenter::MainWindowPresenter(MainWindowView& view) this, SLOT(slotMainToolbarVisibilityChanged(bool)) ); - -#ifdef MF_NER - QObject::connect(nerChooseTagsDialog->getChooseButton(), SIGNAL(clicked()), this, SLOT(handleFindNerEntities())); - QObject::connect(nerResultDialog, SIGNAL(choiceFinished()), this, SLOT(handleFtsNerEntity())); -#endif - // async task 2 GUI events distributor distributor = new AsyncTaskNotificationsDistributor(this); // setup callback for cleanup when it finishes QObject::connect(distributor, SIGNAL(finished()), distributor, SLOT(deleteLater())); distributor->start(); -#ifdef MF_NER - // NER worker - nerWorker = nullptr; -#endif // send signal to components to be updated on a configuration change QObject::connect(configDialog, SIGNAL(saveConfigSignal()), this, SLOT(handleMindPreferences())); @@ -1112,161 +1099,6 @@ void MainWindowPresenter::handleFindNoteByName() } } -#ifdef MF_NER - -void MainWindowPresenter::doActionFindNerPersons() -{ - if(orloj->isFacetActiveOutlineManagement()) { - nerChooseTagsDialog->clearCheckboxes(); - nerChooseTagsDialog->getPersonsCheckbox()->setChecked(true); - nerChooseTagsDialog->show(); - } else { - statusBar->showInfo(tr("Initializing NER and predicting...")); - QMessageBox::critical(&view, tr("NER"), tr("Memory NER not implemented yet.")); - } -} -void MainWindowPresenter::doActionFindNerLocations() -{ - if(orloj->isFacetActiveOutlineManagement()) { - nerChooseTagsDialog->clearCheckboxes(); - nerChooseTagsDialog->getLocationsCheckbox()->setChecked(true); - nerChooseTagsDialog->show(); - } else { - statusBar->showInfo(tr("Initializing NER and predicting...")); - QMessageBox::critical(&view, tr("NER"), tr("Memory NER not implemented yet.")); - } -} -void MainWindowPresenter::doActionFindNerOrganizations() -{ - if(orloj->isFacetActiveOutlineManagement()) { - nerChooseTagsDialog->clearCheckboxes(); - nerChooseTagsDialog->getOrganizationsCheckbox()->setChecked(true); - nerChooseTagsDialog->show(); - } else { - statusBar->showInfo(tr("Initializing NER and predicting...")); - QMessageBox::critical(&view, tr("NER"), tr("Memory NER not implemented yet.")); - } -} -void MainWindowPresenter::doActionFindNerMisc() -{ - if(orloj->isFacetActiveOutlineManagement()) { - nerChooseTagsDialog->clearCheckboxes(); - nerChooseTagsDialog->getMiscCheckbox()->setChecked(true); - nerChooseTagsDialog->show(); - } else { - statusBar->showInfo(tr("Initializing NER and predicting...")); - QMessageBox::critical(&view, tr("NER"), tr("Memory NER not implemented yet.")); - } -} - -NerMainWindowWorkerThread* MainWindowPresenter::startNerWorkerThread( - Mind* m, - OrlojPresenter* o, - int f, - std::vector* r, - QDialog* d) -{ - QThread* thread = new QThread; - NerMainWindowWorkerThread* worker - = new NerMainWindowWorkerThread(thread, m, o, f, r, d); - - // signals - worker->moveToThread(thread); - // TODO implement dialog w/ error handling - QObject::connect(worker, SIGNAL(error(QString)), this, SLOT(errorString(QString))); - QObject::connect(thread, SIGNAL(started()), worker, SLOT(process())); - // open dialog to choose from result(s) - QObject::connect(worker, SIGNAL(finished()), this, SLOT(handleChooseNerEntityResult())); - // worker's finished signal quits thread ~ thread CANNOT be reused - QObject::connect(worker, SIGNAL(finished()), thread, SLOT(quit())); - // schedule thread for automatic deletion by Qt - I delete worker myself - //QObject::connect(worker, SIGNAL(finished()), worker, SLOT(deleteLater())); - QObject::connect(thread, SIGNAL(finished()), thread, SLOT(deleteLater())); - - thread->start(); - - return worker; -} - -// handleFindNerPerson() -> handleChooseNerEntityResult() -> handleFtsNerEntity() -void MainWindowPresenter::handleFindNerEntities() -{ - nerChooseTagsDialog->hide(); - - int entityFilter{}; - entityFilter = - (nerChooseTagsDialog->getPersonsCheckbox()->isChecked()?NerNamedEntityType::PERSON:0) | - (nerChooseTagsDialog->getLocationsCheckbox()->isChecked()?NerNamedEntityType::LOCATION:0) | - (nerChooseTagsDialog->getOrganizationsCheckbox()->isChecked()?NerNamedEntityType::ORGANIZATION:0) | - (nerChooseTagsDialog->getMiscCheckbox()->isChecked()?NerNamedEntityType::MISC:0); - - MF_DEBUG("Named-entity type filter: " << entityFilter << endl); - - vector* result - = new vector{}; - if(mind->isNerInitilized()) { - statusBar->showInfo(tr("Recognizing named entities...")); - - mind->recognizePersons(orloj->getOutlineView()->getCurrentOutline(), entityFilter, *result); - - chooseNerEntityResult(result); - } else { - statusBar->showInfo(tr("Initializing NER and recognizing named entities...")); - - // launch async worker - QDialog* progressDialog - = new QDialog{&view}; - nerWorker - = startNerWorkerThread(mind, orloj, entityFilter, result, progressDialog); - - // show PROGRESS dialog - will be closed by worker - QVBoxLayout* mainLayout = new QVBoxLayout{}; - QLabel* l = new QLabel{tr(" Initializing (the first run only) NER and predicting... ")}; - mainLayout->addWidget(l); - progressDialog->setLayout(mainLayout); - progressDialog->setWindowTitle(tr("Named-entity Recognition")); - //progressDialog->resize(fontMetrics().averageCharWidth()*35, height()); - //progressDialog->setModal(true); - progressDialog->update(); - progressDialog->activateWindow(); - progressDialog->show(); - // dialog is deleted by worker thread - } -} - -void MainWindowPresenter::chooseNerEntityResult(vector* nerEntities) -{ - MF_DEBUG("Showing NER results to choose one entity for FTS..." << endl); - statusBar->showInfo(tr("NER predicition finished")); - - if(nerEntities && nerEntities->size()) { - nerResultDialog->show(*nerEntities); - } else { - QMessageBox::information(&view, tr("Named-entity Recognition"), tr("No named entities recognized.")); - } -} - -void MainWindowPresenter::handleChooseNerEntityResult() -{ - vector* nerEntities = nerWorker->getResult(); - chooseNerEntityResult(nerEntities); - - // cleanup: thread is deleted by Qt (deleteLater() signal) - delete nerEntities; - delete nerWorker; -} - -void MainWindowPresenter::handleFtsNerEntity() -{ - if(nerResultDialog->getChoice().size()) { - executeFts( - nerResultDialog->getChoice(), - false, - orloj->getOutlineView()->getCurrentOutline()); - } -} - -#endif - void MainWindowPresenter::doActionViewRecentNotes() { vector notes{}; diff --git a/app/src/qt/main_window_presenter.h b/app/src/qt/main_window_presenter.h index 8c6c567b..47bc6a1d 100644 --- a/app/src/qt/main_window_presenter.h +++ b/app/src/qt/main_window_presenter.h @@ -30,9 +30,6 @@ #include "main_menu_presenter.h" #include "gear/async_task_notifications_distributor.h" -#ifdef MF_NER - #include "ner_main_window_worker_thread.h" -#endif #include "cli_n_breadcrumbs_presenter.h" #include "orloj_presenter.h" #include "status_bar_presenter.h" @@ -61,8 +58,6 @@ #include "dialogs/terminal_dialog.h" #include "dialogs/export_csv_file_dialog.h" #include "dialogs/export_file_dialog.h" -#include "dialogs/ner_choose_tag_types_dialog.h" -#include "dialogs/ner_result_dialog.h" #include #include @@ -108,9 +103,6 @@ class MainWindowPresenter : public QObject Mind* mind; AsyncTaskNotificationsDistributor* distributor; -#ifdef MF_NER - NerMainWindowWorkerThread* nerWorker; -#endif MarkdownOutlineRepresentation* mdRepresentation; HtmlOutlineRepresentation* htmlRepresentation; @@ -149,8 +141,6 @@ class MainWindowPresenter : public QObject NewFileDialog* newFileDialog; ExportFileDialog* exportOutlineToHtmlDialog; ExportCsvFileDialog* exportMemoryToCsvDialog; - NerChooseTagTypesDialog *nerChooseTagsDialog; - NerResultDialog* nerResultDialog; public: explicit MainWindowPresenter(MainWindowView& view); @@ -183,15 +173,6 @@ class MainWindowPresenter : public QObject // N view void handleNoteViewLinkClicked(const QUrl& url); - // NER - NerMainWindowWorkerThread* startNerWorkerThread( - Mind* m, - OrlojPresenter* o, - int f, - std::vector* r, - QDialog* d - ); - public slots: // mind #ifdef DO_MF_DEBUG @@ -231,16 +212,6 @@ public slots: void doTriggerFindNoteByTag(const m8r::Tag* tag); void doSwitchFindByTagDialog(bool toFindNotesByTag); void handleFindNoteByTag(); -#ifdef MF_NER - void doActionFindNerPersons(); - void doActionFindNerLocations(); - void doActionFindNerOrganizations(); - void doActionFindNerMisc(); - void handleFindNerEntities(); - void chooseNerEntityResult(vector*); - void handleChooseNerEntityResult(); - void handleFtsNerEntity(); -#endif // view void sortAndSaveOrganizersConfig(); void doActionViewOrganizers(); diff --git a/app/src/qt/ner_leaderboard_model.cpp b/app/src/qt/ner_leaderboard_model.cpp deleted file mode 100644 index 0fe62020..00000000 --- a/app/src/qt/ner_leaderboard_model.cpp +++ /dev/null @@ -1,109 +0,0 @@ -/* - ner_leaderboard_model.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_leaderboard_model.h" - -namespace m8r { - -using namespace std; - -NerLeaderboardModel::NerLeaderboardModel(QWidget* parent) - : QStandardItemModel(parent) -{ - setColumnCount(3); - setRowCount(0); -} - -NerLeaderboardModel::~NerLeaderboardModel() -{ -} - -void NerLeaderboardModel::removeAllRows() -{ - QStandardItemModel::clear(); - - QStringList tableHeader; - tableHeader - << tr("Name") - << tr("Type") - << tr("Score"); - - // IMPROVE set tooltips: items w/ tooltips instead of just strings - setHorizontalHeaderLabels(tableHeader); -} - -void NerLeaderboardModel::addRow(string& entityName, NerNamedEntityType entityType, float score) -{ - QList items; - QStandardItem* item; - - QString html{}; - html += QString::fromStdString(entityName); - - // item - item = new QStandardItem(html); - item->setToolTip(html); - items += item; - - html.clear(); - switch(entityType) { - case NerNamedEntityType::PERSON: - html += tr("person"); - break; - case NerNamedEntityType::LOCATION: - html += tr("location"); - break; - case NerNamedEntityType::ORGANIZATION: - html += tr("organization"); - break; - case NerNamedEntityType::MISC: - html += tr("misc"); - break; - } - - item = new QStandardItem(html); - item->setToolTip(html); - items += item; - - html.clear(); - if(score>0.29) { - html += ""; - } - score = ROUND_FLOAT(score, 1000); - html += QString::number(score); - if(score>0.29) { - html += ""; - } - item = new QStandardItem(html); - item->setData(QVariant::fromValue(score)); - items += item; - - appendRow(items); -} - -} // m8r namespace diff --git a/app/src/qt/ner_leaderboard_model.h b/app/src/qt/ner_leaderboard_model.h deleted file mode 100644 index b95b2e7d..00000000 --- a/app/src/qt/ner_leaderboard_model.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - ner_leaderboard_model.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8RUI_NER_LEADERBOARD_MODEL_H -#define M8RUI_NER_LEADERBOARD_MODEL_H - -#include - -#include "../../../lib/src/mind/ai/nlp/ner_named_entity.h" -#include "../../../lib/src/gear/lang_utils.h" - -namespace m8r { - -class NerLeaderboardModel : public QStandardItemModel -{ - Q_OBJECT - -public: - explicit NerLeaderboardModel(QWidget* parent); - NerLeaderboardModel(const NerLeaderboardModel&) = delete; - NerLeaderboardModel(const NerLeaderboardModel&&) = delete; - NerLeaderboardModel &operator=(const NerLeaderboardModel&) = delete; - NerLeaderboardModel &operator=(const NerLeaderboardModel&&) = delete; - ~NerLeaderboardModel(); - - void removeAllRows(); - void addRow(std::string& entityName, NerNamedEntityType entityType, float score); -}; - -} -#endif // M8RUI_NER_LEADERBOARD_MODEL_H diff --git a/app/src/qt/ner_leaderboard_view.cpp b/app/src/qt/ner_leaderboard_view.cpp deleted file mode 100644 index 0009011f..00000000 --- a/app/src/qt/ner_leaderboard_view.cpp +++ /dev/null @@ -1,61 +0,0 @@ -/* - ner_leaderboard_view.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_leaderboard_view.h" - -namespace m8r { - -NerLeaderboardView::NerLeaderboardView(QWidget* parent) - : QTableView(parent) -{ - verticalHeader()->setVisible(false); - // BEFARE this kills performance: verticalHeader()->setSectionResizeMode(QHeaderView::ResizeToContents); - - // IMPORTANT this must b in constructors - causes CPU high consuption loop if in an event handler - verticalHeader()->setSectionResizeMode(QHeaderView::Fixed); - - setSortingEnabled(true); - - setEditTriggers(QAbstractItemView::NoEditTriggers); - setSelectionBehavior(QAbstractItemView::SelectRows); - setSelectionMode(QAbstractItemView::SingleSelection); - - // ensure HTML cells rendering - HtmlDelegate* delegate = new HtmlDelegate(); - setItemDelegate(delegate); -} - -void NerLeaderboardView::resizeEvent(QResizeEvent* event) -{ - MF_DEBUG("NerLeaderboardView::resizeEvent " << event << std::endl); - - if(horizontalHeader()->length() > 0) { - // ensure that 1st column gets the remaining space from others - horizontalHeader()->setSectionResizeMode(0, QHeaderView::Stretch); - } - verticalHeader()->setDefaultSectionSize(fontMetrics().height()*1.5); - - // type - this->setColumnWidth(1, this->fontMetrics().averageCharWidth()*15); - // % - this->setColumnWidth(2, this->fontMetrics().averageCharWidth()*12); - - QTableView::resizeEvent(event); -} - -} // m8r namespace diff --git a/app/src/qt/ner_leaderboard_view.h b/app/src/qt/ner_leaderboard_view.h deleted file mode 100644 index 3fd0d558..00000000 --- a/app/src/qt/ner_leaderboard_view.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - ner_leaderboard_view.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8RUI_NER_LEADERBOARD_VIEW_H -#define M8RUI_NER_LEADERBOARD_VIEW_H - -#include - -#include "html_delegate.h" - -namespace m8r { - -class NerLeaderboardView : public QTableView -{ - Q_OBJECT - -public: - explicit NerLeaderboardView(QWidget* parent); - NerLeaderboardView(const NerLeaderboardView&) = delete; - NerLeaderboardView(const NerLeaderboardView&&) = delete; - NerLeaderboardView &operator=(const NerLeaderboardView&) = delete; - NerLeaderboardView &operator=(const NerLeaderboardView&&) = delete; - virtual ~NerLeaderboardView() override {} - - virtual void resizeEvent(QResizeEvent* event) override; -}; - -} -#endif // M8RUI_NER_LEADERBOARD_VIEW_H diff --git a/app/src/qt/ner_main_window_worker_thread.cpp b/app/src/qt/ner_main_window_worker_thread.cpp deleted file mode 100644 index 99eb6eec..00000000 --- a/app/src/qt/ner_main_window_worker_thread.cpp +++ /dev/null @@ -1,38 +0,0 @@ -/* - ner_main_window_worker_thread.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_main_window_worker_thread.h" - -namespace m8r { - -NerMainWindowWorkerThread::~NerMainWindowWorkerThread() -{ - delete progressDialog; -} - -void NerMainWindowWorkerThread::process() -{ - mind->recognizePersons(orloj->getOutlineView()->getCurrentOutline(), entityFilter, *result); - - progressDialog->hide(); - - MF_DEBUG("NER initialization and prediction WORKER finished" << endl); - emit finished(); -} - -} // m8r namespace diff --git a/app/src/qt/ner_main_window_worker_thread.h b/app/src/qt/ner_main_window_worker_thread.h deleted file mode 100644 index 74ce99de..00000000 --- a/app/src/qt/ner_main_window_worker_thread.h +++ /dev/null @@ -1,91 +0,0 @@ -/* - ner_main_window_worker_thread.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8RUI_NER_MAIN_WINDOW_WORKER_THREAD_H -#define M8RUI_NER_MAIN_WINDOW_WORKER_THREAD_H - -#include - -#include - -#include "orloj_presenter.h" - -namespace m8r { - -class OrlojPresenter; - -/** - * @brief NER worker thread class. - * - * IMPORTANT: remember that QThread subclassing is ANTIPATTERN - QThread is just envelope. - * - * IMPORTANT: NEVER allocate heap objects (using new) in the constructor of the QObject - * class as this allocation is then performed on the main thread and not on the - * new QThread instance, meaning that the newly created object is then owned by the - * main thread and not the QThread instance. This will make your code fail to work. - * Instead, allocate such resources in the main function slot such as process() in - * this case as when that is called the object will be on the new thread instance - * and thus it will own the resource. - */ -class NerMainWindowWorkerThread : public QObject -{ - Q_OBJECT - - // just (parent) thread handle allowing to delete it when worker finishes - QThread* thread; - - Mind* mind; - OrlojPresenter* orloj; - int entityFilter; - std::vector* result; - QDialog* progressDialog; - -public: - explicit NerMainWindowWorkerThread( - QThread* t, - Mind* m, - OrlojPresenter* o, - int f, - std::vector* r, - QDialog* d) - { - this->thread = t; - this->mind = m; - this->orloj = o; - this->entityFilter = f; - this->result = r; - this->progressDialog = d; - } - NerMainWindowWorkerThread(const NerMainWindowWorkerThread&) = delete; - NerMainWindowWorkerThread(const NerMainWindowWorkerThread&&) = delete; - NerMainWindowWorkerThread &operator=(const NerMainWindowWorkerThread&) = delete; - NerMainWindowWorkerThread &operator=(const NerMainWindowWorkerThread&&) = delete; - ~NerMainWindowWorkerThread(); - - std::vector* getResult() { return result; } - -public slots: - void process(); - -signals: - void finished(); - void error(QString error); -}; - -} -#endif // M8RUI_NER_MAIN_WINDOW_WORKER_THREAD_H diff --git a/deps/mitie b/deps/mitie deleted file mode 160000 index 6f43a5b9..00000000 --- a/deps/mitie +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6f43a5b9cb8ddd852a047fb3d2e73b816ea54465 diff --git a/lib/lib.pro b/lib/lib.pro index a44412d0..b42ea8a9 100644 --- a/lib/lib.pro +++ b/lib/lib.pro @@ -41,13 +41,6 @@ win32 { DEPENDPATH += $$PWD/../deps/zlib-win/include } -# DEPRECATED: Mitie (NER library) -mfner { - DEFINES += MF_NER - INCLUDEPATH += $$PWD/../deps/mitie/mitielib/include - DEPENDPATH += $$PWD/../deps/mitie/mitielib/include -} - # debug mfdebug|mfunits { DEFINES += DO_MF_DEBUG @@ -162,13 +155,6 @@ SOURCES += \ src/mind/ai/autolinking/cmark_aho_corasick_block_autolinking_preprocessor.cpp } -# DEPRECATED: Mitie (NER library) -mfner { - SOURCES += \ - src/mind/ai/nlp/named_entity_recognition.cpp \ - src/mind/ai/nlp/ner_named_entity.cpp -} - HEADERS += \ ./src/debug.h \ ./src/exceptions.h \ @@ -303,13 +289,6 @@ HEADERS += \ src/mind/ai/autolinking/cmark_aho_corasick_block_autolinking_preprocessor.h } -# DEPRECATED: Mitie (NER library) -mfner { - HEADERS += \ - src/mind/ai/nlp/named_entity_recognition.h \ - src/mind/ai/nlp/ner_named_entity.h -} - win32 { HEADERS += \ ../deps/dirent/dirent.h \ diff --git a/lib/src/mind/ai/ai.cpp b/lib/src/mind/ai/ai.cpp index 8b2b2c2a..3179c8c6 100644 --- a/lib/src/mind/ai/ai.cpp +++ b/lib/src/mind/ai/ai.cpp @@ -23,9 +23,6 @@ namespace m8r { using namespace std; Ai::Ai(Memory& memory, Mind& mind) -#ifdef MF_NER - : ner{} -#endif { switch(Configuration::getInstance().getAaAlgorithm()) { case Configuration::AssociationAssessmentAlgorithm::BOW: @@ -37,12 +34,6 @@ Ai::Ai(Memory& memory, Mind& mind) default: aa = nullptr; } - -#ifdef MF_NER - // TODO get MODEL location from configuration - static std::string nerModelPath{"/home/dvorka/p/mindforger/lab/ner/MITIE/MITIE-models/english/ner_model.dat"}; - ner.setNerModel(nerModelPath); -#endif } Ai::~Ai() diff --git a/lib/src/mind/ai/ai.h b/lib/src/mind/ai/ai.h index b49c937b..5d4d6b68 100644 --- a/lib/src/mind/ai/ai.h +++ b/lib/src/mind/ai/ai.h @@ -29,9 +29,6 @@ #include "./aa_model.h" #include "./ai_aa_weighted_fts.h" #include "./ai_aa_bow.h" -#ifdef MF_NER - #include "./nlp/named_entity_recognition.h" -#endif namespace m8r { @@ -76,14 +73,6 @@ class Ai // Associations assessment implemenations: AA @ weighted FTS, AA @ BoW AiAssociationsAssessment* aa; -#ifdef MF_NER - /* - * Named-entity recognition (NER) - */ - - NamedEntityRecognition ner; -#endif - /* * Neural network models */ @@ -127,17 +116,6 @@ class Ai return aa->getAssociatedNotes(words, associations, self); } -#ifdef MF_NER - bool isNerInitialized() const { return ner.isInitialized(); } - - /** - * @brief Recognize person names in O. - */ - void recognizePersons(const Outline* outline, int entityFilter, std::vector& result) { - ner.recognizePersons(outline, entityFilter, result); - } -#endif - /** * @brief Clear, but don't deallocate. * @@ -162,17 +140,6 @@ class Ai * @brief Train associations assessment neural network once memory is learned. */ void trainAaNn(); - -public: -#ifdef DO_MF_DEBUG - static void print(const Note* n, std::vector>& leaderboard) { - std::cout << "Note '" << n->getName() << "' AA leaderboard("<< leaderboard.size() <<"):" << std::endl; - int i=1; - for(auto& nn:leaderboard) { - std::cout << " #" << i++ << " '" << nn.first->getName() << "' ~ " << nn.second << std::endl; - } - } -#endif }; } diff --git a/lib/src/mind/ai/nlp/named_entity_recognition.cpp b/lib/src/mind/ai/nlp/named_entity_recognition.cpp deleted file mode 100644 index 18b9085e..00000000 --- a/lib/src/mind/ai/nlp/named_entity_recognition.cpp +++ /dev/null @@ -1,178 +0,0 @@ -/* - named_entity_recognition.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "named_entity_recognition.h" - -namespace m8r { - -using namespace std; - -NamedEntityRecognition::NamedEntityRecognition() - : initilized{false}, nerModel{} -{ -} - -NamedEntityRecognition::~NamedEntityRecognition() -{ -} - -void NamedEntityRecognition::setNerModel(const std::string& nerModel) { - std::lock_guard criticalSection{initMutex}; - - initilized = false; - nerModelPath = nerModel; -} - -// this method is NOT synchronized - callers are synchronized so that race condition is avoided -bool NamedEntityRecognition::loadAndInitNerModel() -{ - if(!initilized) { - // Load MITIE's named entity extractor from disk. Each file in the MITIE-models - // folder begins with a string containing the name of the serialized class. In - // this case classname contains "mitie::named_entity_extractor". It can be used to - // identify what is in any particular file. However, in this example we don't need - // it so it is just ignored. -#ifdef DO_MF_DEBUG - MF_DEBUG("NER loading model: " << nerModelPath << endl); - auto begin = chrono::high_resolution_clock::now(); -#endif - string classname; - dlib::deserialize(nerModelPath) >> classname >> nerModel; - initilized = true; -#ifdef DO_MF_DEBUG - auto end = chrono::high_resolution_clock::now(); - MF_DEBUG("NER model loaded in " << chrono::duration_cast(end-begin).count()/1000.0 << "ms" << endl); -#endif - -#ifdef DO_MF_DEBUG - // print out what kind of tags this tagger can predict. - const std::vector tagstr = nerModel.get_tag_name_strings(); - MF_DEBUG("NER tagger supports "<< tagstr.size() <<" tags:" << endl); - for(unsigned int i = 0; i < tagstr.size(); ++i) { - MF_DEBUG(" " << tagstr[i] << endl); - } -#endif - } - - return true; -} - -vector NamedEntityRecognition::tokenizeFile(const string& filename) -{ - ifstream fin(filename.c_str()); - if(!fin) { - cout << "Unable to load input text file" << endl; - exit(EXIT_FAILURE); - } - - // The conll_tokenizer splits the contents of an istream into a bunch of words and is - // MITIE's default tokenization method. - mitie::conll_tokenizer tok(fin); - std::vector tokens; - string token; - - // Read the tokens out of the file one at a time and store into tokens. - while(tok(token)) { - tokens.push_back(token); - } - - return tokens; -} - -bool NamedEntityRecognition::recognizePersons(vector& result) -{ - UNUSED_ARG(result); - - - - std::lock_guard criticalSection{initMutex}; - - if(loadAndInitNerModel()) { - // ... - } - - return false; -} - -bool NamedEntityRecognition::recognizePersons(const Outline* outline, int entityTypeFilter, vector& result) -{ - std::lock_guard criticalSection{initMutex}; - - if(loadAndInitNerModel()) { - try { - // tokenize data to prepare it for the tagger - MF_DEBUG("NER: tokenizing O " << outline->getKey() << endl); - std::vector tokens = tokenizeFile(outline->getKey()); - - std::vector > chunks; - std::vector chunk_tags; - std::vector chunk_scores; - - // Now detect all the entities in the text file we loaded and print them to the screen. - // The output of this function is a set of "chunks" of tokens, each a named entity. - // Additionally, if it is useful for your application a confidence score for each "chunk" - // is available by using the predict() method. The larger the score the more - // confident MITIE is in the tag. -#ifdef DO_MF_DEBUG - MF_DEBUG("NER predicting..." << endl); - auto begin = chrono::high_resolution_clock::now(); -#endif - nerModel.predict(tokens, chunks, chunk_tags, chunk_scores); -#ifdef DO_MF_DEBUG - auto end = chrono::high_resolution_clock::now(); - MF_DEBUG("NER prediction done in " << chrono::duration_cast(end-begin).count()/1000.0 << "ms" << endl); -#endif - - // If a confidence score is not necessary for your application you can detect entities - // using the operator() method as shown in the following line. - //ner(tokens, chunks, chunk_tags); - - MF_DEBUG("\nNumber of named entities detected: " << chunks.size() << endl); - const std::vector tagstr = nerModel.get_tag_name_strings(); - string entityName{}; - for (unsigned int i = 0; i < chunks.size(); ++i) { - if((1<(1<(chunk_scores[i])}; - result.push_back(entity); - } - } - - return true; - } - catch(std::exception& e) { - cerr << "NRE error: " << e.what() << endl; - } - } - - return false; -} - -} // m8r namespace diff --git a/lib/src/mind/ai/nlp/named_entity_recognition.h b/lib/src/mind/ai/nlp/named_entity_recognition.h deleted file mode 100644 index ffb80ebb..00000000 --- a/lib/src/mind/ai/nlp/named_entity_recognition.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - named_entity_recognition.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8R_NAMED_ENTITY_RECOGNITION_H -#define M8R_NAMED_ENTITY_RECOGNITION_H - -#include -#include - -#include -#include -#include -#include -#include - -#include "../../deps/mitie/mitielib/include/mitie/named_entity_extractor.h" -#include "../../deps/mitie/mitielib/include/mitie/conll_tokenizer.h" -#include "../../deps/mitie/mitielib/include/mitie.h" - -#include "ner_named_entity.h" - -#include "../../../model/outline.h" - -namespace m8r { - -class NamedEntityRecognition -{ -private: - std::mutex initMutex; - bool initilized; - - std::string nerModelPath; - mitie::named_entity_extractor nerModel; - -public: - explicit NamedEntityRecognition(); - NamedEntityRecognition(const NamedEntityRecognition&) = delete; - NamedEntityRecognition(const NamedEntityRecognition&&) = delete; - NamedEntityRecognition &operator=(const NamedEntityRecognition&) = delete; - NamedEntityRecognition &operator=(const NamedEntityRecognition&&) = delete; - ~NamedEntityRecognition(); - - bool isInitialized() const { return initilized; } - - /** - * @brief Set NER model location. - * - * This set path to the method, but it does NOT load and initialize it. - */ - void setNerModel(const std::string& nerModel); - - /** - * @brief NRE persons in memory. - */ - bool recognizePersons(std::vector& result); - - /** - * @brief NRE persons in O. - */ - bool recognizePersons(const Outline* outline, int entityTypeFilter, std::vector& result); - -private: - std::vector tokenizeFile(const std::string& filename); - - /** - * @brief Load and initialize NER model file. - * - * NER file is typically huge (MBs) therefore it is loaded and initialized on demand. - */ - bool loadAndInitNerModel(); -}; - -} -#endif // M8R_NAMED_ENTITY_RECOGNITION_H diff --git a/lib/src/mind/ai/nlp/ner_named_entity.cpp b/lib/src/mind/ai/nlp/ner_named_entity.cpp deleted file mode 100644 index c04c4dd8..00000000 --- a/lib/src/mind/ai/nlp/ner_named_entity.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - ner_named_entity.cpp MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#include "ner_named_entity.h" - -namespace m8r { -} // m8r namespace diff --git a/lib/src/mind/ai/nlp/ner_named_entity.h b/lib/src/mind/ai/nlp/ner_named_entity.h deleted file mode 100644 index a29a940e..00000000 --- a/lib/src/mind/ai/nlp/ner_named_entity.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - ner_named_entity.h MindForger thinking notebook - - Copyright (C) 2016-2024 Martin Dvorak - - This program is free software; you can redistribute it and/or - modify it under the terms of the GNU General Public License - as published by the Free Software Foundation; either version 2 - of the License, or (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ -#ifndef M8R_NER_NAMED_ENTITY_H -#define M8R_NER_NAMED_ENTITY_H - -#include - -namespace m8r { - -enum NerNamedEntityType { - PERSON = 1<<0, - LOCATION = 1<<1, - ORGANIZATION = 1<<2, - MISC = 1<<3 -}; - -struct NerNamedEntity -{ - std::string name; - NerNamedEntityType type; - float score; - - explicit NerNamedEntity(const std::string& n, NerNamedEntityType t, float s) { - this->name = n; - this->type = t; - this->score = s; - } -}; - -} -#endif // M8R_NER_NAMED_ENTITY_H diff --git a/lib/src/mind/mind.cpp b/lib/src/mind/mind.cpp index 1b596a63..4d183efe 100644 --- a/lib/src/mind/mind.cpp +++ b/lib/src/mind/mind.cpp @@ -1408,23 +1408,6 @@ MindStatistics* Mind::getStatistics() return stats; } -/* - * NER - */ - -#ifdef MF_NER - -bool Mind::isNerInitilized() const -{ - return ai->isNerInitialized(); -} - -void Mind::recognizePersons(const Outline* outline, int entityFilter, std::vector& result) { - ai->recognizePersons(outline, entityFilter, result); -} - -#endif - // unique_ptr template BREAKS Qt Developer indentation > stored at EOF unique_ptr> Mind::findOutlineByNameFts(const string& pattern) const { diff --git a/lib/src/mind/mind.h b/lib/src/mind/mind.h index 1ee217ed..2174ea9a 100644 --- a/lib/src/mind/mind.h +++ b/lib/src/mind/mind.h @@ -36,9 +36,6 @@ #include "../config/configuration.h" #include "../representations/representation_interceptor.h" #include "../representations/markdown/markdown_configuration_representation.h" -#ifdef MF_NER - #include "ai/nlp/named_entity_recognition.h" -#endif namespace m8r { @@ -330,17 +327,6 @@ class Mind : public OntologyProvider size_t getTriplesCount() const { return triples.size(); } -#ifdef MF_NER - - /* - * NRE - */ - - bool isNerInitilized() const; - void recognizePersons(const Outline* outline, int entityFilter, std::vector& result); - -#endif - /* * REMEMBERING */ diff --git a/mindforger.pro b/mindforger.pro index 9f0f6f94..22b0835e 100644 --- a/mindforger.pro +++ b/mindforger.pro @@ -32,8 +32,6 @@ # qmake CONFIG+=mfci ... CI build (AppVeyor, ...) w/ build info @ window title # qmake CONFIG+=mfrc ... RC build w/ build info @ window title # qmake CONFIG+=mfunits ... option to run unit tests -# qmake CONFIG+=mfllamacpp ... EXPERIMENTAL option to enable wingman @ llama.cpp -# qmake CONFIG+=mfner ... DEPRECATED: build project w/ NER and link dlib/MITIE # # Warning: DEPRECATED build options will be removed in the next major release. #