Skip to content

Commit

Permalink
feat: define language as attribute instead of entity
Browse files Browse the repository at this point in the history
  • Loading branch information
@marrouchi
marrouchi committed Sep 23, 2024
1 parent c4b2e97 commit c304992
Show file tree
Hide file tree
Showing 18 changed files with 247 additions and 85 deletions.
7 changes: 7 additions & 0 deletions api/src/chat/repositories/message.repository.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { Injectable, Optional } from '@nestjs/common';
import { InjectModel } from '@nestjs/mongoose';
import { Model } from 'mongoose';

import { LanguageService } from '@/i18n/services/language.service';
import { LoggerService } from '@/logger/logger.service';
import { NlpSampleCreateDto } from '@/nlp/dto/nlp-sample.dto';
import { NlpSampleState } from '@/nlp/schemas/types';
Expand All @@ -36,10 +37,13 @@ export class MessageRepository extends BaseRepository<

private readonly logger: LoggerService;

private readonly languageService: LanguageService;

constructor(
@InjectModel(Message.name) readonly model: Model<AnyMessage>,
@Optional() nlpSampleService?: NlpSampleService,
@Optional() logger?: LoggerService,
@Optional() languageService?: LanguageService,
) {
super(
model,
Expand All @@ -49,6 +53,7 @@ export class MessageRepository extends BaseRepository<
);
this.logger = logger;
this.nlpSampleService = nlpSampleService;
this.languageService = languageService;
}

/**
Expand All @@ -72,10 +77,12 @@ export class MessageRepository extends BaseRepository<
'message' in _doc &&
'text' in _doc.message
) {
const defaultLang = await this.languageService?.getDefaultLanguage();
const record: NlpSampleCreateDto = {
text: _doc.message.text,
type: NlpSampleState.inbox,
trained: false,
language: defaultLang.id,
};
try {
await this.nlpSampleService.findOneOrCreate(record, record);
Expand Down
9 changes: 9 additions & 0 deletions api/src/extensions/helpers/nlp/default/__test__/__mock__/base.mock.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ export const baseNlpEntity = {
builtin: true,
};

export const baseLanguage = {
...modelInstance,
title: 'English',
code: 'en',
default: true,
};

export const entitiesMock: NlpEntityFull[] = [
{
...baseNlpEntity,
Expand Down Expand Up @@ -89,6 +96,7 @@ export const samplesMock: NlpSampleFull[] = [
],
trained: false,
type: NlpSampleState.train,
language: baseLanguage,
},
{
...modelInstance,
Expand All @@ -112,5 +120,6 @@ export const samplesMock: NlpSampleFull[] = [
],
trained: false,
type: NlpSampleState.train,
language: baseLanguage,
},
];
2 changes: 1 addition & 1 deletion api/src/i18n/i18n.module.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ export class I18nModule extends NativeI18nModule {
TranslationService,
TranslationSeeder,
]),
exports: exports.concat(I18nService),
exports: exports.concat(I18nService, LanguageService),
};
}
}
41 changes: 39 additions & 2 deletions api/src/i18n/services/language.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,53 @@
* 3. SaaS Restriction: This software, or any derivative of it, may not be used to offer a competing product or service (SaaS) without prior written consent from Hexastack. Offering the software as a service or using it in a commercial cloud environment without express permission is strictly prohibited.
*/

import { Injectable } from '@nestjs/common';
import { CACHE_MANAGER } from '@nestjs/cache-manager';
import { Inject, Injectable } from '@nestjs/common';
import { Cache } from 'cache-manager';

import {
DEFAULT_LANGUAGE_CACHE_KEY,
LANGUAGES_CACHE_KEY,
} from '@/utils/constants/cache';
import { Cacheable } from '@/utils/decorators/cacheable.decorator';
import { BaseService } from '@/utils/generics/base-service';

import { LanguageRepository } from '../repositories/language.repository';
import { Language } from '../schemas/language.schema';

@Injectable()
export class LanguageService extends BaseService<Language> {
constructor(readonly repository: LanguageRepository) {
constructor(
readonly repository: LanguageRepository,
@Inject(CACHE_MANAGER) private readonly cacheManager: Cache,
) {
super(repository);
}

/**
* Retrieves all available languages from the repository.
*
* @returns A promise that resolves to an object where each key is a language code
* and the corresponding value is the `Language` object.
*/
@Cacheable(LANGUAGES_CACHE_KEY)
async getLanguages() {
const languages = await this.findAll();
return languages.reduce((acc, curr) => {
return {
...acc,
[curr.code]: curr,
};
}, {});
}

/**
* Retrieves the default language from the repository.
*
* @returns A promise that resolves to the default `Language` object.
*/
@Cacheable(DEFAULT_LANGUAGE_CACHE_KEY)
async getDefaultLanguage() {
return await this.findOne({ default: true });
}
}
90 changes: 53 additions & 37 deletions api/src/nlp/controllers/nlp-sample.controller.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ import { Test, TestingModule } from '@nestjs/testing';
import { AttachmentRepository } from '@/attachment/repositories/attachment.repository';
import { AttachmentModel } from '@/attachment/schemas/attachment.schema';
import { AttachmentService } from '@/attachment/services/attachment.service';
import { LanguageRepository } from '@/i18n/repositories/language.repository';
import { Language, LanguageModel } from '@/i18n/schemas/language.schema';
import { I18nService } from '@/i18n/services/i18n.service';
import { LanguageService } from '@/i18n/services/language.service';
import { LoggerService } from '@/logger/logger.service';
import { SettingRepository } from '@/setting/repositories/setting.repository';
import { SettingModel } from '@/setting/schemas/setting.schema';
Expand Down Expand Up @@ -57,7 +60,9 @@ describe('NlpSampleController', () => {
let nlpEntityService: NlpEntityService;
let nlpValueService: NlpValueService;
let attachmentService: AttachmentService;
let languageService: LanguageService;
let byeJhonSampleId: string;
let languages: Language[];

beforeAll(async () => {
const module: TestingModule = await Test.createTestingModule({
Expand All @@ -74,6 +79,7 @@ describe('NlpSampleController', () => {
NlpEntityModel,
NlpValueModel,
SettingModel,
LanguageModel,
]),
],
providers: [
Expand All @@ -88,6 +94,8 @@ describe('NlpSampleController', () => {
NlpValueRepository,
NlpSampleService,
NlpSampleEntityService,
LanguageRepository,
LanguageService,
EventEmitter2,
NlpService,
SettingRepository,
Expand Down Expand Up @@ -122,6 +130,8 @@ describe('NlpSampleController', () => {
})
).id;
attachmentService = module.get<AttachmentService>(AttachmentService);
languageService = module.get<LanguageService>(LanguageService);
languages = await languageService.findAll();
});
afterAll(async () => {
await closeInMongodConnection();
Expand All @@ -134,7 +144,7 @@ describe('NlpSampleController', () => {
const pageQuery = getPageQuery<NlpSample>({ sort: ['text', 'desc'] });
const result = await nlpSampleController.findPage(
pageQuery,
['entities'],
['language', 'entities'],
{},
);
const nlpSamples = await nlpSampleService.findAll();
Expand All @@ -146,6 +156,7 @@ describe('NlpSampleController', () => {
entities: nlpSampleEntities.filter((currSampleEntity) => {
return currSampleEntity.sample === currSample.id;
}),
language: languages.find((lang) => lang.id === currSample.language),
};
acc.push(sampleWithEntities);
return acc;
Expand All @@ -163,7 +174,12 @@ describe('NlpSampleController', () => {
['invalidCriteria'],
{},
);
expect(result).toEqualPayload(nlpSampleFixtures);
expect(result).toEqualPayload(
nlpSampleFixtures.map((sample) => ({
...sample,
language: languages[sample.language].id,
})),
);
});
});

Expand All @@ -177,14 +193,19 @@ describe('NlpSampleController', () => {

describe('create', () => {
it('should create nlp sample', async () => {
const enLang = await languageService.findOne({ code: 'en' });
const nlSample: NlpSampleDto = {
text: 'text1',
trained: true,
type: NlpSampleState.test,
entities: [],
language: enLang.id,
};
const result = await nlpSampleController.create(nlSample);
expect(result).toEqualPayload(nlSample);
expect(result).toEqualPayload({
...nlSample,
language: enLang,
});
});
});

Expand All @@ -209,7 +230,10 @@ describe('NlpSampleController', () => {
const result = await nlpSampleController.findOne(yessSample.id, [
'invalidCreteria',
]);
expect(result).toEqualPayload(nlpSampleFixtures[0]);
expect(result).toEqualPayload({
...nlpSampleFixtures[0],
language: languages[nlpSampleFixtures[0].language].id,
});
});

it('should find a nlp sample and populate its entities', async () => {
Expand All @@ -225,6 +249,7 @@ describe('NlpSampleController', () => {
const samplesWithEntities = {
...nlpSampleFixtures[0],
entities: [yessSampleEntity],
language: languages[nlpSampleFixtures[0].language],
};
expect(result).toEqualPayload(samplesWithEntities);
});
Expand All @@ -241,6 +266,9 @@ describe('NlpSampleController', () => {
const yessSample = await nlpSampleService.findOne({
text: 'yess',
});
const frLang = await languageService.findOne({
code: 'fr',
});
const result = await nlpSampleController.updateOne(yessSample.id, {
text: 'updated',
trained: true,
Expand All @@ -251,6 +279,7 @@ describe('NlpSampleController', () => {
value: 'update',
},
],
language: frLang.id,
});
const updatedSample = {
text: 'updated',
Expand All @@ -263,19 +292,25 @@ describe('NlpSampleController', () => {
value: expect.stringMatching(/^[a-z0-9]+$/),
},
],
language: frLang,
};
expect(result.text).toEqual(updatedSample.text);
expect(result.type).toEqual(updatedSample.type);
expect(result.trained).toEqual(updatedSample.trained);
expect(result.entities).toMatchObject(updatedSample.entities);
expect(result.language).toEqualPayload(updatedSample.language);
});

it('should throw exception when nlp sample id not found', async () => {
const frLang = await languageService.findOne({
code: 'fr',
});
await expect(
nlpSampleController.updateOne(byeJhonSampleId, {
text: 'updated',
trained: true,
type: NlpSampleState.test,
language: frLang.id,
}),
).rejects.toThrow(NotFoundException);
});
Expand Down Expand Up @@ -352,7 +387,7 @@ describe('NlpSampleController', () => {
).id;
const mockCsvData: string = [
`text,intent,language`,
`Was kostet dieser bmw,preis,de`,
`How much does a BMW cost?,price,en`,
].join('\n');
jest.spyOn(fs, 'existsSync').mockReturnValueOnce(true);
jest.spyOn(fs, 'readFileSync').mockReturnValueOnce(mockCsvData);
Expand All @@ -361,58 +396,39 @@ describe('NlpSampleController', () => {
const intentEntityResult = await nlpEntityService.findOne({
name: 'intent',
});
const languageEntityResult = await nlpEntityService.findOne({
name: 'language',
});
const preisValueResult = await nlpValueService.findOne({
value: 'preis',
});
const deValueResult = await nlpValueService.findOne({
value: 'de',
const priceValueResult = await nlpValueService.findOne({
value: 'price',
});
const textSampleResult = await nlpSampleService.findOne({
text: 'Was kostet dieser bmw',
text: 'How much does a BMW cost?',
});
const language = await languageService.findOne({
code: 'en',
});
const intentEntity = {
name: 'intent',
lookups: ['trait'],
doc: '',
builtin: false,
};
const languageEntity = {
name: 'language',
lookups: ['trait'],
builtin: false,
doc: '',
};
const preisVlueEntity = await nlpEntityService.findOne({
const priceValueEntity = await nlpEntityService.findOne({
name: 'intent',
});
const preisValue = {
value: 'preis',
expressions: [],
builtin: false,
entity: preisVlueEntity.id,
};
const deValueEntity = await nlpEntityService.findOne({
name: 'language',
});
const deValue = {
value: 'de',
const priceValue = {
value: 'price',
expressions: [],
builtin: false,
entity: deValueEntity.id,
entity: priceValueEntity.id,
};
const textSample = {
text: 'Was kostet dieser bmw',
text: 'How much does a BMW cost?',
trained: false,
type: 'train',
language: language.id,
};

expect(languageEntityResult).toEqualPayload(languageEntity);
expect(intentEntityResult).toEqualPayload(intentEntity);
expect(preisValueResult).toEqualPayload(preisValue);
expect(deValueResult).toEqualPayload(deValue);
expect(priceValueResult).toEqualPayload(priceValue);
expect(textSampleResult).toEqualPayload(textSample);
expect(result).toEqual({ success: true });
});
Expand Down
Loading

0 comments on commit c304992

Please sign in to comment.