From 179de9baf740ea3b696cc8b23777cf5410dc5361 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 14:21:02 +0900 Subject: [PATCH 1/7] chore(deps): add cheerio --- package-lock.json | 213 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + 2 files changed, 215 insertions(+) diff --git a/package-lock.json b/package-lock.json index f978b16..afda8a8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@nestjs/schedule": "^3.0.3", "@nestjs/swagger": "^7.1.16", "axios": "^1.3.4", + "cheerio": "^1.0.0-rc.12", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -40,6 +41,7 @@ "@nestjs/schematics": "^9.0.0", "@nestjs/testing": "^9.0.0", "@prisma/client": "^5.3.1", + "@types/cheerio": "^0.22.35", "@types/cookie-parser": "^1.4.3", "@types/cron": "^2.0.1", "@types/express": "^4.17.13", @@ -3773,6 +3775,15 @@ "@types/node": "*" } }, + "node_modules/@types/cheerio": { + "version": "0.22.35", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz", + "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -4986,6 +4997,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "node_modules/bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -5206,6 +5222,42 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "node_modules/cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + }, + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -5595,6 +5647,32 @@ "node": ">= 8" } }, + "node_modules/css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/dayjs": { "version": "1.11.9", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.9.tgz", @@ -9236,6 +9314,17 @@ "node": ">=8" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -9429,6 +9518,40 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "dependencies": { + "entities": "^4.4.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "dependencies": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/parseley": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", @@ -15011,6 +15134,15 @@ "@types/node": "*" } }, + "@types/cheerio": { + "version": "0.22.35", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz", + "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/connect": { "version": "3.4.35", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.35.tgz", @@ -16025,6 +16157,11 @@ } } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==" + }, "bowser": { "version": "2.11.0", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.11.0.tgz", @@ -16168,6 +16305,33 @@ "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.12", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.12.tgz", + "integrity": "sha512-VqR8m68vM46BNnuZ5NtnGBKIE/DfN0cRIzg9n40EIq9NOv90ayxLBXA8fXC5gquFRGJSTRqBq25Jt2ECLR431Q==", + "requires": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1", + "htmlparser2": "^8.0.1", + "parse5": "^7.0.0", + "parse5-htmlparser2-tree-adapter": "^7.0.0" + } + }, + "cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "requires": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + } + }, "chokidar": { "version": "3.5.3", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.3.tgz", @@ -16467,6 +16631,23 @@ "which": "^2.0.1" } }, + "css-select": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.1.0.tgz", + "integrity": "sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==", + "requires": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + } + }, + "css-what": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.1.0.tgz", + "integrity": "sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==" + }, "dayjs": { "version": "1.11.9", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.9.tgz", @@ -19244,6 +19425,14 @@ "path-key": "^3.0.0" } }, + "nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "requires": { + "boolbase": "^1.0.0" + } + }, "object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -19377,6 +19566,30 @@ "lines-and-columns": "^1.1.6" } }, + "parse5": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.1.2.tgz", + "integrity": "sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==", + "requires": { + "entities": "^4.4.0" + }, + "dependencies": { + "entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==" + } + } + }, + "parse5-htmlparser2-tree-adapter": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.0.0.tgz", + "integrity": "sha512-B77tOZrqqfUfnVcOrUvfdLbz4pu4RopLD/4vmu3HUPswwTA8OH0EMW9BlWR2B0RCoiZRAHEUu7IxeP1Pd1UU+g==", + "requires": { + "domhandler": "^5.0.2", + "parse5": "^7.0.0" + } + }, "parseley": { "version": "0.12.1", "resolved": "https://registry.npmjs.org/parseley/-/parseley-0.12.1.tgz", diff --git a/package.json b/package.json index 5105da7..6510ff8 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "@nestjs/schedule": "^3.0.3", "@nestjs/swagger": "^7.1.16", "axios": "^1.3.4", + "cheerio": "^1.0.0-rc.12", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -52,6 +53,7 @@ "@nestjs/schematics": "^9.0.0", "@nestjs/testing": "^9.0.0", "@prisma/client": "^5.3.1", + "@types/cheerio": "^0.22.35", "@types/cookie-parser": "^1.4.3", "@types/cron": "^2.0.1", "@types/express": "^4.17.13", From 27713a8695494d1a44dc58fa058acf08677a85ab Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 14:21:22 +0900 Subject: [PATCH 2/7] feat: get academic notices --- src/notice/dto/getAllNotice.dto.ts | 4 +- src/notice/notice.module.ts | 18 +++++-- src/notice/notice.service.ts | 77 ++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/notice/dto/getAllNotice.dto.ts b/src/notice/dto/getAllNotice.dto.ts index caa3280..963b67a 100644 --- a/src/notice/dto/getAllNotice.dto.ts +++ b/src/notice/dto/getAllNotice.dto.ts @@ -65,7 +65,7 @@ export class GetAllNoticeQueryDto { @IsString() @IsEnum(['deadline', 'hot', 'recent']) @IsOptional() - orderBy?: string; + orderBy?: 'recent' | 'deadline' | 'hot'; @ApiProperty({ example: 'own', @@ -75,5 +75,5 @@ export class GetAllNoticeQueryDto { @IsString() @IsEnum(['own', 'reminders']) @IsOptional() - my?: string; + my?: 'own' | 'reminders'; } diff --git a/src/notice/notice.module.ts b/src/notice/notice.module.ts index 1ed1c9c..23a1909 100644 --- a/src/notice/notice.module.ts +++ b/src/notice/notice.module.ts @@ -1,15 +1,23 @@ +import { HttpModule } from '@nestjs/axios'; import { Module } from '@nestjs/common'; -import { NoticeController } from './notice.controller'; -import { NoticeService } from './notice.service'; -import { UserModule } from 'src/user/user.module'; import { ConfigModule } from '@nestjs/config'; -import { ImageModule } from 'src/image/image.module'; import { FcmModule } from 'src/global/service/fcm.module'; +import { ImageModule } from 'src/image/image.module'; import { PrismaModule } from 'src/prisma/prisma.module'; +import { UserModule } from 'src/user/user.module'; +import { NoticeController } from './notice.controller'; import { NoticeRepository } from './notice.repository'; +import { NoticeService } from './notice.service'; @Module({ - imports: [ConfigModule, UserModule, ImageModule, FcmModule, PrismaModule], + imports: [ + ConfigModule, + UserModule, + ImageModule, + FcmModule, + PrismaModule, + HttpModule, + ], controllers: [NoticeController], providers: [NoticeService, NoticeRepository], }) diff --git a/src/notice/notice.service.ts b/src/notice/notice.service.ts index 478ef33..98195a0 100644 --- a/src/notice/notice.service.ts +++ b/src/notice/notice.service.ts @@ -1,8 +1,19 @@ +import { HttpService } from '@nestjs/axios'; import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { Cron } from '@nestjs/schedule'; +import cheerio from 'cheerio'; import dayjs from 'dayjs'; import { htmlToText } from 'html-to-text'; +import { + catchError, + filter, + firstValueFrom, + map, + of, + takeUntil, + throwError, +} from 'rxjs'; import { FcmService } from 'src/global/service/fcm.service'; import { ImageService } from 'src/image/image.service'; import { AdditionalNoticeDto } from './dto/additionalNotice.dto'; @@ -18,8 +29,10 @@ export class NoticeService { private readonly noticeRepository: NoticeRepository, private readonly imageService: ImageService, private readonly fcmService: FcmService, + private readonly httpService: HttpService, configService: ConfigService, ) { + this.crawlAcademicNotice(); this.s3Url = `https://s3.${configService.get( 'AWS_S3_REGION', )}.amazonaws.com/${configService.get('AWS_S3_BUCKET_NAME')}/`; @@ -193,4 +206,68 @@ export class NoticeService { ); }); } + + private async getAcademicNoticeList() { + const baseUrl = 'https://www.gist.ac.kr/kr/html/sub05/050209.html'; + const stream = this.httpService.get(baseUrl).pipe( + map((res) => res.data), + map((e) => cheerio.load(e)), + catchError(throwError), + ); + const $ = await firstValueFrom(stream); + const notices = $('table > tbody > tr') + .filter((_, e) => e.type === 'tag' && !e.attribs.class.includes('lstNtc')) + .toArray() + .map( + (e) => + e.type === 'tag' && { + id: Number.parseInt($(e).find('td').first().text().trim()), + title: $(e).find('td').eq(2).text().trim(), + link: `${baseUrl}${$(e).find('td').eq(2).find('a').attr('href')}`, + author: $(e).find('td').eq(3).text().trim(), + category: $(e).find('td').eq(1).text().trim(), + createdAt: $(e).find('td').eq(5).text().trim(), + }, + ); + return notices; + } + + private async getAcademicNotice(link: string) { + const baseUrl = 'https://www.gist.ac.kr/kr/html/sub05/050209.html'; + const stream = this.httpService.get(link).pipe( + map((res) => res.data), + map((e) => cheerio.load(e)), + catchError(throwError), + ); + const $ = await firstValueFrom(stream); + const files = $('.bd_detail_file > ul > li > a') + .toArray() + .map((e) => ({ + href: `${baseUrl}${$(e).attr('href')}`, + name: $(e).text().trim(), + type: $(e).attr('class') as + | 'doc' + | 'hwp' + | 'pdf' + | 'imgs' + | 'xls' + | 'etc', + })); + const content = $('.bd_detail_content').html().trim(); + return { files, content }; + } + + // @Cron('*/30 * * * * *') + async crawlAcademicNotice() { + const notices = await this.getAcademicNoticeList(); + console.log(notices); + // const recentNotice = await this.noticeRepository.getNoticeList({ + // limit: 1, + // orderBy: 'recent', + // tags: ['academic'], + // }); + // console.log(recentNotice); + // console.log(notices.map((n) => n.link)); + // console.log((await this.getAcademicNotice(notices[2].link)).files); + } } From 6e62ad328be72104ec3f40658e233d185300dad0 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 14:45:18 +0900 Subject: [PATCH 3/7] feat: create missing academic notices --- src/notice/notice.service.ts | 45 +++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/src/notice/notice.service.ts b/src/notice/notice.service.ts index 98195a0..1e5b99c 100644 --- a/src/notice/notice.service.ts +++ b/src/notice/notice.service.ts @@ -9,10 +9,11 @@ import { catchError, filter, firstValueFrom, + from, map, - of, - takeUntil, + takeWhile, throwError, + toArray, } from 'rxjs'; import { FcmService } from 'src/global/service/fcm.service'; import { ImageService } from 'src/image/image.service'; @@ -260,14 +261,36 @@ export class NoticeService { // @Cron('*/30 * * * * *') async crawlAcademicNotice() { const notices = await this.getAcademicNoticeList(); - console.log(notices); - // const recentNotice = await this.noticeRepository.getNoticeList({ - // limit: 1, - // orderBy: 'recent', - // tags: ['academic'], - // }); - // console.log(recentNotice); - // console.log(notices.map((n) => n.link)); - // console.log((await this.getAcademicNotice(notices[2].link)).files); + const recentNotice = await this.noticeRepository.getNoticeList({ + limit: 1, + orderBy: 'recent', + tags: ['academic'], + }); + from(notices).pipe( + filter((n) => n.title === recentNotice[0].contents[0].title), + ); + const noticesToCreate$ = from(notices).pipe( + takeWhile((n) => n.title !== recentNotice[0].contents[0].title), + toArray(), + map((n) => n.reverse()), + ); + const noticesToCreate = await firstValueFrom(noticesToCreate$); + for (const noticeMetadata of noticesToCreate) { + const notice = await this.getAcademicNotice(noticeMetadata.link); + const filesList = notice.files + .map((file) => `
  • ${file.name}
    • `) + .join(''); + const filesBody = `
      ${filesList}
    `; + const body = `${notice.files.length ? filesBody : ''}${notice.content}`; + await this.noticeRepository.createNotice( + { + title: noticeMetadata.title, + body, + images: [], + tags: [4], + }, + '1', + ); + } } } From 592aca5dcc4816d92d7b70f36a6ff23b99c104f1 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 14:51:55 +0900 Subject: [PATCH 4/7] feat: academic notice tag --- src/notice/notice.module.ts | 2 ++ src/notice/notice.service.ts | 8 +++++++- src/tag/tag.module.ts | 5 +++-- src/tag/tag.repository.ts | 22 +++++++++++++++++++++- src/tag/tag.service.ts | 6 +++++- 5 files changed, 38 insertions(+), 5 deletions(-) diff --git a/src/notice/notice.module.ts b/src/notice/notice.module.ts index 23a1909..561dd97 100644 --- a/src/notice/notice.module.ts +++ b/src/notice/notice.module.ts @@ -4,6 +4,7 @@ import { ConfigModule } from '@nestjs/config'; import { FcmModule } from 'src/global/service/fcm.module'; import { ImageModule } from 'src/image/image.module'; import { PrismaModule } from 'src/prisma/prisma.module'; +import { TagModule } from 'src/tag/tag.module'; import { UserModule } from 'src/user/user.module'; import { NoticeController } from './notice.controller'; import { NoticeRepository } from './notice.repository'; @@ -17,6 +18,7 @@ import { NoticeService } from './notice.service'; FcmModule, PrismaModule, HttpModule, + TagModule, ], controllers: [NoticeController], providers: [NoticeService, NoticeRepository], diff --git a/src/notice/notice.service.ts b/src/notice/notice.service.ts index 1e5b99c..e2ef3b0 100644 --- a/src/notice/notice.service.ts +++ b/src/notice/notice.service.ts @@ -17,6 +17,7 @@ import { } from 'rxjs'; import { FcmService } from 'src/global/service/fcm.service'; import { ImageService } from 'src/image/image.service'; +import { TagService } from 'src/tag/tag.service'; import { AdditionalNoticeDto } from './dto/additionalNotice.dto'; import { CreateNoticeDto } from './dto/createNotice.dto'; import { ForeignContentDto } from './dto/foreignContent.dto'; @@ -31,6 +32,7 @@ export class NoticeService { private readonly imageService: ImageService, private readonly fcmService: FcmService, private readonly httpService: HttpService, + private readonly tagService: TagService, configService: ConfigService, ) { this.crawlAcademicNotice(); @@ -282,12 +284,16 @@ export class NoticeService { .join(''); const filesBody = `
      ${filesList}
    `; const body = `${notice.files.length ? filesBody : ''}${notice.content}`; + const tags = await this.tagService.findOrCreateTags([ + 'academic', + noticeMetadata.category, + ]); await this.noticeRepository.createNotice( { title: noticeMetadata.title, body, images: [], - tags: [4], + tags: tags.map(({ id }) => id), }, '1', ); diff --git a/src/tag/tag.module.ts b/src/tag/tag.module.ts index 61683ea..a8c2661 100644 --- a/src/tag/tag.module.ts +++ b/src/tag/tag.module.ts @@ -1,12 +1,13 @@ import { Module } from '@nestjs/common'; -import { TagController } from './tag.controller'; -import { TagService } from './tag.service'; import { PrismaModule } from 'src/prisma/prisma.module'; +import { TagController } from './tag.controller'; import { TagRepository } from './tag.repository'; +import { TagService } from './tag.service'; @Module({ imports: [PrismaModule], controllers: [TagController], providers: [TagService, TagRepository], + exports: [TagService], }) export class TagModule {} diff --git a/src/tag/tag.repository.ts b/src/tag/tag.repository.ts index 318f97e..87e95b7 100644 --- a/src/tag/tag.repository.ts +++ b/src/tag/tag.repository.ts @@ -6,9 +6,9 @@ import { NotFoundException, } from '@nestjs/common'; import { Tag } from '@prisma/client'; +import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; import { PrismaService } from 'src/prisma/prisma.service'; import { GetTagDto } from './dto/getTag.dto'; -import { PrismaClientKnownRequestError } from '@prisma/client/runtime/library'; @Injectable() export class TagRepository { @@ -79,4 +79,24 @@ export class TagRepository { throw new InternalServerErrorException('database error'); }); } + + async findOrCreateTags(tags: string[]): Promise { + await this.prismaSerice.tag + .createMany({ + data: tags.map((name) => ({ name })), + skipDuplicates: true, + }) + .catch((err) => { + this.logger.error('findOrCreateTags'); + this.logger.debug(err); + throw new InternalServerErrorException('database error'); + }); + return this.prismaSerice.tag.findMany({ + where: { + name: { + in: tags, + }, + }, + }); + } } diff --git a/src/tag/tag.service.ts b/src/tag/tag.service.ts index 9e36b8f..49dfc21 100644 --- a/src/tag/tag.service.ts +++ b/src/tag/tag.service.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; -import { CreateTagDto } from './dto/createTag.dto'; import { Tag } from '@prisma/client'; +import { CreateTagDto } from './dto/createTag.dto'; import { GetTagDto } from './dto/getTag.dto'; import { TagRepository } from './tag.repository'; @@ -27,4 +27,8 @@ export class TagService { async deleteTag(id: number): Promise { await this.tagRepository.deleteTag(id); } + + async findOrCreateTags(tags: string[]): Promise { + return this.tagRepository.findOrCreateTags(tags); + } } From 52bb8119ae5ddc82844460d5cd97c56f7129f634 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 15:04:44 +0900 Subject: [PATCH 5/7] feat: attach user to academic notice --- src/notice/notice.module.ts | 1 + src/notice/notice.service.ts | 18 +++++++++++------- src/user/user.module.ts | 12 ++++++------ src/user/user.service.ts | 17 +++++++++++++++++ 4 files changed, 35 insertions(+), 13 deletions(-) diff --git a/src/notice/notice.module.ts b/src/notice/notice.module.ts index 561dd97..dca5b08 100644 --- a/src/notice/notice.module.ts +++ b/src/notice/notice.module.ts @@ -19,6 +19,7 @@ import { NoticeService } from './notice.service'; PrismaModule, HttpModule, TagModule, + UserModule, ], controllers: [NoticeController], providers: [NoticeService, NoticeRepository], diff --git a/src/notice/notice.service.ts b/src/notice/notice.service.ts index e2ef3b0..c76858a 100644 --- a/src/notice/notice.service.ts +++ b/src/notice/notice.service.ts @@ -7,17 +7,18 @@ import dayjs from 'dayjs'; import { htmlToText } from 'html-to-text'; import { catchError, - filter, firstValueFrom, from, map, takeWhile, throwError, + timeout, toArray, } from 'rxjs'; import { FcmService } from 'src/global/service/fcm.service'; import { ImageService } from 'src/image/image.service'; import { TagService } from 'src/tag/tag.service'; +import { UserService } from 'src/user/user.service'; import { AdditionalNoticeDto } from './dto/additionalNotice.dto'; import { CreateNoticeDto } from './dto/createNotice.dto'; import { ForeignContentDto } from './dto/foreignContent.dto'; @@ -33,9 +34,9 @@ export class NoticeService { private readonly fcmService: FcmService, private readonly httpService: HttpService, private readonly tagService: TagService, + private readonly userService: UserService, configService: ConfigService, ) { - this.crawlAcademicNotice(); this.s3Url = `https://s3.${configService.get( 'AWS_S3_REGION', )}.amazonaws.com/${configService.get('AWS_S3_BUCKET_NAME')}/`; @@ -213,6 +214,7 @@ export class NoticeService { private async getAcademicNoticeList() { const baseUrl = 'https://www.gist.ac.kr/kr/html/sub05/050209.html'; const stream = this.httpService.get(baseUrl).pipe( + timeout(10000), map((res) => res.data), map((e) => cheerio.load(e)), catchError(throwError), @@ -238,6 +240,7 @@ export class NoticeService { private async getAcademicNotice(link: string) { const baseUrl = 'https://www.gist.ac.kr/kr/html/sub05/050209.html'; const stream = this.httpService.get(link).pipe( + timeout(10000), map((res) => res.data), map((e) => cheerio.load(e)), catchError(throwError), @@ -260,7 +263,7 @@ export class NoticeService { return { files, content }; } - // @Cron('*/30 * * * * *') + @Cron('30 * * * * *') async crawlAcademicNotice() { const notices = await this.getAcademicNoticeList(); const recentNotice = await this.noticeRepository.getNoticeList({ @@ -268,9 +271,6 @@ export class NoticeService { orderBy: 'recent', tags: ['academic'], }); - from(notices).pipe( - filter((n) => n.title === recentNotice[0].contents[0].title), - ); const noticesToCreate$ = from(notices).pipe( takeWhile((n) => n.title !== recentNotice[0].contents[0].title), toArray(), @@ -278,6 +278,7 @@ export class NoticeService { ); const noticesToCreate = await firstValueFrom(noticesToCreate$); for (const noticeMetadata of noticesToCreate) { + console.log(noticeMetadata.title); const notice = await this.getAcademicNotice(noticeMetadata.link); const filesList = notice.files .map((file) => `
  • ${file.name}
    • `) @@ -288,6 +289,9 @@ export class NoticeService { 'academic', noticeMetadata.category, ]); + const user = await this.userService.addTempUser( + `${noticeMetadata.author} (${noticeMetadata.category})`, + ); await this.noticeRepository.createNotice( { title: noticeMetadata.title, @@ -295,7 +299,7 @@ export class NoticeService { images: [], tags: tags.map(({ id }) => id), }, - '1', + user.uuid, ); } } diff --git a/src/user/user.module.ts b/src/user/user.module.ts index b5c51de..c48ef03 100644 --- a/src/user/user.module.ts +++ b/src/user/user.module.ts @@ -1,12 +1,12 @@ -import { Module } from '@nestjs/common'; -import { UserController } from './user.controller'; -import { UserService } from './user.service'; import { HttpModule } from '@nestjs/axios'; +import { Module } from '@nestjs/common'; +import { PrismaModule } from 'src/prisma/prisma.module'; +import { AnonymousStrategy } from './guard/anonymous.strategy'; import { IdPGuard } from './guard/idp.guard'; import { IdPStrategy } from './guard/idp.strategy'; -import { AnonymousStrategy } from './guard/anonymous.strategy'; import { IdpOptionalStrategy } from './guard/idpOptional.strategy'; -import { PrismaModule } from 'src/prisma/prisma.module'; +import { UserController } from './user.controller'; +import { UserService } from './user.service'; @Module({ imports: [HttpModule, PrismaModule], @@ -18,6 +18,6 @@ import { PrismaModule } from 'src/prisma/prisma.module'; AnonymousStrategy, ], controllers: [UserController], - exports: [IdPGuard], + exports: [IdPGuard, UserService], }) export class UserModule {} diff --git a/src/user/user.service.ts b/src/user/user.service.ts index 5acc1cc..459a94a 100644 --- a/src/user/user.service.ts +++ b/src/user/user.service.ts @@ -7,6 +7,7 @@ import { import { ConfigService } from '@nestjs/config'; import { User } from '@prisma/client'; import { AxiosError } from 'axios'; +import crypto from 'crypto'; import { catchError, firstValueFrom } from 'rxjs'; import { PrismaService } from 'src/prisma/prisma.service'; import { LoginDto } from './dto/login.dto'; @@ -206,4 +207,20 @@ export class UserService { }, }); } + + async addTempUser(name: string) { + const user = await this.prismaService.user.findFirst({ + where: { name }, + }); + if (user) { + return user; + } + return this.prismaService.user.create({ + data: { + uuid: crypto.randomUUID(), + name, + consent: false, + }, + }); + } } From 1fe8ff697dbe840d362da1f90c5593e59d91d665 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 15:27:41 +0900 Subject: [PATCH 6/7] feat: add timezone plugin to dayjs --- src/init.ts | 6 ++++++ src/main.ts | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 src/init.ts diff --git a/src/init.ts b/src/init.ts new file mode 100644 index 0000000..bbda1c3 --- /dev/null +++ b/src/init.ts @@ -0,0 +1,6 @@ +import dayjs from 'dayjs'; +import timezone from 'dayjs/plugin/timezone'; +import utc from 'dayjs/plugin/utc'; + +dayjs.extend(utc); +dayjs.extend(timezone); diff --git a/src/main.ts b/src/main.ts index 7114830..4fe2492 100644 --- a/src/main.ts +++ b/src/main.ts @@ -1,7 +1,8 @@ import { NestFactory } from '@nestjs/core'; -import { AppModule } from './app.module'; -import cookieParser from 'cookie-parser'; import { DocumentBuilder, SwaggerModule } from '@nestjs/swagger'; +import cookieParser from 'cookie-parser'; +import { AppModule } from './app.module'; +import './init'; async function bootstrap() { const app = await NestFactory.create(AppModule); From deecf27c849bbf6fd273f23835b6b4f2a3d8a778 Mon Sep 17 00:00:00 2001 From: 2paperstar Date: Thu, 23 Nov 2023 15:29:02 +0900 Subject: [PATCH 7/7] feat: add notice with treating createdAt --- src/notice/notice.repository.ts | 2 ++ src/notice/notice.service.ts | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/notice/notice.repository.ts b/src/notice/notice.repository.ts index 4c7a58b..f9f0d59 100644 --- a/src/notice/notice.repository.ts +++ b/src/notice/notice.repository.ts @@ -203,6 +203,7 @@ export class NoticeRepository { async createNotice( { title, body, deadline, tags, images }: CreateNoticeDto, userUuid: string, + createdAt?: Date, ) { const findedTags = await this.prismaService.tag.findMany({ where: { @@ -240,6 +241,7 @@ export class NoticeRepository { url: image, })), }, + createdAt, }, }) .catch((err) => { diff --git a/src/notice/notice.service.ts b/src/notice/notice.service.ts index c76858a..d269683 100644 --- a/src/notice/notice.service.ts +++ b/src/notice/notice.service.ts @@ -263,7 +263,7 @@ export class NoticeService { return { files, content }; } - @Cron('30 * * * * *') + @Cron('0 */30 * * *') async crawlAcademicNotice() { const notices = await this.getAcademicNoticeList(); const recentNotice = await this.noticeRepository.getNoticeList({ @@ -278,7 +278,6 @@ export class NoticeService { ); const noticesToCreate = await firstValueFrom(noticesToCreate$); for (const noticeMetadata of noticesToCreate) { - console.log(noticeMetadata.title); const notice = await this.getAcademicNotice(noticeMetadata.link); const filesList = notice.files .map((file) => `
  • ${file.name}
    • `) @@ -300,6 +299,7 @@ export class NoticeService { tags: tags.map(({ id }) => id), }, user.uuid, + dayjs(noticeMetadata.createdAt).tz('Asia/Seoul').toDate(), ); } }