From b346081f62ea7eb89c7989fcf7da801646278827 Mon Sep 17 00:00:00 2001 From: Yannis Alexiou Date: Fri, 15 Mar 2019 13:12:43 +0200 Subject: [PATCH] Module update to scrap the new ICSD site --- courses/course.js | 180 +++++++++--------- courses/semester.js | 182 ++++++++++--------- package-lock.json | 394 +++++++++++++++++++--------------------- package.json | 6 +- professors/members.js | 177 ++++-------------- professors/professor.js | 52 ++++-- 6 files changed, 451 insertions(+), 540 deletions(-) diff --git a/courses/course.js b/courses/course.js index 287afa3..ff54428 100644 --- a/courses/course.js +++ b/courses/course.js @@ -1,119 +1,117 @@ const request = require('request'); //Helps us make HTTP calls const cheerio = require('cheerio'); -const generalSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)'; -const firstFixTableSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(2)>td'; -const courseWebsiteSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(5)>td'; -const contentOutlineSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(8)>td>p' -const learningOutcomesSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(11)>td>p'; -const prerequisitesSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(14)>td>p'; -const basicTextbooksSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(17)>td>p'; -const additionalReferencesSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(20)>td>p'; -const teachingMethodSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(23)>td>p'; -const grandingMethodSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(26)>td>p'; -const languageOfInstructionSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(29)>td'; -const modeOfDeliverySelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody>tr:nth-child(32)>td>p'; +const generalSelector = 'div.tab-content'; +const contentOutlineSelector = 'div.tab-content>div:nth-child(3)>div:nth-child(2)>p' +const learningOutcomesSelector = 'div.tab-content>div:nth-child(4)>div:nth-child(2)>'; +const prerequisitesSelector = 'div.tab-content>div:nth-child(5)>div:nth-child(2)>p'; +const basicTextbooksSelector = 'div.tab-content>div:nth-child(6)>div:nth-child(2)>'; +const additionalReferencesSelector = 'div.tab-content>div:nth-child(7)>div:nth-child(2)>'; +const teachingMethodSelector = 'div.tab-content>div:nth-child(8)>div:nth-child(2)>p'; +const grandingMethodSelector = 'div.tab-content>div:nth-child(9)>div:nth-child(2)>p'; +const languageOfInstructionSelector = 'div.tab-content>div:nth-child(10)>div:nth-child(2)'; +const modeOfDeliverySelector = 'div.tab-content>div:nth-child(11)>div:nth-child(2)>p'; function requestCourseDetails(url) { -return new Promise(function (resolve, reject) { - request(url, function (error, res, body) { - if (!error && res.statusCode == 200) { - resolve(body); - } else { - reject(error); - } + return new Promise(function (resolve, reject) { + request(url, function (error, res, body) { + if (!error && res.statusCode == 200) { + resolve(body); + } else { + reject(error); + } + }); }); - }); } async function getAllCourseDetails(url) { - let html = await requestCourseDetails(url); - var allCourseDetailsList = allCourseDetails(html, url); - return allCourseDetailsList + let html = await requestCourseDetails(url); + var allCourseDetailsList = allCourseDetails(html, url); + return allCourseDetailsList } async function getBasicCourseDetails(url) { - let html = await requestCourseDetails(url); - var basicCourseDetailsList = basicCourseData(html, url); - return basicCourseDetailsList; + let html = await requestCourseDetails(url); + var basicCourseDetailsList = basicCourseData(html, url); + return basicCourseDetailsList; } async function getAdvancedCourseDetails(url) { - let html = await requestCourseDetails(url); - var advancedCourseDetailsList = advancedCourseData(html); - return advancedCourseDetailsList; + let html = await requestCourseDetails(url); + var advancedCourseDetailsList = advancedCourseData(html, url); + return advancedCourseDetailsList; } function allCourseDetails(html, url) { - var basicData = basicCourseData(html, url); - var advancedData = advancedCourseData(html); + var basicData = basicCourseData(html, url); + var advancedData = advancedCourseData(html, url); - var finalData = Object.assign(basicData, advancedData); + var finalData = Object.assign(basicData, advancedData); - return finalData; + return finalData; } function basicCourseData(html, url) { - var $ = cheerio.load(html); - var firstFixTable = $('table.mathima>tbody>tr').filter(function() { - var data = $(this); - return data; - }); - - var title = firstFixTable.children().children().eq(1).text(); - var code = firstFixTable.children().children().eq(3).text(); - var semester = firstFixTable.children().children().eq(5).text(); - var ects = firstFixTable.children().children().eq(7).text(); - var theoryHours = firstFixTable.children().children().eq(9).text(); - var labHours = firstFixTable.children().children().eq(11).text(); - var professor = firstFixTable.children().children().eq(13).text(); - - var basicData = { - title: title, - code: code, - semester: semester, - ects: ects, - theoryHours: theoryHours, - labHours: labHours, - professor: professor, - link: url - } - - return basicData; + var $ = cheerio.load(html); + var firstFixTable = $('table.table>tbody').filter(function () { + var data = $(this); + return data; + }); + + var title = firstFixTable.children().children().eq(1).text(); + var code = firstFixTable.children().children().eq(3).text(); + var semester = firstFixTable.children().children().eq(5).text(); + var ects = firstFixTable.children().children().eq(7).text(); + var theoryHours = firstFixTable.children().children().eq(9).text(); + var labHours = firstFixTable.children().children().eq(11).text(); + var professor = firstFixTable.children().children().eq(13).text(); + + var basicData = { + title: title, + code: code, + semester: semester, + ects: ects, + theoryHours: theoryHours, + labHours: labHours, + professor: professor, + link: url + } + + return basicData; } -function advancedCourseData(html) { - var $ = cheerio.load(html); - - var courseWebsite = $(courseWebsiteSelector).text(); - var contentOutline = $(contentOutlineSelector).text(); - var learningOutcomes = $(learningOutcomesSelector).text(); - var prerequisites = $(prerequisitesSelector).text(); - var basicTextbooks = $(basicTextbooksSelector).text(); - var additionalReferences = $(additionalReferencesSelector).text(); - var teachingMethod = $(teachingMethodSelector).text(); - var grandingMethod = $(grandingMethodSelector).text(); - var languageOfInstruction = $(languageOfInstructionSelector).text(); - var modeOfDelivery = $(modeOfDeliverySelector).text(); - - var advancedData = { - courseWebsite: courseWebsite, - contentOutline: contentOutline, - learningOutcomes: learningOutcomes, - prerequisites: prerequisites, - basicTextbooks: basicTextbooks, - additionalReferences: additionalReferences, - teachingMethod: teachingMethod, - grandingMethod: grandingMethod, - languageOfInstruction: languageOfInstruction, - modeOfDelivery: modeOfDelivery - } - - return advancedData; +function advancedCourseData(html, url) { + var $ = cheerio.load(html); + + var courseWebsite = url; + var contentOutline = $(contentOutlineSelector).text(); + var learningOutcomes = $(learningOutcomesSelector).text(); + var prerequisites = $(prerequisitesSelector).text(); + var basicTextbooks = $(basicTextbooksSelector).text(); + var additionalReferences = $(additionalReferencesSelector).text(); + var teachingMethod = $(teachingMethodSelector).text(); + var grandingMethod = $(grandingMethodSelector).text(); + var languageOfInstruction = $(languageOfInstructionSelector).text(); + var modeOfDelivery = $(modeOfDeliverySelector).text(); + + var advancedData = { + courseWebsite: courseWebsite, + contentOutline: contentOutline, + learningOutcomes: learningOutcomes, + prerequisites: prerequisites, + basicTextbooks: basicTextbooks, + additionalReferences: additionalReferences, + teachingMethod: teachingMethod, + grandingMethod: grandingMethod, + languageOfInstruction: languageOfInstruction, + modeOfDelivery: modeOfDelivery + } + + return advancedData; } module.exports = { - getAllCourseDetails, - getBasicCourseDetails, - getAdvancedCourseDetails -}; + getAllCourseDetails, + getBasicCourseDetails, + getAdvancedCourseDetails +}; \ No newline at end of file diff --git a/courses/semester.js b/courses/semester.js index 8a152ae..df3dbe2 100644 --- a/courses/semester.js +++ b/courses/semester.js @@ -3,19 +3,27 @@ const cheerio = require('cheerio'); const course = require('./course'); -const semesterOneURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=1'; -const semesterTwoURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=2'; -const semesterThreeURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=3'; -const semesterFourURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=4'; -const semesterFiveURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=5'; -const semesterSixURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=6'; -const semesterSevenURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=7'; -const semesterEightURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=8'; -const semesterNineURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=9'; -const semesterTenURL = 'http://www.icsd.aegean.gr/icsd/proptyxiaka/mathimata_ana_examino.php?semester=10'; +const coursesUrl = 'http://www.icsd.aegean.gr/icsd/pps'; +//New way +const semesterOneSelector = 'div#exam1>div:nth-child(2)>'; +const semesterTwoSelector = 'div#exam2>div:nth-child(2)>'; +const semesterThreeSelector = 'div#exam3>div:nth-child(2)>'; +const semesterFourSelector = 'div#exam4>div:nth-child(2)>'; +const semesterFiveSelector = 'div#exam5>div:nth-child(2)>'; +const semesterSixSelector = 'div#exam6>div:nth-child(2)>'; +const semesterSevenSelector = 'div#exam7>div:nth-child(2)>'; +const semesterEightSelector = 'div#exam8>div:nth-child(2)>'; +const semesterNineSelector = 'div#exam9>div:nth-child(2)>'; +const semesterTenSelector = 'div#exam10>div:nth-child(2)>'; + +//Append +const compulsorySemesterTableSelector = 'div>div:nth-child(1)>table>tbody'; +const cicleSemesterTableSelector = 'div>div:nth-child(2)'; +const optionalSemesterTableSelector = 'div>div:nth-child(3)>table>tbody'; +const freeCourseSemesterTableSelector = 'div>div:nth-child(4)>table>tbody' function requestTo(url) { -return new Promise(function (resolve, reject) { + return new Promise(function (resolve, reject) { request(url, function (error, res, body) { if (!error && res.statusCode == 200) { resolve(body); @@ -30,16 +38,16 @@ async function checkSemesters(isBasic) { //The final Array var allCourses; - var semesterOneCourses = await getCourseStructOfSemester(semesterOneURL); - var semesterTwoCourses = await getCourseStructOfSemester(semesterTwoURL); - var semesterThreeCourses = await getCourseStructOfSemester(semesterThreeURL); - var semesterFourCourses = await getCourseStructOfSemester(semesterFourURL); - var semesterFiveCourses = await getCourseStructOfSemester(semesterFiveURL); - var semesterSixCourses = await getCourseStructOfSemester(semesterSixURL); - var semesterSevenCourses = await getCourseStructOfSemester(semesterSevenURL); - var semesterEightCourses = await getCourseStructOfSemester(semesterEightURL); - var semesterNineCourses = await getCourseStructOfSemester(semesterNineURL); - var semesterTenCourses = await getCourseStructOfSemester(semesterTenURL); + var semesterOneCourses = await getCourseStructOfSemester(semesterOneSelector); + var semesterTwoCourses = await getCourseStructOfSemester(semesterTwoSelector); + var semesterThreeCourses = await getCourseStructOfSemester(semesterThreeSelector); + var semesterFourCourses = await getCourseStructOfSemester(semesterFourSelector); + var semesterFiveCourses = await getCourseStructOfSemester(semesterFiveSelector); + var semesterSixCourses = await getCourseStructOfSemester(semesterSixSelector); + var semesterSevenCourses = await getCourseStructOfSemester(semesterSevenSelector); + var semesterEightCourses = await getCourseStructOfSemester(semesterEightSelector); + var semesterNineCourses = await getCourseStructOfSemester(semesterNineSelector); + var semesterTenCourses = await getCourseStructOfSemester(semesterTenSelector); allCourses = semesterOneCourses.concat( semesterTwoCourses, @@ -53,6 +61,7 @@ async function checkSemesters(isBasic) { semesterTenCourses ); + //TODO: Change = to == if (isBasic == true) { var basicCourseData = []; for (var i = 0; i < allCourses.length; i++) { @@ -62,8 +71,7 @@ async function checkSemesters(isBasic) { } return basicCourseData; - } - else { + } else { var advancedCourseData = []; for (var i = 0; i < allCourses.length; i++) { var thisAdvancedCourseData = await course.getAllCourseDetails(allCourses[i].link); @@ -73,100 +81,113 @@ async function checkSemesters(isBasic) { } } -async function getCourseStructOfSemester(url) { +async function getCourseStructOfSemester(semesterSelector) { - var semester = identifySemester(url); + var semester = identifySemester(semesterSelector); var thisSemesterCourses; if (semester < 7 || semester == 10) { //Compulsory - thisSemesterCourses = await getCompulsorySemesterCourses(url, semester); - } - else { + thisSemesterCourses = await getCompulsorySemesterCourses(semesterSelector, semester); + } else { //Cycle - thisSemesterCourses = await getCycleSemesterCourses(url, semester) + thisSemesterCourses = await getCycleSemesterCourses(semesterSelector, semester) } return thisSemesterCourses; } -async function getCompulsorySemesterCourses(url, semester) { - let html = await requestTo(url); +async function getCompulsorySemesterCourses(semesterSelector, semester) { + let html = await requestTo(coursesUrl); //The final Array var coursesOfThisSemester; - var tableSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table:nth-child(3)>tbody'; - var thisSemesterCourses = await compulsorySemesterSelector(html, tableSelector, semester); - return thisSemesterCourses; + var tableSelector = semesterSelector + compulsorySemesterTableSelector; + var thisSemesterCompulsoryCourses = await simpleSemesterSelector(html, tableSelector, semester); + var tableSelector = semesterSelector + optionalSemesterTableSelector; + var thisSemesterOptionalCourses = await simpleSemesterSelector(html, tableSelector, semester); + coursesOfThisSemester = thisSemesterCompulsoryCourses.concat( + thisSemesterOptionalCourses + ); + return coursesOfThisSemester; } -function compulsorySemesterSelector(html, selector, semester) { +function simpleSemesterSelector(html, selector, semester) { var $ = cheerio.load(html); - var tableFiltered = $(selector).filter(function() { + var tableFiltered = $(selector).filter(function () { var data = $(this); return data; }); var coursesArray = []; - const icsdDomain = 'http://www.icsd.aegean.gr'; + const icsdDomain = 'http://www.icsd.aegean.gr/icsd/pps_lessons.php?lesson_id='; - coursesOfThisSemester = tableFiltered.children('tr').each(function(i, elem) { + coursesOfThisSemester = tableFiltered.children('tr').each(function (i, elem) { var data = $(this); - var isLesson = data.children().attr('align'); - if (isLesson) { - var row = data.children().children(); - - var code = row.eq(0).text(); - var title = row.eq(1).text(); - var link = row.eq(1).attr('href'); - var theoryHours = row.eq(2).text(); - var labHours = row.eq(3).text(); - var ects = row.eq(4).text(); - - var eachItem = { - code: code, - title: title, - kind: 'Compulsory', - semester: semester, - link: icsdDomain + link, - theoryHours: theoryHours, - labHours: labHours, - ects: ects - } - - coursesArray.push(eachItem); + var row = data.children(); + var code = row.eq(0).text(); + var title = row.eq(1).text(); + var link = row.eq(1).attr('href'); + var theoryHours = row.eq(2).text(); + var labHours = row.eq(3).text(); + var ects = row.eq(4).text(); + + var eachItem = { + code: code, + title: title, + kind: 'Compulsory', + semester: semester, + link: icsdDomain + code, + theoryHours: theoryHours, + labHours: labHours, + ects: ects } + + coursesArray.push(eachItem); }); return coursesArray; } -async function getCycleSemesterCourses(url, semester) { - let html = await requestTo(url); +async function getCycleSemesterCourses(semesterSelector, semester) { + let html = await requestTo(coursesUrl); //The final Array var coursesOfThisSemester; - var tableSelector = 'table.wrapper>tbody>tr>td>center:nth-child(10)>table>tbody>tr:nth-child(1)>td>center>table>tbody>tr>td:nth-child(2)>table>tbody'; + //Semester regular + var tableSelector = semesterSelector + cicleSemesterTableSelector; var thisSemesterCourses = await cycleSemesterSelector(html, tableSelector, semester); - return thisSemesterCourses; + //Semester optional + var tableSelector = semesterSelector + optionalSemesterTableSelector; + var thisSemesterOptionalCourses = await simpleSemesterSelector(html, tableSelector, semester); + //Semester free + var tableSelector = semesterSelector + freeCourseSemesterTableSelector; + var thisSemesterFreeCourses = await simpleSemesterSelector(html, tableSelector, semester); + + var coursesOfThisSemester = thisSemesterCourses.concat( + thisSemesterOptionalCourses, + thisSemesterFreeCourses + ); + return coursesOfThisSemester; } function cycleSemesterSelector(html, selector, semester) { var $ = cheerio.load(html); - var tableFiltered = $(selector).filter(function() { + var tableFiltered = $(selector).filter(function () { var data = $(this); return data; }); var coursesArray = []; - const icsdDomain = 'http://www.icsd.aegean.gr'; + const icsdDomain = 'http://www.icsd.aegean.gr/icsd/pps_lessons.php?lesson_id='; - coursesOfThisSemester = tableFiltered.children('tr').each(function(i, elem) { + tableFiltered.children('table').children('tbody').each(function (i, elem) { var data = $(this); - var isLesson = data.children().attr('align'); - if (isLesson) { - var row = data.children().children(); + data.children('tr').each(function (i, elem) { + var data = $(this); + + var row = data.children(); var code = row.eq(0).text(); var title = row.eq(1).text(); @@ -180,30 +201,27 @@ function cycleSemesterSelector(html, selector, semester) { title: title, kind: 'cycle', semester: semester, - link: icsdDomain + link, + link: icsdDomain + code, theoryHours: theoryHours, labHours: labHours, ects: ects } coursesArray.push(eachItem); - } + }); + }); + + return coursesArray; } function identifySemester(semeterUrl) { - var splittedLink = splitEqual(semeterUrl); - var semester = splittedLink[1]; + //var semeterUrl = 'div#exam1>div:nth-child(2)'; + var semester = semeterUrl.split('exam')[1].split('>')[0]; return semester; } -function splitEqual(receivedMessage) { - var fields = receivedMessage.split('='); - - return fields; -} - module.exports = { checkSemesters -}; +}; \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index a8bbe88..3110083 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,5 +1,5 @@ { - "name": "icsdscrapper", + "name": "icsd-scraper", "version": "1.1.0", "lockfileVersion": 1, "requires": true, @@ -10,20 +10,23 @@ "integrity": "sha512-zkYho6/4wZyX6o9UQ8rd0ReEaiEYNNCqYFIAACe2Tf9DrYlgzWW27OigYHnnztnnZQwVRpwWmZKegFmDpinIsA==" }, "ajv": { - "version": "5.5.2", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", - "integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=", + "version": "6.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.10.0.tgz", + "integrity": "sha512-nffhOpkymDECQyR0mnsUtoCE8RlX38G0rYP+wgLWFyZuUyuuojSSvi/+euOiQBIn63whYwYVIIH1TvE3tu4OEg==", "requires": { - "co": "4.6.0", - "fast-deep-equal": "1.0.0", - "fast-json-stable-stringify": "2.0.0", - "json-schema-traverse": "0.3.1" + "fast-deep-equal": "^2.0.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" } }, "asn1": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz", - "integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y=" + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.4.tgz", + "integrity": "sha512-jxwzQpLQjSmWXgwaCZE9Nz+glAG01yF1QnWgbhGwHI5A6FRIEY6IVqtHhIepHqI7/kyEyQEagBC5mBEFlIYvdg==", + "requires": { + "safer-buffer": "~2.1.0" + } }, "assert-plus": { "version": "1.0.0", @@ -41,17 +44,16 @@ "integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg=" }, "aws4": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz", - "integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4=" + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.8.0.tgz", + "integrity": "sha512-ReZxvNHIOv88FlT7rxcXIIC0fPt4KZqZbOlivyWtXLt8ESx84zd3kMC6iK5jVeS2qt+g7ftS7ye4fi06X5rtRQ==" }, "bcrypt-pbkdf": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz", - "integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=", - "optional": true, + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", + "integrity": "sha1-pDAdOJtqQ/m2f/PKEaP2Y342Dp4=", "requires": { - "tweetnacl": "0.14.5" + "tweetnacl": "^0.14.3" } }, "boolbase": { @@ -59,14 +61,6 @@ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" }, - "boom": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz", - "integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=", - "requires": { - "hoek": "4.2.0" - } - }, "caseless": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz", @@ -77,25 +71,20 @@ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz", "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=", "requires": { - "css-select": "1.2.0", - "dom-serializer": "0.1.0", - "entities": "1.1.1", - "htmlparser2": "3.9.2", - "lodash": "4.17.5", - "parse5": "3.0.3" + "css-select": "~1.2.0", + "dom-serializer": "~0.1.0", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" } }, - "co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" - }, "combined-stream": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz", - "integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=", + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.7.tgz", + "integrity": "sha512-brWl9y6vOB1xYPZcpZde3N9zDByXTosAeMDo4p1wzo6UMOX4vumB+TP1RZ76sfE6Md68Q0NJSrE/gbezd4Ul+w==", "requires": { - "delayed-stream": "1.0.0" + "delayed-stream": "~1.0.0" } }, "core-util-is": { @@ -103,33 +92,15 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, - "cryptiles": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz", - "integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=", - "requires": { - "boom": "5.2.0" - }, - "dependencies": { - "boom": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz", - "integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==", - "requires": { - "hoek": "4.2.0" - } - } - } - }, "css-select": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", "requires": { - "boolbase": "1.0.0", - "css-what": "2.1.0", + "boolbase": "~1.0.0", + "css-what": "2.1", "domutils": "1.5.1", - "nth-check": "1.0.1" + "nth-check": "~1.0.1" } }, "css-what": { @@ -142,7 +113,7 @@ "resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz", "integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=", "requires": { - "assert-plus": "1.0.0" + "assert-plus": "^1.0.0" } }, "delayed-stream": { @@ -155,8 +126,8 @@ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.0.tgz", "integrity": "sha1-BzxpdUbOB4DOI75KKOKT5AvDDII=", "requires": { - "domelementtype": "1.1.3", - "entities": "1.1.1" + "domelementtype": "~1.1.1", + "entities": "~1.1.1" }, "dependencies": { "domelementtype": { @@ -176,7 +147,7 @@ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.1.tgz", "integrity": "sha1-iS5HAAqZvlW783dP/qBWHYh5wlk=", "requires": { - "domelementtype": "1.3.0" + "domelementtype": "1" } }, "domutils": { @@ -184,17 +155,17 @@ "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", "requires": { - "dom-serializer": "0.1.0", - "domelementtype": "1.3.0" + "dom-serializer": "0", + "domelementtype": "1" } }, "ecc-jsbn": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz", - "integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=", - "optional": true, + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", + "integrity": "sha1-OoOpBOVDUyh4dMVkt1SThoSamMk=", "requires": { - "jsbn": "0.1.1" + "jsbn": "~0.1.0", + "safer-buffer": "^2.1.0" } }, "entities": { @@ -203,9 +174,9 @@ "integrity": "sha1-blwtClYhtdra7O+AuQ7ftc13cvA=" }, "extend": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz", - "integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ=" + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" }, "extsprintf": { "version": "1.3.0", @@ -213,9 +184,9 @@ "integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU=" }, "fast-deep-equal": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz", - "integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8=" + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=" }, "fast-json-stable-stringify": { "version": "2.0.0", @@ -228,13 +199,13 @@ "integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE=" }, "form-data": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz", - "integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=", + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.3.tgz", + "integrity": "sha512-1lLKB2Mu3aGP1Q/2eCOx0fNbRMe7XdwktwOruhfqqd0rIJWwN4Dh+E3hrPSlDCXnSR7UtZ1N38rVXm+6+MEhJQ==", "requires": { - "asynckit": "0.4.0", - "combined-stream": "1.0.5", - "mime-types": "2.1.17" + "asynckit": "^0.4.0", + "combined-stream": "^1.0.6", + "mime-types": "^2.1.12" } }, "getpass": { @@ -242,7 +213,7 @@ "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz", "integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=", "requires": { - "assert-plus": "1.0.0" + "assert-plus": "^1.0.0" } }, "har-schema": { @@ -251,41 +222,25 @@ "integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI=" }, "har-validator": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz", - "integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=", - "requires": { - "ajv": "5.5.2", - "har-schema": "2.0.0" - } - }, - "hawk": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz", - "integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==", + "version": "5.1.3", + "resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.1.3.tgz", + "integrity": "sha512-sNvOCzEQNr/qrvJgc3UG/kD4QtlHycrzwS+6mfTrrSq97BvaYcPZZI1ZSqGSPR73Cxn4LKTD4PttRwfU7jWq5g==", "requires": { - "boom": "4.3.1", - "cryptiles": "3.1.2", - "hoek": "4.2.0", - "sntp": "2.1.0" + "ajv": "^6.5.5", + "har-schema": "^2.0.0" } }, - "hoek": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz", - "integrity": "sha512-v0XCLxICi9nPfYrS9RL8HbYnXi9obYAeLbSP00BmnZwCK9+Ih9WOjoZ8YoHCoav2csqn4FOz4Orldsy2dmDwmQ==" - }, "htmlparser2": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.9.2.tgz", "integrity": "sha1-G9+HrMoPP55T+k/M6w9LTLsAszg=", "requires": { - "domelementtype": "1.3.0", - "domhandler": "2.4.1", - "domutils": "1.5.1", - "entities": "1.1.1", - "inherits": "2.0.3", - "readable-stream": "2.3.3" + "domelementtype": "^1.3.0", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^2.0.2" } }, "http-signature": { @@ -293,9 +248,9 @@ "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", "integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=", "requires": { - "assert-plus": "1.0.0", - "jsprim": "1.4.1", - "sshpk": "1.13.1" + "assert-plus": "^1.0.0", + "jsprim": "^1.2.2", + "sshpk": "^1.7.0" } }, "inherits": { @@ -321,8 +276,7 @@ "jsbn": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz", - "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=", - "optional": true + "integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=" }, "json-schema": { "version": "0.2.3", @@ -330,9 +284,9 @@ "integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM=" }, "json-schema-traverse": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz", - "integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A=" + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" }, "json-stringify-safe": { "version": "5.0.1", @@ -351,21 +305,21 @@ } }, "lodash": { - "version": "4.17.5", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.5.tgz", - "integrity": "sha512-svL3uiZf1RwhH+cWrfZn3A4+U58wbP0tGVTLQPbjplZxZ8ROD9VLuNgsRniTlLe7OlSqR79RUehXgpBW/s0IQw==" + "version": "4.17.11", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", + "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" }, "mime-db": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz", - "integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE=" + "version": "1.38.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.38.0.tgz", + "integrity": "sha512-bqVioMFFzc2awcdJZIzR3HjZFX20QhilVS7hytkKrv7xFAn8bM1gzc/FOX2awLISvWe0PV8ptFKcon+wZ5qYkg==" }, "mime-types": { - "version": "2.1.17", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", - "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", + "version": "2.1.22", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.22.tgz", + "integrity": "sha512-aGl6TZGnhm/li6F7yx82bJiBZwgiEa4Hf6CNr8YO+r5UHr53tSTYZb102zyU50DOWWKeOv0uQLRL0/9EiKWCog==", "requires": { - "mime-db": "1.30.0" + "mime-db": "~1.38.0" } }, "nth-check": { @@ -373,20 +327,20 @@ "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.1.tgz", "integrity": "sha1-mSms32KPwsQQmN6rgqxYDPFJquQ=", "requires": { - "boolbase": "1.0.0" + "boolbase": "~1.0.0" } }, "oauth-sign": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz", - "integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM=" + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.9.0.tgz", + "integrity": "sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ==" }, "parse5": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", "requires": { - "@types/node": "9.4.0" + "@types/node": "*" } }, "performance-now": { @@ -399,57 +353,67 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz", "integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M=" }, + "psl": { + "version": "1.1.31", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.1.31.tgz", + "integrity": "sha512-/6pt4+C+T+wZUieKR620OpzN/LlnNKuWjy1iFLQ/UG35JqHlR/89MP1d96dUfkf6Dne3TuLQzOYEYshJ+Hx8mw==" + }, "punycode": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", - "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" + "version": "6.5.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz", + "integrity": "sha512-N5ZAX4/LxJmF+7wN74pUD6qAh9/wnvdQcjq9TZjevvXzSUo7bfmw91saqMjzGS2xq91/odN2dW/WOl7qQHNDGA==" }, "readable-stream": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz", "integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==", "requires": { - "core-util-is": "1.0.2", - "inherits": "2.0.3", - "isarray": "1.0.0", - "process-nextick-args": "1.0.7", - "safe-buffer": "5.1.1", - "string_decoder": "1.0.3", - "util-deprecate": "1.0.2" + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~1.0.6", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.0.3", + "util-deprecate": "~1.0.1" } }, "request": { - "version": "2.83.0", - "resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz", - "integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==", + "version": "2.88.0", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.0.tgz", + "integrity": "sha512-NAqBSrijGLZdM0WZNsInLJpkJokL72XYjUpnB0iwsRgxh7dB6COrHnTBNwN0E+lHDAJzu7kLAkDeY08z2/A0hg==", "requires": { - "aws-sign2": "0.7.0", - "aws4": "1.6.0", - "caseless": "0.12.0", - "combined-stream": "1.0.5", - "extend": "3.0.1", - "forever-agent": "0.6.1", - "form-data": "2.3.1", - "har-validator": "5.0.3", - "hawk": "6.0.2", - "http-signature": "1.2.0", - "is-typedarray": "1.0.0", - "isstream": "0.1.2", - "json-stringify-safe": "5.0.1", - "mime-types": "2.1.17", - "oauth-sign": "0.8.2", - "performance-now": "2.1.0", - "qs": "6.5.1", - "safe-buffer": "5.1.1", - "stringstream": "0.0.5", - "tough-cookie": "2.3.3", - "tunnel-agent": "0.6.0", - "uuid": "3.2.1" + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.0", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.4.3", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + }, + "dependencies": { + "safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + } } }, "safe-buffer": { @@ -457,27 +421,25 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" }, - "sntp": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz", - "integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==", - "requires": { - "hoek": "4.2.0" - } + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "sshpk": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz", - "integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=", + "version": "1.16.1", + "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", + "integrity": "sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg==", "requires": { - "asn1": "0.2.3", - "assert-plus": "1.0.0", - "bcrypt-pbkdf": "1.0.1", - "dashdash": "1.14.1", - "ecc-jsbn": "0.1.1", - "getpass": "0.1.7", - "jsbn": "0.1.1", - "tweetnacl": "0.14.5" + "asn1": "~0.2.3", + "assert-plus": "^1.0.0", + "bcrypt-pbkdf": "^1.0.0", + "dashdash": "^1.12.0", + "ecc-jsbn": "~0.1.1", + "getpass": "^0.1.1", + "jsbn": "~0.1.0", + "safer-buffer": "^2.0.2", + "tweetnacl": "~0.14.0" } }, "string_decoder": { @@ -485,20 +447,23 @@ "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz", "integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==", "requires": { - "safe-buffer": "5.1.1" + "safe-buffer": "~5.1.0" } }, - "stringstream": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz", - "integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg=" - }, "tough-cookie": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz", - "integrity": "sha1-C2GKVWW23qkL80JdBNVe3EdadWE=", + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.4.3.tgz", + "integrity": "sha512-Q5srk/4vDM54WJsJio3XNn6K2sCG+CQ8G5Wz6bZhRZoAe/+TxjWB/GlFAnYEbkYVlON9FMk/fE3h2RLpPXo4lQ==", "requires": { - "punycode": "1.4.1" + "psl": "^1.1.24", + "punycode": "^1.4.1" + }, + "dependencies": { + "punycode": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz", + "integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4=" + } } }, "tunnel-agent": { @@ -506,14 +471,21 @@ "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=", "requires": { - "safe-buffer": "5.1.1" + "safe-buffer": "^5.0.1" } }, "tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=", - "optional": true + "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=" + }, + "uri-js": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.2.2.tgz", + "integrity": "sha512-KY9Frmirql91X2Qgjry0Wd4Y+YTdrdZheS8TFwvkbLWf/G5KNJDCh6pKL5OZctEW4+0Baa5idK2ZQuELRwPznQ==", + "requires": { + "punycode": "^2.1.0" + } }, "util-deprecate": { "version": "1.0.2", @@ -521,18 +493,18 @@ "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" }, "uuid": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz", - "integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA==" + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" }, "verror": { "version": "1.10.0", "resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz", "integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=", "requires": { - "assert-plus": "1.0.0", + "assert-plus": "^1.0.0", "core-util-is": "1.0.2", - "extsprintf": "1.3.0" + "extsprintf": "^1.2.0" } } } diff --git a/package.json b/package.json index 01112cc..3800812 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "icsd-scraper", - "version": "1.1.0", - "description": "A Node.js module to scrap the icsd site and retrieve details about professros and courses.", + "version": "2.0.0", + "description": "A Node.js module to scrap the icsd site and retrieve details about professors and courses.", "main": "index.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1" @@ -10,7 +10,7 @@ "license": "GPL-3.0", "dependencies": { "cheerio": "^1.0.0-rc.2", - "request": "^2.83.0" + "request": "^2.88.0" }, "repository": { "type": "git", diff --git a/professors/members.js b/professors/members.js index d5c5c6b..de228ae 100644 --- a/professors/members.js +++ b/professors/members.js @@ -3,13 +3,10 @@ const cheerio = require('cheerio'); const professorDetails = require('./professor'); -//Links to search -const urlDep = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=dep'; -const urlPd407 = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=pd407'; -const urlPostDoc = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=postdoc'; -const ulrOthers = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=other'; -const urlEedip = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=eedip'; -const urlEtep = 'http://www.icsd.aegean.gr/icsd/prosopiko/members.php?category=etep'; +//New Links +const academicStaff = 'http://www.icsd.aegean.gr/icsd/akadimaiko'; +const labStaff = 'http://www.icsd.aegean.gr/icsd/ergastiriako'; +const researchStaff = 'http://www.icsd.aegean.gr/icsd/erevnitiko'; function requestTo(url) { return new Promise(function (resolve, reject) { @@ -24,109 +21,54 @@ return new Promise(function (resolve, reject) { } async function getAllProfessors() { - console.log("Start searching... 🤓"); + console.log("Start searching... 🧐"); //Final Array - var allProfessors; - - var allDepProfessors = await getDepCategoryProfessors(urlDep); - console.log("I still searching 🧐 \n"); - var allOtherProfessors = await allOtherCategoriesProfessors(); - - allProfessors = allDepProfessors.concat(allOtherProfessors); + var allProfessors = await allProfessorsCategories(); return allProfessors; } - -var getDepCategoryProfessors = async (url) => { - let html = await requestTo(url); - - //The final Array - var professorsOfThisCategory; - - //Creating the Selectors for each Table - var professors = 'table.menu_default>tbody>tr:nth-child(2)>td>table:nth-child(1)>tbody>tr'; - var associateProfessors = 'table.menu_default>tbody>tr:nth-child(2)>td>table:nth-child(3)>tbody>tr'; - var permantAssistantProfessors = 'table.menu_default>tbody>tr:nth-child(2)>td>table:nth-child(5)>tbody>tr'; - var assistantProfessors = 'table.menu_default>tbody>tr:nth-child(2)>td>table:nth-child(7)>tbody>tr'; - var retirementProfessors = 'table.menu_default>tbody>tr:nth-child(2)>td>table:nth-child(9)>tbody>tr'; - - //Make an array for each Selector - var professorsArray = professorDepTable(professors, html); - var associateProfessorsArray = professorDepTable(associateProfessors, html); - var permantAssistantProfessorsArray = professorDepTable(permantAssistantProfessors, html); - var assistantProfessorsArray = professorDepTable(assistantProfessors, html); - var retirementProfessorsArray = professorDepTable(retirementProfessors, html); - - //Concat (Merge) all previous arrays into one - professorsOfThisCategory = professorsArray.concat( - associateProfessorsArray, - permantAssistantProfessorsArray, - assistantProfessorsArray, - retirementProfessorsArray); - - //Search for more detauls for each professor - for (let i = 0; i < professorsOfThisCategory.length; i++) { - - let thisProfessor = professorsOfThisCategory[i] - var professorFullDetails = await getProfessorProfile(thisProfessor); - - //name, offce etc - professorsOfThisCategory[i] = professorFullDetails; - } - - return professorsOfThisCategory; -} - -var allOtherCategoriesProfessors = async () => { +var allProfessorsCategories = async () => { //The final Array - var professorsOfOtherCategories; + var allProfessorsCategories; //Make an array from each page - var pdProfessorsArray = await getOtherCategoryProfessor(urlPd407); - var postDocProfessorsArray = await getOtherCategoryProfessor(urlPostDoc); - var otherProfessorsArray = await getOtherCategoryProfessor(ulrOthers); - var eedipProfessorsArray = await getOtherCategoryProfessor(urlEedip); - var etepProfessorsArray = await getOtherCategoryProfessor(urlEtep); + var academicProfessorsArray = await getProfessorCategory(academicStaff); + var labProfessorsArray = await getProfessorCategory(labStaff); + var researchProfessorsArray = await getProfessorCategory(researchStaff); //Concat (Merge) all previous arrays into one - professorsOfOtherCategories = pdProfessorsArray.concat( - postDocProfessorsArray, - otherProfessorsArray, - eedipProfessorsArray, - etepProfessorsArray); + allProfessorsCategories = academicProfessorsArray.concat( + labProfessorsArray, + researchProfessorsArray); //Search for more detauls for each professor - for (let i = 0; i < professorsOfOtherCategories.length; i++) { + for (let i = 0; i < allProfessorsCategories.length; i++) { - let thisProfessor = professorsOfOtherCategories[i] + let thisProfessor = allProfessorsCategories[i] var professorFullDetails = await getProfessorProfile(thisProfessor); //name, office etc - professorsOfOtherCategories[i] = professorFullDetails; + allProfessorsCategories[i] = professorFullDetails; } - return professorsOfOtherCategories; + return allProfessorsCategories; } -var getOtherCategoryProfessor = async (url) => { +var getProfessorCategory = async (url) => { let html = await requestTo(url); - //The final Array - var professorsOfThisCategory; - - var generalSelector = 'table.menu_default>tbody>tr:nth-child(2)>td>table>tbody>tr'; + var generalSelector = 'div.grid'; - var splittedLink = splitEqual(url); - var academicRank = checkAcademicRank(splittedLink[1]); - var professorsOfThisCategory = otherProfessorsTable(generalSelector, html, academicRank); + //The final Array + var professorsOfThisCategory = professorSectionDiv(generalSelector, html); return professorsOfThisCategory; } -function otherProfessorsTable(selector, html, academicRank) { +function professorSectionDiv(selector, html) { var $ = cheerio.load(html); var fullSideListFiltered = $(selector).filter(function() { var data = $(this); @@ -134,13 +76,15 @@ function otherProfessorsTable(selector, html, academicRank) { }); const professorsProfieArray = []; - const icsdDomain = 'http://www.icsd.aegean.gr'; + const icsdDomain = 'http://www.icsd.aegean.gr/icsd/'; - professorsOfThisCategory = fullSideListFiltered.children('td').each(function(i, elem) { + professorsOfThisCategory = fullSideListFiltered.children('div').each(function(i, elem) { var data = $(this); - var name = data.children().children().text(); - var link = data.children().children().attr('href'); + var name = data.children('a').children('p').eq(0).text(); + var link = data.children('a').attr('href'); + var academicRank = data.attr('data-category'); + academicRank = checkAcademicRank(academicRank); var eachItem = { name: name, @@ -154,69 +98,32 @@ function otherProfessorsTable(selector, html, academicRank) { return professorsProfieArray; } -function professorDepTable(selector, html) { - var $ = cheerio.load(html); - var fullSideListFiltered = $(selector).filter(function() { - var data = $(this); - return data; - }); - - const professorsProfieArray = []; - const icsdDomain = 'http://www.icsd.aegean.gr'; - var academicRank = ''; - - professorsOfThisCategory = fullSideListFiltered.children('td').each(function(i, elem) { - var data = $(this); - - var professors = data.children().text(); - var link = data.children().children().attr('href'); - - //Αν δεν έχει link τότε το professors είναι academicRank - if (!link) { - academicRank = checkAcademicRank(professors); - } - else { - var eachItem = { - name: professors, - link: icsdDomain + link, - academicRank: academicRank - } - - professorsProfieArray.push(eachItem); - } - }); - return professorsProfieArray; -} - function checkAcademicRank(receivedString) { var academicRank = ''; switch (receivedString) { - case 'Καθηγητές': + case 'kathigitis': academicRank = 'Καθηγητής' break; - case 'Αναπληρωτές Καθηγητές': + case 'anaplirotis': academicRank = 'Αναπληρωτής Καθηγητής' break; - case 'Μόνιμοι Επίκουροι Καθήγητες': + case 'mepikouros': academicRank = 'Μόνιμος Επίκουρος Καθηγητής' break; - case 'Επίκουροι Καθηγητές': + case 'epikouros': academicRank = 'Επίκουρος Καθηγητής' break; - case 'Συνταξιοδοτημένα μέλη ΔΕΠ': - academicRank = 'Συνταξιοδοτημένο μέλος ΔΕΠ' - break; - case 'postdoc': + case 'meta': academicRank = 'Μεταδιδάκτορας' break; - case 'other': - academicRank = 'Διδάσκων' - break; case 'eedip': academicRank = 'ΕΔΙΠ/ΕΕΠ' break; - case 'etep': - academicRank = 'ΕΤΕΠ' + case 'phd': + academicRank = 'Υποψήφιος Διδάκτορας' + break; + case 'external': + academicRank = 'Εξωτερικός Συνεργάτης' break; default: academicRank = 'Διδάσκων' @@ -240,12 +147,6 @@ async function getProfessorProfile(thisProfessor) { return eachProfessorFullDetails; } -function splitEqual(receivedMessage) { - var fields = receivedMessage.split('='); - - return fields; -} - module.exports = { getAllProfessors }; diff --git a/professors/professor.js b/professors/professor.js index c3c840d..99301af 100644 --- a/professors/professor.js +++ b/professors/professor.js @@ -1,21 +1,23 @@ const request = require('request'); //Helps us make HTTP calls const cheerio = require('cheerio'); -const nameSelector = 'table#table1>tbody>tr:nth-child(1)>td:nth-child(3)'; -const officeSelector = 'table#table1>tbody>tr:nth-child(2)>td:nth-child(2)'; -const telSelector = 'table#table1>tbody>tr:nth-child(3)>td:nth-child(2)'; -const emailSelector = 'table#table1>tbody>tr:nth-child(4)>td:nth-child(2)'; -const websiteSelector = 'table#table1>tbody>tr:nth-child(5)>td:nth-child(2)'; -const imageSelector = 'table#table1>tbody>tr:nth-child(1)>td:nth-child(1)>center>img'; +const nameSelector = 'ul.m-subheader__breadcrumbs>li:nth-child(7)>a>span>b>i'; +const officeSelector = 'div>div>ul>li:nth-child(4)>a>span'; +const telSelector = 'div>div>ul>li:nth-child(3)>a>span'; +const emailSelector = 'div>div>ul>li:nth-child(2)>a>span'; +const websiteSelector = 'div>div>ul>li:nth-child(5)'; +const imageSelector = 'div.m-card-profile__pic-leo>img'; +const academicLevelSelector = 'a.m-card-profile__email'; +const citationsSelector = 'div>div>ul>li:nth-child(6)>a'; async function getProfessorsDetails(url) { let html = await requestProfessorDetails(url); - var professorDetailsList = professorDetails(html); + var professorDetailsList = professorDetails(html, url); return professorDetailsList } function requestProfessorDetails(url) { -return new Promise(function (resolve, reject) { + return new Promise(function (resolve, reject) { request(url, function (error, res, body) { if (!error && res.statusCode == 200) { resolve(body); @@ -26,25 +28,45 @@ return new Promise(function (resolve, reject) { }); } -function professorDetails(html) { +function professorDetails(html, url) { var $ = cheerio.load(html); + var name = $(nameSelector).text(); - var office = $(officeSelector).text(); - var tel = $(telSelector).text(); - var email = $(emailSelector).text(); - var website = $(websiteSelector).text(); + var office = $(officeSelector).eq(1).text(); + var tel = $(telSelector).eq(2).text(); + var email = $(emailSelector).eq(3).text(); + var website = $(websiteSelector).eq(2).children('a').attr('href'); var image = $(imageSelector).attr('src'); + var citations = $(citationsSelector).eq(1).attr('href'); + var academicLevel = $(academicLevelSelector).text(); + //Normatlize data + //Normatlize website + if (website.indexOf('javascript') !== -1) { + website = url; + } + //Normalize email + if (email.indexOf(' [at] ') !== -1) { + email = email.replace(" [at] ", "@"); + } + if (email.includes(' [dot] ')) { + email = email.replace(' [dot] ', '.') + } + var details = { name: name, office: office, tel: tel, email: email, website: website, - image: image + image: image, + academicLevel: academicLevel, + citations: citations } return details } -module.exports = {getProfessorsDetails}; +module.exports = { + getProfessorsDetails +}; \ No newline at end of file