-
Notifications
You must be signed in to change notification settings - Fork 0
/
index.js
66 lines (58 loc) · 2.2 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
const fetch = require('node-fetch');
const fs = require('fs');
const cheerio = require('cheerio');
const trim = require('trim');
//file name
let path = 'scrappeddata.csv';
//Query set to search Mcycle 2A for sale
var queryURL = 'https://onemotoring.lta.gov.sg/content/onemotoring/home/buying/coe-open-bidding.html'
const scrapedData = [];
const tableHeaders = ["Bidding End Date", "Timestamp", "Category", "Category Description", "Current COE Price", "Quota", "Bids Received"];
fetch(queryURL)
.then(res => res.text())
.then(body => {
//console.log(body)
const $ = cheerio.load(body);
//bidding Ends
const biddingEndDate = $(".coe_details p:first").text().substr(20, 18);
//Bidding Status and date
const biddingTimestamp = $(".coe_details h2:first").text().substr(21, 18);
//getting values of the current bid seen
$("body > div.content_container.app-content > main > div.standard-content > div > div.section.coe_details > div > table > tbody > tr").each((index, element) => {
if (index === 0) {
const ths = $(element).find("th");
$(ths).each((i, element) => {
tableHeaders.push(
$(element)
.text()
.toLowerCase()
);
});
return true;
}
const tds = $(element).find("td");
const tableRow = {};
tableRow[tableHeaders[0]] = biddingEndDate
tableRow[tableHeaders[1]] = biddingTimestamp
$(tds).each((i, element) => {
tableRow[tableHeaders[i + 2]] = $(element).text();
});
scrapedData.push(tableRow);
});
const CSV = arrayToCSV(scrapedData);
console.log(CSV)
writeToFile(CSV)
})
function arrayToCSV(data) {
csv = data.map(row => Object.values(row));
//csv.unshift(Object.keys(data[0]));
return `"${csv.join('"\n"').replace(/,/g, '","')}"`;
}
function writeToFile(value) {
fs.appendFile(path, value + "\n", function (err) {
if (err) {
console.log(err)
};
console.log('Saved!');
});
}