Fix scripts

sefinek · Mar 6, 2024 · 9404506 · 9404506
1 parent f4c4a5e
commit 9404506
Show file tree

Hide file tree

Showing 8 changed files with 103 additions and 137 deletions.
diff --git a/bash/download.sh b/bash/download.sh
@@ -6,7 +6,11 @@
 # List of URLs for blocklists. Each entry includes the URL and the desired path.
 # The path now includes an author-specific folder and a "fork" prefix for the filename.
 urls=(
-  ## Ads
+  ## Abuse
+  "https://blocklistproject.github.io/Lists/abuse.txt abuse/blocklistproject/hosts.fork.txt"
+  "https://urlhaus.abuse.ch/downloads/hostfile abuse/urlhaus.abuse.ch/hostfile.fork.txt"
+
+  ## Advertising
   "https://adaway.org/hosts.txt ads/adaway/hosts.fork.txt"
   "https://blocklistproject.github.io/Lists/ads.txt ads/blocklistproject/hosts.fork.txt"
   "https://blocklistproject.github.io/Lists/youtube.txt ads/blocklistproject/youtube.fork.txt"
@@ -27,6 +31,10 @@ urls=(
   "https://v.firebog.net/hosts/Easylist.txt ads/firebog/Easylist.fork.txt"
   "https://v.firebog.net/hosts/Prigent-Ads.txt ads/firebog/Prigent-Ads.fork.txt"
 
+  ## AMP hosts
+  "https://ente.dev/api/blocklist/google-amp-hosts amp/ente-dev/google-amp-hosts.fork.txt"
+  "https://www.github.developerdan.com/hosts/lists/amp-hosts-extended.txt amp/developerdan/amp-hosts-extended.fork.txt"
+
   ## Tracking and telemetry
   "https://ente.dev/api/blocklist/tv tracking-and-telemetry/ente-dev/tv.fork.txt"
   "https://hostfiles.frogeye.fr/firstparty-trackers-hosts.txt tracking-and-telemetry/frogeye/firstparty-trackers-hosts.txt"
@@ -35,10 +43,6 @@ urls=(
   "https://raw.githubusercontent.com/mitchellkrogza/Stop.Google.Analytics.Ghost.Spam.HOWTO/master/output/domains/INACTIVE/list tracking-and-telemetry/mitchellkrogza/INACTIVE.fork.txt"
   "https://raw.githubusercontent.com/neodevpro/neodevhost/master/host tracking-and-telemetry/neodevpro/host.fork.txt"
 
-  ## AMP hosts
-  "https://ente.dev/api/blocklist/google-amp-hosts amp/ente-dev/google-amp-hosts.fork.txt"
-  "https://www.github.developerdan.com/hosts/lists/amp-hosts-extended.txt amp/developerdan/amp-hosts-extended.fork.txt"
-
   ## Malicious
   "https://blocklistproject.github.io/Lists.malware.txt malicious/blocklistproject/malware.fork.txt"
   "https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt malicious/quidsup/notrack-malware.fork.txt"
@@ -67,10 +71,6 @@ urls=(
   "https://v.firebog.net/hosts/Prigent-Crypto.txt cryptojacking/firebog/Prigent/Crypto.fork.txt"
   "https://zerodot1.gitlab.io/CoinBlockerLists/hosts cryptojacking/zerodot1/CoinBlockerLists-hosts.fork.txt"
 
-  ## Abuse
-  "https://blocklistproject.github.io/Lists/abuse.txt abuse/blocklistproject/hosts.fork.txt"
-  "https://urlhaus.abuse.ch/downloads/hostfile abuse/urlhaus.abuse.ch/hostfile.fork.txt"
-
   ## Fraud
   "https://blocklistproject.github.io/Lists/fraud.txt fraud/blocklistproject/hosts.fork.txt"
 

diff --git a/scripts/functions/sha256.js b/scripts/functions/sha256.js
diff --git a/scripts/functions/sha512.js b/scripts/functions/sha512.js
@@ -0,0 +1,29 @@
+const { promises: fs } = require('node:fs');
+const path = require('node:path');
+const { createHash } = require('node:crypto');
+
+module.exports = async (thisFileName, type) => {
+	const basePath = path.join(__dirname, '../../blocklist/template');
+	const relativePath = path.relative(basePath, path.dirname(thisFileName));
+	const cacheFolder = path.join(__dirname, `../../cache/${type}`, relativePath);
+
+	await fs.mkdir(cacheFolder, { recursive: true }).catch(err => console.error(`❌ Error creating cache folder: ${err}`));
+
+	const cacheFilePath = path.join(cacheFolder, `${path.basename(thisFileName, '.txt')}.sha512`);
+	const hashFromCacheFile = await fs.readFile(cacheFilePath, 'utf8').catch(() => null);
+
+	const buff = await fs.readFile(thisFileName);
+	const hash = createHash('sha512').update(buff).digest('hex');
+	if (hash === hashFromCacheFile) {
+		console.log(`⏭️ ${hash} / ${type}:${path.basename(thisFileName)} / skipped`);
+		return { stop: true };
+	}
+
+	await fs.writeFile(cacheFilePath, hash).catch(err => {
+		console.error(`❌ Error writing cache file ${cacheFilePath}: ${err}`);
+		return { stop: true };
+	});
+
+	console.log(`✅ ${hash} -> ${type}:${path.basename(thisFileName)} / hashed`);
+	return { cacheHash: hash, stop: false };
+};
diff --git a/scripts/generate/0.0.0.0.js b/scripts/generate/0.0.0.0.js
@@ -1,10 +1,12 @@
 const { promises: fs } = require('node:fs');
 const path = require('node:path');
 const date = require('../functions/date.js');
-const sha256 = require('../functions/sha256.js');
+const sha256 = require('../functions/sha512.js');
 
-const convert = async (folderPath = path.join(__dirname, '../../blocklist/template')) => {
-	const generatedPath = path.join(__dirname, '../../blocklist/generated/0.0.0.0');
+const format = '0.0.0.0';
+
+const convert = async (folderPath = path.join(__dirname, '../../blocklist/template'), relativePath = '') => {
+	const generatedPath = path.join(__dirname, `../../blocklist/generated/${format}`, relativePath);
 	try {
 		await fs.access(generatedPath);
 	} catch (err) {
@@ -18,40 +20,32 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 		const thisFileName = path.join(folderPath, file.name);
 
 		// Cache
-		const { cacheHash, stop } = await sha256(thisFileName, '0.0.0.0', file);
+		const { cacheHash, stop } = await sha256(thisFileName, format, file);
 		if (stop) return;
 
 		// Content
 		const fileContent = await fs.readFile(thisFileName, 'utf8');
 		const replacedFile = fileContent
 			.replaceAll(/^(?:127\.0\.0\.1|0\.0\.0\.0) /gmu, '0.0.0.0 ')
-			// grex "#0.0.0.0 " "#127.0.0.1 " "# 0.0.0.0 " "# 127.0.0.1 " ":: "
 			.replaceAll(/#(?: ?127\.0\.0\.1| ?0\.0\.0\.0) |:: /gmu, '# 0.0.0.0 ')
 			.replace(/<Release>/gim, '0.0.0.0 before each domain')
 			.replace(/<Version>/gim, date.timestamp)
 			.replace(/<LastUpdate>/gim, `${date.full} | ${date.now} | ${date.timezone}`);
 
-		const subFolderName = path.basename(path.dirname(thisFileName));
-		const categoryPath = subFolderName === 'template' ? generatedPath : path.join(generatedPath, subFolderName);
-		const fullNewFile = path.join(categoryPath, file.name);
-
-		try {
-			await fs.access(categoryPath);
-		} catch (err) {
-			await fs.mkdir(categoryPath, { recursive: true });
-		}
-
+		const fullNewFile = path.join(generatedPath, file.name);
 		await fs.writeFile(fullNewFile, replacedFile);
-		console.log(`✔️ ${cacheHash || file.name} saved in ${thisFileName}`);
+
+		console.log(`✔️ ${cacheHash || file.name} ++ ${fullNewFile}`);
 	}));
 
 	try {
 		const subdirectories = files.filter(file => file.isDirectory());
 		await Promise.all(subdirectories.map(async subdirectory => {
-			await convert(path.join(folderPath, subdirectory.name));
+			const nextRelativePath = path.join(relativePath, subdirectory.name);
+			await convert(path.join(folderPath, subdirectory.name), nextRelativePath);
 		}));
 	} catch (err) {
-		console.error(`❌ ${folderPath}:`, err);
+		console.error(`❌ Error processing ${folderPath}:`, err);
 	}
 };
 
@@ -62,4 +56,4 @@ const run = async () => {
 
 (async () => await run())();
 
-module.exports = () => run;
+module.exports = run;
diff --git a/scripts/generate/127.0.0.1.js b/scripts/generate/127.0.0.1.js
@@ -1,10 +1,12 @@
 const { promises: fs } = require('node:fs');
 const path = require('node:path');
 const date = require('../functions/date.js');
-const sha256 = require('../functions/sha256.js');
+const sha256 = require('../functions/sha512.js');
 
-const convert = async (folderPath = path.join(__dirname, '../../blocklist/template')) => {
-	const generatedPath = path.join(__dirname, '../../blocklist/generated/127.0.0.1');
+const format = '127.0.0.1';
+
+const convert = async (folderPath = path.join(__dirname, '../../blocklist/template'), relativePath = '') => {
+	const generatedPath = path.join(__dirname, `../../blocklist/generated/${format}`, relativePath);
 	try {
 		await fs.access(generatedPath);
 	} catch (err) {
@@ -18,7 +20,7 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 		const thisFileName = path.join(folderPath, file.name);
 
 		// Cache
-		const { cacheHash, stop } = await sha256(thisFileName, '127.0.0.1', file);
+		const { cacheHash, stop } = await sha256(thisFileName, format, file);
 		if (stop) return;
 
 		// Content
@@ -30,27 +32,20 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 			.replace(/<Version>/gim, date.timestamp.toString())
 			.replace(/<LastUpdate>/gim, `${date.full} | ${date.now} | ${date.timezone}`);
 
-		const subFolderName = path.basename(path.dirname(thisFileName));
-		const categoryPath = subFolderName === 'template' ? generatedPath : path.join(generatedPath, subFolderName);
-		const fullNewFile = path.join(categoryPath, file.name);
-
-		try {
-			await fs.access(categoryPath);
-		} catch (err) {
-			await fs.mkdir(categoryPath, { recursive: true });
-		}
-
+		const fullNewFile = path.join(generatedPath, file.name);
 		await fs.writeFile(fullNewFile, replacedFile);
-		console.log(`✔️ ${cacheHash || file.name} saved in ${thisFileName}`);
+
+		console.log(`✔️ ${cacheHash || file.name} ++ ${fullNewFile}`);
 	}));
 
 	try {
 		const subdirectories = files.filter(file => file.isDirectory());
 		await Promise.all(subdirectories.map(async subdirectory => {
-			await convert(path.join(folderPath, subdirectory.name));
+			const nextRelativePath = path.join(relativePath, subdirectory.name);
+			await convert(path.join(folderPath, subdirectory.name), nextRelativePath);
 		}));
 	} catch (err) {
-		console.error(`❌ ${folderPath}:`, err);
+		console.error(`❌ Error processing ${folderPath}:`, err);
 	}
 };
 
@@ -61,4 +56,4 @@ const run = async () => {
 
 (async () => await run())();
 
-module.exports = () => run;
+module.exports = run;
diff --git a/scripts/generate/adguard.js b/scripts/generate/adguard.js
@@ -1,10 +1,12 @@
 const { promises: fs } = require('node:fs');
 const path = require('node:path');
 const date = require('../functions/date.js');
-const sha256 = require('../functions/sha256.js');
+const sha256 = require('../functions/sha512.js');
 
-const convert = async (folderPath = path.join(__dirname, '../../blocklist/template')) => {
-	const generatedPath = path.join(__dirname, '../../blocklist/generated/adguard');
+const format = 'adguard';
+
+const convert = async (folderPath = path.join(__dirname, '../../blocklist/template'), relativePath = '') => {
+	const generatedPath = path.join(__dirname, `../../blocklist/generated/${format}`, relativePath);
 	try {
 		await fs.access(generatedPath);
 	} catch (err) {
@@ -18,7 +20,7 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 		const thisFileName = path.join(folderPath, file.name);
 
 		// Cache
-		const { cacheHash, stop } = await sha256(thisFileName, 'adguard', file);
+		const { cacheHash, stop } = await sha256(thisFileName, format, file);
 		if (stop) return;
 
 		// Content
@@ -36,27 +38,20 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 			.replace(/<Version>/gim, date.timestamp.toString())
 			.replace(/<LastUpdate>/gim, `${date.full} | ${date.now} | ${date.timezone}`);
 
-		const subFolderName = path.basename(path.dirname(thisFileName));
-		const categoryPath = subFolderName === 'template' ? generatedPath : path.join(generatedPath, subFolderName);
-		const fullNewFile = path.join(categoryPath, file.name);
-
-		try {
-			await fs.access(categoryPath);
-		} catch (err) {
-			await fs.mkdir(categoryPath, { recursive: true });
-		}
-
+		const fullNewFile = path.join(generatedPath, file.name);
 		await fs.writeFile(fullNewFile, replacedFile);
-		console.log(`✔️ ${cacheHash || file.name} saved in ${thisFileName}`);
+
+		console.log(`✔️ ${cacheHash || file.name} ++ ${fullNewFile}`);
 	}));
 
 	try {
 		const subdirectories = files.filter(file => file.isDirectory());
 		await Promise.all(subdirectories.map(async subdirectory => {
-			await convert(path.join(folderPath, subdirectory.name));
+			const nextRelativePath = path.join(relativePath, subdirectory.name);
+			await convert(path.join(folderPath, subdirectory.name), nextRelativePath);
 		}));
 	} catch (err) {
-		console.error(`❌ ${folderPath}:`, err);
+		console.error(`❌ Error processing ${folderPath}:`, err);
 	}
 };
 
@@ -67,4 +62,4 @@ const run = async () => {
 
 (async () => await run())();
 
-module.exports = () => run;
+module.exports = run;
diff --git a/scripts/generate/dnsmasq.js b/scripts/generate/dnsmasq.js
@@ -1,10 +1,12 @@
 const { promises: fs } = require('node:fs');
 const path = require('node:path');
 const date = require('../functions/date.js');
-const sha256 = require('../functions/sha256.js');
+const sha256 = require('../functions/sha512.js');
 
-const convert = async (folderPath = path.join(__dirname, '../../blocklist/template')) => {
-	const generatedPath = path.join(__dirname, '../../blocklist/generated/dnsmasq');
+const format = 'dnsmasq';
+
+const convert = async (folderPath = path.join(__dirname, '../../blocklist/template'), relativePath = '') => {
+	const generatedPath = path.join(__dirname, `../../blocklist/generated/${format}`, relativePath);
 	try {
 		await fs.access(generatedPath);
 	} catch (err) {
@@ -18,7 +20,7 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 		const thisFileName = path.join(folderPath, file.name);
 
 		// Cache
-		const { cacheHash, stop } = await sha256(thisFileName, 'dnsmasq', file);
+		const { cacheHash, stop } = await sha256(thisFileName, format, file);
 		if (stop) return;
 
 		// Content
@@ -37,27 +39,20 @@ const convert = async (folderPath = path.join(__dirname, '../../blocklist/templa
 			.replace(/<Version>/gim, date.timestamp.toString())
 			.replace(/<LastUpdate>/gim, `${date.full} | ${date.now} | ${date.timezone}`);
 
-		const subFolderName = path.basename(path.dirname(thisFileName));
-		const categoryPath = subFolderName === 'template' ? generatedPath : path.join(generatedPath, subFolderName);
-		const fullNewFile = path.join(categoryPath, file.name);
-
-		try {
-			await fs.access(categoryPath);
-		} catch (err) {
-			await fs.mkdir(categoryPath, { recursive: true });
-		}
-
+		const fullNewFile = path.join(generatedPath, file.name);
 		await fs.writeFile(fullNewFile, replacedFile);
-		console.log(`✔️ ${cacheHash || file.name} saved in ${thisFileName}`);
+
+		console.log(`✔️ ${cacheHash || file.name} ++ ${fullNewFile}`);
 	}));
 
 	try {
 		const subdirectories = files.filter(file => file.isDirectory());
 		await Promise.all(subdirectories.map(async subdirectory => {
-			await convert(path.join(folderPath, subdirectory.name));
+			const nextRelativePath = path.join(relativePath, subdirectory.name);
+			await convert(path.join(folderPath, subdirectory.name), nextRelativePath);
 		}));
 	} catch (err) {
-		console.error(`❌ ${folderPath}:`, err);
+		console.error(`❌ Error processing ${folderPath}:`, err);
 	}
 };
 
@@ -68,4 +63,4 @@ const run = async () => {
 
 (async () => await run())();
 
-module.exports = () => run;
+module.exports = run;