diff --git a/index.js b/index.js index ea1cf3a..5cf909f 100644 --- a/index.js +++ b/index.js @@ -14,12 +14,15 @@ const term = require( 'terminal-kit' ).terminal; const Downloader = require("nodejs-file-downloader"); const semver = require('semver'); //const _7z = require('7zip-min'); -const axios = require('axios') +const EasyDl = require("easydl"); +const chalk = require("chalk"); +const bytes = require("bytes"); const platform = os.platform() const shell = platform === 'win32' ? 'powershell.exe' : 'bash'; const L = require("./llama") const A = require("./alpaca") const TorrentDownloader = require("./torrent") +const cliProgress = require("cli-progress"); const exists = s => new Promise(r=>fs.access(s, fs.constants.F_OK, e => r(!e))) const escapeNewLine = (platform, arg) => platform === 'win32' ? arg.replaceAll(/\n/g, "\\n").replaceAll(/\r/g, "\\r") : arg const escapeDoubleQuotes = (platform, arg) => platform === 'win32' ? arg.replaceAll(/"/g, '`"') : arg.replaceAll(/"/g, '\\"') @@ -102,33 +105,97 @@ class Dalai { } return encodedStr; } - down(url, dest, headers) { - return new Promise((resolve, reject) => { - const task = path.basename(dest) - this.startProgress(task) - axios({ - url, - method: 'GET', - responseType: 'stream', - maxContentLength: Infinity, - headers, - onDownloadProgress: progressEvent => { - const progress = (progressEvent.loaded / progressEvent.total) * 100; - this.progress(task, progress) - } + async down(url, dest, headers) { + const task = path.basename(dest) + this.startProgress(task) + + const download = new EasyDl(url, dest, { + connections: 5, + maxRetry: 30, + existBehavior: "overwrite", + httpOptions: { + headers: headers || {} + } + }); + + download.on("progress", (progressReport) => { + this.progress(task, progressReport.total.percentage); + }); - }).then(response => { - const writer = fs.createWriteStream(dest); - response.data.pipe(writer); - writer.on('finish', () => { - this.progressBar.update(1); - term("\n") - resolve() + let recentError = null; + download.on("error", (error) => { + recentError = error; + }); + + await new Promise((accept, reject) => { + download.once("end", () => accept()); + download.once("close", () => reject(recentError || new Error("download failed"))); + + download.start(); + }); + + this.progressBar.update(1); + term("\n"); + } + async multiDownload(items) { + const multibar = new cliProgress.MultiBar({ + clearOnComplete: false, + hideCursor: true, + autopadding: true, + format: `${chalk.bold("{filename}")} ${chalk.yellow("{percentage}%")} ${chalk.cyan("{bar}")} {speed}${chalk.grey("{eta_formatted}")}`, + }, cliProgress.Presets.shades_classic); + + const longestFileName = items.reduce((acc, item) => { + return Math.max(acc, path.basename(item.dest).trim().length); + }, 0); + + async function downloadFile(url, dest, headers) { + const bar = multibar.create(100, 0); + bar.update(0, { + speed: "", + filename: path.basename(dest).trim().padEnd(longestFileName, " "), + }); + + const download = new EasyDl(url, dest, { + connections: 5, + maxRetry: 30, + existBehavior: "overwrite", + httpOptions: { + headers: headers || {} + } + }); + + download.on("progress", (progressReport) => { + bar.update(progressReport.total.percentage, { + speed: Number.isFinite(progressReport.total.speed) ? chalk.blue((bytes(progressReport.total.speed) + "/s").padEnd(10)) + chalk.grey(" | ") : "" }); - }).catch(error => { - reject(error) }); - }) + + let recentError = null; + download.on("error", (error) => { + recentError = error; + }); + + await new Promise((accept, reject) => { + download.once("end", () => accept()); + download.once("close", () => reject(recentError || new Error("download failed"))); + + download.start(); + }); + + bar.update(100); + bar.stop(); + } + + const downloads = []; + for (const item of items) { + const { url, dest, headers } = item; + downloads.push(downloadFile(url, dest, headers)); + } + + await Promise.all(downloads); + multibar.stop(); + term("\n"); } async python () { // install self-contained python => only for windows for now diff --git a/llama.js b/llama.js index 6e54625..0ed76f7 100644 --- a/llama.js +++ b/llama.js @@ -138,6 +138,7 @@ npx dalai install 7B 13B const resolvedPath = path.resolve(this.home, "models", model) await fs.promises.mkdir(resolvedPath, { recursive: true }).catch((e) => { }) + const filesToDownload = []; for(let file of files) { if (fs.existsSync(path.resolve(resolvedPath, file))) { console.log(`Skip file download, it already exists: ${file}`) @@ -145,12 +146,18 @@ npx dalai install 7B 13B } const url = `https://agi.gpt4.org/llama/LLaMA/${model}/${file}` - await this.root.down(url, path.resolve(resolvedPath, file), { - "User-Agent": "Mozilla/5.0" - }) + filesToDownload.push({ + url, + dest: path.resolve(resolvedPath, file), + headers: { + "User-Agent": "Mozilla/5.0" + } + }); } + await this.root.multiDownload(filesToDownload); const files2 = ["tokenizer_checklist.chk", "tokenizer.model"] + const filesToDownload2 = []; for(let file of files2) { // if (fs.existsSync(path.resolve(this.home, "models", file))) { // console.log(`Skip file download, it already exists: ${file}`) @@ -158,10 +165,15 @@ npx dalai install 7B 13B // } const url = `https://agi.gpt4.org/llama/LLaMA/${file}` const dir = path.resolve(this.home, "models") - await this.root.down(url, path.resolve(dir, file), { - "User-Agent": "Mozilla/5.0" - }) + filesToDownload2.push({ + url, + dest: path.resolve(dir, file), + headers: { + "User-Agent": "Mozilla/5.0" + } + }); } + await this.root.multiDownload(filesToDownload2); } } diff --git a/package.json b/package.json index 8839c99..8a23d6e 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,10 @@ "postinstall": "node setup" }, "dependencies": { - "axios": "^1.3.4", + "bytes": "^3.1.2", + "chalk": "^4.1.2", + "cli-progress": "^3.12.0", + "easydl": "^1.0.3", "ejs": "^3.1.8", "express": "^4.18.2", "isomorphic-git": "^1.22.0",