Merge branch 'main' into mpopovac-txfusion-example-noninline-missing-…

…libraries Merge main
matter-labs · Sep 8, 2023 · a0612b3 · a0612b3
2 parents b5c2485 + e88444f
commit a0612b3
Show file tree

Hide file tree

Showing 13 changed files with 239 additions and 43 deletions.
diff --git a/packages/hardhat-zksync-deploy/CHANGELOG.md b/packages/hardhat-zksync-deploy/CHANGELOG.md
@@ -1,9 +1,16 @@
 # @matterlabs/hardhat-zksync-deploy
 
+## 0.6.5
+
+### Patch Changes
+
+- Updated dependencies [a079196]
+  - @matterlabs/hardhat-zksync-solc@0.4.2
+
 ## 0.6.4
 
 ### Patch Changes
 
-- 6507daa: 
+- 6507daa:
   - Use configured wallet provider in the deployer class
   - Prevent duplicated bytecodes in factory deps
diff --git a/packages/hardhat-zksync-deploy/package.json b/packages/hardhat-zksync-deploy/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@matterlabs/hardhat-zksync-deploy",
-  "version": "0.6.5-alpha.1",
+  "version": "0.6.5",
   "description": "Hardhat plugin to deploy smart contracts into the zkSync network",
   "repository": "github:matter-labs/hardhat-zksync",
   "homepage": "https://github.com/matter-labs/hardhat-zksync/tree/main/packages/hardhat-zksync-deploy",
@@ -32,7 +32,7 @@
     "README.md"
   ],
   "dependencies": {
-    "@matterlabs/hardhat-zksync-solc": "0.4.2-alpha.1",
+    "@matterlabs/hardhat-zksync-solc": "0.4.2",
     "chalk": "4.1.2",
     "ts-morph": "^19.0.0"
   },

diff --git a/packages/hardhat-zksync-solc/CHANGELOG.md b/packages/hardhat-zksync-solc/CHANGELOG.md
@@ -1,5 +1,12 @@
 # @matterlabs/hardhat-zksync-solc
 
+## 0.4.2
+
+### Patch Changes
+
+- a079196: - Added detect-missing-library mode
+  - Added release URL as primary download source for zkvyper compiler
+
 ## 0.4.1
 
 ### Patch Changes

diff --git a/packages/hardhat-zksync-solc/package.json b/packages/hardhat-zksync-solc/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@matterlabs/hardhat-zksync-solc",
-  "version": "0.4.2-alpha.1",
+  "version": "0.4.2",
   "description": "Hardhat plugin to compile smart contracts for the zkSync network",
   "repository": "github:matter-labs/hardhat-zksync",
   "homepage": "https://github.com/matter-labs/hardhat-zksync/tree/main/packages/hardhat-zksync-solc",

diff --git a/packages/hardhat-zksync-solc/src/compile/downloader.ts b/packages/hardhat-zksync-solc/src/compile/downloader.ts
@@ -3,10 +3,9 @@ import fsExtra from "fs-extra";
 import chalk from "chalk";
 import { spawnSync } from 'child_process';
 
-import { download } from 'hardhat/internal/util/download';
-import { Mutex } from 'hardhat/internal/vendor/await-semaphore';
+import { performance } from 'perf_hooks';
 
-import { getZksolcUrl, isURL, isVersionInRange, saltFromUrl } from "../utils";
+import { download, getZksolcUrl, isURL, isVersionInRange, saltFromUrl } from "../utils";
 import { 
     COMPILER_BINARY_CORRUPTION_ERROR, 
     COMPILER_VERSION_INFO_FILE_DOWNLOAD_ERROR, 
@@ -157,20 +156,36 @@ export class ZksolcCompilerDownloader {
         const url = `${ZKSOLC_BIN_VERSION_INFO}/version.json`;
         const downloadPath = this._getCompilerVersionInfoPath(compilersDir);
 
-        await download(url, downloadPath, 30000);
+        await download(url, downloadPath, 'hardhat-zksync', 'compiler-version-info', 30000);
     }
 
     private async _downloadCompiler(): Promise<string> {
-        let url = this._configCompilerPath;
-        if (!this._isCompilerPathURL) {
-            url = getZksolcUrl(ZKSOLC_BIN_REPOSITORY, this._version);
-        }
-
         const downloadPath = this.getCompilerPath();
-        await download(url, downloadPath, 30000);
+
+        const url = this._getCompilerUrl(true);
+        try {
+            await this._attemptDownload(url, downloadPath);
+        } catch (e: any) {
+            if (!this._isCompilerPathURL) {
+                const fallbackUrl = this._getCompilerUrl(false);
+                await this._attemptDownload(fallbackUrl, downloadPath);
+            }
+        }
+
         return downloadPath;
     }
 
+    private _getCompilerUrl(useGithubRelease: boolean): string {
+        if (this._isCompilerPathURL) {
+            return this._configCompilerPath;
+        }
+        return getZksolcUrl(ZKSOLC_BIN_REPOSITORY, this._version, useGithubRelease);
+    }
+
+    private async _attemptDownload(url: string, downloadPath: string): Promise<void> {
+        return download(url, downloadPath, 'hardhat-zksync', this._version, 30000);
+    }
+
     private static async _readCompilerVersionInfo(compilerVersionInfoPath: string): Promise<CompilerVersionInfo> {
         return fsExtra.readJSON(compilerVersionInfoPath);
     }

diff --git a/packages/hardhat-zksync-solc/src/compile/index.ts b/packages/hardhat-zksync-solc/src/compile/index.ts
@@ -1,6 +1,7 @@
 import { ZkSolcConfig } from '../types';
 import { compileWithBinary } from './binary';
 import { HardhatDocker, Image } from '@nomiclabs/hardhat-docker';
+import semver from 'semver';
 import {
     validateDockerIsInstalled,
     createDocker,
@@ -12,6 +13,7 @@ import {
 import { CompilerInput } from 'hardhat/types';
 import { ZkSyncSolcPluginError } from '../errors';
 import { findMissingLibraries, mapMissingLibraryDependencies, writeLibrariesToFile } from '../utils';
+import { DETECT_MISSING_LIBRARY_MODE_COMPILER_VERSION } from '../constants';
 
 export async function compile(zksolcConfig: ZkSolcConfig, input: CompilerInput, solcPath?: string) {
     let compiler: ICompiler;
@@ -38,21 +40,23 @@ export class BinaryCompiler implements ICompiler {
 
     public async compile(input: CompilerInput, config: ZkSolcConfig) {
         // Check for missing libraries
-        const zkSolcOutput = await compileWithBinary(input, config, this.solcPath, true);
+        if (semver.gte(config.version, DETECT_MISSING_LIBRARY_MODE_COMPILER_VERSION)) {
+            const zkSolcOutput = await compileWithBinary(input, config, this.solcPath, true);
 
-        const missingLibraries = findMissingLibraries(zkSolcOutput);
-        if (missingLibraries.size > 0) {
-            if (!config.settings.missingLibrariesPath) {
-                throw new ZkSyncSolcPluginError('Missing libraries path is not specified');
-            }
-
-            const missingLibraryDependencies = mapMissingLibraryDependencies(zkSolcOutput, missingLibraries);
-            // Write missing libraries to file
-            const missingLibrariesPath = config.settings.missingLibrariesPath!;
-            await writeLibrariesToFile(missingLibrariesPath, missingLibraryDependencies);
+            const missingLibraries = findMissingLibraries(zkSolcOutput);
+            if (missingLibraries.size > 0) {
+                if (!config.settings.missingLibrariesPath) {
+                    throw new ZkSyncSolcPluginError('Missing libraries path is not specified');
+                }
+                
+                const missingLibraryDependencies = mapMissingLibraryDependencies(zkSolcOutput, missingLibraries);
+                // Write missing libraries to file
+                const missingLibrariesPath = config.settings.missingLibrariesPath!;
+                await writeLibrariesToFile(missingLibrariesPath, missingLibraryDependencies);
 
-            config.settings.areLibrariesMissing = true;
-            return zkSolcOutput;
+                config.settings.areLibrariesMissing = true;
+                return zkSolcOutput;
+            }
         }
 
         // Make sure that libraries are not missing.

diff --git a/packages/hardhat-zksync-solc/src/constants.ts b/packages/hardhat-zksync-solc/src/constants.ts
@@ -4,6 +4,7 @@ export const PLUGIN_NAME = '@matterlabs/hardhat-zksync-solc';
 export const ZK_ARTIFACT_FORMAT_VERSION = 'hh-zksolc-artifact-1';
 export const ZKSOLC_BIN_REPOSITORY = 'https://github.com/matter-labs/zksolc-bin';
 export const ZKSOLC_BIN_VERSION_INFO = `https://raw.githubusercontent.com/matter-labs/zksolc-bin/main`;
+export const DETECT_MISSING_LIBRARY_MODE_COMPILER_VERSION = '1.3.14';
 
 export const defaultZkSolcConfig: ZkSolcConfig = {
     version: 'latest',
@@ -43,7 +44,7 @@ export const COMPILING_INFO_MESSAGE = (zksolcVersion: string, solcVersion: strin
 
 export const MISSING_LIBRARIES_NOTICE = 'zksolc compiler detected missing libraries! For more details, visit: https://era.zksync.io/docs/tools/hardhat/compiling-libraries.html.';
 export const COMPILE_AND_DEPLOY_LIBRARIES_INSTRUCTIONS = 'To compile and deploy libraries, please run: `yarn hardhat deploy-zksync:libraries --private-key <PRIVATE_KEY>`';
-export const MISSING_LIBRARY_LINK = 'For more details on how to use deploy-zksync:libraries task from hardhat-zksync-deploy plugin, visit: <LINK TO ADD>.';
+export const MISSING_LIBRARY_LINK = 'For more details on how to use deploy-zksync:libraries task from hardhat-zksync-deploy plugin, visit: https://era.zksync.io/docs/tools/hardhat/hardhat-zksync-deploy.html.';
 
 export const SOLCJS_EXECUTABLE_CODE = `#!/usr/bin/env node
 "use strict";var __createBinding=this&&this.__createBinding||(Object.create?function(e,t,r,o){void 0===o&&(o=r);var i=Object.getOwnPropertyDescriptor(t,r);i&&("get"in i?t.__esModule:!i.writable&&!i.configurable)||(i={enumerable:!0,get:function(){return t[r]}}),Object.defineProperty(e,o,i)}:function(e,t,r,o){void 0===o&&(o=r),e[o]=t[r]}),__setModuleDefault=this&&this.__setModuleDefault||(Object.create?function(e,t){Object.defineProperty(e,"default",{enumerable:!0,value:t})}:function(e,t){e.default=t}),__importStar=this&&this.__importStar||function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var r in e)"default"!==r&&Object.prototype.hasOwnProperty.call(e,r)&&__createBinding(t,e,r);return __setModuleDefault(t,e),t},__importDefault=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(exports,"__esModule",{value:!0}),exports._loadCompilerSources=void 0;const os_1=__importDefault(require("os")),fs_1=__importDefault(require("fs")),path_1=__importDefault(require("path"));function packageExists(e){try{return require.resolve(e),!0}catch(e){return!1}}function findPackagePath(e,t){let r=t,o=r+"/node_modules/"+e;for(;"/"!==r;){if(packageExists(o))return o;r=path_1.default.dirname(r),o=r+"/node_modules/"+e}}async function getSolc(e,t){var r=findPackagePath("solc/wrapper",t);const{default:o}=await Promise.resolve().then(()=>__importStar(require(r)));return o(_loadCompilerSources(e))}function _loadCompilerSources(e){const t=module.constructor;if(void 0===t._extensions)return require(e);var r=t._extensions[".js"];t._extensions[".js"]=function(e,t){var r=fs_1.default.readFileSync(t,"utf8");Object.getPrototypeOf(e)._compile.call(e,r,t)};e=require(e);return t._extensions[".js"]=r,e}exports._loadCompilerSources=_loadCompilerSources;async function readStdin(){return new Promise(e=>{let t="";process.stdin.on("data",e=>t+=e),process.stdin.on("end",()=>e(t))})}!async function(){var e;const t=await getSolc("SOLCJS_PATH","WORKING_DIR");process.argv.includes("--version")?(e=await t.version(),process.stdout.write("solc, the solidity compiler commandline interface"+os_1.default.EOL),process.stdout.write("Version: "+e+os_1.default.EOL)):(e=await readStdin(),e=t.compile(e),process.stdout.write(e))}();`;
diff --git a/packages/hardhat-zksync-solc/src/utils.ts b/packages/hardhat-zksync-solc/src/utils.ts
@@ -7,6 +7,12 @@ import { CompilerVersionInfo } from './compile/downloader';
 import fse from 'fs-extra';
 import lockfile from 'proper-lockfile';
 import { ZkSyncSolcPluginError } from './errors';
+import fs from "fs";
+import path from "path";
+import util from "util";
+import type { Dispatcher } from "undici";
+
+const TEMP_FILE_PREFIX = "tmp-";
 
 export function filterSupportedOutputSelections(outputSelection: CompilerOutputSelection, zkCompilerVersion: string): CompilerOutputSelection {
     const filteredOutputSelection: CompilerOutputSelection = {};
@@ -78,13 +84,18 @@ export function saltFromUrl(url: string): string {
     return sha1(url);
 }
 
-export function getZksolcUrl(repo: string, version: string): string {
+export function getZksolcUrl(repo: string, version: string, isRelease: boolean = true): string {
     // @ts-ignore
     const platform = { darwin: 'macosx', linux: 'linux', win32: 'windows' }[process.platform];
     // @ts-ignore
     const toolchain = { linux: '-musl', win32: '-gnu', darwin: '' }[process.platform];
     const arch = process.arch == 'x64' ? 'amd64' : process.arch;
     const ext = process.platform == 'win32' ? '.exe' : '';
+
+    if (isRelease) {
+        return `${repo}/releases/download/v${version}/zksolc-${platform}-${arch}${toolchain}-v${version}${ext}`;
+    }
+
     return `${repo}/raw/main/${platform}-${arch}/zksolc-${platform}-${arch}${toolchain}-v${version}${ext}`;
 }
 
@@ -186,4 +197,57 @@ export const writeLibrariesToFile = async (path: string, libraries: any[]): Prom
     } finally {
         await lockfile.unlock(path);
     }
-};
+};
+
+function resolveTempFileName(filePath: string): string {
+    const { dir, ext, name } = path.parse(filePath);
+
+    return path.format({
+        dir,
+        ext,
+        name: `${TEMP_FILE_PREFIX}${name}`,
+    });
+}
+
+export async function download(
+    url: string,
+    filePath: string,
+    userAgent: string,
+    version: string,
+    timeoutMillis = 10000,
+    extraHeaders: { [name: string]: string } = {}
+) {
+    const { pipeline } = await import("stream");
+    const { getGlobalDispatcher, request } = await import("undici");
+    const streamPipeline = util.promisify(pipeline);
+
+    let dispatcher: Dispatcher = getGlobalDispatcher();
+
+    // Fetch the url
+    const response = await request(url, {
+        dispatcher,
+        headersTimeout: timeoutMillis,
+        maxRedirections: 10,
+        method: "GET",
+        headers: {
+            ...extraHeaders,
+            "User-Agent": `${userAgent} ${version}`,
+        },
+    });
+
+    if (response.statusCode >= 200 && response.statusCode <= 299) {
+        const tmpFilePath = resolveTempFileName(filePath);
+        await fse.ensureDir(path.dirname(filePath));
+
+        await streamPipeline(response.body, fs.createWriteStream(tmpFilePath));
+        return fse.move(tmpFilePath, filePath, { overwrite: true });
+    }
+
+    // undici's response bodies must always be consumed to prevent leaks
+    const text = await response.body.text();
+
+    // eslint-disable-next-line
+    throw new Error(
+        `Failed to download ${url} - ${response.statusCode} received. ${text}`
+    );
+}
diff --git a/packages/hardhat-zksync-vyper/CHANGELOG.md b/packages/hardhat-zksync-vyper/CHANGELOG.md
@@ -1,10 +1,16 @@
 # @matterlabs/hardhat-zksync-vyper
 
+## 0.2.1
+
+### Patch Changes
+
+- 3407adc: Add release URL as primary download source for zkvyper compiler
+
 ## 0.2.0
 
 ### Minor Changes
 
-- 6ff144f: 
+- 6ff144f:
   - Enhanced zkvyper compiler version checking mechanism.
   - Improved error handling for incorrect zkvyper compiler versions.
   - Optimized validation process for zkvyper compiler configuration.

diff --git a/packages/hardhat-zksync-vyper/package.json b/packages/hardhat-zksync-vyper/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@matterlabs/hardhat-zksync-vyper",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Hardhat plugin to compile Vyper smart contracts for the zkSync network",
   "repository": "github:matter-labs/hardhat-zksync",
   "homepage": "https://github.com/matter-labs/hardhat-zksync/tree/main/packages/hardhat-zksync-vyper",

diff --git a/packages/hardhat-zksync-vyper/src/compile/downloader.ts b/packages/hardhat-zksync-vyper/src/compile/downloader.ts
@@ -3,9 +3,7 @@ import fsExtra from "fs-extra";
 import chalk from "chalk";
 import { spawnSync } from 'child_process';
 
-import { download } from 'hardhat/internal/util/download';
-
-import { getZkvyperUrl, isURL, isVersionInRange } from "../utils";
+import { download, getZkvyperUrl, isURL, isVersionInRange } from "../utils";
 import { 
     COMPILER_BINARY_CORRUPTION_ERROR, 
     COMPILER_VERSION_INFO_FILE_DOWNLOAD_ERROR, 
@@ -148,20 +146,37 @@ export class ZkVyperCompilerDownloader {
         const url = `${ZKVYPER_BIN_VERSION_INFO}/version.json`;
         const downloadPath = this._getCompilerVersionInfoPath(compilersDir);
 
-        await download(url, downloadPath);
+        await download(url, downloadPath, 'hardhat-zksync-zkvyper', 'compiler-version-info', 30000);
     }
 
     private async _downloadCompiler(): Promise<string> {
-        let url = this._configCompilerPath;
-        if (!isURL(url)) {
-            url = getZkvyperUrl(ZKVYPER_BIN_REPOSITORY, this._version);
+        const downloadPath = this.getCompilerPath();
+
+        const url = this._getCompilerUrl(true);
+        try {
+            await this._attemptDownload(url, downloadPath);
+        } catch (e: any) {
+            if (!isURL(this._configCompilerPath)) {
+                const fallbackUrl = this._getCompilerUrl(false);
+                await this._attemptDownload(fallbackUrl, downloadPath);
+            }
         }
 
-        const downloadPath = this.getCompilerPath();
-        await download(url, downloadPath);
         return downloadPath;
     }
 
+    private _getCompilerUrl(useGithubRelease: boolean): string {
+        if (isURL(this._configCompilerPath)) {
+            return this._configCompilerPath;
+        }
+
+        return getZkvyperUrl(ZKVYPER_BIN_REPOSITORY, this._version, useGithubRelease);
+    }
+
+    private async _attemptDownload(url: string, downloadPath: string): Promise<void> {
+        return download(url, downloadPath, 'hardhat-zksync-zkvyper', this._version, 30000);
+    }
+
     private static async _readCompilerVersionInfo(compilerVersionInfoPath: string): Promise<CompilerVersionInfo> {
         return fsExtra.readJSON(compilerVersionInfoPath);
     }