From 99a29b422ced52224428c773c8c1f0ece4757970 Mon Sep 17 00:00:00 2001 From: Afshin Arani Date: Thu, 13 Apr 2023 16:32:28 +0330 Subject: [PATCH 1/2] Backend/Utxo: support native segwit Previously, native segwit was not widly supported so it was necessary to do segwit over P2SH, these days native segwit is supported by most wallets and with it's lower fee is the recommended choice. Lightning protocol is even dropping support for using P2SH shutdown scripts [1]. This commit adds support for native segwit (P2WPKH) while keeping the support for spending funds in users's old P2SH wallets. [1] https://github.com/lightning/bolts/commit/8f2104e3b6829b7c8ed190359326ac93eedc9ff5 --- .../ElectrumIntegrationTests.fs | 4 +- src/GWallet.Backend/Config.fs | 2 + src/GWallet.Backend/ServerManager.fs | 2 +- .../UtxoCoin/ElectrumClient.fs | 29 +++++++- .../UtxoCoin/UtxoCoinAccount.fs | 66 ++++++++++++++++--- 5 files changed, 88 insertions(+), 15 deletions(-) diff --git a/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs b/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs index 696f2a5b5..a8f63aa65 100644 --- a/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs +++ b/src/GWallet.Backend.Tests/ElectrumIntegrationTests.fs @@ -176,14 +176,14 @@ type ElectrumIntegrationTests() = let currency = Currency.BTC let argument = GetScriptHash currency CheckElectrumServersConnection ElectrumServerSeedList.DefaultBtcList currency - (ElectrumClient.GetBalance argument) BalanceAssertion + (ElectrumClient.GetBalances (List.singleton argument)) BalanceAssertion [] member __.``can connect (just check balance) to some electrum LTC servers``() = let currency = Currency.LTC let argument = GetScriptHash currency CheckElectrumServersConnection ElectrumServerSeedList.DefaultLtcList currency - (ElectrumClient.GetBalance argument) BalanceAssertion + (ElectrumClient.GetBalances (List.singleton argument)) BalanceAssertion [] member __.``can get list UTXOs of an address from some electrum BTC servers``() = diff --git a/src/GWallet.Backend/Config.fs b/src/GWallet.Backend/Config.fs index 476f63c99..50668dcc5 100644 --- a/src/GWallet.Backend/Config.fs +++ b/src/GWallet.Backend/Config.fs @@ -34,6 +34,8 @@ module Config = // balances, so you might find discrepancies (e.g. the donut-chart-view) let internal NoNetworkBalanceForDebuggingPurposes = false + let internal UseNativeSegwit = false + let IsWindowsPlatform() = RuntimeInformation.IsOSPlatform OSPlatform.Windows diff --git a/src/GWallet.Backend/ServerManager.fs b/src/GWallet.Backend/ServerManager.fs index dbe42a612..958ff3d6e 100644 --- a/src/GWallet.Backend/ServerManager.fs +++ b/src/GWallet.Backend/ServerManager.fs @@ -117,7 +117,7 @@ module ServerManager = failwith <| SPrintF1 "Currency %A not UTXO?" currency let utxoFunc electrumServer = async { - let! bal = UtxoCoin.ElectrumClient.GetBalance scriptHash electrumServer + let! bal = UtxoCoin.ElectrumClient.GetBalances (List.singleton scriptHash) electrumServer return bal.Confirmed |> decimal } UtxoCoin.Server.GetServerFuncs utxoFunc servers |> Some diff --git a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs index 594725962..568d5947e 100644 --- a/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs +++ b/src/GWallet.Backend/UtxoCoin/ElectrumClient.fs @@ -51,7 +51,7 @@ module ElectrumClient = | { Encrypted = false; Protocol = Tcp port } -> Init electrumServer.ServerInfo.NetworkPath port - let GetBalance (scriptHash: string) (stratumServer: Async) = async { + let GetBalances (scriptHashes: List) (stratumServer: Async) = async { // FIXME: we should rather implement this method in terms of: // - querying all unspent transaction outputs (X) -> block heights included // - querying transaction history (Y) -> block heights included @@ -67,8 +67,31 @@ module ElectrumClient = // [ see https://www.youtube.com/watch?v=hjYCXOyDy7Y&feature=youtu.be&t=1171 for more information ] // * -> although that would be fixing only half of the problem, we also need proof of completeness let! stratumClient = stratumServer - let! balanceResult = stratumClient.BlockchainScriptHashGetBalance scriptHash - return balanceResult.Result + let rec innerGetBalances (scriptHashes: List) (result: BlockchainScriptHashGetBalanceInnerResult) = + async { + match scriptHashes with + | scriptHash::otherScriptHashes -> + let! balanceHash = stratumClient.BlockchainScriptHashGetBalance scriptHash + + return! + innerGetBalances + otherScriptHashes + { + result with + Unconfirmed = result.Unconfirmed + balanceHash.Result.Unconfirmed + Confirmed = result.Confirmed + balanceHash.Result.Confirmed + } + | [] -> + return result + } + + return! + innerGetBalances + scriptHashes + { + Unconfirmed = 0L + Confirmed = 0L + } } let GetUnspentTransactionOutputs scriptHash (stratumServer: Async) = async { diff --git a/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs b/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs index 56cf8a76a..5001d5512 100644 --- a/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs +++ b/src/GWallet.Backend/UtxoCoin/UtxoCoinAccount.fs @@ -72,12 +72,23 @@ module Account = // TODO: measure how long does it take to get the script hash and if it's too long, cache it at app startup? BitcoinAddress.Create(publicAddress, GetNetwork currency) |> GetElectrumScriptHashFromAddress - let internal GetPublicAddressFromPublicKey currency (publicKey: PubKey) = + let internal GetSegwitP2shPublicAddressFromPublicKey currency (publicKey: PubKey) = + publicKey + .GetScriptPubKey(ScriptPubKeyType.SegwitP2SH) + .GetDestinationAddress(GetNetwork currency) + .ToString() + + let internal GetNativeSegwitPublicAddressFromPublicKey currency (publicKey: PubKey) = publicKey .GetScriptPubKey(ScriptPubKeyType.Segwit) - .Hash - .GetAddress(GetNetwork currency) + .GetDestinationAddress(GetNetwork currency) .ToString() + + let internal GetPublicAddressFromPublicKey = + if Config.UseNativeSegwit then + GetNativeSegwitPublicAddressFromPublicKey + else + GetSegwitP2shPublicAddressFromPublicKey let internal GetPublicAddressFromNormalAccountFile (currency: Currency) (accountFile: FileRepresentation): string = let pubKey = PubKey(accountFile.Name) @@ -139,11 +150,17 @@ module Account = (mode: ServerSelectionMode) (cancelSourceOption: Option) : Async = - let scriptHashHex = GetElectrumScriptHashFromPublicAddress account.Currency account.PublicAddress + let scriptHashesHex = + [ + GetNativeSegwitPublicAddressFromPublicKey account.Currency account.PublicKey + |> GetElectrumScriptHashFromPublicAddress account.Currency + GetSegwitP2shPublicAddressFromPublicKey account.Currency account.PublicKey + |> GetElectrumScriptHashFromPublicAddress account.Currency + ] let querySettings = QuerySettings.Balance(mode,(BalanceMatchWithCacheOrInitialBalance account.PublicAddress account.Currency)) - let balanceJob = ElectrumClient.GetBalance scriptHashHex + let balanceJob = ElectrumClient.GetBalances scriptHashesHex Server.Query account.Currency querySettings balanceJob cancelSourceOption let private GetBalancesFromServer (account: IUtxoAccount) @@ -176,9 +193,21 @@ module Account = let txHash = uint256 inputOutpointInfo.TransactionHash let scriptPubKeyInBytes = NBitcoin.DataEncoders.Encoders.Hex.DecodeData inputOutpointInfo.DestinationInHex let scriptPubKey = Script(scriptPubKeyInBytes) + // We convert the scriptPubKey to address temporarily to compare it with + // our own addresses, we could compare scriptPubKeys directly but we would + // need functions that return scriptPubKey of our addresses instead of a + // string. + let sourceAddress = scriptPubKey.GetDestinationAddress(GetNetwork account.Currency).ToString() let coin = Coin(txHash, uint32 inputOutpointInfo.OutputIndex, Money(inputOutpointInfo.ValueInSatoshis), scriptPubKey) - coin.ToScriptCoin account.PublicKey.WitHash.ScriptPubKey :> ICoin + if sourceAddress = GetSegwitP2shPublicAddressFromPublicKey account.Currency account.PublicKey then + coin.ToScriptCoin(account.PublicKey.WitHash.ScriptPubKey) :> ICoin + elif sourceAddress = GetNativeSegwitPublicAddressFromPublicKey account.Currency account.PublicKey then + coin :> ICoin + else + //We filter utxos based on scriptPubKey when retrieving from electrum + //so this is unreachable. + failwith "Unreachable: unrecognized scriptPubKey" let private CreateTransactionAndCoinsToBeSigned (account: IUtxoAccount) (transactionInputs: List) @@ -294,9 +323,28 @@ module Account = else newAcc,tail - let job = GetElectrumScriptHashFromPublicAddress account.Currency account.PublicAddress - |> ElectrumClient.GetUnspentTransactionOutputs - let! utxos = Server.Query account.Currency (QuerySettings.Default ServerSelectionMode.Fast) job None + let currency = account.Currency + + let getUtxos (publicAddress: string) = + async { + let job = GetElectrumScriptHashFromPublicAddress currency publicAddress + |> ElectrumClient.GetUnspentTransactionOutputs + + return! Server.Query currency (QuerySettings.Default ServerSelectionMode.Fast) job None + } + + let! utxos = + async { + let! nativeSegwitUtxos = + GetNativeSegwitPublicAddressFromPublicKey currency account.PublicKey + |> getUtxos + + let! legacySegwitUtxos = + GetSegwitP2shPublicAddressFromPublicKey currency account.PublicKey + |> getUtxos + + return Seq.concat [ nativeSegwitUtxos; legacySegwitUtxos ] + } if not (utxos.Any()) then failwith "No UTXOs found!" From 6498d0acee277aefbfbdadca41fea32530dde67f Mon Sep 17 00:00:00 2001 From: Afshin Arani Date: Tue, 31 Oct 2023 13:23:42 +0330 Subject: [PATCH 2/2] CI: create snap beta for native segwit by default This commit aims to create beta snap packages to allow testing experimental features like native segwit. This commit also updates Fsdk inside our scripts because the `GatherOrGetDefaultPrefix` function in the previous version did not allow additional arguments beside --prefix. A flag called --auto was also added for bypassing the "press any key" in bump.fsx. Co-authored-by: Mehrshad --- .github/workflows/CI.yml | 50 +++++++++++++++++++++++++++++++++++ ReadMe.md | 2 +- scripts/bump.fsx | 22 ++++++++++----- scripts/configure.fsx | 26 ++++++++++++++++-- scripts/find.fsx | 2 +- scripts/fsx | 2 +- scripts/sanitycheck.fsx | 2 +- scripts/snap_build.sh | 2 +- scripts/snap_bump.fsx | 33 +++++++++++++++++++++++ scripts/snap_release.fsx | 13 ++++++--- src/GWallet.Backend/Config.fs | 7 ++++- 11 files changed, 143 insertions(+), 18 deletions(-) create mode 100644 scripts/snap_bump.fsx diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fba5af706..aaf81ed89 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -534,3 +534,53 @@ jobs: run: | sudo apt update ./scripts/snap_release.sh + + snap_pkg_beta: + + needs: + - conventions + + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v1 + - name: Install snap tools + run: | + sudo apt update + ./scripts/install_snapcraft.sh + + # hack to disable msbuild detection + # NOTE: you might think an easier way to do this would be use container:\nimage: ubuntu22.04 and then not install msbuild, + # but that doesn't work because we get the following error when trying to install snapcraft via `snap install --classic`: + # > error: cannot communicate with server: Post "http://localhost/v2/snaps/snapcraft": dial unix /run/snapd.socket: connect: no such file or directory + - name: HACK to emulate msbuild uninstall + run: sudo rm `which msbuild` + + - name: Bump snap version + run: | + git submodule foreach git fetch --all && git submodule sync --recursive && git submodule update --init --recursive + dotnet fsi ./scripts/snap_bump.fsx + + - name: Generate snap package + run: | + # retry 3 times because of flakey nuget; TODO: remove retry when we migrate to .NET6 (removing LEGACY_FRAMEWORK support) + ./scripts/snap_build.sh --native-segwit || ./scripts/snap_build.sh --native-segwit || ./scripts/snap_build.sh --native-segwit || ./scripts/snap_build.sh --native-segwit + + - name: Install snap + # dangerous because it's a local snap (not one from the SnapStore) + run: sudo snap install --dangerous *.snap + + - name: Test snap + run: geewallet --version + + - uses: actions/upload-artifact@v3 + name: Upload snap package as artifact + with: + name: snap_beta + path: ./*.snap + + - name: Upload snap package to Snap Store + env: + SNAPCRAFT_LOGIN: ${{ secrets.SNAPCRAFT_LOGIN }} + run: | + sudo apt update + ./scripts/snap_release.sh beta diff --git a/ReadMe.md b/ReadMe.md index ae8fe90fc..d72f96ec8 100644 --- a/ReadMe.md +++ b/ReadMe.md @@ -63,7 +63,7 @@ In the development side of things, we advocate for simplicity: This list is the (intended) order of preference for new features: - Migration from Xamarin.Forms to MAUI (in progress, see the PR#199 or its successor). -- Switch from SegWit to native-SegWit (Bech32): PR#211. +- Make native-SegWit (Bech32) be default for bitcoin (right now this is the case for the beta version of our snap package). - Support for payment-channels & state-channels (in BTC/LTC via lightning, see 'lightning' branch; and in ETH/ETC/DAI via Connext?). - Explore better stablecoin approach than L1-DAI because DAI is not 100% decentralized (e.g. LUSD better?) and because L1 is too high fees (e.g. sDAI better? see https://twitter.com/koeppelmann/status/1736766570825654630 ). - Automatic reminders for Seed/password checks to make sure user has not forgotten them (see https://twitter.com/takinbo/status/1201529679519330305 ). diff --git a/scripts/bump.fsx b/scripts/bump.fsx index 405da8ffc..101b2ba5a 100755 --- a/scripts/bump.fsx +++ b/scripts/bump.fsx @@ -4,7 +4,7 @@ open System open System.IO #if !LEGACY_FRAMEWORK -#r "nuget: Fsdk, Version=0.6.0--date20230812-0646.git-2268d50" +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" #else #r "System.Configuration" open System.Configuration @@ -21,15 +21,18 @@ let IsStable miniVersion = (int miniVersion % 2) = 0 let args = Misc.FsxOnlyArguments() + +let isAuto = List.contains "--auto" args + let suppliedVersion = if args.Length > 0 then - if args.Length > 1 then - Console.Error.WriteLine "Only one argument supported, not more" + if args.Length > 2 then + Console.Error.WriteLine "Only two argument supported, not more" Environment.Exit 1 failwith "Unreachable" else let full = Version(args.Head) - if not (IsStable full.Build) then + if (not isAuto) && not (IsStable full.Build) then Console.Error.WriteLine "Mini-version (previous-to-last number, e.g. 2 in 0.1.2.3) should be an even (stable) number" Environment.Exit 2 failwith "Unreachable" @@ -263,14 +266,19 @@ if not replaceScript.Exists then GitDiff() Console.WriteLine "Bumping..." -RunUpdateServers() +if not isAuto then + RunUpdateServers() let fullUnstableVersion,newFullStableVersion = Bump true -GitCommit fullUnstableVersion newFullStableVersion -GitTag newFullStableVersion +if not isAuto then + GitCommit fullUnstableVersion newFullStableVersion + GitTag newFullStableVersion Console.WriteLine (sprintf "Version bumped to %s." (newFullStableVersion.ToString())) +if isAuto then + Environment.Exit 0 + if isReleaseManual then Console.WriteLine "Release binaries now and press any key when you finish." Console.ReadKey true |> ignore diff --git a/scripts/configure.fsx b/scripts/configure.fsx index 28c0ebcec..b3638aabd 100644 --- a/scripts/configure.fsx +++ b/scripts/configure.fsx @@ -4,7 +4,7 @@ open System open System.IO #if !LEGACY_FRAMEWORK -#r "nuget: Fsdk, Version=0.6.0--date20230812-0646.git-2268d50" +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" #else #r "System.Configuration" open System.Configuration @@ -187,6 +187,18 @@ let fsxRunner = fsxRunnerBinText buildConfigFile.Name +let AddToDefinedConstants (constant: string) (configMap: Map) = + let configKey = "DefineConstants" + + match configMap.TryFind configKey with + | None -> + configMap + |> Map.add configKey constant + | Some previousConstants -> + configMap + |> Map.add configKey (sprintf "%s;%s" previousConstants constant) + + let configFileToBeWritten = let initialConfigFile = Map.empty.Add("Prefix", prefix.FullName) @@ -195,11 +207,21 @@ let configFileToBeWritten = | Some theTool -> initialConfigFile.Add("LegacyBuildTool", theTool) | None -> initialConfigFile - let finalConfigFile = + let configFileStageThree = match buildTool with | Some theTool -> configFileStageTwo.Add("BuildTool", theTool) | None -> configFileStageTwo + let finalConfigFile = + let nativeSegwitEnabled = + Misc.FsxOnlyArguments() + |> List.contains "--native-segwit" + if nativeSegwitEnabled then + configFileStageThree + |> AddToDefinedConstants "NATIVE_SEGWIT" + else + configFileStageThree + finalConfigFile let lines = diff --git a/scripts/find.fsx b/scripts/find.fsx index f39abb2a6..16a16b4ec 100755 --- a/scripts/find.fsx +++ b/scripts/find.fsx @@ -5,7 +5,7 @@ open System.IO open System.Linq #if !LEGACY_FRAMEWORK -#r "nuget: Fsdk, Version=0.6.0--date20230812-0646.git-2268d50" +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" #else #r "System.Configuration" #load "fsx/Fsdk/Misc.fs" diff --git a/scripts/fsx b/scripts/fsx index 5488853b1..2737eea87 160000 --- a/scripts/fsx +++ b/scripts/fsx @@ -1 +1 @@ -Subproject commit 5488853b17fedb44707e8459480297b618cffad0 +Subproject commit 2737eea87f6bd4becda5c172c208314af044184a diff --git a/scripts/sanitycheck.fsx b/scripts/sanitycheck.fsx index 6d83643af..3506ba182 100755 --- a/scripts/sanitycheck.fsx +++ b/scripts/sanitycheck.fsx @@ -14,7 +14,7 @@ open System.Xml.Linq open System.Xml.XPath #if !LEGACY_FRAMEWORK -#r "nuget: Fsdk, Version=0.6.0--date20230812-0646.git-2268d50" +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" #else #r "System.Configuration" open System.Configuration diff --git a/scripts/snap_build.sh b/scripts/snap_build.sh index 2a6c07e3a..3196aa8db 100755 --- a/scripts/snap_build.sh +++ b/scripts/snap_build.sh @@ -8,7 +8,7 @@ DEBIAN_FRONTEND=noninteractive sudo apt install -y fsharp build-essential pkg-co # just in case this is a retry-run, we want to clean artifacts from previous try rm -rf ./staging -./configure.sh --prefix=./staging +./configure.sh --prefix=./staging "$@" make make install diff --git a/scripts/snap_bump.fsx b/scripts/snap_bump.fsx new file mode 100644 index 000000000..6be82a814 --- /dev/null +++ b/scripts/snap_bump.fsx @@ -0,0 +1,33 @@ +#!/usr/bin/env fsharpi + +open System +open System.IO + +#if !LEGACY_FRAMEWORK +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" +#else +#r "System.Configuration" +open System.Configuration + +#load "fsx/Fsdk/Misc.fs" +#load "fsx/Fsdk/Process.fs" +#load "fsx/Fsdk/Git.fs" +#endif +open Fsdk +open Fsdk.Process + +#load "fsxHelper.fs" +open GWallet.Scripting + +let currentVersion = Misc.GetCurrentVersion(FsxHelper.RootDir) + +let newVersion = + // e.g. to bump from 0.7.x.y to 0.9.x.y + Version(currentVersion.Major, currentVersion.Minor + 2, currentVersion.Build, currentVersion.Revision).ToString() + +Process.Execute( + { + Command = "dotnet" + Arguments = sprintf "fsi %s %s --auto" (Path.Combine(FsxHelper.ScriptsDir.FullName, "bump.fsx")) newVersion + }, Echo.All +).UnwrapDefault() |> ignore diff --git a/scripts/snap_release.fsx b/scripts/snap_release.fsx index b8cf74b5e..06cfa08b0 100755 --- a/scripts/snap_release.fsx +++ b/scripts/snap_release.fsx @@ -15,7 +15,7 @@ open System.Xml.Linq open System.Xml.XPath #if !LEGACY_FRAMEWORK -#r "nuget: Fsdk, Version=0.6.0--date20230812-0646.git-2268d50" +#r "nuget: Fsdk, Version=0.6.0--date20231031-0834.git-2737eea" #else #r "System.Configuration" open System.Configuration @@ -149,8 +149,15 @@ Process.Execute({ Command = "snapcraft"; Arguments = "login --with snapcraft.log Console.WriteLine "Login successfull. Upload starting..." let snapPush = - // the 'stable' and 'candidate' channels require 'stable' grade in the yaml - let channel = "stable" + let channel = + match Misc.FsxOnlyArguments() with + | [ channel ] -> + channel + | [] -> + // the 'stable' and 'candidate' channels require 'stable' grade in the yaml + "stable" + | _ -> + failwith "Invalid arguments" Process.Execute( { diff --git a/src/GWallet.Backend/Config.fs b/src/GWallet.Backend/Config.fs index 50668dcc5..17c213bd9 100644 --- a/src/GWallet.Backend/Config.fs +++ b/src/GWallet.Backend/Config.fs @@ -34,7 +34,12 @@ module Config = // balances, so you might find discrepancies (e.g. the donut-chart-view) let internal NoNetworkBalanceForDebuggingPurposes = false - let internal UseNativeSegwit = false + let internal UseNativeSegwit = +#if NATIVE_SEGWIT + true +#else + false +#endif let IsWindowsPlatform() = RuntimeInformation.IsOSPlatform OSPlatform.Windows