diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml index 6110009..736aaf3 100644 --- a/.github/workflows/cd.yaml +++ b/.github/workflows/cd.yaml @@ -30,28 +30,55 @@ jobs: target/release/grin-wallet-${{ github.ref_name }}-linux-x86_64.tar.gz target/release/grin-wallet-${{ github.ref_name }}-linux-x86_64-sha256sum.txt - macos-release: - name: macOS Release + macos-release-x86: + name: macOS Release - x86_64 runs-on: macos-latest steps: + - name: Install Rust Target + run: rustup target add x86_64-apple-darwin - name: Checkout uses: actions/checkout@v6 with: submodules: recursive - name: Build - run: cargo build --release + run: cargo build --release --target x86_64-apple-darwin - name: Archive - working-directory: target/release + working-directory: target/x86_64-apple-darwin/release run: tar -czvf grin-wallet-${{ github.ref_name }}-macos-x86_64.tar.gz grin-wallet - name: Create Checksum - working-directory: target/release + working-directory: target/x86_64-apple-darwin/release run: openssl sha256 grin-wallet-${{ github.ref_name }}-macos-x86_64.tar.gz > grin-wallet-${{ github.ref_name }}-macos-x86_64-sha256sum.txt - name: Release uses: softprops/action-gh-release@v1 with: - files: | - target/release/grin-wallet-${{ github.ref_name }}-macos-x86_64.tar.gz - target/release/grin-wallet-${{ github.ref_name }}-macos-x86_64-sha256sum.txt + files: | + target/x86_64-apple-darwin/release/grin-wallet-${{ github.ref_name }}-macos-x86_64.tar.gz + target/x86_64-apple-darwin/release/grin-wallet-${{ github.ref_name }}-macos-x86_64-sha256sum.txt + + macos-release-arm64: + name: macOS Release - arm64 + runs-on: macos-latest + steps: + - name: Install Rust Target + run: rustup target add aarch64-apple-darwin + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + - name: Build + run: cargo build --release --target aarch64-apple-darwin + - name: Archive + working-directory: target/aarch64-apple-darwin/release + run: tar -czvf grin-wallet-${{ github.ref_name }}-macos-arm64.tar.gz grin-wallet + - name: Create Checksum + working-directory: target/aarch64-apple-darwin/release + run: openssl sha256 grin-wallet-${{ github.ref_name }}-macos-arm64.tar.gz > grin-wallet-${{ github.ref_name }}-macos-arm64-sha256sum.txt + - name: Release + uses: softprops/action-gh-release@v1 + with: + files: | + target/aarch64-apple-darwin/release/grin-wallet-${{ github.ref_name }}-macos-arm64.tar.gz + target/aarch64-apple-darwin/release/grin-wallet-${{ github.ref_name }}-macos-arm64-sha256sum.txt windows-release: name: Windows Release diff --git a/.gitmodules b/.gitmodules index e974e83..3b2b4f5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "grin"] path = grin - branch = staging + branch = master url = https://github.com/mimblewimble/grin diff --git a/Cargo.lock b/Cargo.lock index 1f4930f..873c248 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2257,7 +2257,7 @@ checksum = "9985c9503b412198aa4197559e9a318524ebc4519c229bfa05a535828c950b9d" [[package]] name = "grin_api" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "bytes 1.12.0", "easy-jsonrpc-mw", @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "grin_chain" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "bit-vec", "bitflags 1.3.2", @@ -2313,7 +2313,7 @@ dependencies = [ [[package]] name = "grin_core" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "blake2-rfc", "byteorder", @@ -2339,7 +2339,7 @@ dependencies = [ [[package]] name = "grin_keychain" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "blake2-rfc", "byteorder", @@ -2360,7 +2360,7 @@ dependencies = [ [[package]] name = "grin_p2p" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "bitflags 1.3.2", "built", @@ -2383,7 +2383,7 @@ dependencies = [ [[package]] name = "grin_pool" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "blake2-rfc", "chrono", @@ -2415,7 +2415,7 @@ dependencies = [ [[package]] name = "grin_store" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "byteorder", "chrono", @@ -2437,7 +2437,7 @@ dependencies = [ [[package]] name = "grin_util" -version = "5.5.1-alpha.0" +version = "5.5.1" dependencies = [ "anyhow", "backtrace", @@ -2457,7 +2457,7 @@ dependencies = [ [[package]] name = "grin_wallet" -version = "5.4.1" +version = "5.5.0" dependencies = [ "built", "clap", @@ -2485,7 +2485,7 @@ dependencies = [ [[package]] name = "grin_wallet_api" -version = "5.4.1" +version = "5.5.0" dependencies = [ "base64 0.12.3", "chrono", @@ -2510,7 +2510,7 @@ dependencies = [ [[package]] name = "grin_wallet_config" -version = "5.4.1" +version = "5.5.0" dependencies = [ "dirs 2.0.2", "grin_core", @@ -2526,7 +2526,7 @@ dependencies = [ [[package]] name = "grin_wallet_controller" -version = "5.4.1" +version = "5.5.0" dependencies = [ "easy-jsonrpc-mw", "ed25519-dalek", @@ -2557,7 +2557,7 @@ dependencies = [ [[package]] name = "grin_wallet_impls" -version = "5.4.1" +version = "5.5.0" dependencies = [ "arti-client", "base64 0.12.3", @@ -2604,7 +2604,7 @@ dependencies = [ [[package]] name = "grin_wallet_libwallet" -version = "5.4.1" +version = "5.5.0" dependencies = [ "age", "base64 0.9.3", @@ -2627,6 +2627,7 @@ dependencies = [ "log", "num-bigint 0.2.6", "rand 0.6.5", + "rayon", "regex", "serde", "serde_derive", @@ -2642,7 +2643,7 @@ dependencies = [ [[package]] name = "grin_wallet_util" -version = "5.4.1" +version = "5.5.0" dependencies = [ "data-encoding", "ed25519-dalek", diff --git a/Cargo.toml b/Cargo.toml index 18dfbf0..1b22005 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Simple, private and scalable cryptocurrency implementation based on the MimbleWimble chain format." license = "Apache-2.0" @@ -29,11 +29,11 @@ semver = "1.0.28" rustyline = "9" lazy_static = "1" -grin_wallet_api = { path = "./api", version = "5.4.1" } -grin_wallet_impls = { path = "./impls", version = "5.4.1" } -grin_wallet_libwallet = { path = "./libwallet", version = "5.4.1" } -grin_wallet_controller = { path = "./controller", version = "5.4.1" } -grin_wallet_config = { path = "./config", version = "5.4.1" } +grin_wallet_api = { path = "./api", version = "5.5.0" } +grin_wallet_impls = { path = "./impls", version = "5.5.0" } +grin_wallet_libwallet = { path = "./libwallet", version = "5.5.0" } +grin_wallet_controller = { path = "./controller", version = "5.5.0" } +grin_wallet_config = { path = "./config", version = "5.5.0" } ##### Grin Imports @@ -51,4 +51,4 @@ built = { version = "0.8.0", features = ["git2"]} url = "2.1" serde = "1" serde_json = "1" -easy-jsonrpc-mw = "0.5.4" \ No newline at end of file +easy-jsonrpc-mw = "0.5.4" diff --git a/api/Cargo.toml b/api/Cargo.toml index ee4a43c..c797161 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_api" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Grin Wallet API" license = "Apache-2.0" @@ -22,10 +22,10 @@ ring = "0.17.14" base64 = "0.12" ed25519-dalek = "2" -grin_wallet_libwallet = { path = "../libwallet", version = "5.4.1" } -grin_wallet_config = { path = "../config", version = "5.4.1" } -grin_wallet_impls = { path = "../impls", version = "5.4.1" } -grin_wallet_util = { path = "../util", version = "5.4.1" } +grin_wallet_libwallet = { path = "../libwallet", version = "5.5.0" } +grin_wallet_config = { path = "../config", version = "5.5.0" } +grin_wallet_impls = { path = "../impls", version = "5.5.0" } +grin_wallet_util = { path = "../util", version = "5.5.0" } ##### Grin Imports diff --git a/api/src/foreign_rpc.rs b/api/src/foreign_rpc.rs index 78a44ea..d16e877 100644 --- a/api/src/foreign_rpc.rs +++ b/api/src/foreign_rpc.rs @@ -376,7 +376,15 @@ pub fn run_doctest_foreign( util::init_test_logger(); let _ = fs::remove_dir_all(test_dir); - global::set_local_chain_type(ChainTypes::AutomatedTesting); + global::set_global_chain_type(ChainTypes::AutomatedTesting); + + let _ = fs::create_dir_all(test_dir); + let config = initial_setup_wallet( + &ChainTypes::AutomatedTesting, + Some(PathBuf::from(test_dir)), + false, + ) + .unwrap(); let _ = fs::create_dir_all(test_dir); let config = initial_setup_wallet( diff --git a/api/src/owner.rs b/api/src/owner.rs index 6eee54b..d9a8132 100644 --- a/api/src/owner.rs +++ b/api/src/owner.rs @@ -477,6 +477,69 @@ where ) } + /// Returns max available value to send for selecting all outputs. + /// + /// # Arguments + /// * `keychain_mask` - Wallet secret mask to XOR against the stored wallet seed before using, if + /// being used. + /// * `refresh_from_node` - If true, the wallet will attempt to contact + /// a node (via the [`NodeClient`](../grin_wallet_libwallet/types/trait.NodeClient.html) + /// provided during wallet instantiation). If `false`, the results will + /// contain transaction information that may be out-of-date (from the last time + /// the wallet's output set was refreshed against the node). + /// Note this setting is ignored if the updater process is running via a call to + /// [`start_updater`](struct.Owner.html#method.start_updater) + /// * `minimum_confirmations` - The minimum number of confirmations an output + /// should have before it's included in the 'amount_currently_spendable' total + /// + /// # Returns + /// * `(bool, u64, u64, u32)` - A tuple: + /// * The first `bool` element indicates whether the data was successfully + /// refreshed from the node (note this may be false even if the `refresh_from_node` + /// argument was set to `true`). + /// * The second `u64` is amount to send. + /// * The third element is fee + /// * The fourth is maximum input count + /// + /// # Example + /// Set up as in [`new`](struct.Owner.html#method.new) method above. + /// ``` + /// # grin_wallet_api::doctest_helper_setup_doc_env!(wallet, wallet_config); + /// + /// let api_owner = Owner::new(wallet.clone(), None, std::path::PathBuf::from("grin-wallet.toml")); + /// let update_from_node = true; + /// let minimum_confirmations = 10; + /// let result = api_owner.estimate_max_sendable(None, update_from_node, minimum_confirmations); + /// + /// if let Ok((was_updated, amount, fee, max_inputs)) = result { + /// //... + /// } + /// ``` + + pub fn estimate_max_sendable( + &self, + keychain_mask: Option<&SecretKey>, + refresh_from_node: bool, + minimum_confirmations: u64, + ) -> Result<(bool, u64, u64, u32), Error> { + let tx = { + let t = self.status_tx.lock(); + t.clone() + }; + let refresh_from_node = match self.updater_running.load(Ordering::Relaxed) { + true => false, + false => refresh_from_node, + }; + + owner::estimate_max_sendable( + self.wallet_inst.clone(), + keychain_mask, + &tx, + refresh_from_node, + minimum_confirmations, + ) + } + /// Returns a list of [Transaction Log Entries](../grin_wallet_libwallet/types/struct.TxLogEntry.html) /// from the active account in the wallet. /// @@ -608,21 +671,34 @@ where refresh_from_node: bool, minimum_confirmations: u64, ) -> Result<(bool, WalletInfo), Error> { + let (updated, _, info) = self.retrieve_summary_info_with_refresh_status( + keychain_mask, + refresh_from_node, + minimum_confirmations, + )?; + Ok((updated, info)) + } + + /// Retrieve summary info and whether refresh was skipped. + pub fn retrieve_summary_info_with_refresh_status( + &self, + keychain_mask: Option<&SecretKey>, + refresh_from_node: bool, + minimum_confirmations: u64, + ) -> Result<(bool, bool, WalletInfo), Error> { let tx = { let t = self.status_tx.lock(); t.clone() }; - let refresh_from_node = match self.updater_running.load(Ordering::Relaxed) { - true => false, - false => refresh_from_node, - }; - owner::retrieve_summary_info( + let refresh_skipped = refresh_from_node && self.updater_running.load(Ordering::Relaxed); + let (updated, info) = owner::retrieve_summary_info( self.wallet_inst.clone(), keychain_mask, &tx, - refresh_from_node, + refresh_from_node && !refresh_skipped, minimum_confirmations, - ) + )?; + Ok((updated, refresh_skipped, info)) } /// Initiates a new transaction as the sender, creating a new diff --git a/api/src/owner_rpc.rs b/api/src/owner_rpc.rs index 6bf9d8d..e2d6616 100644 --- a/api/src/owner_rpc.rs +++ b/api/src/owner_rpc.rs @@ -231,6 +231,51 @@ pub trait OwnerRpc { tx_id: Option, ) -> Result<(bool, Vec), Error>; + /** + Networked version of [Owner::estimate_max_sendable](struct.Owner.html#method.estimate_max_sendable). + + # Json rpc example + + ``` + # grin_wallet_api::doctest_helper_json_rpc_owner_assert_response!( + # r#" + { + "jsonrpc": "2.0", + "method": "estimate_max_sendable", + "params": { + "token": "d202964900000000d302964900000000d402964900000000d502964900000000", + "refresh_from_node": true, + "minimum_confirmations": 1 + }, + "id": 1 + } + # "# + # , + # r#" + { + "id": 1, + "jsonrpc": "2.0", + "result": { + "Ok": [ + true, + 59987500000, + 12500000, + 1 + ] + } + } + # "# + # , 4, false, false, false, false); + ``` + */ + + fn estimate_max_sendable( + &self, + token: Token, + refresh_from_node: bool, + minimum_confirmations: u64, + ) -> Result<(bool, u64, u64, u32), Error>; + /** Networked version of [Owner::retrieve_txs](struct.Owner.html#method.retrieve_txs). @@ -464,7 +509,7 @@ pub trait OwnerRpc { ) -> Result<(bool, WalletInfo), Error>; /** - ;Networked version of [Owner::init_send_tx](struct.Owner.html#method.init_send_tx). + Networked version of [Owner::init_send_tx](struct.Owner.html#method.init_send_tx). ``` # grin_wallet_api::doctest_helper_json_rpc_owner_assert_response!( @@ -481,6 +526,7 @@ pub trait OwnerRpc { "max_outputs": 500, "num_change_outputs": 1, "selection_strategy_is_use_all": true, + "refresh_outputs_from_node": true, "target_slate_version": null, "payment_proof_recipient_address": "tgrin1xtxavwfgs48ckf3gk8wwgcndmn0nt4tvkl8a7ltyejjcy2mc6nfs9gm2lp", "ttl_blocks": null, @@ -641,6 +687,7 @@ pub trait OwnerRpc { "max_outputs": 500, "num_change_outputs": 1, "selection_strategy_is_use_all": true, + "refresh_outputs_from_node": true, "target_slate_version": null, "payment_proof_recipient_address": null, "ttl_blocks": null, @@ -1163,54 +1210,54 @@ pub trait OwnerRpc { fn node_height(&self, token: Token) -> Result; /** - Initializes the secure JSON-RPC API. This function must be called and a shared key - established before any other OwnerAPI JSON-RPC function can be called. + Initializes the secure JSON-RPC API. This function must be called and a shared key + established before any other OwnerAPI JSON-RPC function can be called. - The shared key will be derived using ECDH with the provided public key on the secp256k1 curve. This - function will return its public key used in the derivation, which the caller should multiply by its - private key to derive the shared key. + The shared key will be derived using ECDH with the provided public key on the secp256k1 curve. This + function will return its public key used in the derivation, which the caller should multiply by its + private key to derive the shared key. - Once the key is established, all further requests and responses are encrypted and decrypted with the - following parameters: - * AES-256 in GCM mode with 128-bit tags and 96 bit nonces - * 12 byte nonce which must be included in each request/response to use on the decrypting side - * Empty vector for additional data - * Suffix length = AES-256 GCM mode tag length = 16 bytes - * + Once the key is established, all further requests and responses are encrypted and decrypted with the + following parameters: + * AES-256 in GCM mode with 128-bit tags and 96 bit nonces + * 12 byte nonce which must be included in each request/response to use on the decrypting side + * Empty vector for additional data + * Suffix length = AES-256 GCM mode tag length = 16 bytes + * - Fully-formed JSON-RPC requests (as documented) should be encrypted using these parameters, encoded - into base64 and included with the one-time nonce in a request for the `encrypted_request_v3` method - as follows: + Fully-formed JSON-RPC requests (as documented) should be encrypted using these parameters, encoded + into base64 and included with the one-time nonce in a request for the `encrypted_request_v3` method + as follows: - ``` - # let s = r#" - { - "jsonrpc": "2.0", - "method": "encrypted_request_v3", - "id": "1", - "params": { - "nonce": "ef32...", - "body_enc": "e0bcd..." - } - } - # "#; - ``` + ``` + # let s = r#" + { + "jsonrpc": "2.0", + "method": "encrypted_request_v3", + "id": "1", + "params": { + "nonce": "ef32...", + "body_enc": "e0bcd..." + } + } + # "#; + ``` - With a typical response being: + With a typical response being: - ``` - # let s = r#"{ - { - "jsonrpc": "2.0", - "method": "encrypted_response_v3", - "id": "1", - "Ok": { - "nonce": "340b...", - "body_enc": "3f09c..." - } - } - # }"#; - ``` + ``` + # let s = r#"{ + { + "jsonrpc": "2.0", + "method": "encrypted_response_v3", + "id": "1", + "Ok": { + "nonce": "340b...", + "body_enc": "3f09c..." + } + } + # }"#; + ``` */ @@ -2106,6 +2153,20 @@ where ) } + fn estimate_max_sendable( + &self, + token: Token, + refresh_from_node: bool, + minimum_confirmations: u64, + ) -> Result<(bool, u64, u64, u32), Error> { + Owner::estimate_max_sendable( + self, + (&token.keychain_mask).as_ref(), + refresh_from_node, + minimum_confirmations, + ) + } + fn retrieve_txs( &self, token: Token, @@ -2595,7 +2656,15 @@ pub fn run_doctest_owner( util::init_test_logger(); let _ = fs::remove_dir_all(test_dir); - global::set_local_chain_type(ChainTypes::AutomatedTesting); + global::set_global_chain_type(ChainTypes::AutomatedTesting); + + let _ = fs::create_dir_all(test_dir); + let config = initial_setup_wallet( + &ChainTypes::AutomatedTesting, + Some(PathBuf::from(test_dir)), + false, + ) + .unwrap(); let _ = fs::create_dir_all(test_dir); let config = initial_setup_wallet( diff --git a/config/Cargo.toml b/config/Cargo.toml index a50f1e4..5d7e339 100644 --- a/config/Cargo.toml +++ b/config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_config" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Configuration for grin wallet , a simple, private and scalable cryptocurrency implementation based on the MimbleWimble chain format." license = "Apache-2.0" @@ -27,4 +27,3 @@ grin_util = { path = "../grin/util"} [dev-dependencies] tempfile = "3.1" - diff --git a/controller/Cargo.toml b/controller/Cargo.toml index 32e03bc..5de94f2 100644 --- a/controller/Cargo.toml +++ b/controller/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_controller" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Controllers for grin wallet instantiation" license = "Apache-2.0" @@ -25,11 +25,11 @@ lazy_static = "1" thiserror = "1" qr_code = "1.1.0" -grin_wallet_util = { path = "../util", version = "5.4.1" } -grin_wallet_api = { path = "../api", version = "5.4.1" } -grin_wallet_impls = { path = "../impls", version = "5.4.1" } -grin_wallet_libwallet = { path = "../libwallet", version = "5.4.1" } -grin_wallet_config = { path = "../config", version = "5.4.1" } +grin_wallet_util = { path = "../util", version = "5.5.0" } +grin_wallet_api = { path = "../api", version = "5.5.0" } +grin_wallet_impls = { path = "../impls", version = "5.5.0" } +grin_wallet_libwallet = { path = "../libwallet", version = "5.5.0" } +grin_wallet_config = { path = "../config", version = "5.5.0" } ##### Grin Imports diff --git a/controller/src/command.rs b/controller/src/command.rs index d99c092..e2d0de0 100644 --- a/controller/src/command.rs +++ b/controller/src/command.rs @@ -335,6 +335,21 @@ pub struct SendArgs { pub slatepack_qr: bool, } +fn max_retry_args(mut init_args: InitTxArgs, amount: u64, max_inputs: u32) -> InitTxArgs { + init_args.amount = amount; + init_args.max_outputs = max_inputs; + init_args.selection_strategy_is_use_all = true; + init_args +} + +fn estimate_strategies(use_max_amount: bool) -> &'static [&'static str] { + if use_max_amount { + &["all"] + } else { + &["smallest", "all"] + } +} + pub fn send( owner_api: &mut Owner, keychain_mask: Option<&SecretKey>, @@ -359,16 +374,24 @@ where let mut slate = Slate::blank(2, false); let mut amount = args.amount; + let (info_updated, update_skipped, wallet_info) = owner_api + .retrieve_summary_info_with_refresh_status( + keychain_mask, + true, + args.minimum_confirmations, + )?; if args.use_max_amount { - let (_, wallet_info) = - owner_api.retrieve_summary_info(keychain_mask, true, args.minimum_confirmations)?; amount = wallet_info.amount_currently_spendable; } + if !info_updated && !update_skipped { + warn!("Wallet info update failed: node connection error"); + } if args.estimate_selection_strategies { - let strategies = vec!["smallest", "all"] - .into_iter() + let strategies = estimate_strategies(args.use_max_amount) + .iter() + .copied() .map(|strategy| { - let init_args = InitTxArgs { + let mut init_args = InitTxArgs { src_acct_name: None, amount, amount_includes_fee: Some(args.amount_includes_fee), @@ -376,10 +399,29 @@ where max_outputs: args.max_outputs as u32, num_change_outputs: args.change_outputs as u32, selection_strategy_is_use_all: strategy == "all", + refresh_outputs_from_node: !info_updated, estimate_only: Some(true), ..Default::default() }; - let slate = owner_api.init_send_tx(keychain_mask, init_args)?; + let result = owner_api.init_send_tx(keychain_mask, init_args.clone()); + let slate = match result { + Ok(s) => s, + Err(e) => match e { + libwallet::Error::BigAmountError(a, _fee, max_inputs) => { + debug!("{}", e); + if args.use_max_amount { + amount = a; + init_args = max_retry_args(init_args, amount, max_inputs); + owner_api.init_send_tx(keychain_mask, init_args)? + } else { + return Err(grin_wallet_libwallet::Error::from(e)); + } + } + _ => { + return Err(grin_wallet_libwallet::Error::from(e)); + } + }, + }; Ok((strategy, slate.amount, slate.fee_fields)) }) .collect::, grin_wallet_libwallet::Error>>()?; @@ -390,7 +432,7 @@ where .as_deref() .map(SlatepackAddress::try_from) .transpose()?; - let init_args = InitTxArgs { + let mut init_args = InitTxArgs { src_acct_name: None, amount, amount_includes_fee: Some(args.amount_includes_fee), @@ -398,6 +440,7 @@ where max_outputs: args.max_outputs as u32, num_change_outputs: args.change_outputs as u32, selection_strategy_is_use_all: args.selection_strategy == "all", + refresh_outputs_from_node: !info_updated, target_slate_version: args.target_slate_version, payment_proof_recipient_address, ttl_blocks: args.ttl_blocks, @@ -405,23 +448,46 @@ where late_lock: Some(args.late_lock), ..Default::default() }; - let result = owner_api.init_send_tx(keychain_mask, init_args); - slate = match result { - Ok(s) => { - info!( - "Tx created: {} grin to {} (strategy '{}')", - core::amount_to_hr_string(amount, false), - dest.as_ref() - .map(ToString::to_string) - .unwrap_or_else(|| "no destination".to_string()), - args.selection_strategy, - ); - s - } - Err(e) => { - info!("Tx not created: {}", e); - return Err(Error::from(e)); - } + let init_send_tx = |init_args: InitTxArgs| -> Result { + let result = owner_api.init_send_tx(keychain_mask, init_args.clone()); + let slate = match result { + Ok(s) => { + info!( + "Tx created: {} grin to {} (strategy '{}')", + core::amount_to_hr_string(init_args.amount, false), + dest.as_ref() + .map(ToString::to_string) + .unwrap_or_else(|| "no destination".to_string()), + args.selection_strategy, + ); + s + } + Err(e) => return Err(e), + }; + Ok(slate) + }; + slate = match init_send_tx(init_args.clone()) { + Ok(s) => s, + Err(e) => match e { + libwallet::Error::BigAmountError(a, _fee, max_inputs) => { + debug!("{}", e); + if args.use_max_amount { + amount = a; + init_args = max_retry_args(init_args, amount, max_inputs); + init_send_tx(init_args).map_err(|e| { + info!("Tx not created: {}", e); + Error::from(e) + })? + } else { + info!("Tx not created: {}", e); + return Err(Error::from(e)); + } + } + _ => { + info!("Tx not created: {}", e); + return Err(Error::from(e)); + } + }, }; } @@ -1073,6 +1139,16 @@ where .transpose()?; let mut slate = args.slate.clone(); + // Refresh wallet state from node. + let (info_updated, update_skipped, _) = owner_api.retrieve_summary_info_with_refresh_status( + keychain_mask, + true, + args.minimum_confirmations, + )?; + if !info_updated && !update_skipped { + warn!("Wallet info update failed: node connection error"); + } + if args.estimate_selection_strategies { let strategies = vec!["smallest", "all"] .into_iter() @@ -1084,13 +1160,14 @@ where max_outputs: args.max_outputs as u32, num_change_outputs: 1u32, selection_strategy_is_use_all: strategy == "all", + refresh_outputs_from_node: !info_updated, estimate_only: Some(true), ..Default::default() }; - let slate = owner_api.init_send_tx(keychain_mask, init_args).unwrap(); - (strategy, slate.amount, slate.fee_fields) + let slate = owner_api.init_send_tx(keychain_mask, init_args)?; + Ok((strategy, slate.amount, slate.fee_fields)) }) - .collect(); + .collect::, libwallet::Error>>()?; display::estimate(slate.amount, strategies, dark_scheme); } else { let init_args = InitTxArgs { @@ -1100,6 +1177,7 @@ where max_outputs: args.max_outputs as u32, num_change_outputs: 1u32, selection_strategy_is_use_all: args.selection_strategy == "all", + refresh_outputs_from_node: !info_updated, ttl_blocks: args.ttl_blocks, send_args: None, ..Default::default() @@ -1898,3 +1976,29 @@ where Ok(()) } + +#[cfg(test)] +mod send_tests { + use super::*; + + #[test] + fn max_retry_updates_args() { + let args = InitTxArgs { + amount: 100, + max_outputs: 500, + selection_strategy_is_use_all: false, + ..Default::default() + }; + let args = max_retry_args(args, 42, 7); + + assert_eq!(args.amount, 42); + assert_eq!(args.max_outputs, 7); + assert!(args.selection_strategy_is_use_all); + } + + #[test] + fn max_estimate_uses_all() { + assert_eq!(estimate_strategies(true), &["all"]); + assert_eq!(estimate_strategies(false), &["smallest", "all"]); + } +} diff --git a/controller/tests/accounts.rs b/controller/tests/accounts.rs index 35413bd..9a47e41 100644 --- a/controller/tests/accounts.rs +++ b/controller/tests/accounts.rs @@ -256,6 +256,15 @@ fn accounts_test_impl(test_dir: &'static str) -> Result<(), libwallet::Error> { }, )?; + // Use the requested account's stored height + { + wallet_inst!(wallet1, w); + assert_eq!(w.parent_key_id(), ExtKeychain::derive_key_id(2, 1, 0, 0, 0)); + let account2 = ExtKeychain::derive_key_id(2, 2, 0, 0, 0); + let account2_info = libwallet::retrieve_info(w, &account2, 1)?; + assert_eq!(account2_info.last_confirmed_height, 12); + } + // other account should be untouched { wallet_inst!(wallet1, w); diff --git a/controller/tests/common/mod.rs b/controller/tests/common/mod.rs index f9b7141..28115a5 100644 --- a/controller/tests/common/mod.rs +++ b/controller/tests/common/mod.rs @@ -90,7 +90,7 @@ pub fn clean_output_dir(test_dir: &str) { pub fn setup(test_dir: &str) { util::init_test_logger(); clean_output_dir(test_dir); - global::set_local_chain_type(ChainTypes::AutomatedTesting); + setup_global_chain_type(); } /// Some tests require the global chain_type to be configured due to threads being spawned internally. @@ -98,7 +98,7 @@ pub fn setup(test_dir: &str) { /// leaks across multiple tests and will likely have unintended consequences. #[allow(dead_code)] pub fn setup_global_chain_type() { - global::init_global_chain_type(global::ChainTypes::AutomatedTesting); + global::init_global_chain_type(ChainTypes::AutomatedTesting); } pub fn create_wallet_proxy( diff --git a/controller/tests/transaction.rs b/controller/tests/transaction.rs index 55734dc..6c3d8e5 100644 --- a/controller/tests/transaction.rs +++ b/controller/tests/transaction.rs @@ -21,6 +21,7 @@ extern crate grin_wallet_libwallet as libwallet; use grin_core as core; use self::core::core::transaction; +use self::core::core::Transaction; use self::core::global; use self::libwallet::{InitTxArgs, OutputStatus, Slate, SlateState}; use impls::test_framework::{self, LocalWalletClient}; @@ -677,6 +678,128 @@ fn tx_rollback(test_dir: &'static str) -> Result<(), libwallet::Error> { Ok(()) } +fn big_amount_error(test_dir: &'static str) -> Result<(), libwallet::Error> { + // Create a new proxy to simulate server and wallet responses + let mut wallet_proxy = create_wallet_proxy(test_dir); + let chain = wallet_proxy.chain.clone(); + let stopper = wallet_proxy.running.clone(); + + create_wallet_and_add!( + client1, + wallet1, + mask1_i, + test_dir, + "wallet1", + None, + &mut wallet_proxy, + true + ); + let mask1 = (&mask1_i).as_ref(); + println!("Mask1: {:?}", mask1); + + // Set the wallet proxy listener running + thread::spawn(move || { + if let Err(e) = wallet_proxy.run() { + error!("Wallet Proxy error: {}", e); + } + }); + + let outputs_num = 255; + + let reward = core::consensus::REWARD; + let cm = global::coinbase_maturity(); + // mine a few blocks + let _ = + test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), mask1, outputs_num, false); + + let min_confirmations = 1; + + // Check wallet 1 content are as expected + wallet::controller::owner_single_use( + wallet1.clone(), + mask1, + PathBuf::from(test_dir), + |api, m| { + let (wallet1_refreshed, wallet1_info) = + api.retrieve_summary_info(m, true, min_confirmations)?; + println!( + "Wallet 1 Info Pre-Transaction, after {} blocks: {:?}", + wallet1_info.last_confirmed_height, wallet1_info + ); + assert!(wallet1_refreshed); + assert_eq!( + wallet1_info.amount_currently_spendable, + (wallet1_info.last_confirmed_height - cm) * reward + ); + assert_eq!(wallet1_info.amount_immature, cm * reward); + Ok(()) + }, + )?; + + // test selecting max amount + let result = wallet::controller::owner_single_use( + wallet1.clone(), + mask1, + PathBuf::from(test_dir), + |api, m| { + api.init_send_tx( + m, + InitTxArgs { + amount: (reward * outputs_num as u64) - (cm * reward), + amount_includes_fee: Some(true), + minimum_confirmations: 0, + max_outputs: 500, + num_change_outputs: 1, + selection_strategy_is_use_all: true, + refresh_outputs_from_node: true, + ..InitTxArgs::default() + }, + )?; + Ok(()) + }, + ); + + println!("{:?}", result); + + let max_tx_weight = global::max_tx_weight(); + let tx_weight = Transaction::weight_by_iok(outputs_num as u64, 1, 1); + + assert!(tx_weight > max_tx_weight); + + assert!(result.is_err()); + assert!(matches!( + result.as_ref().err().unwrap(), + libwallet::Error::BigAmountError { .. } + )); + + match result { + Ok(_) => {} + Err(e) => match e { + libwallet::Error::BigAmountError(a, fee, num_inputs) => { + wallet::controller::owner_single_use( + wallet1.clone(), + mask1, + PathBuf::from(test_dir), + |api, m| { + let (_, e_a, e_fee, e_num_inputs) = + api.estimate_max_sendable(m, true, min_confirmations)?; + assert_eq!(e_a, a - fee); + assert_eq!(e_fee, fee); + assert_eq!(e_num_inputs, num_inputs); + Ok(()) + }, + )?; + } + _ => {} + }, + } + + // let logging finish + stopper.store(false, Ordering::Relaxed); + thread::sleep(Duration::from_millis(200)); + Ok(()) +} + #[test] fn db_wallet_basic_transaction_api() { let test_dir = "test_output/basic_transaction_api"; @@ -696,3 +819,13 @@ fn db_wallet_tx_rollback() { } clean_output_dir(test_dir); } + +#[test] +fn db_wallet_big_amount_error() { + let test_dir = "test_output/big_amount_error"; + setup(test_dir); + if let Err(e) = big_amount_error(test_dir) { + panic!("Libwallet Error: {}", e); + } + clean_output_dir(test_dir); +} diff --git a/controller/tests/updater_thread.rs b/controller/tests/updater_thread.rs index ea8273d..2beebab 100644 --- a/controller/tests/updater_thread.rs +++ b/controller/tests/updater_thread.rs @@ -19,9 +19,6 @@ extern crate grin_wallet_controller as wallet; extern crate grin_wallet_impls as impls; extern crate grin_wallet_libwallet as libwallet; -// use crate::libwallet::api_impl::owner_updater::{start_updater_log_thread, StatusMessage}; -// use grin_wallet_util::grin_core as core; - use impls::test_framework::{self, LocalWalletClient}; use std::path::PathBuf; use std::sync::atomic::Ordering; @@ -30,7 +27,7 @@ use std::time::Duration; #[macro_use] mod common; -use common::{clean_output_dir, create_wallet_proxy, setup, setup_global_chain_type}; +use common::{clean_output_dir, create_wallet_proxy, setup}; /// updater thread test impl fn updater_thread_test_impl(test_dir: &'static str) -> Result<(), libwallet::Error> { @@ -121,10 +118,6 @@ fn updater_thread_test_impl(test_dir: &'static str) -> Result<(), libwallet::Err #[test] fn updater_thread() { - // The "updater" kicks off a new thread so we need to ensure the global chain_type - // is set for this to work correctly. - setup_global_chain_type(); - let test_dir = "test_output/updater_thread"; setup(test_dir); if let Err(e) = updater_thread_test_impl(test_dir) { diff --git a/doc/samples/v3_api_node/package-lock.json b/doc/samples/v3_api_node/package-lock.json index 5d6ddb0..0649ef6 100644 --- a/doc/samples/v3_api_node/package-lock.json +++ b/doc/samples/v3_api_node/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.1", "license": "ISC", "dependencies": { - "jayson": "^3.2.0" + "jayson": "^4.3.0" } }, "node_modules/@types/connect": { @@ -20,35 +20,38 @@ "@types/node": "*" } }, - "node_modules/@types/express-serve-static-core": { - "version": "4.17.2", - "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.17.2.tgz", - "integrity": "sha512-El9yMpctM6tORDAiBwZVLMcxoTMcqqRO9dVyYcn7ycLWbvR8klrDn8CAOwRfZujZtWD7yS/mshTdz43jMOejbg==", - "dependencies": { - "@types/node": "*", - "@types/range-parser": "*" - } - }, - "node_modules/@types/lodash": { - "version": "4.14.149", - "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.149.tgz", - "integrity": "sha512-ijGqzZt/b7BfzcK9vTrS6MFljQRPn5BFWOx8oE0GYxribu6uV+aA9zZuXI1zc/etK9E8nrgdoF2+LgUw7+9tJQ==" - }, "node_modules/@types/node": { - "version": "12.12.27", - "resolved": "https://registry.npmjs.org/@types/node/-/node-12.12.27.tgz", - "integrity": "sha512-odQFl/+B9idbdS0e8IxDl2ia/LP8KZLXhV3BUeI98TrZp0uoIzQPhGd+5EtzHmT0SMOIaPd7jfz6pOHLWTtl7A==" + "version": "12.20.55", + "resolved": "https://registry.npmjs.org/@types/node/-/node-12.20.55.tgz", + "integrity": "sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==", + "license": "MIT" }, - "node_modules/@types/range-parser": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@types/range-parser/-/range-parser-1.2.3.tgz", - "integrity": "sha512-ewFXqrQHlFsgc09MK5jP5iR7vumV/BYayNC6PgJO2LPe8vrnNFyjQjSppfEngITi0qvfKtzFvgKymGheFM9UOA==" + "node_modules/@types/ws": { + "version": "7.4.7", + "resolved": "https://registry.npmjs.org/@types/ws/-/ws-7.4.7.tgz", + "integrity": "sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==", + "license": "MIT", + "dependencies": { + "@types/node": "*" + } }, "node_modules/commander": { "version": "2.20.3", "resolved": "https://registry.npmjs.org/commander/-/commander-2.20.3.tgz", "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==" }, + "node_modules/delay": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/es6-promise": { "version": "4.2.8", "resolved": "https://registry.npmjs.org/es6-promise/-/es6-promise-4.2.8.tgz", @@ -70,25 +73,39 @@ "node": "> 0.1.90" } }, + "node_modules/isomorphic-ws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/isomorphic-ws/-/isomorphic-ws-4.0.1.tgz", + "integrity": "sha512-BhBvN2MBpWTaSHdWRb/bwdZJ1WaehQ2L1KngkCkfLUGF0mAWAT1sQUQacEmQ0jXkFw/czDXPNQSL5u2/Krsz1w==", + "license": "MIT", + "peerDependencies": { + "ws": "*" + } + }, "node_modules/jayson": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/jayson/-/jayson-3.2.0.tgz", - "integrity": "sha512-DZQnwA57GcStw4soSYB2VntWXFfoWvmSarlaWePDYOWhjxT72PBM4atEBomaTaS1uqk3jFC9UO9AyWjlujo3xw==", + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/jayson/-/jayson-4.3.0.tgz", + "integrity": "sha512-AauzHcUcqs8OBnCHOkJY280VaTiCm57AbuO7lqzcw7JapGj50BisE3xhksye4zlTSR1+1tAz67wLTl8tEH1obQ==", + "license": "MIT", "dependencies": { - "@types/connect": "^3.4.32", - "@types/express-serve-static-core": "^4.16.9", - "@types/lodash": "^4.14.139", - "@types/node": "^12.7.7", - "commander": "^2.12.2", + "@types/connect": "^3.4.33", + "@types/node": "^12.12.54", + "@types/ws": "^7.4.4", + "commander": "^2.20.3", + "delay": "^5.0.0", "es6-promisify": "^5.0.0", "eyes": "^0.1.8", + "isomorphic-ws": "^4.0.1", "json-stringify-safe": "^5.0.1", - "JSONStream": "^1.3.1", - "lodash": "^4.17.15", - "uuid": "^3.2.1" + "stream-json": "^1.9.1", + "uuid": "^8.3.2", + "ws": "^7.5.10" }, "bin": { "jayson": "bin/jayson.js" + }, + "engines": { + "node": ">=8" } }, "node_modules/json-stringify-safe": { @@ -96,46 +113,53 @@ "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" }, - "node_modules/jsonparse": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", - "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=", - "engines": [ - "node >= 0.2.0" - ] + "node_modules/stream-chain": { + "version": "2.2.5", + "resolved": "https://registry.npmjs.org/stream-chain/-/stream-chain-2.2.5.tgz", + "integrity": "sha512-1TJmBx6aSWqZ4tx7aTpBDXK0/e2hhcNSTV8+CbFJtDjbb+I1mZ8lHit0Grw9GRT+6JbIrrDd8esncgBi8aBXGA==", + "license": "BSD-3-Clause" }, - "node_modules/JSONStream": { - "version": "1.3.5", - "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz", - "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==", + "node_modules/stream-json": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/stream-json/-/stream-json-1.9.1.tgz", + "integrity": "sha512-uWkjJ+2Nt/LO9Z/JyKZbMusL8Dkh97uUBTv3AJQ74y07lVahLY4eEFsPsE97pxYBwr8nnjMAIch5eqI0gPShyw==", + "license": "BSD-3-Clause", "dependencies": { - "jsonparse": "^1.2.0", - "through": ">=2.2.7 <3" - }, - "bin": { - "JSONStream": "bin.js" - }, - "engines": { - "node": "*" + "stream-chain": "^2.2.5" } }, - "node_modules/lodash": { - "version": "4.18.1", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.18.1.tgz", - "integrity": "sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==" - }, - "node_modules/through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "node_modules/uuid": { - "version": "3.4.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz", - "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", - "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).", + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-11.1.1.tgz", + "integrity": "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", "bin": { - "uuid": "bin/uuid" + "uuid": "dist/esm/bin/uuid" + } + }, + "node_modules/ws": { + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.5.13.tgz", + "integrity": "sha512-rsKI6xDBFVf4r/x8XyChGK04QR/XHroxs/jUcoWvtEZM8TPU/X/uIY9B1CsSzYws9ZJb/6bbBu7dPhFW00CAoA==", + "license": "MIT", + "engines": { + "node": ">=8.3.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": "^5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } } } } diff --git a/doc/samples/v3_api_node/package.json b/doc/samples/v3_api_node/package.json index 1126955..fb89fc9 100644 --- a/doc/samples/v3_api_node/package.json +++ b/doc/samples/v3_api_node/package.json @@ -9,6 +9,9 @@ "author": "", "license": "ISC", "dependencies": { - "jayson": "^3.2.0" + "jayson": "^4.3.0" + }, + "overrides": { + "uuid": "^11.1.1" } } diff --git a/grin b/grin index bad44b4..d5f3235 160000 --- a/grin +++ b/grin @@ -1 +1 @@ -Subproject commit bad44b4e9bd7e37417d80cc469bcba316249bd93 +Subproject commit d5f32350ab548736c17c1db7ff51457917fe4e96 diff --git a/impls/Cargo.toml b/impls/Cargo.toml index da64641..ca0ea1e 100644 --- a/impls/Cargo.toml +++ b/impls/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_impls" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Concrete types derived from libwallet traits" license = "Apache-2.0" @@ -47,9 +47,9 @@ hyper-util = { version = "0.1.20", features = ["tokio"] } http-body-util = "0.1.3" bytes = "1.11.1" -grin_wallet_util = { path = "../util", version = "5.4.1" } -grin_wallet_config = { path = "../config", version = "5.4.1" } -grin_wallet_libwallet = { path = "../libwallet", version = "5.4.1" } +grin_wallet_util = { path = "../util", version = "5.5.0" } +grin_wallet_config = { path = "../config", version = "5.5.0" } +grin_wallet_libwallet = { path = "../libwallet", version = "5.5.0" } ##### Grin Imports @@ -59,4 +59,4 @@ grin_chain = { path = "../grin/chain"} grin_util = { path = "../grin/util"} grin_api = { path = "../grin/api"} -##### \ No newline at end of file +##### diff --git a/impls/src/node_clients/http.rs b/impls/src/node_clients/http.rs index b7e29c6..8b7ed0e 100644 --- a/impls/src/node_clients/http.rs +++ b/impls/src/node_clients/http.rs @@ -24,7 +24,7 @@ use crate::libwallet::{NodeClient, NodeVersionInfo}; use crate::util::secp::pedersen; use crate::util::ToHex; use futures::stream::FuturesUnordered; -use futures::TryStreamExt; +use futures::StreamExt; use std::collections::HashMap; use std::env; use std::net::SocketAddr; @@ -259,17 +259,27 @@ impl NodeClient for HTTPNodeClient { reqs.push(build_request("get_outputs", p)); } - let mut tasks = Vec::with_capacity(params.len()); - for req in &reqs { - tasks.push(cl.post_async::( - url.as_str(), - req, - api_secret.clone(), - )); - } + let mut outputs = Vec::new(); - let task: FuturesUnordered<_> = tasks.into_iter().collect(); - task.try_collect().await + let max_num_requests = 16; + for req_chunks in reqs.chunks(max_num_requests) { + let mut tasks = vec![]; + for req in req_chunks { + tasks.push(cl.post_async::( + url.as_str(), + req, + api_secret.clone(), + )); + } + let mut task: FuturesUnordered<_> = tasks.into_iter().collect(); + while let Some(item) = task.next().await { + match item { + Ok(i) => outputs.push(i), + Err(e) => return Err(e), + } + } + } + Ok(outputs) }; let rt = RUNTIME.clone(); diff --git a/libwallet/Cargo.toml b/libwallet/Cargo.toml index fd8842b..18bd829 100644 --- a/libwallet/Cargo.toml +++ b/libwallet/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_libwallet" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Simple, private and scalable cryptocurrency implementation based on the MimbleWimble chain format." license = "Apache-2.0" @@ -15,6 +15,7 @@ mwixnet-test = [] [dependencies] blake2-rfc = "0.2" rand = "0.6" +rayon = "1.12" serde = "1" serde_derive = "1" serde_with = { version = "1", features = ["chrono"] } @@ -42,8 +43,8 @@ num-bigint = "0.2" chacha20 = "0.8.1" hmac = { version = "0.12.0", features = ["std"]} -grin_wallet_util = { path = "../util", version = "5.4.1" } -grin_wallet_config = { path = "../config", version = "5.4.1" } +grin_wallet_util = { path = "../util", version = "5.5.0" } +grin_wallet_config = { path = "../config", version = "5.5.0" } ##### Grin Imports diff --git a/libwallet/src/api_impl/foreign.rs b/libwallet/src/api_impl/foreign.rs index 7a9cc63..d9e0857 100644 --- a/libwallet/src/api_impl/foreign.rs +++ b/libwallet/src/api_impl/foreign.rs @@ -99,7 +99,7 @@ where ret_slate.tx = Some(Slate::empty_transaction()); - let height = w.last_confirmed_height()?; + let height = w.last_confirmed_height_for_parent(&parent_key_id)?; let keychain = w.keychain(keychain_mask)?; let context = tx::add_output_to_slate( diff --git a/libwallet/src/api_impl/owner.rs b/libwallet/src/api_impl/owner.rs index a888467..e5744aa 100644 --- a/libwallet/src/api_impl/owner.rs +++ b/libwallet/src/api_impl/owner.rs @@ -19,6 +19,7 @@ use uuid::Uuid; use crate::api_impl::foreign::finalize_tx as foreign_finalize; use crate::contract::proofs::{InvoiceProof, ProofWitness}; +use crate::grin_core::core::amount_to_hr_string; use crate::grin_core::core::hash::Hashed; use crate::grin_core::core::{FeeFields, Output, OutputFeatures, Transaction}; use crate::grin_core::libtx::proof; @@ -298,6 +299,65 @@ where )) } +/// Calculate max amount to send. +pub fn estimate_max_sendable<'a, L, C, K>( + wallet_inst: Arc>>>, + keychain_mask: Option<&SecretKey>, + status_send_channel: &Option>, + refresh_from_node: bool, + minimum_confirmations: u64, +) -> Result<(bool, u64, u64, u32), Error> +where + L: WalletLCProvider<'a, C, K>, + C: NodeClient + 'a, + K: Keychain + 'a, +{ + let validated = if refresh_from_node { + update_wallet_state( + wallet_inst.clone(), + keychain_mask, + status_send_channel, + false, + )? + } else { + false + }; + + wallet_lock!(wallet_inst, w); + + let parent_key_id = w.parent_key_id(); + let wallet_info = updater::retrieve_info(w, &parent_key_id, minimum_confirmations)?; + let current_height = w.last_confirmed_height_for_parent(&parent_key_id)?; + let max_outputs = 500; + let change_outputs = 1; + let (amount, fee, input_count) = match selection::select_coins_and_fee( + w, + wallet_info.amount_currently_spendable, + true, + current_height, + minimum_confirmations, + max_outputs, + change_outputs, + true, + &parent_key_id, + ) { + Ok((coins, _total, amount, fee)) => (amount, fee, coins.len() as u32), + Err(e) => match e { + Error::BigAmountError(amount, fee, input_count) => { + let amount = amount.checked_sub(fee).ok_or(Error::GenericError(format!( + "Transaction amount {} is too small to include fee {}, send lower amount", + amount_to_hr_string(amount, true), + amount_to_hr_string(fee, true) + )))?; + (amount, fee, input_count) + } + _ => return Err(e), + }, + }; + + Ok((validated, amount, fee, input_count)) +} + /// Retrieve txs pub fn retrieve_txs<'a, L, C, K>( wallet_inst: Arc>>>, @@ -656,6 +716,7 @@ where args.max_outputs as usize, args.num_change_outputs as usize, args.selection_strategy_is_use_all, + args.refresh_outputs_from_node, &parent_key_id, )?; slate.amount = total; @@ -684,6 +745,7 @@ where args.max_outputs as usize, args.num_change_outputs as usize, args.selection_strategy_is_use_all, + args.refresh_outputs_from_node, &parent_key_id, true, use_test_rng, @@ -842,6 +904,7 @@ where args.max_outputs as usize, args.num_change_outputs as usize, args.selection_strategy_is_use_all, + args.refresh_outputs_from_node, &parent_key_id, false, use_test_rng, diff --git a/libwallet/src/api_impl/owner_updater.rs b/libwallet/src/api_impl/owner_updater.rs index d46cdaa..9201c6e 100644 --- a/libwallet/src/api_impl/owner_updater.rs +++ b/libwallet/src/api_impl/owner_updater.rs @@ -58,6 +58,7 @@ pub fn start_updater_log_thread( let _ = thread::Builder::new() .name("wallet-updater-status".to_string()) .spawn(move || { + let mut last_scan_percent = 0; while let Ok(m) = rx.recv() { // save to our message queue to be read by other consumers { @@ -72,10 +73,19 @@ pub fn start_updater_log_thread( StatusMessage::UpdatingTransactions(s) => debug!("{}", s), StatusMessage::FullScanWarn(s) => warn!("{}", s), StatusMessage::Scanning(s, m) => { + if m == 0 { + last_scan_percent = 0; + } debug!("{}", s); - warn!("Scanning - {}% complete", m); + if last_scan_percent < m { + warn!("Scanning - {}% complete", m); + last_scan_percent = m; + } + } + StatusMessage::ScanningComplete(s) => { + warn!("{}", s); + last_scan_percent = 0; } - StatusMessage::ScanningComplete(s) => warn!("{}", s), StatusMessage::UpdateWarning(s) => warn!("{}", s), } } diff --git a/libwallet/src/api_impl/types.rs b/libwallet/src/api_impl/types.rs index d87a7bd..0575fab 100644 --- a/libwallet/src/api_impl/types.rs +++ b/libwallet/src/api_impl/types.rs @@ -68,6 +68,9 @@ pub struct InitTxArgs { /// as many outputs as are needed to meet the amount, (and no more) starting with the smallest /// value outputs. pub selection_strategy_is_use_all: bool, + /// Flag to refresh outputs from node. + #[serde(default = "default_refresh_outputs_from_node")] + pub refresh_outputs_from_node: bool, /// Optionally set the output target slate version (acceptable /// down to the minimum slate version compatible with the current. If `None` the slate /// is generated with the latest version. @@ -117,6 +120,7 @@ impl Default for InitTxArgs { max_outputs: 500, num_change_outputs: 1, selection_strategy_is_use_all: true, + refresh_outputs_from_node: true, target_slate_version: None, ttl_blocks: None, estimate_only: Some(false), @@ -350,6 +354,10 @@ pub struct BuiltOutput { pub output: Output, } +fn default_refresh_outputs_from_node() -> bool { + true +} + /// Update transaction slate state. pub fn update_tx_slate_state( wallet: &mut WalletBackend, @@ -385,3 +393,20 @@ where } Ok(()) } + +#[cfg(test)] +mod tests { + use super::InitTxArgs; + + #[test] + fn legacy_refresh_default() { + let mut value = serde_json::to_value(InitTxArgs::default()).unwrap(); + value + .as_object_mut() + .unwrap() + .remove("refresh_outputs_from_node"); + + let args: InitTxArgs = serde_json::from_value(value).unwrap(); + assert!(args.refresh_outputs_from_node); + } +} diff --git a/libwallet/src/backend.rs b/libwallet/src/backend.rs index 5833563..31378f0 100644 --- a/libwallet/src/backend.rs +++ b/libwallet/src/backend.rs @@ -17,6 +17,7 @@ use std::fs; use std::fs::File; use std::io::{Read, Write}; use std::path::Path; +use uuid::Uuid; use crate::blake2::blake2b::{Blake2b, Blake2bResult}; @@ -324,9 +325,8 @@ where Ok(items.into_iter()) } - /// Get an (Optional) tx log entry by parent key id and log id. Keyed to match - /// save_tx_log_entry (parent_id + numeric log id), unlike the unused uuid form. - pub fn get_tx_log_entry( + /// Get an (Optional) tx log entry by parent key id and log id. + pub fn get_tx_log_entry_by_id( &self, parent_id: Identifier, log_id: u32, @@ -337,6 +337,13 @@ where .map_err(|e| e.into()) } + /// Get an (Optional) tx log entry by uuid. + pub fn get_tx_log_entry(&self, u: &Uuid) -> Result, Error> { + self.db + .get_ser(Some(TX_LOG_ENTRY_PREFIX), u.as_bytes(), None) + .map_err(|e| e.into()) + } + /// Iterate over all tx log data stored by the backend. pub fn tx_log_iter( &self, @@ -498,13 +505,22 @@ where self.next_child_for(&parent_key_id, keychain_mask) } - /// Last verified height of outputs directly descending from the given parent key. + /// Last verified height of outputs directly descending from the current parent key. pub fn last_confirmed_height(&mut self) -> Result { + let parent_key_id = self.parent_key_id.clone(); + self.last_confirmed_height_for_parent(&parent_key_id) + } + + /// Last verified height of outputs directly descending from the given parent key. + pub(crate) fn last_confirmed_height_for_parent( + &mut self, + parent_key_id: &Identifier, + ) -> Result { let batch = self.db.batch()?; let last_confirmed_height = batch .get_ser( Some(CONFIRMED_HEIGHT_PREFIX), - &self.parent_key_id.to_bytes(), + &parent_key_id.to_bytes(), None, )? .unwrap_or_else(|| 0); diff --git a/libwallet/src/contract/actions/revoke.rs b/libwallet/src/contract/actions/revoke.rs index 52268a2..08993a3 100644 --- a/libwallet/src/contract/actions/revoke.rs +++ b/libwallet/src/contract/actions/revoke.rs @@ -79,7 +79,7 @@ where // Cancel the original tx only if it is still in a cancellable state. On a resumed revoke // it is already a *Cancelled type (and the inputs are Unspent), so we skip straight to // re-spending them. - let revoked = w.get_tx_log_entry(parent_key_id.clone(), tx_id)?; + let revoked = w.get_tx_log_entry_by_id(parent_key_id.clone(), tx_id)?; let revoked_slate_id = revoked.as_ref().and_then(|e| e.tx_slate_id); let needs_cancel = match revoked.as_ref() { Some(e) => matches!( diff --git a/libwallet/src/contract/utils.rs b/libwallet/src/contract/utils.rs index b5eaf48..df50915 100644 --- a/libwallet/src/contract/utils.rs +++ b/libwallet/src/contract/utils.rs @@ -214,7 +214,7 @@ where // We create a new entry with log_id=0 and but replace it with the real id before committing create_tx_log_entry(slate, context.get_net_change()?, parent_key_id.clone(), 0)? } else { - w.get_tx_log_entry(parent_key_id.clone(), context.log_id.unwrap())? + w.get_tx_log_entry_by_id(parent_key_id.clone(), context.log_id.unwrap())? .unwrap() } }; diff --git a/libwallet/src/error.rs b/libwallet/src/error.rs index 1863d2d..c9248ca 100644 --- a/libwallet/src/error.rs +++ b/libwallet/src/error.rs @@ -14,7 +14,7 @@ //! Error types for libwallet -use crate::grin_core::core::{committed, transaction}; +use crate::grin_core::core::{amount_to_hr_string, committed, transaction}; use crate::grin_core::libtx; use crate::grin_keychain; use crate::grin_util::secp; @@ -37,6 +37,14 @@ pub enum Error { needed_disp: String, }, + /// Big amount error. + #[error( + "Amount too large for a single transaction; can send at most {} using {2} inputs. Send that amount \ + to yourself using the 'all' selection strategy and a maximum of {2} inputs to consolidate outputs", + amount_to_hr_string(*.0, true) + )] + BigAmountError(u64, u64, u32), + /// Fee error #[error("Fee Error: {0}")] Fee(String), @@ -396,3 +404,24 @@ impl From for Error { Error::SlatepackAddress(format!("{}", error)) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn big_amount_message() { + let message = Error::BigAmountError(3_000_000_000, 12_500_000, 2).to_string(); + assert!(message.contains("3.0")); + assert_eq!( + message + .as_bytes() + .windows("2 inputs".len()) + .filter(|&w| w == "2 inputs".as_bytes()) + .count(), + 2 + ); + assert!(message.contains("'all' selection strategy")); + assert!(message.contains("consolidate outputs")); + } +} diff --git a/libwallet/src/internal/scan.rs b/libwallet/src/internal/scan.rs index 85f0e9a..fb5ec73 100644 --- a/libwallet/src/internal/scan.rs +++ b/libwallet/src/internal/scan.rs @@ -28,6 +28,7 @@ use crate::internal::{keys, updater}; use crate::types::*; use crate::{wallet_lock, Error, OutputCommitMapping}; use blake2_rfc::blake2b::blake2b; +use rayon::prelude::*; use std::cmp; use std::collections::HashMap; use std::sync::mpsc::Sender; @@ -54,18 +55,6 @@ struct OutputResult { pub is_coinbase: bool, } -#[derive(Debug, Clone)] -/// Collect stats in case we want to just output a single tx log entry -/// for restored non-coinbase outputs -struct RestoredTxStats { - /// - pub log_id: u32, - /// - pub amount_credited: u64, - /// - pub num_outputs: usize, -} - fn identify_utxo_outputs( keychain: &K, outputs: Vec<(pedersen::Commitment, pedersen::RangeProof, bool, u64, u64)>, @@ -75,49 +64,74 @@ fn identify_utxo_outputs( where K: Keychain, { - let mut wallet_outputs: Vec = Vec::new(); - let legacy_builder = proof::LegacyProofBuilder::new(keychain); let builder = proof::ProofBuilder::new(keychain); let legacy_version = HeaderVersion(1); - for output in outputs.iter() { - let (commit, proof, is_coinbase, height, mmr_index) = output; - // attempt to unwind message from the RP and get a value - // will fail if it's not ours - let info = { - // Before HF+2wk, try legacy rewind first - let info_legacy = - if valid_header_version(height.saturating_sub(2 * WEEK_HEIGHT), legacy_version) { - proof::rewind(keychain.secp(), &legacy_builder, *commit, None, *proof)? - } else { - None + let chain_type = global::get_chain_type(); + + let identified = outputs + .par_iter() + .map( + |output| -> Result, Error> { + // set chain type for thread + global::set_local_chain_type(chain_type); + + let (commit, proof, is_coinbase, height, mmr_index) = output; + // attempt to unwind message from the RP and get a value + // will fail if it's not ours + let info = { + // Before HF+2wk, try legacy rewind first + let info_legacy = if valid_header_version( + height.saturating_sub(2 * WEEK_HEIGHT), + legacy_version, + ) { + proof::rewind(keychain.secp(), &legacy_builder, *commit, None, *proof)? + } else { + None + }; + + // If legacy didn't work, try new rewind + if info_legacy.is_none() { + proof::rewind(keychain.secp(), &builder, *commit, None, *proof)? + } else { + info_legacy + } }; - // If legacy didn't work, try new rewind - if info_legacy.is_none() { - proof::rewind(keychain.secp(), &builder, *commit, None, *proof)? - } else { - info_legacy - } - }; + let (amount, key_id, switch) = match info { + Some(i) => i, + None => return Ok(None), + }; - let (amount, key_id, switch) = match info { - Some(i) => i, - None => { - continue; - } - }; + let lock_height = if *is_coinbase { + *height + global::coinbase_maturity() + } else { + *height + }; - let lock_height = if *is_coinbase { - *height + global::coinbase_maturity() - } else { - *height - }; + Ok(Some(( + OutputResult { + commit: *commit, + key_id: key_id.clone(), + n_child: key_id.to_path().last_path_index(), + value: amount, + height: *height, + lock_height, + is_coinbase: *is_coinbase, + mmr_index: *mmr_index, + }, + switch, + ))) + }, + ) + .collect::, Error>>()?; + let mut wallet_outputs = Vec::new(); + for (output, switch) in identified.into_iter().flatten() { let msg = format!( "Output found: {:?}, amount: {:?}, key_id: {:?}, mmr_index: {},", - commit, amount, key_id, mmr_index, + output.commit, output.value, output.key_id, output.mmr_index, ); if let Some(ref s) = status_send_channel { @@ -131,16 +145,7 @@ where } } - wallet_outputs.push(OutputResult { - commit: *commit, - key_id: key_id.clone(), - n_child: key_id.to_path().last_path_index(), - value: amount, - height: *height, - lock_height, - is_coinbase: *is_coinbase, - mmr_index: *mmr_index, - }); + wallet_outputs.push(output); } Ok(wallet_outputs) } @@ -266,7 +271,7 @@ where result_vec.append(&mut identify_utxo_outputs( keychain, - outputs.clone(), + outputs, status_send_channel, perc_complete, )?); @@ -285,86 +290,72 @@ where Ok((result_vec, last_retrieved_return_index, perc_complete)) } -fn restore_missing_output<'a, L, C, K>( +const MISSING_OUTPUTS_BATCH_SIZE: usize = 1000; + +fn restore_missing_outputs<'a, L, C, K>( wallet_inst: Arc>>>, keychain_mask: Option<&SecretKey>, - output: OutputResult, + outputs: Vec, found_parents: &mut HashMap, - tx_stats: &mut Option<&mut HashMap>, + status_send_channel: &Option>, + perc_complete: u8, ) -> Result<(), Error> where L: WalletLCProvider<'a, C, K>, C: NodeClient + 'a, K: Keychain + 'a, { - wallet_lock!(wallet_inst, w); + for o_chunks in outputs.chunks(MISSING_OUTPUTS_BATCH_SIZE) { + wallet_lock!(wallet_inst, w); + let mut batch = w.batch(keychain_mask)?; + for output in o_chunks { + let msg = format!( + "Confirmed output for {} with ID {} ({:?}, index {}) exists in UTXO set but not in wallet. \ + Restoring.", + output.value, output.key_id, output.commit, output.mmr_index + ); + if let Some(ref s) = status_send_channel { + let _ = s.send(StatusMessage::Scanning(msg, perc_complete)); + } - let commit = w.calc_commit_for_cache(keychain_mask, output.value, &output.key_id)?; - let mut batch = w.batch(keychain_mask)?; + let parent_key_id = output.key_id.parent_path(); + if !found_parents.contains_key(&parent_key_id) { + found_parents.insert(parent_key_id.clone(), 0); + } - let parent_key_id = output.key_id.parent_path(); - if !found_parents.contains_key(&parent_key_id) { - found_parents.insert(parent_key_id.clone(), 0); - if let Some(ref mut s) = tx_stats { - s.insert( - parent_key_id.clone(), - RestoredTxStats { - log_id: batch.next_tx_log_id(&parent_key_id)?, - amount_credited: 0, - num_outputs: 0, - }, - ); + let log_id = batch.next_tx_log_id(&parent_key_id)?; + let entry_type = match output.is_coinbase { + true => TxLogEntryType::ConfirmedCoinbase, + false => TxLogEntryType::TxReceived, + }; + let mut t = TxLogEntry::new(parent_key_id.clone(), entry_type, log_id); + t.confirmed = true; + t.amount_credited = output.value; + t.num_outputs = 1; + t.update_confirmation_ts(); + batch.save_tx_log_entry(t, &parent_key_id)?; + + batch.save(OutputData { + root_key_id: parent_key_id.clone(), + key_id: output.key_id.clone(), + n_child: output.n_child, + mmr_index: Some(output.mmr_index), + commit: Some(output.commit.0.to_vec().to_hex()), + value: output.value, + status: OutputStatus::Unspent, + height: output.height, + lock_height: output.lock_height, + is_coinbase: output.is_coinbase, + tx_log_entry: Some(log_id), + })?; + + let max_child_index = *found_parents.get(&parent_key_id).unwrap(); + if output.n_child >= max_child_index { + found_parents.insert(parent_key_id, output.n_child); + } } + batch.commit()?; } - - let log_id = if tx_stats.is_none() || output.is_coinbase { - let log_id = batch.next_tx_log_id(&parent_key_id)?; - let entry_type = match output.is_coinbase { - true => TxLogEntryType::ConfirmedCoinbase, - false => TxLogEntryType::TxReceived, - }; - let mut t = TxLogEntry::new(parent_key_id.clone(), entry_type, log_id); - t.confirmed = true; - t.amount_credited = output.value; - t.num_outputs = 1; - t.update_confirmation_ts(); - batch.save_tx_log_entry(t, &parent_key_id)?; - log_id - } else if let Some(ref mut s) = tx_stats { - let ts = s.get(&parent_key_id).unwrap().clone(); - s.insert( - parent_key_id.clone(), - RestoredTxStats { - log_id: ts.log_id, - amount_credited: ts.amount_credited + output.value, - num_outputs: ts.num_outputs + 1, - }, - ); - ts.log_id - } else { - 0 - }; - - let _ = batch.save(OutputData { - root_key_id: parent_key_id.clone(), - key_id: output.key_id, - n_child: output.n_child, - mmr_index: Some(output.mmr_index), - commit: commit, - value: output.value, - status: OutputStatus::Unspent, - height: output.height, - lock_height: output.lock_height, - is_coinbase: output.is_coinbase, - tx_log_entry: Some(log_id), - }); - - let max_child_index = *found_parents.get(&parent_key_id).unwrap(); - if output.n_child >= max_child_index { - found_parents.insert(parent_key_id, output.n_child); - } - - batch.commit()?; Ok(()) } @@ -570,7 +561,7 @@ where let _ = s.send(StatusMessage::Scanning(msg, perc_complete)); } o.status = OutputStatus::Unspent; - // any transactions associated with this should be cancelled + // any transactions associated with this should be canceled cancel_tx_log_entry(wallet_inst.clone(), keychain_mask, &o)?; wallet_lock!(wallet_inst, w); let mut batch = w.batch(keychain_mask)?; @@ -580,24 +571,17 @@ where let mut found_parents: HashMap = HashMap::new(); + debug!("scan: restoring {} missing outputs", missing_outs.len()); + // Restore missing outputs, adding transaction for it back to the log - for m in missing_outs.into_iter() { - let msg = format!( - "Confirmed output for {} with ID {} ({:?}, index {}) exists in UTXO set but not in wallet. \ - Restoring.", - m.value, m.key_id, m.commit, m.mmr_index - ); - if let Some(ref s) = status_send_channel { - let _ = s.send(StatusMessage::Scanning(msg, perc_complete)); - } - restore_missing_output( - wallet_inst.clone(), - keychain_mask, - m, - &mut found_parents, - &mut None, - )?; - } + restore_missing_outputs( + wallet_inst.clone(), + keychain_mask, + missing_outs, + &mut found_parents, + status_send_channel, + perc_complete, + )?; if delete_unconfirmed { // Unlock locked outputs diff --git a/libwallet/src/internal/selection.rs b/libwallet/src/internal/selection.rs index 75d298d..010b724 100644 --- a/libwallet/src/internal/selection.rs +++ b/libwallet/src/internal/selection.rs @@ -29,6 +29,8 @@ use crate::slate::Slate; use crate::types::*; use crate::util::OnionV3Address; use crate::{address, WalletBackend}; +use grin_core::core::Transaction; +use grin_core::global; use std::collections::HashMap; use std::convert::TryInto; @@ -98,7 +100,7 @@ where context.amount = slate.amount; // Store our private identifiers for each input - for input in inputs { + for input in &inputs { context.add_input(&input.key_id, &input.mmr_index, input.value); } @@ -487,6 +489,30 @@ where }; } } + + let input_len = coins.len(); + let output_len = if total == amount_with_fee { + 1 + } else { + change_outputs + 1 + }; + let tx_weight = Transaction::weight_by_iok(input_len as u64, output_len as u64, 1u64); + let max_tx_weight = global::max_tx_weight(); + if tx_weight > max_tx_weight { + let (max_amount, max_inputs) = max_spendable_amount( + &coins, + output_len as u64, + max_tx_weight, + amount_includes_fee, + ); + error!( + "Transaction weight {}, exceeds global max_tx_weight {}, can send maximum {}, send such amount to yourself for outputs consolidation", + tx_weight, max_tx_weight, amount_to_hr_string(max_amount, true) + ); + let fee = tx_fee(max_inputs as usize, 1, 1); + return Err(Error::BigAmountError(max_amount, fee, max_inputs)); + } + // If original amount includes fee, the new amount should // be reduced, to accommodate the fee. let new_amount = match amount_includes_fee { @@ -498,6 +524,64 @@ where Ok((coins, total, new_amount, fee)) } +fn max_spendable_amount( + outputs: &[OutputData], + output_len: u64, + max_tx_weight: u64, + amount_includes_fee: bool, +) -> (u64, u32) { + let mut values: Vec = outputs + .iter() + .map(|output| output.value) + .enumerate() + .take_while(|(index, _)| { + Transaction::weight_by_iok(*index as u64 + 1, output_len, 1) <= max_tx_weight + }) + .map(|(_, value)| value) + .collect(); + + // sort outputs by decreasing value to calculate the best fee + values.sort_by(|a, b| b.cmp(a)); + + let mut total_value = 0; + let mut amount = 0; + let mut inputs = 0; + for (index, value) in values.into_iter().enumerate() { + let inputs_len = index + 1; + total_value += value; + if total_value > tx_fee(inputs_len, 1, 1) { + amount = total_value; + inputs = inputs_len as u32; + } + } + + let amount = if amount_includes_fee { + amount + } else { + amount.saturating_sub(tx_fee(inputs as usize, 1, 1)) + }; + (amount, inputs) +} + +fn eligible_outputs( + wallet: &WalletBackend, + current_height: u64, + minimum_confirmations: u64, + parent_key_id: &Identifier, +) -> Result, Error> +where + C: NodeClient, + K: Keychain, +{ + Ok(wallet + .iter()? + .filter(|output| { + output.root_key_id == *parent_key_id + && output.eligible_to_spend(current_height, minimum_confirmations) + }) + .collect()) +} + /// Selects inputs and change for a transaction pub fn inputs_and_change( coins: &[OutputData], @@ -591,13 +675,8 @@ where K: Keychain, { // first find all eligible outputs based on number of confirmations - let mut eligible = wallet - .iter()? - .filter(|out| { - out.root_key_id == *parent_key_id - && out.eligible_to_spend(current_height, minimum_confirmations) - }) - .collect::>(); + let mut eligible = + eligible_outputs(wallet, current_height, minimum_confirmations, parent_key_id)?; let max_available = eligible.len(); @@ -696,8 +775,11 @@ where slate.add_participant_info(&keychain, &context, None)?; let mut parts = vec![]; - for (id, _, value) in &context.get_inputs() { - let input = wallet.iter()?.find(|out| out.key_id == *id); + for (id, mmr_index, value) in &context.get_inputs() { + let input = match wallet.get(id, mmr_index) { + Ok(o) => Some(o), + Err(_) => wallet.iter()?.find(|out| out.key_id == *id), + }; if let Some(i) = input { if i.is_coinbase { parts.push(build::coinbase_input(*value, i.key_id.clone())); @@ -717,3 +799,135 @@ where slate.tx_or_err_mut()?.offset = slate.offset.clone(); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + fn output(value: u64) -> OutputData { + OutputData { + root_key_id: Identifier::zero(), + key_id: Identifier::zero(), + n_child: 0, + commit: None, + mmr_index: None, + value, + status: OutputStatus::Unspent, + height: 0, + lock_height: 0, + is_coinbase: false, + tx_log_entry: None, + } + } + + #[test] + fn max_spendable_uses_all_outputs() { + global::set_local_accept_fee_base(1); + + let selected = vec![output(100), output(200)]; + let eligible = vec![output(100), output(200), output(10000)]; + let output_len = 1; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + assert_eq!( + max_spendable_amount(&selected, output_len, max_weight, true), + (300, 2) + ); + assert_eq!( + max_spendable_amount(&eligible, output_len, max_weight, true), + (300, 2) + ); + + let max_weight = Transaction::weight_by_iok(3, output_len, 1); + assert_eq!( + max_spendable_amount(&eligible, output_len, max_weight, true), + (10300, 3) + ); + } + + #[test] + fn max_spendable_fee_amount() { + global::set_local_accept_fee_base(1); + + // Can not make txs when fee equals amount and not enough outputs. + { + let fee = tx_fee(2, 1, 1); + let coins = vec![output(13), output(13)]; + let output_len = 2; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + assert_eq!(fee, coins.iter().map(|o| o.value).sum::()); + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, false), + (0, 0) + ); + } + + // Can not make txs when fee more than amount and not enough outputs. + { + let fee = tx_fee(2, 1, 1); + let coins = vec![output(12), output(13)]; + let output_len = 2; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + assert!(fee > coins.iter().map(|o| o.value).sum::()); + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, false), + (0, 0) + ); + } + + // Select not all outputs to cover fee. + { + let fee = tx_fee(3, 1, 1); + let coins = vec![output(fee), output(1), output(1)]; + let output_len = 2; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, false), + (2, 2) + ); + } + } + + #[test] + fn max_spendable_excludes_fee() { + global::set_local_accept_fee_base(500_000); + let coins = vec![output(1_000_000_000), output(2_000_000_000)]; + let output_len = 2; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, false), + (3_000_000_000 - tx_fee(2, 1, 1), 2) + ); + } + + #[test] + fn covers_fee_with_multiple_outputs() { + global::set_local_accept_fee_base(1); + let coins = vec![output(20), output(20)]; + let output_len = 1; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, true), + (40, 2) + ); + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, false), + (40 - tx_fee(2, 1, 1), 2) + ); + } + + #[test] + fn sorts_large_values() { + global::set_local_accept_fee_base(1); + let high_value = 1_u64 << 63; + let coins = vec![output(1), output(high_value)]; + let output_len = 1; + let max_weight = Transaction::weight_by_iok(2, output_len, 1); + + assert_eq!( + max_spendable_amount(&coins, output_len, max_weight, true), + (high_value + 1, 2) + ); + } +} diff --git a/libwallet/src/internal/tx.rs b/libwallet/src/internal/tx.rs index 0b5c075..931223e 100644 --- a/libwallet/src/internal/tx.rs +++ b/libwallet/src/internal/tx.rs @@ -99,6 +99,7 @@ pub fn estimate_send_tx( max_outputs: usize, num_change_outputs: usize, selection_strategy_is_use_all: bool, + refresh_outputs: bool, parent_key_id: &Identifier, ) -> Result< ( @@ -114,7 +115,9 @@ where // Get lock height let current_height = wallet.w2n_client().get_chain_tip()?.0; // ensure outputs we're selecting are up to date - updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + if refresh_outputs { + updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + } // Sender selects outputs into a new slate and save our corresponding keys in // a transaction context. The secret key in our transaction context will be @@ -147,6 +150,7 @@ pub fn add_inputs_to_slate( max_outputs: usize, num_change_outputs: usize, selection_strategy_is_use_all: bool, + refresh_outputs: bool, parent_key_id: &Identifier, is_initiator: bool, use_test_rng: bool, @@ -156,8 +160,10 @@ where C: NodeClient, K: Keychain, { - // sender should always refresh outputs - updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + // Refresh unless the caller already updated the wallet state. + if refresh_outputs { + updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + } // Sender selects outputs into a new slate and save our corresponding keys in // a transaction context. The secret key in our transaction context will be @@ -260,8 +266,10 @@ where C: NodeClient, K: Keychain, { - // sender should always refresh outputs - updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + // Refresh unless the caller already updated the wallet state. + if init_tx_args.refresh_outputs_from_node { + updater::refresh_outputs(wallet, keychain_mask, parent_key_id, false)?; + } // we're just going to run a selection to get the potential fee, // but this won't be locked diff --git a/libwallet/src/internal/updater.rs b/libwallet/src/internal/updater.rs index 9049a9e..f9dbf9c 100644 --- a/libwallet/src/internal/updater.rs +++ b/libwallet/src/internal/updater.rs @@ -91,7 +91,7 @@ where } /// Apply advanced filtering to resultset from retrieve_txs below -pub fn apply_advanced_tx_list_filtering( +fn apply_advanced_tx_list_filtering( wallet: &mut WalletBackend, parent_key_id: Option<&Identifier>, query_args: &RetrieveTxQueryArgs, @@ -417,6 +417,7 @@ where /// Refreshes the outputs in a wallet with the latest information /// from a node +/// Also removes stale unconfirmed coinbase outputs across all accounts pub fn refresh_outputs( wallet: &mut WalletBackend, keychain_mask: Option<&SecretKey>, @@ -432,8 +433,12 @@ where Ok(()) } -/// build a local map of wallet outputs keyed by commit -/// and a list of outputs we want to query the node for +/// Build a local map of wallet outputs keyed by commit. +/// The map keys identify outputs to query from the node. +/// If `update_all` is `false`, select outputs involved in outstanding +/// transactions for the account and outputs without a transaction log entry. +/// Returns mapping of output commit to tuple of derived key for output, +/// PMMR index, tx entry log identifier and check if output is unspent pub fn map_wallet_outputs( wallet: &mut WalletBackend, keychain_mask: Option<&SecretKey>, @@ -525,7 +530,7 @@ where } /// Apply refreshed API output data to the wallet -pub fn apply_api_outputs( +fn apply_api_outputs( wallet: &mut WalletBackend, keychain_mask: Option<&SecretKey>, wallet_outputs: &HashMap, Option, bool)>, @@ -542,7 +547,7 @@ where // api output (if it exists) and refresh it in-place in the wallet. // Note: minimizing the time we spend holding the wallet lock. { - let last_confirmed_height = wallet.last_confirmed_height()?; + let last_confirmed_height = wallet.last_confirmed_height_for_parent(parent_key_id)?; // If the server height is less than our confirmed height, don't apply // these changes as the chain is syncing, incorrect or forking if height < last_confirmed_height { @@ -819,7 +824,7 @@ where C: NodeClient, K: Keychain, { - let current_height = wallet.last_confirmed_height()?; + let current_height = wallet.last_confirmed_height_for_parent(parent_key_id)?; let outputs = wallet .iter()? .filter(|out| out.root_key_id == *parent_key_id); @@ -896,7 +901,7 @@ where //TODO: Split up the output creation and the wallet insertion /// Build a coinbase output and the corresponding kernel -pub fn receive_coinbase( +fn receive_coinbase( wallet: &mut WalletBackend, keychain_mask: Option<&SecretKey>, block_fees: &BlockFees, diff --git a/libwallet/src/lib.rs b/libwallet/src/lib.rs index 5f30b3d..2c7c8ac 100644 --- a/libwallet/src/lib.rs +++ b/libwallet/src/lib.rs @@ -75,6 +75,9 @@ pub use api_impl::types::{ }; pub use backend::{WalletBackend, WalletBatch}; pub use internal::scan::scan; +pub use internal::updater::{ + map_wallet_outputs, refresh_outputs, retrieve_info, retrieve_outputs, retrieve_txs, +}; pub use slate_versions::ser as dalek_ser; pub use types::{ AcctPathMapping, BlockIdentifier, CbData, Context, NodeClient, NodeVersionInfo, OutputData, diff --git a/libwallet/src/types.rs b/libwallet/src/types.rs index 8bfe295..b0e4738 100644 --- a/libwallet/src/types.rs +++ b/libwallet/src/types.rs @@ -26,7 +26,7 @@ use crate::grin_core::{global, ser}; use crate::grin_keychain::{Identifier, Keychain}; use crate::grin_util::logger::LoggingConfig; use crate::grin_util::secp::key::{PublicKey, SecretKey}; -use crate::grin_util::secp::{self, pedersen, Secp256k1, Signature}; +use crate::grin_util::secp::{pedersen, Secp256k1, Signature}; use crate::grin_util::{ToHex, ZeroingString}; use crate::slate::PaymentMemo; use crate::slate_versions::ser as dalek_ser; @@ -39,8 +39,8 @@ use rand::thread_rng; use serde; use serde_json; use std::collections::HashMap; -use std::fmt; use std::time::Duration; +use std::{cmp, fmt}; use uuid::Uuid; /// Combined trait to allow dynamic wallet dispatch @@ -360,6 +360,10 @@ impl fmt::Display for OutputStatus { } } +/// Maximum private transaction context size. +/// Verified against `max_tx_weight()` below. +const MAX_CONTEXT_SIZE: usize = 4 * 1024 * 1024; + #[derive(Serialize, Deserialize, Clone, Debug)] /// Holds the context for a single aggsig transaction pub struct Context { @@ -404,7 +408,7 @@ pub struct Context { impl Context { /// Create a new context with defaults pub fn new( - secp: &secp::Secp256k1, + secp: &Secp256k1, parent_key_id: &Identifier, use_test_rng: bool, is_initiator: bool, @@ -426,7 +430,7 @@ impl Context { /// Create a new context with a specific excess pub fn with_excess( - secp: &secp::Secp256k1, + secp: &Secp256k1, sec_key: SecretKey, parent_key_id: &Identifier, use_test_rng: bool, @@ -506,13 +510,17 @@ impl Context { impl ser::Writeable for Context { fn write(&self, writer: &mut W) -> Result<(), ser::Error> { - writer.write_bytes(&serde_json::to_vec(self).map_err(|_| ser::Error::CorruptedData)?) + let data = serde_json::to_vec(self).map_err(|_| ser::Error::CorruptedData)?; + if data.len() > MAX_CONTEXT_SIZE { + return Err(ser::Error::TooLargeReadErr); + } + writer.write_bytes(&data) } } impl ser::Readable for Context { fn read(reader: &mut R) -> Result { - let data = reader.read_bytes_len_prefix()?; + let data = read_bytes_len_prefix(reader)?; serde_json::from_slice(&data[..]).map_err(|_| ser::Error::CorruptedData) } } @@ -933,6 +941,33 @@ impl ViewWalletOutputResult { } } +fn read_bytes_len_prefix(reader: &mut R) -> Result, ser::Error> { + let len = reader.read_u64()?; + if len > MAX_CONTEXT_SIZE as u64 { + return Err(ser::Error::TooLargeReadErr); + } + let mut len = len as usize; + match reader.read_limit() { + None => reader.read_fixed_bytes(len), + Some(limit) => { + if limit == 0 { + return reader.read_fixed_bytes(len); + } + let mut data = vec![]; + loop { + if len == 0 { + break; + } + let read_size = cmp::min(len, limit); + let read_data = reader.read_fixed_bytes(read_size)?; + data.extend_from_slice(&read_data); + len -= read_size; + } + Ok(data) + } + } +} + /// Serializes an Option to and from a string pub mod option_duration_as_secs { use serde::de::Error; @@ -970,6 +1005,8 @@ pub mod option_duration_as_secs { #[cfg(test)] mod tests { use super::*; + use grin_core::ser::{DeserializationMode, ProtocolVersion, Readable, Reader, StreamingReader}; + use grin_keychain::{ExtKeychain, ExtKeychainPath}; use serde_json::Value; #[derive(Serialize, Deserialize, PartialEq, Clone, Debug)] @@ -1011,4 +1048,94 @@ mod tests { let none2 = serde_json::from_str::("{}").unwrap(); assert_eq!(none, none2); } + + #[test] + fn big_context_read() { + let protocol_ver = ProtocolVersion(3); + + let create_context = || { + let parent = ExtKeychainPath::new(1, 1, 0, 0, 0).to_identifier(); + let sender_keychain = ExtKeychain::from_random_seed(true).unwrap(); + + let mut context = Context::new(sender_keychain.secp(), &parent, false, true); + for i in 0..3000 { + let key_id = ExtKeychain::derive_key_id(1, 1, i, 0, 0); + context.add_output(&key_id, &None, i as u64); + } + context + }; + + { + let context = create_context(); + let ser_value = ser::ser_vec(&context, protocol_ver).unwrap(); + let mut value = ser_value.as_slice(); + assert!(value.len() > 100_000); + let context = ser::deserialize::( + &mut value, + protocol_ver, + DeserializationMode::Full, + ); + assert!(context.is_ok()); + } + + // StreamingReader has no per-read limit and reads the complete Context. + let context = create_context(); + let ser_value = ser::ser_vec(&context, protocol_ver).unwrap(); + let mut value = ser_value.as_slice(); + let mut streaming_reader = StreamingReader::new(&mut value, protocol_ver); + assert!(streaming_reader.read_limit().is_none()); + let res = Context::read(&mut streaming_reader); + assert_eq!(res.unwrap().get_outputs().len(), 3000); + } + + #[test] + fn context_size_limit() { + let protocol_ver = ProtocolVersion(3); + let oversized_len = ((MAX_CONTEXT_SIZE + 1) as u64).to_be_bytes(); + let mut oversized_prefix = oversized_len.as_slice(); + let read = ser::deserialize::( + &mut oversized_prefix, + protocol_ver, + DeserializationMode::Full, + ); + assert_eq!(read.unwrap_err(), ser::Error::TooLargeReadErr); + + let parent = ExtKeychainPath::new(1, 1, 0, 0, 0).to_identifier(); + let sender_keychain = ExtKeychain::from_random_seed(true).unwrap(); + let mut context = Context::new(sender_keychain.secp(), &parent, false, true); + context.late_lock_args = Some(InitTxArgs { + src_acct_name: Some("x".repeat(MAX_CONTEXT_SIZE)), + ..Default::default() + }); + + assert_eq!( + ser::ser_vec(&context, protocol_ver).unwrap_err(), + ser::Error::TooLargeReadErr + ); + } + + #[test] + fn max_weight_context_size() { + global::set_local_chain_type(global::ChainTypes::Mainnet); + let max_tx_weight = global::max_tx_weight(); + let max_inputs = (1..) + .take_while(|inputs| Transaction::weight_by_iok(*inputs, 1, 1) <= max_tx_weight) + .last() + .unwrap(); + + let parent = ExtKeychainPath::new(1, 1, 0, 0, 0).to_identifier(); + let sender_keychain = ExtKeychain::from_random_seed(true).unwrap(); + let mut context = Context::new(sender_keychain.secp(), &parent, true, true); + for i in 0..max_inputs { + let key_id = ExtKeychain::derive_key_id(1, 1, i as u32, 0, 0); + context.add_input(&key_id, &Some(u64::MAX), u64::MAX); + } + + let size = serde_json::to_vec(&context).unwrap().len(); + println!("max-weight Context: {size} bytes ({max_inputs} inputs)"); + assert!( + size <= MAX_CONTEXT_SIZE, + "max-weight Context is {size} bytes, limit is {MAX_CONTEXT_SIZE}" + ); + } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 7833061..85cbe74 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -132,7 +132,7 @@ pub fn clean_output_dir(test_dir: &str) { pub fn setup(test_dir: &str) { util::init_test_logger(); clean_output_dir(test_dir); - global::set_local_chain_type(ChainTypes::AutomatedTesting); + setup_global_chain_type(); } /// Some tests require the global chain_type to be configured. diff --git a/util/Cargo.toml b/util/Cargo.toml index 7552d89..898522d 100644 --- a/util/Cargo.toml +++ b/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "grin_wallet_util" -version = "5.4.1" +version = "5.5.0" authors = ["Grin Developers "] description = "Util, for generic utilities and to re-export grin crates" license = "Apache-2.0"