Merge upstream staging into contracts

Bring the branch current with upstream staging (= master + #748). The earlier
"Merge branch master into contracts" used a stale local master snapshot, which
left the branch ~41 commits behind; this merges the current tip, including the
grin 5.4.0 dependency bump (#745) and the tx-delete / retrieve_txs fixes.

Only Cargo.lock conflicted (regenerated). Full cargo test --workspace green
(52 binaries / 185 tests).

.gitignore

@@ -11,3 +11,5 @@ grin.log
 wallet.seed
 test_output
 .idea/
+*.vs
+.gitattributes

Cargo.toml

@@ -9,7 +9,7 @@ keywords = [ "crypto", "grin", "mimblewimble" ]
 readme = "README.md"
 exclude = ["**/*.grin", "**/*.grin2"]
 build = "src/build/build.rs"
-edition = "2018"
+edition = "2021"
 
 [[bin]]
 name = "grin-wallet"
@@ -41,10 +41,10 @@ grin_wallet_util = { path = "./util", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_keychain = "5.3.3"
-grin_util = "5.3.3"
-grin_api = "5.3.3"
+grin_core = "5.4.0"
+grin_keychain = "5.4.0"
+grin_util = "5.4.0"
+grin_api = "5.4.0"
 
 # For beta release
 
@@ -68,7 +68,7 @@ grin_api = "5.3.3"
 ###### 
 
 [build-dependencies]
-built = { version = "0.7", features = ["git2"]}
+built = { version = "0.8.0", features = ["git2"]}
 
 [dev-dependencies]
 url = "2.1"

api/Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 exclude = ["**/*.grin", "**/*.grin2"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 log = "0.4"
@@ -30,9 +30,9 @@ grin_wallet_util = { path = "../util", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_keychain = "5.3.3"
-grin_util = "5.3.3"
+grin_core = "5.4.0"
+grin_keychain = "5.4.0"
+grin_util = "5.4.0"
 
 # For beta release
 

api/src/foreign.rs

@@ -150,7 +150,7 @@ where
 	/// // These traits can be replaced with alternative implementations if desired
 	///
 	/// let mut wallet = Box::new(DefaultWalletImpl::<'static, HTTPNodeClient>::new(node_client.clone()).unwrap())
-	///     as Box<WalletInst<'static, DefaultLCProvider<HTTPNodeClient, ExtKeychain>, HTTPNodeClient, ExtKeychain>>;
+	///     as Box<dyn WalletInst<'static, DefaultLCProvider<HTTPNodeClient, ExtKeychain>, HTTPNodeClient, ExtKeychain>>;
 	///
 	/// // Wallet LifeCycle Provider provides all functions init wallet and work with seeds, etc...
 	/// let lc = wallet.lc_provider().unwrap();
@@ -540,7 +540,7 @@ macro_rules! doctest_helper_setup_doc_env_foreign {
 			DefaultWalletImpl::<'static, HTTPNodeClient>::new(node_client.clone()).unwrap(),
 		)
 			as Box<
-				WalletInst<
+				dyn WalletInst<
 					'static,
 					DefaultLCProvider<HTTPNodeClient, ExtKeychain>,
 					HTTPNodeClient,

api/src/owner.rs

@@ -158,7 +158,7 @@ where
 	/// // These traits can be replaced with alternative implementations if desired
 	///
 	/// let mut wallet = Box::new(DefaultWalletImpl::<'static, HTTPNodeClient>::new(node_client.clone()).unwrap())
-	///     as Box<WalletInst<'static, DefaultLCProvider<HTTPNodeClient, ExtKeychain>, HTTPNodeClient, ExtKeychain>>;
+	///     as Box<dyn WalletInst<'static, DefaultLCProvider<HTTPNodeClient, ExtKeychain>, HTTPNodeClient, ExtKeychain>>;
 	///
 	/// // Wallet LifeCycle Provider provides all functions init wallet and work with seeds, etc...
 	/// let lc = wallet.lc_provider().unwrap();
@@ -2719,7 +2719,7 @@ macro_rules! doctest_helper_setup_doc_env {
 			DefaultWalletImpl::<'static, HTTPNodeClient>::new(node_client.clone()).unwrap(),
 		)
 			as Box<
-				WalletInst<
+				dyn WalletInst<
 					'static,
 					DefaultLCProvider<HTTPNodeClient, ExtKeychain>,
 					HTTPNodeClient,

config/Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 workspace = ".."
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 rand = "0.6"
@@ -21,8 +21,8 @@ grin_wallet_util = { path = "../util", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_util = "5.3.3"
+grin_core = "5.4.0"
+grin_util = "5.4.0"
 
 # For beta release
 

config/src/comments.rs

@@ -333,10 +333,9 @@ pub fn migrate_comments(
 					None
 				}
 				false => {
-					let comments: String =
-						vec_old_conf.iter().map(|s| s.chars()).flatten().collect();
+					let comments: String = vec_old_conf.iter().flat_map(|s| s.chars()).collect();
 					let key = get_key(line_nospace);
-					match !(key == "NOT_FOUND") {
+					match key != "NOT_FOUND" {
 						true => {
 							vec_old_conf.clear();
 							hm_key_cmt_old.insert(key.clone(), comments);
@@ -356,10 +355,10 @@ pub fn migrate_comments(
 		.filter_map(|line| {
 			let line_nospace = line.trim();
 			let is_ascii_control = line_nospace.chars().all(|x| x.is_ascii_control());
-			match !(line.contains("#") || is_ascii_control) {
+			match !line.contains("#") && !is_ascii_control {
 				true => {
 					let key = get_key(line_nospace);
-					match !(key == "NOT_FOUND") {
+					match key != "NOT_FOUND" {
 						true => Some((key, line_nospace.to_string())),
 						false => None,
 					}

config/src/config.rs

@@ -14,16 +14,6 @@
 
 //! Configuration file management
 
-use dirs;
-use rand::distributions::{Alphanumeric, Distribution};
-use rand::thread_rng;
-use std::env;
-use std::fs::{self, File};
-use std::io::prelude::*;
-use std::io::BufReader;
-use std::path::PathBuf;
-use toml;
-
 use crate::comments::{insert_comments, migrate_comments};
 use crate::core::global;
 use crate::types::{
@@ -31,11 +21,20 @@ use crate::types::{
 };
 use crate::types::{TorConfig, WalletConfig};
 use crate::util::logger::LoggingConfig;
+use rand::distributions::{Alphanumeric, Distribution};
+use rand::thread_rng;
+use std::env;
+use std::fs::{self, File};
+use std::io::prelude::*;
+use std::io::BufReader;
+use std::path::{Path, PathBuf};
+use toml;
 
 /// Wallet configuration file name
 pub const WALLET_CONFIG_FILE_NAME: &str = "grin-wallet.toml";
 const WALLET_LOG_FILE_NAME: &str = "grin-wallet.log";
-const GRIN_HOME: &str = ".grin";
+/// .grin folder, usually in home/.grin
+pub const GRIN_HOME: &str = ".grin";
 /// Wallet data directory
 pub const GRIN_WALLET_DIR: &str = "wallet_data";
 /// Node API secret
@@ -43,31 +42,85 @@ pub const API_SECRET_FILE_NAME: &str = ".foreign_api_secret";
 /// Owner API secret
 pub const OWNER_API_SECRET_FILE_NAME: &str = ".owner_api_secret";
 
-fn get_grin_path(
+/// Function to locate the wallet dir and grin-wallet.toml in the order
+/// a) config in top-dir if provided, b) in working dir, c) default dir
+/// Function to get wallet dir and create dirs if not existing
+pub fn get_wallet_path(
 	chain_type: &global::ChainTypes,
 	create_path: bool,
 ) -> Result<PathBuf, ConfigError> {
-	// Check if grin dir exists
-	let mut grin_path = match dirs::home_dir() {
+	// A - Detect grin-wallet.toml in working dir
+	let mut config_path = std::env::current_dir()?;
+	config_path.push(WALLET_CONFIG_FILE_NAME);
+	if create_path == false && config_path.exists() {
+		config_path.pop();
+		println!("Detected 'grin-wallet.toml' in working dir - opening associated wallet");
+		return Ok(config_path);
+	};
+	// B - Select home directory
+	let mut wallet_path = match dirs::home_dir() {
 		Some(p) => p,
 		None => PathBuf::new(),
 	};
-	grin_path.push(GRIN_HOME);
-	grin_path.push(chain_type.shortname());
+	wallet_path.push(GRIN_HOME);
+	wallet_path.push(chain_type.shortname());
 	// Create if the default path doesn't exist
-	if !grin_path.exists() && create_path {
-		fs::create_dir_all(grin_path.clone())?;
+	if !wallet_path.exists() && create_path {
+		fs::create_dir_all(wallet_path.clone())?;
 	}
-
-	if !grin_path.exists() {
+	// Throw an error if the path still does not exist
+	if !wallet_path.exists() {
 		Err(ConfigError::PathNotFoundError(String::from(
-			grin_path.to_str().unwrap(),
+			wallet_path.to_str().unwrap(),
 		)))
 	} else {
-		Ok(grin_path)
+		Ok(wallet_path)
 	}
 }
 
+/// Smart function to detect the nodes .foreign_api_secret file in the order
+/// a) top-dir, b) home directory - create directory if needed
+pub fn get_node_path(
+	data_path: Option<PathBuf>,
+	chain_type: &global::ChainTypes,
+) -> Result<PathBuf, ConfigError> {
+	let node_path = match data_path {
+		// 1) A If top dir provided and api_secret exist, return top dir
+		Some(path) => {
+			let mut node_path = path;
+			node_path.push(GRIN_HOME);
+			node_path.push(chain_type.shortname());
+			node_path.push(API_SECRET_FILE_NAME);
+			if node_path.exists() {
+				node_path.pop();
+				Ok(node_path)
+			// 1) B If top dir exists, but no api_secret, return home dir
+			} else {
+				let mut node_path = match dirs::home_dir() {
+					Some(p) => p,
+					None => PathBuf::new(),
+				};
+				node_path.push(GRIN_HOME);
+				node_path.push(chain_type.shortname());
+				Ok(node_path)
+			}
+		}
+		// 2) If there is no top_dir provided, always return home dir
+		None => {
+			let mut node_path = match dirs::home_dir() {
+				Some(p) => p,
+				None => PathBuf::new(),
+			};
+			node_path.push(GRIN_HOME);
+			node_path.push(chain_type.shortname());
+			Ok(node_path)
+		}
+	};
+	node_path
+}
+
+/// Checks if config in current working dir
+#[allow(dead_code)]
 fn check_config_current_dir(path: &str) -> Option<PathBuf> {
 	let p = env::current_dir();
 	let mut c = match p {
@@ -122,7 +175,7 @@ fn check_api_secret_file(
 ) -> Result<(), ConfigError> {
 	let grin_path = match data_path {
 		Some(p) => p,
-		None => get_grin_path(chain_type, false)?,
+		None => get_node_path(data_path, chain_type)?,
 	};
 	let mut api_secret_path = grin_path;
 	api_secret_path.push(file_name);
@@ -133,55 +186,99 @@ fn check_api_secret_file(
 	}
 }
 
-/// Handles setup and detection of paths for wallet
+/// Initial wallet setup does the following
+/// 1) Load wallet config if run without 'init' 2) create wallet if run with 'init''
+/// Try in thiss order a) current dir as template, b) in top path, or c) .grin home
+/// - load default config values
+/// - update the wallet and node dir to the correct paths
+/// - if grin-wallet.toml exists, but the wallet data dir does not, load config and continue wallet generation
+/// - Automatically detect grin-wallet.toml in current directory
 pub fn initial_setup_wallet(
 	chain_type: &global::ChainTypes,
-	data_path: Option<PathBuf>,
+	mut data_path: Option<PathBuf>,
 	create_path: bool,
 ) -> Result<GlobalWalletConfig, ConfigError> {
-	if create_path {
-		if let Some(p) = data_path.clone() {
-			fs::create_dir_all(p)?;
+	// Fixing the input path when run with -here or -t (top-dir)
+	// - Fix top-dir path to  compensate for bug on Linux to handle "\"
+	// - Convert top-dir path to be always absolute for config generation
+	// - Fix for Windows 10/11 to strip the '\\?\' prefix added to the path
+	if let Some(p) = &data_path {
+		if let Some(p_str) = p.to_str() {
+			let fixed_str = p_str.replace("\\", "/");
+			let fixed_path = PathBuf::from(fixed_str);
+			if create_path {
+				fs::create_dir_all(&fixed_path)?;
+			}
+			let absolute_path = if fixed_path.is_absolute() {
+				fixed_path.canonicalize()?
+			} else {
+				env::current_dir()?.join(&fixed_path).canonicalize()?
+			};
+			let absolute_path =
+				std::path::PathBuf::from(absolute_path.to_str().unwrap().replace(r"\\?\", ""));
+			data_path = Some(absolute_path); // Store the updated path
 		}
 	}
-	// Use config file if current directory if it exists, .grin home otherwise
-	let (path, config) = if let Some(p) = check_config_current_dir(WALLET_CONFIG_FILE_NAME) {
-		let mut path = p.clone();
-		path.pop();
-		(path, GlobalWalletConfig::new(p.to_str().unwrap())?)
-	} else {
-		// Check if grin dir exists
-		let grin_path = match data_path {
-			Some(p) => p,
-			None => get_grin_path(chain_type, create_path)?,
-		};
 
-		// Get path to default config file
-		let mut config_path = grin_path.clone();
-		config_path.push(WALLET_CONFIG_FILE_NAME);
+	// Get wallet data_dir path if none provided
+	let wallet_path = match data_path {
+		Some(p) => p,
+		None => get_wallet_path(chain_type, create_path)?,
+	};
+	println!("Wallet path: {}", wallet_path.display());
+	// Get path to the node directory,
+	let node_path = get_node_path(Some(wallet_path.clone()), chain_type)?;
 
-		// Return defaults if file doesn't exist
-		match config_path.clone().exists() {
-			false => {
-				let mut default_config = GlobalWalletConfig::for_chain(chain_type);
-				default_config.config_file_path = Some(config_path);
-				// update paths relative to current dir
-				default_config.update_paths(&grin_path);
-				(grin_path, default_config)
+	// Get config path and data path
+	let mut config_path = wallet_path.clone();
+	config_path.push(WALLET_CONFIG_FILE_NAME);
+	let mut data_dir = wallet_path.clone();
+	data_dir.push(GRIN_WALLET_DIR);
+	// Check if a config exists in theworking dir, if so load it
+	let (path, config) = match config_path.clone().exists() {
+		// If the config does not exist, load default and updated node and wallet dir
+		false => {
+			let mut default_config = GlobalWalletConfig::for_chain(chain_type);
+			default_config.config_file_path = Some(config_path.clone());
+			default_config.update_paths(&wallet_path, &node_path);
+
+			// Write config file
+			let res =
+				default_config.write_to_file(config_path.to_str().unwrap(), false, None, None);
+
+			if let Err(e) = res {
+				let msg = format!(
+					"Error creating config file as ({}): {}",
+					config_path.to_str().unwrap(),
+					e
+				);
+				return Err(ConfigError::SerializationError(msg));
 			}
-			true => {
-				let mut path = config_path.clone();
-				path.pop();
-				(
-					path,
-					GlobalWalletConfig::new(config_path.to_str().unwrap())?,
-				)
+
+			(wallet_path, default_config)
+		}
+
+		// Return config if not run with init
+		true => {
+			// If run with init and seed does not yet exists, continue, else throw error
+			if data_dir.exists() && create_path == true {
+				let msg = format!(
+					"{} already exists in the target directory ({}). Please remove it first",
+					config_path.to_str().unwrap(),
+					data_dir.to_str().unwrap(),
+				);
+				return Err(ConfigError::SerializationError(msg));
+			} else {
+				let config = GlobalWalletConfig::new(config_path.to_str().unwrap())?;
+				(wallet_path, config)
 			}
 		}
 	};
 
+	// Check API secrets, if ok, return config
 	check_api_secret_file(chain_type, Some(path.clone()), OWNER_API_SECRET_FILE_NAME)?;
 	check_api_secret_file(chain_type, Some(path), API_SECRET_FILE_NAME)?;
+
 	Ok(config)
 }
 
@@ -211,7 +308,7 @@ impl GlobalWalletConfig {
 	pub fn for_chain(chain_type: &global::ChainTypes) -> GlobalWalletConfig {
 		let mut defaults_conf = GlobalWalletConfig::default();
 		let defaults = &mut defaults_conf.members.as_mut().unwrap().wallet;
-		defaults.chain_type = Some(chain_type.clone());
+		defaults.chain_type = Some(*chain_type);
 
 		match *chain_type {
 			global::ChainTypes::Mainnet => {}
@@ -268,21 +365,22 @@ impl GlobalWalletConfig {
 	}
 
 	/// Update paths
-	pub fn update_paths(&mut self, wallet_home: &PathBuf) {
-		let mut wallet_path = wallet_home.clone();
-		wallet_path.push(GRIN_WALLET_DIR);
-		self.members.as_mut().unwrap().wallet.data_file_dir =
-			wallet_path.to_str().unwrap().to_owned();
-		let mut secret_path = wallet_home.clone();
-		secret_path.push(OWNER_API_SECRET_FILE_NAME);
-		self.members.as_mut().unwrap().wallet.api_secret_path =
-			Some(secret_path.to_str().unwrap().to_owned());
-		let mut node_secret_path = wallet_home.clone();
+	pub fn update_paths(&mut self, wallet_home: &PathBuf, node_home: &Path) {
+		let mut data_file_dir = wallet_home.to_path_buf();
+		let mut node_secret_path = node_home.to_path_buf();
+		let mut secret_path = wallet_home.to_path_buf();
+		let mut log_path = wallet_home.to_path_buf();
+		let tor_path = wallet_home.to_path_buf();
 		node_secret_path.push(API_SECRET_FILE_NAME);
+		data_file_dir.push(GRIN_WALLET_DIR);
+		secret_path.push(OWNER_API_SECRET_FILE_NAME);
+		log_path.push(WALLET_LOG_FILE_NAME);
+		self.members.as_mut().unwrap().wallet.data_file_dir =
+			data_file_dir.to_str().unwrap().to_owned();
 		self.members.as_mut().unwrap().wallet.node_api_secret_path =
 			Some(node_secret_path.to_str().unwrap().to_owned());
-		let mut log_path = wallet_home.clone();
-		log_path.push(WALLET_LOG_FILE_NAME);
+		self.members.as_mut().unwrap().wallet.api_secret_path =
+			Some(secret_path.to_str().unwrap().to_owned());
 		self.members
 			.as_mut()
 			.unwrap()
@@ -290,7 +388,6 @@ impl GlobalWalletConfig {
 			.as_mut()
 			.unwrap()
 			.log_file_path = log_path.to_str().unwrap().to_owned();
-		let tor_path = wallet_home.clone();
 		self.members
 			.as_mut()
 			.unwrap()
@@ -339,7 +436,7 @@ impl GlobalWalletConfig {
 	) -> Result<String, ConfigError> {
 		let config: GlobalWalletConfigMembers =
 			toml::from_str(&GlobalWalletConfig::fix_warning_level(config_str.clone())).unwrap();
-		if config.config_file_version != None {
+		if config.config_file_version.is_some() {
 			return Ok(config_str);
 		}
 		let adjusted_config = GlobalWalletConfigMembers {
@@ -347,7 +444,7 @@ impl GlobalWalletConfig {
 			tor: Some(TorConfig {
 				bridge: TorBridgeConfig::default(),
 				proxy: TorProxyConfig::default(),
-				..config.tor.unwrap_or(TorConfig::default())
+				..config.tor.unwrap_or_default()
 			}),
 			..config
 		};

controller/Cargo.toml

@@ -8,11 +8,11 @@ repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 exclude = ["**/*.grin", "**/*.grin2"]
 #build = "src/build/build.rs"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 futures = "0.3"
-hyper = "0.13"
+hyper = "0.14"
 rand = "0.7"
 serde = "1"
 serde_derive = "1"
@@ -39,10 +39,10 @@ grin_wallet_config = { path = "../config", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_keychain = "5.3.3"
-grin_util = "5.3.3"
-grin_api = "5.3.3"
+grin_core = "5.4.0"
+grin_keychain = "5.4.0"
+grin_util = "5.4.0"
+grin_api = "5.4.0"
 
 # For beta release
 
@@ -72,7 +72,7 @@ remove_dir_all = "0.7"
 ##### Grin Imports
 
 # For Release
-grin_chain = "5.3.3"
+grin_chain = "5.4.0"
 
 # For beta release
 

controller/src/command.rs

@@ -77,6 +77,7 @@ pub struct InitArgs {
 	pub restore: bool,
 }
 
+/// Write config (default if None), initiate the wallet
 pub fn init<L, C, K>(
 	owner_api: &mut Owner<L, C, K>,
 	_g_args: &GlobalArgs,

controller/tests/check.rs

@@ -26,6 +26,7 @@ use grin_wallet_libwallet as libwallet;
 use impls::test_framework::{self, LocalWalletClient};
 use impls::{PathToSlate, SlatePutter as _};
 use libwallet::{InitTxArgs, NodeClient};
+use std::cmp;
 use std::sync::atomic::Ordering;
 use std::thread;
 use std::time::Duration;
@@ -847,6 +848,200 @@ fn output_scanning_impl(test_dir: &'static str) -> Result<(), libwallet::Error>
 	Ok(())
 }
 
+fn multi_batch_scan_impl(test_dir: &'static str) -> Result<(), libwallet::Error> {
+	let mut wallet_proxy = create_wallet_proxy(test_dir);
+	let chain = wallet_proxy.chain.clone();
+	let stopper = wallet_proxy.running.clone();
+
+	create_wallet_and_add!(
+		client1,
+		wallet1,
+		mask1_i,
+		test_dir,
+		"wallet1",
+		None,
+		&mut wallet_proxy,
+		false
+	);
+	let mask1 = (&mask1_i).as_ref();
+
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!("Wallet Proxy error: {}", e);
+		}
+	});
+
+	let reward = consensus::REWARD;
+	let bh = 12u64;
+	let _ =
+		test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), mask1, bh as usize, false);
+
+	let mut output_commits = vec![];
+	wallet::controller::owner_single_use(Some(wallet1.clone()), mask1, None, |api, m| {
+		let (_, info) = api.retrieve_summary_info(m, true, 1)?;
+		assert_eq!(info.last_confirmed_height, bh);
+		assert_eq!(info.total, bh * reward);
+		output_commits = api.retrieve_outputs(m, false, true, None)?.1;
+		Ok(())
+	})?;
+
+	{
+		wallet_inst!(wallet1, w);
+		let mut batch = w.batch(mask1)?;
+		batch.delete(&output_commits[8].output.key_id, &None)?;
+		batch.commit()?;
+	}
+
+	let tip = {
+		wallet_inst!(wallet1, w);
+		w.w2n_client().get_chain_tip()?
+	};
+
+	let start_height = 1;
+	let batch_size = 5;
+	let mut total_pmmr_range = None;
+	let mut batches = 0;
+	let status_send_channel = None;
+	for h in (start_height..tip.0).step_by((batch_size + 1) as usize) {
+		let batch_end_height = cmp::min(tip.0, h + batch_size);
+		let (mut info, range) = libwallet::scan(
+			wallet1.clone(),
+			mask1,
+			false,
+			h,
+			batch_end_height,
+			start_height,
+			tip.0,
+			total_pmmr_range,
+			&status_send_channel,
+		)?;
+		info.hash = if batch_end_height == tip.0 {
+			tip.1.clone()
+		} else {
+			"".to_owned()
+		};
+		total_pmmr_range = Some(range);
+		batches += 1;
+
+		wallet_inst!(wallet1, w);
+		let mut batch = w.batch(mask1)?;
+		batch.save_last_scanned_block(info)?;
+		batch.commit()?;
+	}
+	assert!(batches > 1);
+
+	wallet::controller::owner_single_use(Some(wallet1.clone()), mask1, None, |api, m| {
+		let (_, restored_outputs) = api.retrieve_outputs(m, false, true, None)?;
+		let (_, info) = api.retrieve_summary_info(m, false, 1)?;
+		assert_eq!(info.total, bh * reward);
+		assert_eq!(restored_outputs.len(), bh as usize);
+		Ok(())
+	})?;
+
+	stopper.store(false, Ordering::Relaxed);
+	thread::sleep(Duration::from_millis(200));
+	Ok(())
+}
+
+fn restore_corrupted_outputs_across_batches_impl(
+	test_dir: &'static str,
+) -> Result<(), libwallet::Error> {
+	let mut wallet_proxy = create_wallet_proxy(test_dir);
+	let chain = wallet_proxy.chain.clone();
+	let stopper = wallet_proxy.running.clone();
+
+	create_wallet_and_add!(
+		client1,
+		wallet1,
+		mask1_i,
+		test_dir,
+		"wallet1",
+		None,
+		&mut wallet_proxy,
+		false
+	);
+	let mask1 = (&mask1_i).as_ref();
+
+	thread::spawn(move || {
+		if let Err(e) = wallet_proxy.run() {
+			error!("Wallet Proxy error: {}", e);
+		}
+	});
+
+	let reward = consensus::REWARD;
+	let bh = 12u64;
+	let _ =
+		test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), mask1, bh as usize, false);
+
+	let mut output_commits = vec![];
+	wallet::controller::owner_single_use(Some(wallet1.clone()), mask1, None, |api, m| {
+		let (_, info) = api.retrieve_summary_info(m, true, 1)?;
+		assert_eq!(info.total, bh * reward);
+		output_commits = api.retrieve_outputs(m, false, true, None)?.1;
+		Ok(())
+	})?;
+
+	{
+		wallet_inst!(wallet1, w);
+		let mut batch = w.batch(mask1)?;
+		batch.delete(&output_commits[2].output.key_id, &None)?;
+		batch.delete(&output_commits[8].output.key_id, &None)?;
+		batch.commit()?;
+	}
+
+	let tip = {
+		wallet_inst!(wallet1, w);
+		w.w2n_client().get_chain_tip()?
+	};
+	let status_send_channel = None;
+	let (info, _) = libwallet::scan(
+		wallet1.clone(),
+		mask1,
+		false,
+		1,
+		6,
+		1,
+		tip.0,
+		None,
+		&status_send_channel,
+	)?;
+	{
+		wallet_inst!(wallet1, w);
+		{
+			let mut batch = w.batch(mask1)?;
+			batch.save_last_scanned_block(info)?;
+			batch.commit()?;
+		}
+		let last_scanned = w.last_scanned_block()?;
+		assert_eq!(last_scanned.height, 6);
+		assert_eq!(last_scanned.hash, "");
+	}
+
+	wallet::controller::owner_single_use(Some(wallet1.clone()), mask1, None, |api, m| {
+		let (_, outputs_after_first_batch) = api.retrieve_outputs(m, false, false, None)?;
+		assert_eq!(outputs_after_first_batch.len(), (bh - 1) as usize);
+
+		let (_, info) = api.retrieve_summary_info(m, true, 1)?;
+		let (_, restored_outputs) = api.retrieve_outputs(m, false, true, None)?;
+		let unique_keys = restored_outputs
+			.iter()
+			.map(|o| o.output.key_id.clone())
+			.collect::<std::collections::HashSet<_>>();
+
+		assert_eq!(info.total, bh * reward);
+		assert_eq!(restored_outputs.len(), bh as usize);
+		assert_eq!(unique_keys.len(), bh as usize);
+		assert!(restored_outputs
+			.iter()
+			.all(|o| o.output.status == libwallet::OutputStatus::Unspent));
+		Ok(())
+	})?;
+
+	stopper.store(false, Ordering::Relaxed);
+	thread::sleep(Duration::from_millis(200));
+	Ok(())
+}
+
 #[test]
 fn scan() {
 	let test_dir = "test_output/scan";
@@ -876,3 +1071,23 @@ fn output_scanning() {
 	}
 	clean_output_dir(test_dir);
 }
+
+#[test]
+fn multi_batch_scan() {
+	let test_dir = "test_output/multi_batch_scan";
+	setup(test_dir);
+	if let Err(e) = multi_batch_scan_impl(test_dir) {
+		panic!("Libwallet Error: {}", e);
+	}
+	clean_output_dir(test_dir);
+}
+
+#[test]
+fn restore_corrupted_outputs_across_batches() {
+	let test_dir = "test_output/restore_corrupted_outputs_across_batches";
+	setup(test_dir);
+	if let Err(e) = restore_corrupted_outputs_across_batches_impl(test_dir) {
+		panic!("Libwallet Error: {}", e);
+	}
+	clean_output_dir(test_dir);
+}

controller/tests/common/mod.rs

@@ -102,8 +102,12 @@ pub fn setup_global_chain_type() {
 
 pub fn create_wallet_proxy(
 	test_dir: &str,
-) -> WalletProxy<DefaultLCProvider<LocalWalletClient, ExtKeychain>, LocalWalletClient, ExtKeychain>
-{
+) -> WalletProxy<
+	'_,
+	DefaultLCProvider<'_, LocalWalletClient, ExtKeychain>,
+	LocalWalletClient,
+	ExtKeychain,
+> {
 	WalletProxy::new(test_dir)
 }
 

controller/tests/transaction.rs

@@ -442,7 +442,7 @@ fn tx_rollback(test_dir: &'static str) -> Result<(), libwallet::Error> {
 	// few values to keep things shorter
 	let reward = core::consensus::REWARD;
 	let cm = global::coinbase_maturity(); // assume all testing precedes soft fork height
-									  // mine a few blocks
+									   // mine a few blocks
 	let _ = test_framework::award_blocks_to_wallet(&chain, wallet1.clone(), mask1, 5, false);
 
 	let amount = 30_000_000_000;

impls/Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 exclude = ["**/*.grin", "**/*.grin2"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 blake2-rfc = "0.2"
@@ -43,12 +43,12 @@ grin_wallet_libwallet = { path = "../libwallet", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_keychain = "5.3.3"
-grin_chain = "5.3.3"
-grin_util = "5.3.3"
-grin_api = "5.3.3"
-grin_store = "5.3.3"
+grin_core = "5.4.0"
+grin_keychain = "5.4.0"
+grin_chain = "5.4.0"
+grin_util = "5.4.0"
+grin_api = "5.4.0"
+grin_store = "5.4.0"
 
 # For beta release
 

impls/src/backends/lmdb.rs

@@ -726,6 +726,16 @@ where
 		Ok(())
 	}
 
+	fn delete_tx_log_entry(&mut self, tx_id: u32, parent_id: &Identifier) -> Result<(), Error> {
+		let tx_log_key = to_key_u64(
+			TX_LOG_ENTRY_PREFIX,
+			&mut parent_id.to_bytes().to_vec(),
+			tx_id as u64,
+		);
+		self.db.borrow().as_ref().unwrap().delete(&tx_log_key)?;
+		Ok(())
+	}
+
 	fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error> {
 		let acct_key = to_key(
 			ACCOUNT_PATH_MAPPING_PREFIX,

impls/src/lifecycle/default.rs

@@ -27,6 +27,15 @@ use crate::LMDBBackend;
 use grin_util::logger::LoggingConfig;
 use std::fs;
 use std::path::PathBuf;
+use std::path::MAIN_SEPARATOR;
+
+// Helper fuction to format paths according to OS, avoids bugs on Linux
+pub fn fmt_path(path: String) -> String {
+	let sep = &MAIN_SEPARATOR.to_string();
+	let path = path.replace("/", &sep);
+	let path = path.replace("\\", &sep);
+	path
+}
 
 pub struct DefaultLCProvider<'a, C, K>
 where
@@ -64,7 +73,13 @@ where
 	}
 
 	fn get_top_level_directory(&self) -> Result<String, Error> {
-		Ok(self.data_dir.to_owned())
+		let sep = &MAIN_SEPARATOR.to_string();
+		let data_dir = self
+			.data_dir
+			.to_owned()
+			.replace("/", &sep)
+			.replace("\\", &sep);
+		Ok(data_dir)
 	}
 
 	fn create_config(
@@ -87,11 +102,12 @@ where
 				None => None,
 			},
 		};
-		let wallet = match wallet_config {
-			Some(w) => w,
+		// Check if config was provided, if not load default and set update to "true"
+		let (wallet, update) = match wallet_config {
+			Some(w) => (w, false),
 			None => match default_config.members.as_ref() {
-				Some(m) => m.clone().wallet,
-				None => WalletConfig::default(),
+				Some(m) => (m.clone().wallet, true),
+				None => (WalletConfig::default(), true),
 			},
 		};
 		let tor = match tor_config {
@@ -113,13 +129,6 @@ where
 		let mut config_file_name = PathBuf::from(self.data_dir.clone());
 		config_file_name.push(file_name);
 
-		// create top level dir if it doesn't exist
-		let dd = PathBuf::from(self.data_dir.clone());
-		if !dd.exists() {
-			// try create
-			fs::create_dir_all(dd)?;
-		}
-
 		let mut data_dir_name = PathBuf::from(self.data_dir.clone());
 		data_dir_name.push(GRIN_WALLET_DIR);
 
@@ -132,15 +141,26 @@ where
 			return Err(Error::Lifecycle(msg));
 		}
 
-		// just leave as is if file exists but there's no data dir
+		// If config exists but the datadir return ok
 		if config_file_name.exists() {
 			return Ok(());
 		}
+		// default settings are updated if no config was provided, no support for top_dir/here
+		let mut abs_path_node = std::env::current_dir()?;
+		abs_path_node.push(self.data_dir.clone());
+		let mut absolute_path_wallet = std::env::current_dir()?;
+		absolute_path_wallet.push(self.data_dir.clone());
 
-		let mut abs_path = std::env::current_dir()?;
-		abs_path.push(self.data_dir.clone());
-
-		default_config.update_paths(&abs_path);
+		// if no config provided, update defaults
+		if update == true {
+			// create top level dir if it doesn't exist
+			let dd = PathBuf::from(self.data_dir.clone());
+			if !dd.exists() {
+				// try create
+				fs::create_dir_all(dd)?;
+				default_config.update_paths(&abs_path_node, &absolute_path_wallet);
+			}
+		};
 		let res =
 			default_config.write_to_file(config_file_name.to_str().unwrap(), false, None, None);
 		if let Err(e) = res {
@@ -178,7 +198,7 @@ where
 	) -> Result<(), Error> {
 		let mut data_dir_name = PathBuf::from(self.data_dir.clone());
 		data_dir_name.push(GRIN_WALLET_DIR);
-		let data_dir_name = data_dir_name.to_str().unwrap();
+		let data_dir_name = fmt_path((data_dir_name.to_str().unwrap()).to_string());
 		let exists = WalletSeed::seed_file_exists(&data_dir_name);
 		if !test_mode {
 			if let Ok(true) = exists {
@@ -226,7 +246,7 @@ where
 	) -> Result<Option<SecretKey>, Error> {
 		let mut data_dir_name = PathBuf::from(self.data_dir.clone());
 		data_dir_name.push(GRIN_WALLET_DIR);
-		let data_dir_name = data_dir_name.to_str().unwrap();
+		let data_dir_name = fmt_path(data_dir_name.to_str().unwrap().to_string());
 		let mut wallet: LMDBBackend<'a, C, K> =
 			match LMDBBackend::new(&data_dir_name, self.node_client.clone()) {
 				Err(e) => {
@@ -271,7 +291,7 @@ where
 	) -> Result<ZeroingString, Error> {
 		let mut data_dir_name = PathBuf::from(self.data_dir.clone());
 		data_dir_name.push(GRIN_WALLET_DIR);
-		let data_dir_name = data_dir_name.to_str().unwrap();
+		let data_dir_name = fmt_path(data_dir_name.display().to_string());
 		let wallet_seed = WalletSeed::from_file(&data_dir_name, password)
 			.map_err(|_| Error::Lifecycle("Error opening wallet seed file".into()))?;
 		let res = wallet_seed

impls/src/lifecycle/seed.rs

@@ -12,16 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::blake2;
 use core::num::NonZeroU32;
+use rand::{thread_rng, Rng};
+use serde_json;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::Path;
 use std::path::MAIN_SEPARATOR;
 
-use crate::blake2;
-use rand::{thread_rng, Rng};
-use serde_json;
-
 use ring::aead;
 use ring::pbkdf2;
 
@@ -57,6 +56,13 @@ impl WalletSeed {
 		self.0.to_vec().to_hex()
 	}
 
+	// Helper fuction to format paths according to OS, avoids bugs on Linux
+	pub fn fmt_path(path: String) -> String {
+		let sep = &MAIN_SEPARATOR.to_string();
+		let path = path.replace("/", &sep);
+		let path = path.replace("\\", &sep);
+		path
+	}
 	pub fn to_mnemonic(&self) -> Result<String, Error> {
 		let result = mnemonic::from_entropy(&self.0);
 		match result {
@@ -97,7 +103,12 @@ impl WalletSeed {
 	}
 
 	pub fn seed_file_exists(data_file_dir: &str) -> Result<bool, Error> {
-		let seed_file_path = &format!("{}{}{}", data_file_dir, MAIN_SEPARATOR, SEED_FILE,);
+		let seed_file_path = &format!(
+			"{}{}{}",
+			Self::fmt_path(data_file_dir.to_string()),
+			MAIN_SEPARATOR,
+			SEED_FILE,
+		);
 		debug!("Seed file path: {}", seed_file_path);
 		if Path::new(seed_file_path).exists() {
 			Ok(true)
@@ -132,7 +143,12 @@ impl WalletSeed {
 		word_list: util::ZeroingString,
 		password: util::ZeroingString,
 	) -> Result<(), Error> {
-		let seed_file_path = &format!("{}{}{}", data_file_dir, MAIN_SEPARATOR, SEED_FILE,);
+		let seed_file_path = &format!(
+			"{}{}{}",
+			Self::fmt_path(data_file_dir.to_string()),
+			MAIN_SEPARATOR,
+			SEED_FILE,
+		);
 		debug!("data file dir: {}", data_file_dir);
 		if let Ok(true) = WalletSeed::seed_file_exists(data_file_dir) {
 			debug!("seed file exists");
@@ -166,10 +182,15 @@ impl WalletSeed {
 		// create directory if it doesn't exist
 		fs::create_dir_all(data_file_dir).map_err(|_| Error::IO)?;
 
-		let seed_file_path = &format!("{}{}{}", data_file_dir, MAIN_SEPARATOR, SEED_FILE,);
-
+		let seed_file_path = &format!(
+			"{}{}{}",
+			Self::fmt_path(data_file_dir.to_string()),
+			MAIN_SEPARATOR,
+			SEED_FILE,
+		);
+		let data_file_dir = Self::fmt_path(data_file_dir.to_string());
 		warn!("Generating wallet seed file at: {}", seed_file_path);
-		let exists = WalletSeed::seed_file_exists(data_file_dir)?;
+		let exists = WalletSeed::seed_file_exists(&data_file_dir)?;
 		if exists && !test_mode {
 			let msg = format!("Wallet seed already exists at: {}", data_file_dir);
 			error!("{}", msg);
@@ -196,7 +217,12 @@ impl WalletSeed {
 		// create directory if it doesn't exist
 		fs::create_dir_all(data_file_dir).map_err(|_| Error::IO)?;
 
-		let seed_file_path = &format!("{}{}{}", data_file_dir, MAIN_SEPARATOR, SEED_FILE,);
+		let seed_file_path = &format!(
+			"{}{}{}",
+			Self::fmt_path(data_file_dir.to_string()),
+			MAIN_SEPARATOR,
+			SEED_FILE,
+		);
 
 		debug!("Using wallet seed file at: {}", seed_file_path);
 
@@ -219,7 +245,12 @@ impl WalletSeed {
 	}
 
 	pub fn delete_seed_file(data_file_dir: &str) -> Result<(), Error> {
-		let seed_file_path = &format!("{}{}{}", data_file_dir, MAIN_SEPARATOR, SEED_FILE,);
+		let seed_file_path = &format!(
+			"{}{}{}",
+			Self::fmt_path(data_file_dir.to_string()),
+			MAIN_SEPARATOR,
+			SEED_FILE,
+		);
 		if Path::new(seed_file_path).exists() {
 			debug!("Deleting wallet seed file at: {}", seed_file_path);
 			fs::remove_file(seed_file_path).map_err(|_| Error::IO)?;

impls/src/node_clients/http.rs

@@ -142,7 +142,8 @@ impl NodeClient for HTTPNodeClient {
 						verified: Some(false),
 					});
 				} else {
-					error!("Unable to contact Node to get version info: {}", e);
+					error!("Unable to contact Node to get version info: {}, check your node is running", e);
+					warn!("Warning: a) Node is offline, or b) 'node_api_secret_path' in 'grin-wallet.toml' is set incorrectly");
 					return None;
 				}
 			}

integration/Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mimblewimble/grin"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 workspace = ".."
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 hyper = "0.12"

libwallet/Cargo.toml

@@ -8,7 +8,7 @@ repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 exclude = ["**/*.grin", "**/*.grin2"]
 #build = "src/build/build.rs"
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 blake2-rfc = "0.2"
@@ -47,10 +47,10 @@ grin_wallet_config = { path = "../config", version = "5.4.0-alpha.1" }
 ##### Grin Imports
 
 # For Release
-grin_core = "5.3.3"
-grin_keychain = "5.3.3"
-grin_util = "5.3.3"
-grin_store = "5.3.3"
+grin_core = "5.4.0"
+grin_keychain = "5.4.0"
+grin_util = "5.4.0"
+grin_store = "5.4.0"
 
 # For beta release
 

libwallet/src/api_impl/owner.rs

@@ -14,6 +14,7 @@
 
 //! Generic implementation of owner API functions
 
+use std::cmp;
 use uuid::Uuid;
 
 use crate::api_impl::foreign::finalize_tx as foreign_finalize;
@@ -1122,25 +1123,35 @@ where
 		w.w2n_client().get_chain_tip()?
 	};
 
-	let start_height = match start_height {
-		Some(h) => h,
-		None => 1,
-	};
+	let start_height = start_height.unwrap_or_else(|| 1);
 
-	let mut info = scan::scan(
-		wallet_inst.clone(),
-		keychain_mask,
-		delete_unconfirmed,
-		start_height,
-		tip.0,
-		status_send_channel,
-	)?;
-	info.hash = tip.1;
+	// Scan every 10k heights to save data between batches in case of interruption.
+	let mut total_pmmr_range = None;
+	for h in (start_height..tip.0).step_by(10001) {
+		let batch_end_height = cmp::min(tip.0, h + 10000);
+		let (mut info, range) = scan::scan(
+			wallet_inst.clone(),
+			keychain_mask,
+			delete_unconfirmed,
+			h,
+			batch_end_height,
+			start_height,
+			tip.0,
+			total_pmmr_range,
+			status_send_channel,
+		)?;
+		info.hash = if batch_end_height == tip.0 {
+			tip.1.clone()
+		} else {
+			"".to_owned()
+		};
+		total_pmmr_range = Some(range);
 
-	wallet_lock!(wallet_inst, w);
-	let mut batch = w.batch(keychain_mask)?;
-	batch.save_last_scanned_block(info)?;
-	batch.commit()?;
+		wallet_lock!(wallet_inst, w);
+		let mut batch = w.batch(keychain_mask)?;
+		batch.save_last_scanned_block(info)?;
+		batch.commit()?;
+	}
 
 	Ok(())
 }
@@ -1180,6 +1191,9 @@ where
 	}
 }
 
+/// Wallet scan window in blocks (48 hours).
+pub const REORG_RESCAN_WINDOW: u64 = 24 * 60 * 2;
+
 /// Experimental, wrap the entire definition of how a wallet's state is updated
 pub fn update_wallet_state<'a, L, C, K>(
 	wallet_inst: Arc<Mutex<Box<dyn WalletInst<'a, L, C, K>>>>,
@@ -1258,23 +1272,24 @@ where
 	let last_scanned_block = {
 		wallet_lock!(wallet_inst, w);
 		match w.init_status()? {
-			WalletInitStatus::InitNeedsScanning => ScannedBlockInfo {
-				height: 0,
-				hash: "".to_owned(),
-				start_pmmr_index: 0,
-				last_pmmr_index: 0,
-			},
 			WalletInitStatus::InitNoScanning => ScannedBlockInfo {
 				height: tip.clone().0,
 				hash: tip.clone().1,
 				start_pmmr_index: 0,
 				last_pmmr_index: 0,
 			},
-			WalletInitStatus::InitComplete => w.last_scanned_block()?,
+			_ => w.last_scanned_block()?,
 		}
 	};
 
-	let start_index = last_scanned_block.height.saturating_sub(100);
+	let start_height = last_scanned_block
+		.height
+		.saturating_sub(REORG_RESCAN_WINDOW);
+
+	debug!(
+		"update_wallet_state: last_scanned_block: {:?}",
+		last_scanned_block
+	);
 
 	if last_scanned_block.height == 0 {
 		let msg = "This wallet has not been scanned against the current chain. Beginning full scan... (this first scan may take a while, but subsequent scans will be much quicker)".to_string();
@@ -1283,18 +1298,28 @@ where
 		}
 	}
 
-	let mut info = scan::scan(
-		wallet_inst.clone(),
-		keychain_mask,
-		false,
-		start_index,
-		tip.0,
-		status_send_channel,
-	)?;
+	// Scan every 10k heights to save data between batches in case of interruption.
+	let mut total_pmmr_range = None;
+	for h in (start_height..tip.0).step_by(10001) {
+		let batch_end_height = cmp::min(tip.0, h + 10000);
+		let (mut info, range) = scan::scan(
+			wallet_inst.clone(),
+			keychain_mask,
+			false,
+			h,
+			batch_end_height,
+			start_height,
+			tip.0,
+			total_pmmr_range,
+			status_send_channel,
+		)?;
+		info.hash = if batch_end_height == tip.0 {
+			tip.1.clone()
+		} else {
+			"".to_owned()
+		};
+		total_pmmr_range = Some(range);
 
-	info.hash = tip.1;
-
-	{
 		wallet_lock!(wallet_inst, w);
 		let mut batch = w.batch(keychain_mask)?;
 		batch.save_last_scanned_block(info)?;

libwallet/src/internal/scan.rs

@@ -229,26 +229,30 @@ where
 fn collect_chain_outputs<'a, C, K>(
 	keychain: &K,
 	client: C,
+	batch_start_index: u64,
+	batch_end_index: u64,
 	start_index: u64,
-	end_index: Option<u64>,
+	end_index: u64,
 	status_send_channel: &Option<Sender<StatusMessage>>,
-) -> Result<(Vec<OutputResult>, u64), Error>
+) -> Result<(Vec<OutputResult>, u64, u8), Error>
 where
 	C: NodeClient + 'a,
 	K: Keychain + 'a,
 {
 	let batch_size = 1000;
 	let start_index_stat = start_index;
-	let mut start_index = start_index;
+	let mut start_index = batch_start_index;
 	let mut result_vec: Vec<OutputResult> = vec![];
 	let last_retrieved_return_index;
+
+	let mut perc_complete;
 	loop {
 		let (highest_index, last_retrieved_index, outputs) =
-			client.get_outputs_by_pmmr_index(start_index, end_index, batch_size)?;
+			client.get_outputs_by_pmmr_index(start_index, Some(batch_end_index), batch_size)?;
 
-		let range = highest_index as f64 - start_index_stat as f64;
+		let range = end_index as f64 - start_index_stat as f64;
 		let progress = last_retrieved_index as f64 - start_index_stat as f64;
-		let perc_complete = cmp::min(((progress / range) * 100.0) as u8, 99);
+		perc_complete = cmp::min(((progress / range) * 100.0) as u8, 99);
 
 		let msg = format!(
 			"Checking {} outputs, up to index {}. (Highest index: {})",
@@ -264,16 +268,21 @@ where
 			keychain,
 			outputs.clone(),
 			status_send_channel,
-			perc_complete as u8,
+			perc_complete,
 		)?);
 
+		debug!(
+			"collect_chain_outputs: start_index {}, last_pmmr_index {}",
+			start_index, last_retrieved_index
+		);
+
 		if highest_index <= last_retrieved_index {
 			last_retrieved_return_index = last_retrieved_index;
 			break;
 		}
 		start_index = last_retrieved_index + 1;
 	}
-	Ok((result_vec, last_retrieved_return_index))
+	Ok((result_vec, last_retrieved_return_index, perc_complete))
 }
 
 ///
@@ -451,17 +460,30 @@ where
 	Ok(chain_outs)
 }
 
-/// Check / repair wallet contents by scanning against chain
-/// assume wallet contents have been freshly updated with contents
-/// of latest block
+/// Scan chain outputs and repair the wallet state where needed.
+///
+/// This is the low-level worker used by the owner API. Callers should normally
+/// use the owner scan/update methods instead of calling this directly, unless
+/// they need to drive a scan in batches and save progress between those batches.
+///
+/// `batch_start_height` and `batch_end_height` describe the part of the chain
+/// scanned by this call. `start_height` and `end_height` describe the full scan
+/// range, so progress and PMMR bounds still reflect the whole scan.
+///
+/// The returned `ScannedBlockInfo` is the progress marker to persist after the
+/// batch. The returned PMMR range can be passed into the next batch to avoid
+/// looking up the full range again.
 pub fn scan<'a, L, C, K>(
 	wallet_inst: Arc<Mutex<Box<dyn WalletInst<'a, L, C, K>>>>,
 	keychain_mask: Option<&SecretKey>,
 	delete_unconfirmed: bool,
+	batch_start_height: u64,
+	batch_end_height: u64,
 	start_height: u64,
 	end_height: u64,
+	total_pmmr_range: Option<(u64, u64)>,
 	status_send_channel: &Option<Sender<StatusMessage>>,
-) -> Result<ScannedBlockInfo, Error>
+) -> Result<(ScannedBlockInfo, (u64, u64)), Error>
 where
 	L: WalletLCProvider<'a, C, K>,
 	C: NodeClient + 'a,
@@ -469,30 +491,46 @@ where
 {
 	// First, get a definitive list of outputs we own from the chain
 	if let Some(ref s) = status_send_channel {
-		let _ = s.send(StatusMessage::Scanning("Starting UTXO scan".to_owned(), 0));
+		if start_height == batch_start_height {
+			let _ = s.send(StatusMessage::Scanning("Starting UTXO scan".to_owned(), 0));
+		}
 	}
 	let (client, keychain) = {
 		wallet_lock!(wallet_inst, w);
 		(w.w2n_client().clone(), w.keychain(keychain_mask)?)
 	};
 
-	// Retrieve the actual PMMR index range we're looking for
-	let pmmr_range = client.height_range_to_pmmr_indices(start_height, Some(end_height))?;
+	// Retrieve total PMMR index range we're looking for
+	let total_pmmr_range = if let Some(total_range) = total_pmmr_range {
+		total_range
+	} else {
+		client.height_range_to_pmmr_indices(start_height, Some(end_height))?
+	};
 
-	let (chain_outs, last_index) = collect_chain_outputs(
+	// Retrieve current batch PMMR index range
+	let pmmr_range =
+		client.height_range_to_pmmr_indices(batch_start_height, Some(batch_end_height))?;
+
+	debug!(
+		"scan: from: {}, to: {}",
+		batch_start_height, batch_end_height
+	);
+
+	let (chain_outs, last_index, perc_complete) = collect_chain_outputs(
 		&keychain,
 		client,
 		pmmr_range.0,
-		Some(pmmr_range.1),
+		pmmr_range.1,
+		total_pmmr_range.0,
+		total_pmmr_range.1,
 		status_send_channel,
 	)?;
 	let msg = format!(
 		"Identified {} wallet_outputs as belonging to this wallet",
 		chain_outs.len(),
 	);
-
 	if let Some(ref s) = status_send_channel {
-		let _ = s.send(StatusMessage::Scanning(msg, 99));
+		let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 	}
 
 	// Now, get all outputs owned by this wallet (regardless of account)
@@ -530,7 +568,7 @@ where
 			o.value, o.key_id, m.1.commit,
 		);
 		if let Some(ref s) = status_send_channel {
-			let _ = s.send(StatusMessage::Scanning(msg, 99));
+			let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 		}
 		o.status = OutputStatus::Unspent;
 		// any transactions associated with this should be cancelled
@@ -551,7 +589,7 @@ where
 				m.value, m.key_id, m.commit, m.mmr_index
 			);
 		if let Some(ref s) = status_send_channel {
-			let _ = s.send(StatusMessage::Scanning(msg, 99));
+			let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 		}
 		restore_missing_output(
 			wallet_inst.clone(),
@@ -572,7 +610,7 @@ where
 				o.value, o.key_id, m.1.commit,
 			);
 			if let Some(ref s) = status_send_channel {
-				let _ = s.send(StatusMessage::Scanning(msg, 99));
+				let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 			}
 			o.status = OutputStatus::Unspent;
 			cancel_tx_log_entry(wallet_inst.clone(), keychain_mask, &o)?;
@@ -595,7 +633,7 @@ where
 				o.value, o.key_id, m.commit,
 			);
 			if let Some(ref s) = status_send_channel {
-				let _ = s.send(StatusMessage::Scanning(msg, 99));
+				let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 			}
 			cancel_tx_log_entry(wallet_inst.clone(), keychain_mask, &o)?;
 			wallet_lock!(wallet_inst, w);
@@ -616,7 +654,7 @@ where
 			let label = format!("{}_{}", label_base, acct_index);
 			let msg = format!("Setting account {} at path {}", label, path);
 			if let Some(ref s) = status_send_channel {
-				let _ = s.send(StatusMessage::Scanning(msg, 99));
+				let _ = s.send(StatusMessage::Scanning(msg, perc_complete));
 			}
 			keys::set_acct_path(&mut **w, keychain_mask, &label, path)?;
 			acct_index += 1;
@@ -631,15 +669,20 @@ where
 	}
 
 	if let Some(ref s) = status_send_channel {
-		let _ = s.send(StatusMessage::ScanningComplete(
-			"Scanning Complete".to_owned(),
-		));
+		if end_height == batch_end_height {
+			let _ = s.send(StatusMessage::ScanningComplete(
+				"Scanning Complete".to_owned(),
+			));
+		}
 	}
 
-	Ok(ScannedBlockInfo {
-		height: end_height,
-		hash: "".to_owned(),
-		start_pmmr_index: pmmr_range.0,
-		last_pmmr_index: last_index,
-	})
+	Ok((
+		ScannedBlockInfo {
+			height: batch_end_height,
+			hash: "".to_owned(),
+			start_pmmr_index: pmmr_range.0,
+			last_pmmr_index: last_index,
+		},
+		total_pmmr_range,
+	))
 }

libwallet/src/internal/updater.rs

@@ -96,6 +96,7 @@ where
 /// Apply advanced filtering to resultset from retrieve_txs below
 pub fn apply_advanced_tx_list_filtering<'a, T: ?Sized, C, K>(
 	wallet: &mut T,
+	parent_key_id: Option<&Identifier>,
 	query_args: &RetrieveTxQueryArgs,
 ) -> Vec<TxLogEntry>
 where
@@ -107,6 +108,10 @@ where
 	let txs_iter: Box<dyn Iterator<Item = TxLogEntry>> = Box::new(
 		wallet
 			.tx_log_iter()
+			.filter(|tx_entry| match parent_key_id {
+				Some(k) => tx_entry.parent_key_id == *k,
+				None => true,
+			})
 			.filter(|tx_entry| {
 				if let Some(v) = query_args.exclude_cancelled {
 					if v {
@@ -357,7 +362,7 @@ where
 	// Adding in new transaction list query logic. If `tx_id` or `tx_slate_id`
 	// is provided, then `query_args` is ignored and old logic is followed.
 	if query_args.is_some() && tx_id.is_none() && tx_slate_id.is_none() {
-		txs = apply_advanced_tx_list_filtering(wallet, &query_args.unwrap())
+		txs = apply_advanced_tx_list_filtering(wallet, parent_key_id, &query_args.unwrap())
 	} else {
 		txs = wallet
 			.tx_log_iter()

libwallet/src/mwixnet/onion/crypto/dalek.rs

@@ -117,11 +117,13 @@ impl Writeable for DalekPublicKey {
 }
 
 /// Encapsulates an ed25519_dalek::Signature and provides (de-)serialization
+#[allow(dead_code)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct DalekSignature(Signature);
 
 impl DalekSignature {
 	/// Convert hex string to DalekSignature.
+	#[allow(dead_code)]
 	pub fn from_hex(hex: &str) -> Result<Self, DalekError> {
 		let bytes = grin_util::from_hex(hex)
 			.map_err(|_| DalekError::HexError(format!("failed to decode {}", hex)))?;
@@ -131,6 +133,7 @@ impl DalekSignature {
 	}
 
 	/// Verifies DalekSignature
+	#[allow(dead_code)]
 	pub fn verify(&self, pk: &DalekPublicKey, msg: &[u8]) -> Result<(), DalekError> {
 		pk.as_ref()
 			.verify(&msg, &self.0)

libwallet/src/types.rs

@@ -297,6 +297,9 @@ where
 	/// save a tx log entry
 	fn save_tx_log_entry(&mut self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>;
 
+	/// delete a tx log entry
+	fn delete_tx_log_entry(&mut self, tx_id: u32, parent_id: &Identifier) -> Result<(), Error>;
+
 	/// save an account label -> path mapping
 	fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error>;
 

rustfmt.toml

@@ -1,2 +1,2 @@
 hard_tabs = true
-edition = "2018"
+edition = "2021"

src/bin/grin-wallet.rs

@@ -24,12 +24,12 @@ use crate::util::init_logger;
 use clap::App;
 use grin_core as core;
 use grin_util as util;
+use grin_wallet::cmd;
 use grin_wallet_config as config;
 use grin_wallet_impls::HTTPNodeClient;
 use std::env;
 use std::path::PathBuf;
-
-use grin_wallet::cmd;
+use std::path::MAIN_SEPARATOR;
 
 // include build information
 pub mod built_info {
@@ -55,6 +55,13 @@ pub fn info_strings() -> (String, String) {
 	)
 }
 
+// Helper fuction to format paths according to OS, avoids bugs on Linux
+pub fn fmt_path(path: String) -> String {
+	let sep = &MAIN_SEPARATOR.to_string();
+	let path = path.replace("/", &sep).replace("\\", &sep);
+	path
+}
+
 fn log_build_info() {
 	let (basic_info, detailed_info) = info_strings();
 	info!("{}", basic_info);
@@ -82,11 +89,11 @@ fn real_main() -> i32 {
 
 	let mut current_dir = None;
 	let mut create_path = false;
-
 	if args.is_present("top_level_dir") {
 		let res = args.value_of("top_level_dir");
 		match res {
 			Some(d) => {
+				let d = fmt_path(d.to_owned().to_string()); // Fix for fs to work with paths on Linux
 				current_dir = Some(PathBuf::from(d));
 			}
 			None => {
@@ -138,7 +145,6 @@ fn real_main() -> i32 {
 		"Using wallet configuration file at {}",
 		config.config_file_path.as_ref().unwrap().to_str().unwrap()
 	);
-
 	log_build_info();
 
 	global::init_global_chain_type(
@@ -154,9 +160,7 @@ fn real_main() -> i32 {
 	);
 
 	global::init_global_accept_fee_base(config.members.as_ref().unwrap().wallet.accept_fee_base());
-
 	let wallet_config = config.clone().members.unwrap().wallet;
 	let node_client = HTTPNodeClient::new(&wallet_config.check_node_api_http_addr, None).unwrap();
-
 	cmd::wallet_command(&args, config, node_client)
 }

src/cmd/wallet_args.rs

@@ -25,7 +25,7 @@ use grin_core as core;
 use grin_core::core::amount_to_hr_string;
 use grin_keychain as keychain;
 use grin_wallet_api::Owner;
-use grin_wallet_config::{config_file_exists, TorConfig, WalletConfig};
+use grin_wallet_config::{TorConfig, WalletConfig};
 use grin_wallet_controller::{command, Error};
 use grin_wallet_impls::{DefaultLCProvider, DefaultWalletImpl};
 use grin_wallet_libwallet::{self, Slate, SlatepackAddress, SlatepackArmor};
@@ -344,7 +344,7 @@ pub fn parse_init_args<L, C, K>(
 	config: &WalletConfig,
 	g_args: &command::GlobalArgs,
 	args: &ArgMatches,
-	test_mode: bool,
+	_test_mode: bool,
 ) -> Result<command::InitArgs, ParseError>
 where
 	DefaultWalletImpl<'static, C>: WalletInst<'static, L, C, K>,
@@ -352,10 +352,6 @@ where
 	C: NodeClient + 'static,
 	K: keychain::Keychain + 'static,
 {
-	if config_file_exists(&config.data_file_dir) && !test_mode {
-		return Err(ParseError::WalletExists(config.data_file_dir.clone()));
-	}
-
 	let list_length = match args.is_present("short_wordlist") {
 		false => 32,
 		true => 16,
@@ -1185,7 +1181,7 @@ where
 	node_client.set_node_api_secret(global_wallet_args.node_api_secret.clone());
 
 	// legacy hack to avoid the need for changes in existing grin-wallet.toml files
-	// remove `wallet_data` from end of path as
+	// remove `wallet_data` from end of path
 	// new lifecycle provider assumes grin_wallet.toml is in root of data directory
 	let mut top_level_wallet_dir = PathBuf::from(wallet_config.clone().data_file_dir);
 	if top_level_wallet_dir.ends_with(GRIN_WALLET_DIR) {

tests/common/mod.rs

@@ -165,7 +165,7 @@ pub fn config_command_wallet(
 				.to_owned(),
 		))?;
 	}
-	default_config.update_paths(&current_dir);
+	default_config.update_paths(&current_dir, &current_dir);
 	default_config
 		.write_to_file(config_file_name.to_str().unwrap(), false, None, None)
 		.unwrap_or_else(|e| {
@@ -494,4 +494,5 @@ impl From<grin_wallet_controller::Error> for WalletAPIReturnError {
 }
 
 #[derive(Clone, Debug, Serialize, Deserialize)]
+#[allow(dead_code)]
 pub struct RetrieveSummaryInfoResp(pub bool, pub WalletInfo);

util/Cargo.toml

@@ -7,7 +7,7 @@ license = "Apache-2.0"
 repository = "https://github.com/mimblewimble/grin-wallet"
 keywords = [ "crypto", "grin", "mimblewimble" ]
 workspace = ".."
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 rand = "0.6"
@@ -21,7 +21,7 @@ thiserror = "1"
 ##### Grin Imports
 
 # For Release
-grin_util = "5.3.3"
+grin_util = "5.4.0"
 
 # For beta release