Spv proof (#1207)

* feat(unit_tests): add begin of unit test for spv proof

* feat(utxo): continue test

* feat(unit_tests): very wip

* feat(utxo): merkle tree verification now work; need full refactoring

* feat(spv): start spv validation module

* feat(spv): add spv proof

* feat(spv): add merkle proof unit test for a single element

* feat(spv): add complex merkle proof inclusion unit test

* feat(spv): fix some cargo warnings

* feat(spv): simplify error check for validate_spv_proof

- native client should not return error on validate_spv_proof since there is no verification

* feat(spv): complete the first step of the spv proof validation

* feat(spv): complete the unit test for spv proof validation in utxo module

* feat(spv_validation): add vin and vout check for spv proof validation

* feat(spv_validation): match exact error from the spv - will customize later

* feat(utxo): start utxo block header storage + sync with dev

* feat(utxo): continue block header storage interface

* feat(utxo): revert sled - will switch to sqllite

* feat(utxo): start utxo sql block header storage trait implementation

* feat(sql): implement partially add_block_headers_to_storage and the unit test

* feat(sql): add get_block header functions

* feat(sql): add extra function for block insertion

* feat(lib_bitcoin): add raw header type

* feat(spv): Integrate raw block header into spv validation

* feat(spv): remove old comments

* feat(spv): Adding raw header into utxo_common spv validation

* feat(spv): make slp great again

* feat(spv): returning more concrete error types

* feat(toolchain): update to `nightly-2022-02-01` to fix osx compilation

- rust-lang/rust#93414

* feat(cargo): fix warnings

* feat(fmt): rust fmt

* feat(utxo_block_header): add a box dyn for block header storage to utxo common fields

* Revert "feat(utxo_block_header): add a box dyn for block header storage to utxo common fields"

This reverts commit 8914b76.

* feat(utxo): add empty utxo_indexedb_block_header_storage.rs

* feat(utxo): start an empty loop for downloading headers

* feat(indexed_db_block_header): do not use todo to prevent crash at runtime

* feat(block_header_loop): add block header loop in arc builder

* feat(utxo): add a function to retrieve a storage from the ctx for block header

* feat(utxo): start logic of block header downloading, very WIP

* feat(header_validation): add partial header validation + params for enabling

* feat(header_validation): add unexpected difficulty change check

* feat(header_validation): add unit test for validate headers

* feat(header_validation): document the validate_headers function

* feat(header_validation): use appropriate error for validate_headers

* feat(storage): rework storage to make it persistent

* feat(storage): use the block header storage as optional and fix unit tests

* feat(storage): make the conf into coins settings

* feat(storage): implement header from storage or rpc + within validation

* feat(improvements): remove non-used function

* feat(storage): add to storage after validation in retrieve header from storage

* feat(fix_review): first fixes batch

* feat(fix_review): sql fixes + rename `from_address` to `sender_address`

* feat(fix_review): simplify the way to get height, use into_iter + find

* feat(fix_review): add a get_tx_height function + remove more marketcoinops in the arc builder

* feat(fix_review): simplify download_loop with a `try_loop` macro

* feat(fix_review): improve get_tx_height

* feat(fix_review): continue review improvements, move retrieve_last_headers to electrum

* feat(fix_review): remove dead code comment

* feat(spv): next batch of fixes

* feat(review): remove utxo_wrapper_block_header_storage.rs

* feat(review): next batche of fixes

* feat(review): refactor block_header_from_storage_or_rpc

* feat(wasm): use explicitly instant wasm bindgen

* feat(spv): fix wasm tests

* feat(spv): fix unit test compilation

* feat(eth): ignore polygon unit test (unstable)

* feat(unit tests): increase to 6 seconds for the docker unit test

* feat(utxo): simplify block_headers_storage, add error

* feat(review): batch of review fixes

* feat(review): another batch of review fixes

* feat(eth_test): remove the loop

* feat(eth_test): remove the loop

* feat(spv_proof): review fixes

* feat(spv_proof): review fixes

* feat(spv): fix wrong copy paste description

Author: Milerius

Cargo.lock

@@ -88,7 +88,7 @@ num-rational = { version = "0.2", features = ["serde", "bigint", "bigint-std"] }
 num-traits = "0.2"
 rpc = { path = "mm2src/mm2_bitcoin/rpc" }
 rpc_task = { path = "mm2src/rpc_task" }
-parking_lot = { version = "0.11", features = ["nightly"] }
+parking_lot = { version = "0.12.0", features = ["nightly"] }
 parity-util-mem = "0.9"
 # AP: portfolio RPCs are not documented and not used as of now
 # so the crate is disabled to speed up the entire removal of C code
@@ -110,6 +110,7 @@ ser_error = { path = "mm2src/derives/ser_error" }
 ser_error_derive = { path = "mm2src/derives/ser_error_derive" }
 serialization = { path = "mm2src/mm2_bitcoin/serialization" }
 serialization_derive = { path = "mm2src/mm2_bitcoin/serialization_derive" }
+spv_validation = { path = "mm2src/mm2_bitcoin/spv_validation" }
 sp-runtime-interface = { version = "3.0.0", default-features = false, features = ["disable_target_static_assertions"] }
 sp-trie = { version = "3.0", default-features = false }
 
@@ -124,6 +125,7 @@ wasm-bindgen = { version = "0.2.50", features = ["serde-serialize", "nightly"] }
 wasm-bindgen-futures = { version = "0.4.1" }
 wasm-bindgen-test = { version = "0.3.1" }
 web-sys = { version = "0.3.55", features = ["console"] }
+instant = {version = "0.1.12", features = ["wasm-bindgen"]}
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 dirs = { version = "1" }

Cargo.toml

@@ -15,8 +15,8 @@ The current state can be considered as very early alpha.
 1. (Optional) OSX: run `LIBRARY_PATH=/usr/local/opt/openssl/lib`
 1. Run
     ```
-    rustup install nightly-2021-12-16
-    rustup default nightly-2021-12-16
+    rustup install nightly-2022-02-01
+    rustup default nightly-2022-02-01
     rustup component add rustfmt-preview
     ```
 1. Run `cargo build` (or `cargo build -vv` to get verbose build output).

README.md

@@ -13,7 +13,7 @@ doctest = false
 
 [dependencies]
 async-std = { version = "1.5", features = ["unstable"] }
-async-trait = "0.1"
+async-trait = "0.1.52"
 base64 = "0.10.0"
 bigdecimal = { version = "0.1.0", features = ["serde"] }
 bip32 = { version = "0.2.2", default-features = false, features = ["alloc", "secp256k1-ffi"] }
@@ -52,7 +52,7 @@ lightning-invoice = "0.12.0"
 metrics = "0.12"
 mocktopus = "0.7.0"
 num-traits = "0.2"
-parking_lot = { version = "0.11", features = ["nightly"] }
+parking_lot = { version = "0.12.0", features = ["nightly"] }
 primitives = { path = "../mm2_bitcoin/primitives" }
 prost = "0.8"
 rand = { version = "0.7", features = ["std", "small_rng"] }
@@ -69,6 +69,7 @@ serde_derive = "1.0"
 serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] }
 serialization = { path = "../mm2_bitcoin/serialization" }
 serialization_derive = { path = "../mm2_bitcoin/serialization_derive" }
+spv_validation = { path = "../mm2_bitcoin/spv_validation" }
 sha2 = "0.8"
 sha3 = "0.8"
 utxo_signer = { path = "utxo_signer" }

mm2src/coins/Cargo.toml

@@ -1210,7 +1210,7 @@ fn polygon_check_if_my_payment_sent() {
     let request = json!({
         "method": "enable",
         "coin": "MATIC",
-        "urls": ["https://polygon-rpc.com"],
+        "urls": ["https://polygon-mainnet.g.alchemy.com/v2/9YYl6iMLmXXLoflMPHnMTC4Dcm2L2tFH"],
         "swap_contract_address": "0x9130b257d37a52e52f21054c4da3450c72f595ce",
     });
 

mm2src/coins/eth/eth_tests.rs

@@ -15,7 +15,7 @@ bitcoin = "0.27.1"
 common = { path = "../../common" }
 lightning = "0.0.104"
 libc = "0.2"
-parking_lot = { version = "0.11", features = ["nightly"] }
+parking_lot = { version = "0.12.0", features = ["nightly"] }
 secp256k1 = { version = "0.20" }
 serde_json = "1.0"
 

mm2src/coins/lightning_persister/Cargo.toml

@@ -248,6 +248,7 @@ pub struct ValidatePaymentInput {
     pub secret_hash: Vec<u8>,
     pub amount: BigDecimal,
     pub swap_contract_address: Option<BytesJson>,
+    pub confirmations: u64,
 }
 
 /// Swap operations (mostly based on the Hash/Time locked transactions implemented by coin wallets).

mm2src/coins/lp_coins.rs

@@ -131,6 +131,7 @@ fn test_validate_maker_payment() {
         secret_hash: vec![1; 20],
         amount: correct_amount.clone(),
         swap_contract_address: coin.swap_contract_address(),
+        confirmations: 1,
     };
 
     coin.validate_maker_payment(input.clone()).wait().unwrap();
@@ -851,6 +852,7 @@ fn test_validate_maker_payment_malicious() {
         secret_hash,
         amount,
         swap_contract_address: coin.swap_contract_address(),
+        confirmations: 1,
     };
     let error = coin
         .validate_maker_payment(input)

mm2src/coins/qrc20/qrc20_tests.rs

@@ -7,14 +7,12 @@ use common::{async_blocking, PagingOptionsEnum};
 use db_common::sqlite::rusqlite::types::Type;
 use db_common::sqlite::rusqlite::{Connection, Error as SqlError, Row, ToSql, NO_PARAMS};
 use db_common::sqlite::sql_builder::SqlBuilder;
-use db_common::sqlite::{offset_by_id, validate_table_name};
+use db_common::sqlite::{offset_by_id, string_from_row, validate_table_name, CHECK_TABLE_EXISTS_SQL};
 use rpc::v1::types::Bytes as BytesJson;
 use serde_json::{self as json};
 use std::convert::TryInto;
 use std::sync::{Arc, Mutex};
 
-const CHECK_TABLE_EXISTS_SQL: &str = "SELECT name FROM sqlite_master WHERE type='table' AND name=?1;";
-
 fn tx_history_table(ticker: &str) -> String { ticker.to_owned() + "_tx_history" }
 
 fn tx_cache_table(ticker: &str) -> String { ticker.to_owned() + "_tx_cache" }
@@ -194,17 +192,9 @@ where
     P::Item: ToSql,
     F: FnOnce(&Row<'_>) -> Result<T, SqlError>,
 {
-    let maybe_result = conn.query_row(query, params, map_fn);
-    if let Err(SqlError::QueryReturnedNoRows) = maybe_result {
-        return Ok(None);
-    }
-
-    let result = maybe_result?;
-    Ok(Some(result))
+    db_common::sqlite::query_single_row(conn, query, params, map_fn).map_err(MmError::new)
 }
 
-fn string_from_row(row: &Row<'_>) -> Result<String, SqlError> { row.get(0) }
-
 fn tx_details_from_row(row: &Row<'_>) -> Result<TransactionDetails, SqlError> {
     let json_string: String = row.get(0)?;
     json::from_str(&json_string).map_err(|e| SqlError::FromSqlConversionFailure(0, Type::Text, Box::new(e)))

mm2src/coins/sql_tx_history_storage.rs

@@ -29,13 +29,12 @@ mod bchd_pb;
 pub mod qtum;
 pub mod rpc_clients;
 pub mod slp;
+pub mod utxo_block_header_storage;
 pub mod utxo_builder;
 pub mod utxo_common;
 pub mod utxo_standard;
 pub mod utxo_withdraw;
 
-#[cfg(not(target_arch = "wasm32"))] pub mod tx_cache;
-
 use async_trait::async_trait;
 use bigdecimal::BigDecimal;
 use bitcoin::network::constants::Network as BitcoinNetwork;
@@ -69,6 +68,7 @@ use rpc::v1::types::{Bytes as BytesJson, Transaction as RpcTransaction, H256 as
 use script::{Builder, Script, SignatureVersion, TransactionInputSigner};
 use serde_json::{self as json, Value as Json};
 use serialization::{serialize, serialize_with_flags, SERIALIZE_TRANSACTION_WITNESS};
+use spv_validation::types::SPVError;
 use std::array::TryFromSliceError;
 use std::collections::{HashMap, HashSet};
 use std::convert::TryInto;
@@ -95,6 +95,13 @@ use super::{BalanceError, BalanceFut, BalanceResult, CoinsContext, DerivationMet
 use crate::coin_balance::{EnableCoinScanPolicy, HDAddressBalanceScanner};
 use crate::hd_wallet::{HDAccountOps, HDAccountsMutex, HDAddress, HDWalletCoinOps, HDWalletOps, InvalidBip44ChainError};
 use crate::hd_wallet_storage::{HDAccountStorageItem, HDWalletCoinStorage, HDWalletStorageError, HDWalletStorageResult};
+use crate::utxo::utxo_block_header_storage::BlockHeaderStorageError;
+use utxo_block_header_storage::BlockHeaderStorage;
+#[cfg(not(target_arch = "wasm32"))] pub mod tx_cache;
+#[cfg(target_arch = "wasm32")]
+pub mod utxo_indexedb_block_header_storage;
+#[cfg(not(target_arch = "wasm32"))]
+pub mod utxo_sql_block_header_storage;
 
 #[cfg(test)] pub mod utxo_tests;
 #[cfg(target_arch = "wasm32")] pub mod utxo_wasm_tests;
@@ -510,6 +517,7 @@ pub struct UtxoCoinFields {
     pub history_sync_state: Mutex<HistorySyncState>,
     /// Path to the TX cache directory
     pub tx_cache_directory: Option<PathBuf>,
+    pub block_headers_storage: Option<BlockHeaderStorage>,
     /// The cache of recently send transactions used to track the spent UTXOs and replace them with new outputs
     /// The daemon needs some time to update the listunspent list for address which makes it return already spent UTXOs
     /// This cache helps to prevent UTXO reuse in such cases
@@ -545,6 +553,56 @@ impl From<UnsupportedAddr> for WithdrawError {
     fn from(e: UnsupportedAddr) -> Self { WithdrawError::InvalidAddress(e.to_string()) }
 }
 
+#[derive(Debug)]
+pub enum GetTxHeightError {
+    HeightNotFound,
+}
+
+impl From<GetTxHeightError> for SPVError {
+    fn from(e: GetTxHeightError) -> Self {
+        match e {
+            GetTxHeightError::HeightNotFound => SPVError::InvalidHeight,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum GetBlockHeaderError {
+    StorageError(BlockHeaderStorageError),
+    RpcError(JsonRpcError),
+    SerializationError(serialization::Error),
+    InvalidResponse(String),
+    SPVError(SPVError),
+    NativeNotSupported(String),
+    Internal(String),
+}
+
+impl From<JsonRpcError> for GetBlockHeaderError {
+    fn from(err: JsonRpcError) -> Self { GetBlockHeaderError::RpcError(err) }
+}
+
+impl From<UtxoRpcError> for GetBlockHeaderError {
+    fn from(e: UtxoRpcError) -> Self {
+        match e {
+            UtxoRpcError::Transport(e) | UtxoRpcError::ResponseParseError(e) => GetBlockHeaderError::RpcError(e),
+            UtxoRpcError::InvalidResponse(e) => GetBlockHeaderError::InvalidResponse(e),
+            UtxoRpcError::Internal(e) => GetBlockHeaderError::Internal(e),
+        }
+    }
+}
+
+impl From<SPVError> for GetBlockHeaderError {
+    fn from(e: SPVError) -> Self { GetBlockHeaderError::SPVError(e) }
+}
+
+impl From<serialization::Error> for GetBlockHeaderError {
+    fn from(err: serialization::Error) -> Self { GetBlockHeaderError::SerializationError(err) }
+}
+
+impl From<BlockHeaderStorageError> for GetBlockHeaderError {
+    fn from(err: BlockHeaderStorageError) -> Self { GetBlockHeaderError::StorageError(err) }
+}
+
 impl UtxoCoinFields {
     pub fn transaction_preimage(&self) -> TransactionInputSigner {
         let lock_time = if self.conf.ticker == "KMD" {
@@ -1031,6 +1089,14 @@ pub struct UtxoMergeParams {
     pub max_merge_at_once: usize,
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct UtxoBlockHeaderVerificationParams {
+    pub difficulty_check: bool,
+    pub constant_difficulty: bool,
+    pub blocks_limit_to_check: NonZeroU64,
+    pub check_every: f64,
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct UtxoActivationParams {
     pub mode: UtxoRpcMode,
@@ -1053,6 +1119,7 @@ pub enum UtxoFromLegacyReqErr {
     UnexpectedMethod,
     InvalidElectrumServers(json::Error),
     InvalidMergeParams(json::Error),
+    InvalidBlockHeaderVerificationParams(json::Error),
     InvalidRequiredConfs(json::Error),
     InvalidRequiresNota(json::Error),
     InvalidAddressFormat(json::Error),

mm2src/coins/utxo.rs

@@ -124,14 +124,13 @@ impl QtumCoin {
         let delegation_output = self.remove_delegation_output(QRC20_GAS_LIMIT_DEFAULT, QRC20_GAS_PRICE_DEFAULT)?;
         let outputs = vec![delegation_output];
         let my_address = self.my_address().map_to_mm(DelegationError::InternalError)?;
-        Ok(self
-            .generate_delegation_transaction(
-                outputs,
-                my_address,
-                QRC20_GAS_LIMIT_DEFAULT,
-                TransactionType::RemoveDelegation,
-            )
-            .await?)
+        self.generate_delegation_transaction(
+            outputs,
+            my_address,
+            QRC20_GAS_LIMIT_DEFAULT,
+            TransactionType::RemoveDelegation,
+        )
+        .await
     }
 
     async fn am_i_currently_staking(&self) -> Result<Option<String>, MmError<StakingInfosError>> {
@@ -252,14 +251,13 @@ impl QtumCoin {
 
         let outputs = vec![delegation_output];
         let my_address = self.my_address().map_to_mm(DelegationError::InternalError)?;
-        Ok(self
-            .generate_delegation_transaction(
-                outputs,
-                my_address,
-                QRC20_GAS_LIMIT_DELEGATION,
-                TransactionType::StakingDelegation,
-            )
-            .await?)
+        self.generate_delegation_transaction(
+            outputs,
+            my_address,
+            QRC20_GAS_LIMIT_DELEGATION,
+            TransactionType::StakingDelegation,
+        )
+        .await
     }
 
     async fn generate_delegation_transaction(

mm2src/coins/utxo/qtum_delegation.rs

@@ -1025,7 +1025,7 @@ impl Into<BlockHeaderNonce> for ElectrumNonce {
 
 #[derive(Debug, Deserialize)]
 pub struct ElectrumBlockHeadersRes {
-    count: u64,
+    pub count: u64,
     pub hex: BytesJson,
     #[allow(dead_code)]
     max: u64,
@@ -1044,8 +1044,8 @@ pub struct ElectrumBlockHeaderV12 {
 }
 
 impl ElectrumBlockHeaderV12 {
-    pub fn hash(&self) -> H256Json {
-        let block_header = BlockHeader {
+    fn as_block_header(&self) -> BlockHeader {
+        BlockHeader {
             version: self.version as u32,
             previous_header_hash: self.prev_block_hash.into(),
             merkle_root_hash: self.merkle_root.into(),
@@ -1064,7 +1064,17 @@ impl ElectrumBlockHeaderV12 {
             n_height: None,
             n_nonce_u64: None,
             mix_hash: None,
-        };
+        }
+    }
+
+    pub fn as_hex(&self) -> String {
+        let block_header = self.as_block_header();
+        let serialized = serialize(&block_header);
+        hex::encode(serialized)
+    }
+
+    pub fn hash(&self) -> H256Json {
+        let block_header = self.as_block_header();
         BlockHeader::hash(&block_header).into()
     }
 }
@@ -1658,6 +1668,49 @@ impl ElectrumClient {
         rpc_func!(self, "blockchain.block.headers", start_height, count)
     }
 
+    pub fn retrieve_last_headers(
+        &self,
+        blocks_limit_to_check: NonZeroU64,
+        block_height: u64,
+    ) -> UtxoRpcFut<(HashMap<u64, BlockHeader>, Vec<BlockHeader>)> {
+        let (from, count) = {
+            let from = if block_height < blocks_limit_to_check.get() {
+                0
+            } else {
+                block_height - blocks_limit_to_check.get()
+            };
+            (from, blocks_limit_to_check)
+        };
+        Box::new(
+            self.blockchain_block_headers(from, count)
+                .map_to_mm_fut(UtxoRpcError::from)
+                .and_then(move |headers| {
+                    let (block_registry, block_headers) = {
+                        if headers.count == 0 {
+                            return MmError::err(UtxoRpcError::Internal("No headers available".to_string()));
+                        }
+                        let len = CompactInteger::from(headers.count);
+                        let mut serialized = serialize(&len).take();
+                        serialized.extend(headers.hex.0.into_iter());
+                        let mut reader = Reader::new_with_coin_variant(serialized.as_slice(), CoinVariant::Standard);
+                        let maybe_block_headers = reader.read_list::<BlockHeader>();
+                        let block_headers = match maybe_block_headers {
+                            Ok(headers) => headers,
+                            Err(e) => return MmError::err(UtxoRpcError::InvalidResponse(format!("{:?}", e))),
+                        };
+                        let mut block_registry: HashMap<u64, BlockHeader> = HashMap::new();
+                        let mut starting_height = from;
+                        for block_header in &block_headers {
+                            block_registry.insert(starting_height, block_header.clone());
+                            starting_height += 1;
+                        }
+                        (block_registry, block_headers)
+                    };
+                    Ok((block_registry, block_headers))
+                }),
+        )
+    }
+
     /// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#blockchain-transaction-get-merkle
     pub fn blockchain_transaction_get_merkle(&self, txid: H256Json, height: u64) -> RpcRes<TxMerkleBranch> {
         rpc_func!(self, "blockchain.transaction.get_merkle", txid, height)