diff --git a/app/app.rs b/app/app.rs index a9f9be6..a47477d 100644 --- a/app/app.rs +++ b/app/app.rs @@ -61,7 +61,7 @@ fn update_wallet(node: &Node, wallet: &Wallet) -> Result<(), Error> { let addresses = wallet.get_addresses()?; let unconfirmed_utxos = node.get_unconfirmed_utxos_by_addresses(&addresses)?; - let utxos = node.get_utxos_by_addresses(&addresses)?; + let utxos = node.get_utxos_by_addresses(&addresses, 0)?; let confirmed_outpoints: Vec<_> = wallet.get_utxos()?.into_keys().collect(); let confirmed_spent = node .get_spent_utxos(&confirmed_outpoints)? diff --git a/lib/archive.rs b/lib/archive.rs index 0033d57..d06987d 100644 --- a/lib/archive.rs +++ b/lib/archive.rs @@ -7,6 +7,7 @@ use std::{ use bitcoin::{self, hashes::Hash as _}; use fallible_iterator::{FallibleIterator, IteratorExt}; use heed::types::SerdeBincode; +use serde::{Deserialize, Serialize}; use sneed::{ DatabaseUnique, EnvError, RoTxn, RwTxn, RwTxnError, UnitKey, db::{self, error::Error as DbError}, @@ -14,12 +15,16 @@ use sneed::{ }; use crate::types::{ - Block, BlockHash, BmmResult, Body, Header, Tip, Txid, VERSION, Version, + Address, AddressTxidKey, Block, BlockHash, BmmResult, Body, + FilledTransaction, Header, MerkleProof, Tip, Txid, VERSION, Version, proto::mainchain, }; #[allow(clippy::duplicated_attributes)] #[derive(thiserror::Error, transitive::Transitive, Debug)] +#[transitive(from(db::error::Delete, DbError))] +#[transitive(from(db::error::IterInit, DbError))] +#[transitive(from(db::error::IterItem, DbError))] #[transitive(from(db::error::Put, DbError))] #[transitive(from(db::error::TryGet, DbError))] #[transitive(from(env::error::CreateDb, EnvError))] @@ -73,6 +78,19 @@ pub enum Error { NoMainHeight(bitcoin::BlockHash), #[error("no tx with txid {0}")] NoTx(Txid), + #[error( + "txdb filled transaction length mismatch: expected {expected}, actual {actual}" + )] + TxDbFilledTxLenMismatch { expected: usize, actual: usize }, +} + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct FilledTxEntry { + pub tx: FilledTransaction, + pub merkle_proof: MerkleProof, + pub block_hash: BlockHash, + pub block_height: u32, + pub unix_stamp: u64, } #[derive(Clone)] @@ -148,11 +166,13 @@ pub struct Archive { SerdeBincode, SerdeBincode>, >, + /// Capped tx cache, keyed by (address, txid). + txdb: DatabaseUnique>, _version: DatabaseUnique>, } impl Archive { - pub const NUM_DBS: u32 = 14; + pub const NUM_DBS: u32 = 15; pub fn new(env: &sneed::Env) -> Result { let mut rwtxn = env.write_txn()?; @@ -224,6 +244,7 @@ impl Archive { let total_work = DatabaseUnique::create(env, &mut rwtxn, "total_work")?; let txid_to_inclusions = DatabaseUnique::create(env, &mut rwtxn, "txid_to_inclusions")?; + let txdb = DatabaseUnique::create(env, &mut rwtxn, "txdb")?; rwtxn.commit()?; Ok(Self { block_hash_to_height, @@ -239,6 +260,7 @@ impl Archive { successors, total_work, txid_to_inclusions, + txdb, _version: version, }) } @@ -672,6 +694,125 @@ impl Archive { Ok(inclusions) } + pub fn prune_txdb_block( + &self, + rwtxn: &mut RwTxn, + block_hash: BlockHash, + ) -> Result { + let keys = { + let mut keys = Vec::new(); + let mut iter = self.txdb.iter(rwtxn).map_err(DbError::from)?; + while let Some((key, entry)) = iter.next().map_err(DbError::from)? { + if entry.block_hash == block_hash { + keys.push(key); + } + } + keys + }; + for key in &keys { + let _ = self.txdb.delete(rwtxn, key)?; + } + Ok(keys.len()) + } + + pub fn prune_txdb_older_than( + &self, + rwtxn: &mut RwTxn, + cutoff_unix: u64, + ) -> Result { + let keys = { + let mut keys = Vec::new(); + let mut iter = self.txdb.iter(rwtxn).map_err(DbError::from)?; + while let Some((key, entry)) = iter.next().map_err(DbError::from)? { + if entry.unix_stamp < cutoff_unix { + keys.push(key); + } + } + keys + }; + for key in &keys { + let _ = self.txdb.delete(rwtxn, key)?; + } + Ok(keys.len()) + } + + pub fn put_txdb_for_connected_block( + &self, + rwtxn: &mut RwTxn, + block_hash: BlockHash, + block_height: u32, + body: &Body, + filled_txs: &[FilledTransaction], + unix_stamp: u64, + ) -> Result<(), Error> { + if body.transactions.len() != filled_txs.len() { + return Err(Error::TxDbFilledTxLenMismatch { + expected: body.transactions.len(), + actual: filled_txs.len(), + }); + } + for (tx_index, filled_tx) in filled_txs.iter().enumerate() { + let txid = filled_tx.txid(); + debug_assert_eq!(body.transactions[tx_index].txid(), txid); + let merkle_proof = Body::compute_tx_merkle_proof( + &body.coinbase, + &body.transactions, + tx_index, + ); + debug_assert_eq!(merkle_proof.leaf_index, tx_index + 1); + + let mut addresses = HashSet::new(); + addresses.extend( + filled_tx.spent_utxos.iter().map(|output| output.address), + ); + addresses.extend( + filled_tx + .transaction + .outputs + .iter() + .map(|output| output.address), + ); + + for address in addresses { + let key = AddressTxidKey::new(address, txid); + let entry = FilledTxEntry { + tx: filled_tx.clone(), + merkle_proof: merkle_proof.clone(), + block_hash, + block_height, + unix_stamp, + }; + self.txdb.put(rwtxn, &key, &entry)?; + } + } + Ok(()) + } + + pub fn get_txdb_entries_by_address( + &self, + rotxn: &RoTxn, + addresses: &HashSet
, + height_threshold: u32, + ) -> Result, Error> { + let mut entries = Vec::new(); + for address in addresses { + let start = AddressTxidKey::start(*address); + let end = AddressTxidKey::end(*address); + let mut iter = self + .txdb + .range(rotxn, &(start..=end)) + .map_err(DbError::from)?; + while let Some((_key, entry)) = + iter.next().map_err(DbError::from)? + { + if entry.block_height >= height_threshold { + entries.push(entry); + } + } + } + Ok(entries) + } + /// Store a block body. The header must already exist. pub fn put_body( &self, diff --git a/lib/authorization.rs b/lib/authorization.rs index 8467c43..1b7feda 100644 --- a/lib/authorization.rs +++ b/lib/authorization.rs @@ -7,8 +7,8 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use crate::types::{ - Address, AuthorizedTransaction, Body, GetAddress, Transaction, Verify, - VerifyingKey, + ADDRESS_SIZE, Address, AuthorizedTransaction, Body, GetAddress, + Transaction, Verify, VerifyingKey, }; pub use ed25519_dalek::{SignatureError, Signer, SigningKey, Verifier}; @@ -147,7 +147,7 @@ impl Verify for Authorization { pub fn get_address(verifying_key: &VerifyingKey) -> Address { let mut hasher = blake3::Hasher::new(); let mut reader = hasher.update(&verifying_key.to_bytes()).finalize_xof(); - let mut output: [u8; 20] = [0; 20]; + let mut output: [u8; ADDRESS_SIZE] = [0; ADDRESS_SIZE]; reader.fill(&mut output); Address(output) } diff --git a/lib/node/mod.rs b/lib/node/mod.rs index 0365d20..51deedc 100644 --- a/lib/node/mod.rs +++ b/lib/node/mod.rs @@ -519,7 +519,7 @@ where .try_get(&rotxn, &outpoint_key) .map_err(state::Error::from)? { - spent.push((*outpoint, output)); + spent.push((*outpoint, output.output)); } } Ok(spent) @@ -564,9 +564,14 @@ where pub fn get_utxos_by_addresses( &self, addresses: &HashSet
, + height_threshold: u32, ) -> Result, Error> { let rotxn = self.env.read_txn()?; - let utxos = self.state.get_utxos_by_addresses(&rotxn, addresses)?; + let utxos = self.state.get_utxos_by_addresses( + &rotxn, + addresses, + height_threshold, + )?; Ok(utxos) } diff --git a/lib/node/net_task.rs b/lib/node/net_task.rs index 37b3920..89414f0 100644 --- a/lib/node/net_task.rs +++ b/lib/node/net_task.rs @@ -5,7 +5,7 @@ use std::{ collections::{HashMap, HashSet}, net::SocketAddr, sync::Arc, - time::Duration, + time::{Duration, SystemTime, UNIX_EPOCH}, }; use error_fatality::{Nested as _, Split}; @@ -129,6 +129,15 @@ impl From for Error { } } +const TXDB_PRUNE_INTERVAL_BLOCKS: u32 = 144 * 7; +const TXDB_RETENTION_SECS: u64 = 28 * 24 * 60 * 60; + +fn unix_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |duration| duration.as_secs()) +} + fn connect_tip_( rwtxn: &mut RwTxn<'_>, archive: &Archive, @@ -139,18 +148,30 @@ fn connect_tip_( two_way_peg_data: &mainchain::TwoWayPegData, ) -> Result<(), Error> { let block_hash = header.hash(); + let prevalidated = state.prevalidate_block(rwtxn, header, body)?; + let block_height = prevalidated.next_height; + let merkle_root = prevalidated.computed_merkle_root; + let filled_txs = prevalidated.filled_transactions.clone(); + state.connect_prevalidated_block(rwtxn, header, body, prevalidated)?; if tracing::enabled!(tracing::Level::DEBUG) { - let merkle_root = - Body::compute_merkle_root(&body.coinbase, &body.transactions); - let height = state.try_get_height(rwtxn)?; - state.apply_block(rwtxn, header, body)?; - tracing::debug!(?height, %merkle_root, %block_hash, "connected body") - } else { - state.apply_block(rwtxn, header, body)?; + tracing::debug!(height = block_height, %merkle_root, %block_hash, "connected body") } let () = state.connect_two_way_peg_data(rwtxn, two_way_peg_data)?; let () = archive.put_header(rwtxn, header)?; let () = archive.put_body(rwtxn, block_hash, body)?; + let unix_stamp = unix_now(); + let () = archive.put_txdb_for_connected_block( + rwtxn, + block_hash, + block_height, + body, + &filled_txs, + unix_stamp, + )?; + if block_height > 0 && block_height % TXDB_PRUNE_INTERVAL_BLOCKS == 0 { + let cutoff_unix = unix_stamp.saturating_sub(TXDB_RETENTION_SECS); + let _ = archive.prune_txdb_older_than(rwtxn, cutoff_unix)?; + } for transaction in &body.transactions { let () = mempool.delete(rwtxn, transaction.txid())?; } @@ -250,6 +271,7 @@ fn disconnect_tip_( } }; let () = state.disconnect_two_way_peg_data(rwtxn, &two_way_peg_data)?; + let _ = archive.prune_txdb_block(rwtxn, tip_block_hash)?; let () = state.disconnect_tip(rwtxn, &tip_header, &tip_body)?; for transaction in tip_body.authorized_transactions().iter().rev() { mempool.put(rwtxn, transaction)?; diff --git a/lib/state/block.rs b/lib/state/block.rs index f99fcc4..7d40a19 100644 --- a/lib/state/block.rs +++ b/lib/state/block.rs @@ -6,8 +6,8 @@ use sneed::{RoTxn, RwTxn}; use crate::{ state::{Error, PrevalidatedBlock, State, amm, dutch_auction, error}, types::{ - AmountOverflowError, Authorization, BitAssetId, BlockHash, Body, - FilledOutput, FilledOutputContent, GetAddress as _, + AddressOutPointKey, AmountOverflowError, Authorization, BitAssetId, + BlockHash, Body, FilledOutput, FilledOutputContent, GetAddress as _, GetBitcoinValue as _, Hash, Header, InPoint, OutPoint, OutPointKey, OutputContent, SpentOutput, TxData, Verify as _, }, @@ -207,8 +207,6 @@ pub fn connect_prevalidated( .sum::() + body.coinbase.len(); - // Use Vec + sort_unstable instead of individual DB operations for better performance - let mut utxo_deletes: Vec = Vec::with_capacity(total_inputs); let mut stxo_puts: Vec<(OutPointKey, SpentOutput)> = Vec::with_capacity(total_inputs); let mut utxo_puts: Vec<(OutPointKey, FilledOutput)> = @@ -263,7 +261,6 @@ pub fn connect_prevalidated( vin: vin as u32, }, }; - utxo_deletes.push(key); stxo_puts.push((key, spent_output)); } @@ -358,21 +355,20 @@ pub fn connect_prevalidated( } // Sort all vectors in parallel for optimal cursor access - utxo_deletes.par_sort_unstable(); stxo_puts.par_sort_unstable_by_key(|(key, _)| *key); utxo_puts.par_sort_unstable_by_key(|(key, _)| *key); - // Apply all database operations using pre-sorted keys for optimal B-tree access - for key in &utxo_deletes { - state.utxos.delete(rwtxn, key)?; - } - - for (key, spent_output) in &stxo_puts { - state.stxos.put(rwtxn, key, spent_output)?; + for (key, output) in stxo_puts { + if !state.spend_utxo(rwtxn, key, output)? { + return Err(error::NoUtxo { + outpoint: key.to_outpoint(), + } + .into()); + } } - for (key, filled_output) in &utxo_puts { - state.utxos.put(rwtxn, key, filled_output)?; + for (key, filled_output) in utxo_puts { + state.put_utxo(rwtxn, key, filled_output, prevalidated.next_height)?; } // Update tip and height using precomputed values @@ -433,7 +429,7 @@ pub fn connect( content: filled_content, memo: output.memo.clone(), }; - state.utxos.put(rwtxn, &outpoint_key, &filled_output)?; + state.put_utxo(rwtxn, outpoint_key, filled_output, height)?; } for transaction in &body.transactions { let filled_tx = state.fill_transaction(rwtxn, transaction)?; @@ -451,20 +447,21 @@ pub fn connect( vin: vin as u32, }, }; - state.utxos.delete(rwtxn, &input_key)?; - state.stxos.put(rwtxn, &input_key, &spent_output)?; + if !state.spend_utxo(rwtxn, input_key, spent_output)? { + return Err(error::NoUtxo { outpoint: *input }.into()); + } } let Some(filled_outputs) = filled_tx.filled_outputs() else { let err = error::FillTxOutputContents(Box::new(filled_tx)); return Err(err.into()); }; - for (vout, filled_output) in filled_outputs.iter().enumerate() { + for (vout, filled_output) in filled_outputs.into_iter().enumerate() { let outpoint = OutPoint::Regular { txid, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - state.utxos.put(rwtxn, &outpoint_key, filled_output)?; + state.put_utxo(rwtxn, outpoint_key, filled_output, height)?; } match &transaction.data { None => (), @@ -646,13 +643,15 @@ pub fn disconnect_tip( } // delete UTXOs, last-to-first tx.outputs.iter().enumerate().rev().try_for_each( - |(vout, _output)| { + |(vout, output)| { let outpoint = OutPoint::Regular { txid, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(output.address, outpoint_key); + if state.delete_utxo(rwtxn, &address_key)? { Ok::<_, Error>(()) } else { Err(error::NoUtxo { outpoint }.into()) @@ -662,13 +661,7 @@ pub fn disconnect_tip( // unspend STXOs, last-to-first tx.inputs.iter().rev().try_for_each(|outpoint| { let outpoint_key = OutPointKey::from_outpoint(outpoint); - if let Some(spent_output) = - state.stxos.try_get(rwtxn, &outpoint_key)? - { - state.stxos.delete(rwtxn, &outpoint_key)?; - state - .utxos - .put(rwtxn, &outpoint_key, &spent_output.output)?; + if state.unspend_utxo(rwtxn, &outpoint_key)? { Ok(()) } else { Err(Error::NoStxo { @@ -678,20 +671,24 @@ pub fn disconnect_tip( }) })?; // delete coinbase UTXOs, last-to-first - body.coinbase.iter().enumerate().rev().try_for_each( - |(vout, _output)| { + body.coinbase + .iter() + .enumerate() + .rev() + .try_for_each(|(vout, output)| { let outpoint = OutPoint::Coinbase { merkle_root: header.merkle_root, vout: vout as u32, }; let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(output.address, outpoint_key); + if state.delete_utxo(rwtxn, &address_key)? { Ok::<_, Error>(()) } else { Err(error::NoUtxo { outpoint }.into()) } - }, - )?; + })?; match (header.prev_side_hash, height) { (None, 0) => { state.tip.delete(rwtxn, &())?; diff --git a/lib/state/mod.rs b/lib/state/mod.rs index d5a71e2..751369c 100644 --- a/lib/state/mod.rs +++ b/lib/state/mod.rs @@ -10,12 +10,12 @@ use sneed::{DatabaseUnique, RoDatabaseUnique, RoTxn, RwTxn, UnitKey}; use crate::{ authorization::Authorization, types::{ - Address, AmountOverflowError, Authorized, AuthorizedTransaction, - BitAssetId, BlockHash, Body, FilledOutput, FilledTransaction, - GetAddress as _, GetBitcoinValue as _, Header, InPoint, M6id, OutPoint, - OutPointKey, SpentOutput, Transaction, TxData, VERSION, Verify as _, - Version, WithdrawalBundle, WithdrawalBundleStatus, - proto::mainchain::TwoWayPegData, + Address, AddressOutPointKey, AmountOverflowError, Authorized, + AuthorizedTransaction, BitAssetId, BlockHash, Body, FilledOutput, + FilledTransaction, GetAddress as _, GetBitcoinValue as _, Header, + InPoint, M6id, OutPoint, OutPointKey, SpentOutput, Transaction, TxData, + VERSION, Verify as _, Version, WithdrawalBundle, + WithdrawalBundleStatus, proto::mainchain::TwoWayPegData, }, util::Watchable, }; @@ -68,6 +68,18 @@ type WithdrawalBundlesDb = DatabaseUnique< )>, >; +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct UtxoEntry { + pub created_height: u32, + pub output: FilledOutput, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct SpentUtxoEntry { + pub created_height: u32, + pub output: SpentOutput, +} + #[derive(Clone)] pub struct State { /// Current tip @@ -80,7 +92,9 @@ pub struct State { /// Associates Dutch auction sequence numbers with auction state dutch_auctions: dutch_auction::Db, utxos: DatabaseUnique>, - stxos: DatabaseUnique>, + utxos_by_address: + DatabaseUnique>, + stxos: DatabaseUnique>, /// Pending withdrawal bundle. MUST exist in withdrawal_bundles pending_withdrawal_bundle: DatabaseUnique>, /// Latest failed (known) withdrawal bundle @@ -104,7 +118,7 @@ pub struct State { } impl State { - pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12; + pub const NUM_DBS: u32 = bitassets::Dbs::NUM_DBS + 12 + 1; pub fn new(env: &sneed::Env) -> Result { let mut rwtxn = env.write_txn()?; @@ -115,6 +129,8 @@ impl State { let dutch_auctions = DatabaseUnique::create(env, &mut rwtxn, "dutch_auctions")?; let utxos = DatabaseUnique::create(env, &mut rwtxn, "utxos")?; + let utxos_by_address = + DatabaseUnique::create(env, &mut rwtxn, "utxos_by_address")?; let stxos = DatabaseUnique::create(env, &mut rwtxn, "stxos")?; let pending_withdrawal_bundle = DatabaseUnique::create( env, @@ -147,6 +163,7 @@ impl State { bitassets, dutch_auctions, utxos, + utxos_by_address, stxos, pending_withdrawal_bundle, latest_failed_withdrawal_bundle, @@ -180,10 +197,79 @@ impl State { pub fn stxos( &self, - ) -> &RoDatabaseUnique> { + ) -> &RoDatabaseUnique> { &self.stxos } + fn put_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: OutPointKey, + output: FilledOutput, + created_height: u32, + ) -> Result { + let entry = UtxoEntry { + created_height, + output, + }; + let address_key = + AddressOutPointKey::new(entry.output.address, outpoint_key); + self.utxos_by_address.put(rwtxn, &address_key, &entry)?; + self.utxos.put(rwtxn, &outpoint_key, &entry.output)?; + Ok(address_key) + } + + fn delete_utxo( + &self, + rwtxn: &mut RwTxn, + address_key: &AddressOutPointKey, + ) -> Result { + self.utxos_by_address.delete(rwtxn, &address_key)?; + Ok(self.utxos.delete(rwtxn, &address_key.outpoint_key())?) + } + + fn spend_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: OutPointKey, + spent_output: SpentOutput, + ) -> Result { + let address_key = + AddressOutPointKey::new(spent_output.output.address, outpoint_key); + + let Some(entry) = self.utxos_by_address.try_get(rwtxn, &address_key)? + else { + return Ok(false); + }; + + let entry = SpentUtxoEntry { + created_height: entry.created_height, + output: spent_output, + }; + + self.stxos.put(rwtxn, &outpoint_key, &entry)?; + self.delete_utxo(rwtxn, &address_key) + } + + fn unspend_utxo( + &self, + rwtxn: &mut RwTxn, + outpoint_key: &OutPointKey, + ) -> Result { + let Some(entry) = self.stxos.try_get(rwtxn, outpoint_key)? else { + return Ok(false); + }; + + self.put_utxo( + rwtxn, + *outpoint_key, + entry.output.output, + entry.created_height, + )?; + + Ok(self.stxos.delete(rwtxn, outpoint_key)?) + } + pub fn withdrawal_bundle_event_blocks( &self, ) -> &RoDatabaseUnique< @@ -222,13 +308,23 @@ impl State { &self, rotxn: &RoTxn, addresses: &HashSet
, + height_threshold: u32, ) -> Result, Error> { - let utxos: HashMap = self - .utxos - .iter(rotxn)? - .filter(|(_, output)| Ok(addresses.contains(&output.address))) - .map(|(key, output)| Ok((key.to_outpoint(), output))) - .collect()?; + let mut utxos = HashMap::new(); + for address in addresses { + let start = AddressOutPointKey::start(*address); + let end = AddressOutPointKey::end(*address); + let mut iter = self + .utxos_by_address + .range(rotxn, &(start..=end)) + .map_err(sneed::db::Error::from)?; + while let Some((key, entry)) = iter.next()? { + if entry.created_height >= height_threshold { + let outpoint = key.outpoint_key().to_outpoint(); + utxos.insert(outpoint, entry.output); + } + } + } Ok(utxos) } @@ -300,13 +396,13 @@ impl State { .try_get(rotxn, &key)? .ok_or(Error::NoStxo { outpoint: *input })?; assert_eq!( - stxo.inpoint, + stxo.output.inpoint, InPoint::Regular { txid, vin: vin as u32 } ); - spent_utxos.push(stxo.output); + spent_utxos.push(stxo.output.output); } spent_utxos.reverse(); Ok(FilledTransaction { @@ -656,7 +752,8 @@ impl State { let mut total_deposit_stxo_value = bitcoin::Amount::ZERO; let mut total_withdrawal_stxo_value = bitcoin::Amount::ZERO; self.stxos.iter(rotxn)?.map_err(Error::from).for_each( - |(outpoint_key, spent_output)| { + |(outpoint_key, spent_entry)| { + let spent_output = spent_entry.output; let outpoint = outpoint_key.to_outpoint(); if let OutPoint::Deposit(_) = outpoint { total_deposit_stxo_value = total_deposit_stxo_value diff --git a/lib/state/two_way_peg_data.rs b/lib/state/two_way_peg_data.rs index c68035a..8bdb810 100644 --- a/lib/state/two_way_peg_data.rs +++ b/lib/state/two_way_peg_data.rs @@ -12,10 +12,11 @@ use crate::{ rollback::{HeightStamped, RollBack}, }, types::{ - AggregatedWithdrawal, AmountOverflowError, FilledOutput, - FilledOutputContent, InPoint, M6id, OutPoint, OutPointKey, SpentOutput, - WithdrawalBundle, WithdrawalBundleEvent, WithdrawalBundleEventStatus, - WithdrawalBundleStatus, WithdrawalOutputContent, + AddressOutPointKey, AggregatedWithdrawal, AmountOverflowError, + FilledOutput, FilledOutputContent, InPoint, M6id, OutPoint, + OutPointKey, SpentOutput, WithdrawalBundle, WithdrawalBundleEvent, + WithdrawalBundleEventStatus, WithdrawalBundleStatus, + WithdrawalOutputContent, proto::mainchain::{BlockEvent, TwoWayPegData}, }, }; @@ -139,19 +140,13 @@ fn connect_withdrawal_bundle_submitted( return Err(err.into()); } }; - for (outpoint, spend_output) in bundle.spend_utxos() { - let outpoint_key = OutPointKey::from_outpoint(outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + for (outpoint, output) in bundle.spend_utxos() { + if !spend_withdrawal_utxo(state, rwtxn, m6id, outpoint, output)? { return Err(error::NoUtxo { outpoint: *outpoint, } .into()); - }; - let spent_output = SpentOutput { - output: spend_output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; + } } assert_eq!( bundle_status.latest().value, @@ -320,17 +315,15 @@ fn connect_withdrawal_bundle_confirmed( .map(|(key, output)| Ok((key.into(), output))) .collect()?; for (outpoint, output) in &utxos { - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put( - rwtxn, - &OutPointKey::from(outpoint), - &spent_output, - )?; + if !spend_withdrawal_utxo( + state, rwtxn, m6id, outpoint, output, + )? { + return Err(error::NoUtxo { + outpoint: *outpoint, + } + .into()); + } } - state.utxos.clear(rwtxn)?; bundle = WithdrawalBundleInfo::UnknownConfirmed { spend_utxos: utxos, }; @@ -355,8 +348,9 @@ fn connect_withdrawal_bundle_confirmed( "Unexpected withdrawal bundle confirmed, marking bundle UTXOs as spent" ); for (outpoint, output) in bundle.spend_utxos() { - let outpoint_key = OutPointKey::from(outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + if !spend_withdrawal_utxo( + state, rwtxn, m6id, outpoint, output, + )? { return Err( Error::UnexpectedWithdrawalBundleInsolvency { event_block_hash: *event_block_hash, @@ -365,11 +359,6 @@ fn connect_withdrawal_bundle_confirmed( }, ); } - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; } } } @@ -416,10 +405,13 @@ fn connect_withdrawal_bundle_failed( ) { break 'known; } - for (outpoint, output) in bundle.spend_utxos() { + for outpoint in bundle.spend_utxos().keys() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - state.stxos.delete(rwtxn, &outpoint_key)?; - state.utxos.put(rwtxn, &outpoint_key, output)?; + if !state.unspend_utxo(rwtxn, &outpoint_key)? { + return Err(Error::NoStxo { + outpoint: *outpoint, + }); + }; } let latest_failed_m6id = if let Some(mut latest_failed_m6id) = state.latest_failed_withdrawal_bundle.try_get(rwtxn, &())? @@ -499,7 +491,7 @@ fn connect_2wpd_event( let outpoint = OutPoint::Deposit(deposit.outpoint); let output = deposit.output.clone(); let outpoint_key = OutPointKey::from_outpoint(&outpoint); - state.utxos.put(rwtxn, &outpoint_key, &output)?; + state.put_utxo(rwtxn, outpoint_key, output, block_height)?; *latest_deposit_block_hash = Some(event_block_hash); } BlockEvent::WithdrawalBundle(withdrawal_bundle_event) => { @@ -641,14 +633,13 @@ fn disconnect_withdrawal_bundle_submitted( && bundle_status.latest().value == WithdrawalBundleStatus::Pending { - for (outpoint, output) in bundle.spend_utxos().iter().rev() { + for outpoint in bundle.spend_utxos().keys().rev() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; - state.utxos.put(rwtxn, &outpoint_key, output)?; + } } state.pending_withdrawal_bundle.put(rwtxn, &(), &m6id)?; } @@ -701,26 +692,24 @@ fn disconnect_withdrawal_bundle_confirmed( prev_bundle_status.latest().value, WithdrawalBundleStatus::SubmittedUnexpected ) { - for (outpoint, output) in bundle.spend_utxos() { + for outpoint in bundle.spend_utxos().keys() { let outpoint_key = OutPointKey::from(outpoint); - state.utxos.put(rwtxn, &outpoint_key, output)?; - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; + } } } } WithdrawalBundleInfo::UnknownConfirmed { spend_utxos } => { - for (outpoint, output) in spend_utxos { + for outpoint in spend_utxos.keys() { let outpoint_key = OutPointKey::from_outpoint(outpoint); - state.utxos.put(rwtxn, &outpoint_key, output)?; - if !state.stxos.delete(rwtxn, &outpoint_key)? { + if !state.unspend_utxo(rwtxn, &outpoint_key)? { return Err(Error::NoStxo { outpoint: *outpoint, }); - }; + } } bundle = WithdrawalBundleInfo::Unknown; } @@ -770,18 +759,13 @@ fn disconnect_withdrawal_bundle_failed( break 'known; } for (outpoint, output) in bundle.spend_utxos().iter().rev() { - let outpoint_key = OutPointKey::from_outpoint(outpoint); - let spent_output = SpentOutput { - output: output.clone(), - inpoint: InPoint::Withdrawal { m6id }, - }; - state.stxos.put(rwtxn, &outpoint_key, &spent_output)?; - if state.utxos.delete(rwtxn, &outpoint_key)? { + if !spend_withdrawal_utxo(state, rwtxn, m6id, outpoint, output)? + { return Err(error::NoUtxo { outpoint: *outpoint, } .into()); - }; + } } let (prev_latest_failed_m6id, latest_failed_m6id) = state .latest_failed_withdrawal_bundle @@ -809,6 +793,22 @@ fn disconnect_withdrawal_bundle_failed( Ok(()) } +fn spend_withdrawal_utxo( + state: &State, + rwtxn: &mut RwTxn, + m6id: M6id, + outpoint: &OutPoint, + output: &FilledOutput, +) -> Result { + let inpoint = InPoint::Withdrawal { m6id }; + let outpoint_key = OutPointKey::from_outpoint(outpoint); + let spent_output = SpentOutput { + output: output.clone(), + inpoint, + }; + state.spend_utxo(rwtxn, outpoint_key, spent_output) +} + fn disconnect_withdrawal_bundle_event( state: &State, rwtxn: &mut RwTxn, @@ -856,7 +856,9 @@ fn disconnect_event( BlockEvent::Deposit(deposit) => { let outpoint = OutPoint::Deposit(deposit.outpoint); let outpoint_key = OutPointKey::from_outpoint(&outpoint); - if !state.utxos.delete(rwtxn, &outpoint_key)? { + let address_key = + AddressOutPointKey::new(deposit.output.address, outpoint_key); + if !state.delete_utxo(rwtxn, &address_key)? { return Err(error::NoUtxo { outpoint }.into()); } *latest_deposit_block_hash = Some(event_block_hash); diff --git a/lib/types/address.rs b/lib/types/address.rs index b43c901..baa069e 100644 --- a/lib/types/address.rs +++ b/lib/types/address.rs @@ -7,11 +7,13 @@ use utoipa::ToSchema; use crate::types::THIS_SIDECHAIN; +pub const ADDRESS_SIZE: usize = 20; + #[derive(Debug, Error)] pub enum AddressParseError { #[error("bs58 error")] Bs58(#[from] bitcoin::base58::InvalidCharacterError), - #[error("wrong address length {0} != 20")] + #[error("wrong address length {0} != {ADDRESS_SIZE}")] WrongLength(usize), } @@ -20,10 +22,10 @@ pub enum AddressParseError { )] #[repr(transparent)] #[schema(value_type = String)] -pub struct Address(pub [u8; 20]); +pub struct Address(pub [u8; ADDRESS_SIZE]); impl Address { - pub const ALL_ZEROS: Self = Self([0; 20]); + pub const ALL_ZEROS: Self = Self([0; ADDRESS_SIZE]); pub fn as_base58(&self) -> String { bitcoin::base58::encode(&self.0) @@ -50,8 +52,8 @@ impl std::fmt::Debug for Address { } } -impl From<[u8; 20]> for Address { - fn from(other: [u8; 20]) -> Self { +impl From<[u8; ADDRESS_SIZE]> for Address { + fn from(other: [u8; ADDRESS_SIZE]) -> Self { Self(other) } } @@ -74,7 +76,8 @@ impl<'de> Deserialize<'de> for Address { if deserializer.is_human_readable() { DisplayFromStr::deserialize_as(deserializer) } else { - <[u8; 20] as Deserialize>::deserialize(deserializer).map(Self) + <[u8; ADDRESS_SIZE] as Deserialize>::deserialize(deserializer) + .map(Self) } } } diff --git a/lib/types/hashes.rs b/lib/types/hashes.rs index 18b87b3..8b1aa0d 100644 --- a/lib/types/hashes.rs +++ b/lib/types/hashes.rs @@ -147,6 +147,34 @@ impl utoipa::ToSchema for MerkleRoot { } } +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct MerkleProof { + pub leaf_index: usize, + pub siblings: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +pub struct MerkleProofNode { + pub hash: Hash, + pub is_left: bool, +} + +impl MerkleProof { + pub fn verify(&self, leaf: Hash, root: MerkleRoot) -> bool { + let mut hash = leaf; + + for sibling in &self.siblings { + hash = if sibling.is_left { + hash_with_scratch_buffer(&(sibling.hash, hash)) + } else { + hash_with_scratch_buffer(&(hash, sibling.hash)) + }; + } + + hash == Hash::from(root) + } +} + #[derive( BorshDeserialize, BorshSerialize, diff --git a/lib/types/mod.rs b/lib/types/mod.rs index dbf6b90..3c5622c 100644 --- a/lib/types/mod.rs +++ b/lib/types/mod.rs @@ -21,20 +21,20 @@ pub mod proto; pub mod schema; mod transaction; -pub use address::Address; +pub use address::{ADDRESS_SIZE, Address}; pub use bitasset_data::{BitAssetData, BitAssetDataUpdates, Update}; pub use hashes::{ - AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleRoot, - Txid, + AssetId, BitAssetId, BlockHash, DutchAuctionId, Hash, M6id, MerkleProof, + MerkleProofNode, MerkleRoot, Txid, }; pub use keys::{EncryptionPubKey, VerifyingKey}; pub use transaction::{ - AmmBurn, AmmMint, AmmSwap, AssetOutput, AssetOutputContent, Authorized, - AuthorizedTransaction, BitcoinOutput, BitcoinOutputContent, - DutchAuctionBid, DutchAuctionCollect, DutchAuctionParams, FilledOutput, - FilledOutputContent, FilledTransaction, InPoint, OutPoint, OutPointKey, - Output, OutputContent, PointedOutput, SpentOutput, Transaction, TxData, - TxInputs, WithdrawalOutputContent, + AddressOutPointKey, AddressTxidKey, AmmBurn, AmmMint, AmmSwap, AssetOutput, + AssetOutputContent, Authorized, AuthorizedTransaction, BitcoinOutput, + BitcoinOutputContent, DutchAuctionBid, DutchAuctionCollect, + DutchAuctionParams, FilledOutput, FilledOutputContent, FilledTransaction, + InPoint, OutPoint, OutPointKey, Output, OutputContent, PointedOutput, + SpentOutput, Transaction, TxData, TxInputs, WithdrawalOutputContent, }; pub const THIS_SIDECHAIN: u8 = 4; @@ -365,8 +365,62 @@ impl Body { coinbase: &[Output], txs: &[Transaction], ) -> MerkleRoot { - // FIXME: Compute actual merkle root instead of just a hash. - hashes::hash_with_scratch_buffer(&(coinbase, txs)).into() + let mut leaves = Vec::with_capacity(txs.len() + 1); + leaves.push(hashes::hash_with_scratch_buffer(&coinbase)); + leaves.extend(txs.iter().map(hashes::hash_with_scratch_buffer)); + Body::compute_cbmt_tree(&leaves)[0].into() + } + + // https://github.com/nervosnetwork/merkle-tree/blob/5d1898263e7167560fdaa62f09e8d52991a1c712/README.md#tree-struct + fn compute_cbmt_tree(leaves: &[Hash]) -> Vec { + let n = leaves.len(); + let mut nodes = vec![Hash::default(); 2 * n - 1]; + + nodes[n - 1..].copy_from_slice(leaves); + + for idx in (0..n - 1).rev() { + nodes[idx] = hashes::hash_with_scratch_buffer(&( + nodes[2 * idx + 1], + nodes[2 * idx + 2], + )); + } + + nodes + } + + pub fn compute_tx_merkle_proof( + coinbase: &[Output], + txs: &[Transaction], + tx_idx: usize, + ) -> MerkleProof { + let mut leaves = Vec::with_capacity(txs.len() + 1); + leaves.push(hashes::hash_with_scratch_buffer(&coinbase)); + leaves.extend(txs.iter().map(hashes::hash_with_scratch_buffer)); + Body::compute_cbmt_proof(&leaves, tx_idx + 1) + } + + fn compute_cbmt_proof(leaves: &[Hash], leaf_index: usize) -> MerkleProof { + let n = leaves.len(); + let nodes = Body::compute_cbmt_tree(leaves); + let mut idx = leaf_index + n - 1; + let mut siblings = Vec::new(); + + while idx > 0 { + let sibling_idx = (idx + 1) ^ 1; + let sibling_idx = sibling_idx - 1; + + siblings.push(MerkleProofNode { + hash: nodes[sibling_idx], + is_left: sibling_idx < idx, + }); + + idx = (idx - 1) / 2; + } + + MerkleProof { + leaf_index, + siblings, + } } pub fn get_inputs(&self) -> Vec { @@ -558,3 +612,51 @@ pub(crate) static VERSION: LazyLock = LazyLock::new(|| { const VERSION_STR: &str = env!("CARGO_PKG_VERSION"); semver::Version::parse(VERSION_STR).unwrap().into() }); + +#[cfg(test)] +mod merkle_tests { + use super::*; + + fn tx(memo: &[u8]) -> Transaction { + let mut tx = Transaction::default(); + tx.memo = memo.to_vec(); + tx + } + + #[test] + fn tx_merkle_proof_verifies_membership() { + let coinbase = Vec::new(); + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&txs[2]); + + assert!(proof.verify(leaf, root)); + } + + #[test] + fn tx_merkle_proof_rejects_non_member() { + let coinbase = Vec::new(); + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; + let other = tx(b"other"); + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&other); + + assert!(!proof.verify(leaf, root)); + } + + #[test] + fn tx_merkle_proof_rejects_wrong_position() { + let coinbase = Vec::new(); + let txs = vec![tx(b"tx-0"), tx(b"tx-1"), tx(b"tx-2"), tx(b"tx-3")]; + + let root = Body::compute_merkle_root(&coinbase, &txs); + let proof = Body::compute_tx_merkle_proof(&coinbase, &txs, 2); + let leaf = hashes::hash_with_scratch_buffer(&txs[1]); + + assert!(!proof.verify(leaf, root)); + } +} diff --git a/lib/types/transaction/mod.rs b/lib/types/transaction/mod.rs index dad7bfc..0af1b0c 100644 --- a/lib/types/transaction/mod.rs +++ b/lib/types/transaction/mod.rs @@ -1,5 +1,6 @@ use std::{ borrow::Borrow, + borrow::Cow, cmp::Ordering, collections::{HashMap, HashSet}, io::Cursor, @@ -17,7 +18,7 @@ use crate::{ types::{ AmountOverflowError, BitAssetData, BitAssetDataUpdates, GetAddress, GetBitcoinValue, - address::Address, + address::{ADDRESS_SIZE, Address}, hashes::{ self, AssetId, BitAssetId, DutchAuctionId, Hash, M6id, MerkleRoot, Txid, @@ -114,6 +115,7 @@ impl std::fmt::Display for OutPoint { } const OUTPOINT_KEY_SIZE: usize = 37; +const ADDRESS_OUTPOINT_KEY_SIZE: usize = ADDRESS_SIZE + OUTPOINT_KEY_SIZE; /// Fixed-width key for OutPoint based on its canonical Borsh encoding. #[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] @@ -202,8 +204,8 @@ impl<'a> BytesEncode<'a> for OutPointKey { #[inline] fn bytes_encode( item: &'a Self::EItem, - ) -> Result, BoxedError> { - Ok(std::borrow::Cow::Borrowed(item.as_ref())) + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) } } @@ -224,6 +226,164 @@ impl<'a> BytesDecode<'a> for OutPointKey { } } +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub struct AddressOutPointKey([u8; ADDRESS_OUTPOINT_KEY_SIZE]); + +impl AddressOutPointKey { + #[inline] + pub fn new(address: Address, outpoint: OutPointKey) -> Self { + let mut bytes = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + bytes[ADDRESS_SIZE..].copy_from_slice(outpoint.as_bytes()); + Self(bytes) + } + + #[inline] + pub fn start(address: Address) -> Self { + let mut bytes = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(bytes) + } + + #[inline] + pub fn end(address: Address) -> Self { + let mut bytes = [0xffu8; ADDRESS_OUTPOINT_KEY_SIZE]; + bytes[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(bytes) + } + + #[inline] + pub fn address(&self) -> Address { + let mut address = [0u8; ADDRESS_SIZE]; + address.copy_from_slice(&self.0[..ADDRESS_SIZE]); + Address(address) + } + + #[inline] + pub fn outpoint_key(&self) -> OutPointKey { + ::bytes_decode(&self.0[ADDRESS_SIZE..]) + .expect("AddressOutPointKey should contain a valid OutPointKey") + } +} + +impl Ord for AddressOutPointKey { + #[inline] + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl PartialOrd for AddressOutPointKey { + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl AsRef<[u8]> for AddressOutPointKey { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl<'a> BytesEncode<'a> for AddressOutPointKey { + type EItem = AddressOutPointKey; + + #[inline] + fn bytes_encode( + item: &'a Self::EItem, + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) + } +} + +impl<'a> BytesDecode<'a> for AddressOutPointKey { + type DItem = AddressOutPointKey; + + #[inline] + fn bytes_decode(bytes: &'a [u8]) -> Result { + ::bytes_decode(&bytes[ADDRESS_SIZE..])?; + let mut key = [0u8; ADDRESS_OUTPOINT_KEY_SIZE]; + key.copy_from_slice(bytes); + Ok(Self(key)) + } +} + +const TXID_SIZE: usize = blake3::OUT_LEN; +const ADDR_TXID_DB_KEY_SIZE: usize = ADDRESS_SIZE + TXID_SIZE; + +/// Fixed-width txdb key: address || txid. +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct AddressTxidKey([u8; ADDR_TXID_DB_KEY_SIZE]); + +impl AddressTxidKey { + #[inline] + pub fn new(address: Address, txid: Txid) -> Self { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + key[ADDRESS_SIZE..].copy_from_slice(txid.as_slice()); + Self(key) + } + + #[inline] + pub fn start(address: Address) -> Self { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(key) + } + + #[inline] + pub fn end(address: Address) -> Self { + let mut key = [0xffu8; ADDR_TXID_DB_KEY_SIZE]; + key[..ADDRESS_SIZE].copy_from_slice(&address.0); + Self(key) + } + + #[inline] + pub fn address(&self) -> Address { + let mut address = [0u8; ADDRESS_SIZE]; + address.copy_from_slice(&self.0[..ADDRESS_SIZE]); + Address(address) + } + + #[inline] + pub fn txid(&self) -> Txid { + let mut txid = [0u8; TXID_SIZE]; + txid.copy_from_slice(&self.0[ADDRESS_SIZE..]); + Txid(txid) + } +} + +impl AsRef<[u8]> for AddressTxidKey { + #[inline] + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl<'a> BytesEncode<'a> for AddressTxidKey { + type EItem = AddressTxidKey; + + #[inline] + fn bytes_encode( + item: &'a Self::EItem, + ) -> Result, BoxedError> { + Ok(Cow::Borrowed(item.as_ref())) + } +} + +impl<'a> BytesDecode<'a> for AddressTxidKey { + type DItem = AddressTxidKey; + + #[inline] + fn bytes_decode(bytes: &'a [u8]) -> Result { + let mut key = [0u8; ADDR_TXID_DB_KEY_SIZE]; + key.copy_from_slice(bytes); + Ok(Self(key)) + } +} + #[cfg(test)] mod tests { use super::{OUTPOINT_KEY_SIZE, OutPoint, OutPointKey};