Merge branch 'master' into alexaggh/remove_jaeger_everywhere

.github/workflows/cmd-tests.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  test:
+  test-cmd-bot:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.gitlab/pipeline/zombienet.yml

@@ -1,7 +1,7 @@
 .zombienet-refs:
   extends: .build-refs
   variables:
-    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.111"
+    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.112"
     PUSHGATEWAY_URL: "http://zombienet-prometheus-pushgateway.managed-monitoring:9091/metrics/job/zombie-metrics"
     DEBUG: "zombie,zombie::network-node,zombie::kube::client::logs"
     ZOMBIE_PROVIDER: "k8s"

Cargo.toml

@@ -1319,7 +1319,7 @@ tikv-jemalloc-ctl = { version = "0.5.0" }
 tikv-jemallocator = { version = "0.5.0" }
 time = { version = "0.3" }
 tiny-keccak = { version = "2.0.2" }
-tokio = { version = "1.37.0", default-features = false }
+tokio = { version = "1.40.0", default-features = false }
 tokio-retry = { version = "0.3.0" }
 tokio-stream = { version = "0.1.14" }
 tokio-test = { version = "0.4.2" }

bridges/relays/lib-substrate-relay/Cargo.toml

@@ -22,7 +22,6 @@ num-traits = { workspace = true, default-features = true }
 rbtag = { workspace = true }
 structopt = { workspace = true }
 strum = { features = ["derive"], workspace = true, default-features = true }
-rustc-hex = { workspace = true }
 thiserror = { workspace = true }
 
 # Bridge dependencies

bridges/relays/lib-substrate-relay/src/cli/bridge.rs

@@ -23,12 +23,9 @@ use crate::{
 	parachains::SubstrateParachainsPipeline,
 };
 use bp_parachains::{RelayBlockHash, RelayBlockHasher, RelayBlockNumber};
-use codec::{Codec, EncodeLike};
-use messages_relay::Labeled;
 use relay_substrate_client::{
 	Chain, ChainWithRuntimeVersion, ChainWithTransactions, Parachain, RelayChain,
 };
-use std::fmt::Debug;
 
 /// Minimal bridge representation that can be used from the CLI.
 /// It connects a source chain to a target chain.
@@ -102,22 +99,7 @@ where
 /// Bridge representation that can be used from the CLI for relaying messages.
 pub trait MessagesCliBridge: CliBridgeBase {
 	/// The Source -> Destination messages synchronization pipeline.
-	type MessagesLane: SubstrateMessageLane<
-		SourceChain = Self::Source,
-		TargetChain = Self::Target,
-		LaneId = Self::LaneId,
-	>;
-	/// Lane identifier type.
-	type LaneId: Clone
-		+ Copy
-		+ Debug
-		+ Codec
-		+ EncodeLike
-		+ Send
-		+ Sync
-		+ Labeled
-		+ TryFrom<Vec<u8>>
-		+ Default;
+	type MessagesLane: SubstrateMessageLane<SourceChain = Self::Source, TargetChain = Self::Target>;
 
 	/// Optional messages delivery transaction limits that the messages relay is going
 	/// to use. If it returns `None`, limits are estimated using `TransactionPayment` API
@@ -128,4 +110,5 @@ pub trait MessagesCliBridge: CliBridgeBase {
 }
 
 /// An alias for lane identifier type.
-pub type MessagesLaneIdOf<B> = <B as MessagesCliBridge>::LaneId;
+pub type MessagesLaneIdOf<B> =
+	<<B as MessagesCliBridge>::MessagesLane as SubstrateMessageLane>::LaneId;

bridges/relays/lib-substrate-relay/src/cli/relay_headers_and_messages/mod.rs

@@ -31,7 +31,6 @@ pub mod relay_to_relay;
 pub mod relay_to_parachain;
 
 use async_trait::async_trait;
-use codec::{Codec, EncodeLike};
 use std::{fmt::Debug, marker::PhantomData, sync::Arc};
 use structopt::StructOpt;
 
@@ -47,7 +46,6 @@ use crate::{
 	HeadersToRelay, TaggedAccount, TransactionParams,
 };
 use bp_runtime::BalanceOf;
-use messages_relay::Labeled;
 use relay_substrate_client::{
 	AccountIdOf, AccountKeyPairOf, Chain, ChainWithBalances, ChainWithMessages,
 	ChainWithRuntimeVersion, ChainWithTransactions,
@@ -239,20 +237,9 @@ where
 		+ ChainWithRuntimeVersion;
 
 	/// Left to Right bridge.
-	type L2R: MessagesCliBridge<Source = Self::Left, Target = Self::Right, LaneId = Self::LaneId>;
+	type L2R: MessagesCliBridge<Source = Self::Left, Target = Self::Right>;
 	/// Right to Left bridge
-	type R2L: MessagesCliBridge<Source = Self::Right, Target = Self::Left, LaneId = Self::LaneId>;
-	/// Lane identifier type.
-	type LaneId: Clone
-		+ Copy
-		+ Debug
-		+ Codec
-		+ EncodeLike
-		+ Send
-		+ Sync
-		+ Labeled
-		+ TryFrom<Vec<u8>>
-		+ Default;
+	type R2L: MessagesCliBridge<Source = Self::Right, Target = Self::Left>;
 
 	/// Construct new bridge.
 	fn new(params: <Self::Base as Full2WayBridgeBase>::Params) -> anyhow::Result<Self>;
@@ -303,7 +290,7 @@ where
 			self.mut_base().start_on_demand_headers_relayers().await?;
 
 		// add balance-related metrics
-		let lanes: Vec<Self::LaneId> = self
+		let lanes_l2r: Vec<MessagesLaneIdOf<Self::L2R>> = self
 			.base()
 			.common()
 			.shared
@@ -312,28 +299,48 @@ where
 			.cloned()
 			.map(HexLaneId::try_convert)
 			.collect::<Result<Vec<_>, HexLaneId>>()
-			.expect("");
+			.map_err(|e| {
+				anyhow::format_err!("Conversion failed for L2R lanes with error: {:?}!", e)
+			})?;
+		let lanes_r2l: Vec<MessagesLaneIdOf<Self::R2L>> = self
+			.base()
+			.common()
+			.shared
+			.lane
+			.iter()
+			.cloned()
+			.map(HexLaneId::try_convert)
+			.collect::<Result<Vec<_>, HexLaneId>>()
+			.map_err(|e| {
+				anyhow::format_err!("Conversion failed for R2L lanes with error: {:?}!", e)
+			})?;
 		{
 			let common = self.mut_base().mut_common();
 			crate::messages::metrics::add_relay_balances_metrics::<
 				_,
 				Self::Right,
 				MessagesLaneIdOf<Self::L2R>,
-			>(common.left.client.clone(), &common.metrics_params, &common.left.accounts, &lanes)
+			>(
+				common.left.client.clone(), &common.metrics_params, &common.left.accounts, &lanes_l2r
+			)
 			.await?;
 			crate::messages::metrics::add_relay_balances_metrics::<
 				_,
 				Self::Left,
 				MessagesLaneIdOf<Self::R2L>,
 			>(
-				common.right.client.clone(), &common.metrics_params, &common.right.accounts, &lanes
+				common.right.client.clone(),
+				&common.metrics_params,
+				&common.right.accounts,
+				&lanes_r2l,
 			)
 			.await?;
 		}
 
 		// Need 2x capacity since we consider both directions for each lane
-		let mut message_relays = Vec::with_capacity(lanes.len() * 2);
-		for lane in lanes {
+		let mut message_relays =
+			Vec::with_capacity(lanes_l2r.len().saturating_add(lanes_r2l.len()));
+		for lane in lanes_l2r {
 			let left_to_right_messages =
 				crate::messages::run::<<Self::L2R as MessagesCliBridge>::MessagesLane, _, _>(
 					self.left_to_right().messages_relay_params(
@@ -346,7 +353,8 @@ where
 				.map_err(|e| anyhow::format_err!("{}", e))
 				.boxed();
 			message_relays.push(left_to_right_messages);
-
+		}
+		for lane in lanes_r2l {
 			let right_to_left_messages =
 				crate::messages::run::<<Self::R2L as MessagesCliBridge>::MessagesLane, _, _>(
 					self.right_to_left().messages_relay_params(

prdoc/pr_5741.prdoc

@@ -0,0 +1,25 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: make RPC endpoint `chainHead_v1_storage` faster
+
+doc:
+  - audience: Node Operator
+    description: |
+      The RPC endpoint `chainHead_v1_storage` now relies solely on backpressure to
+      determine how quickly to serve back values instead of handing back a fixed number
+      of entries and then expecting the client to ask for more. This should improve the 
+      throughput for bigger storage queries significantly.
+
+      Benchmarks using subxt on localhost:
+        - Iterate over 10 accounts on westend-dev -> ~2-3x faster
+        - Fetch 1024 storage values (i.e, not descedant values) -> ~50x faster
+        - Fetch 1024 descendant values -> ~500x faster
+
+crates: 
+  - name: sc-rpc-spec-v2
+    bump: major
+  - name: sc-rpc-server
+    bump: patch
+  - name: sc-service
+    bump: major

prdoc/pr_5845.prdoc

@@ -0,0 +1,15 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Fix compilation after renaming some of benchmarks in pallet_revive.
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      Changed the "instr" benchmark so that it should no longer return to little weight. It is still bogus but at least benchmarking should not work.
+
+crates:
+  - name: pallet-revive
+    bump: patch
+  - name: pallet-revive-fixtures 
+    bump: major

substrate/client/rpc-servers/src/lib.rs

@@ -255,8 +255,9 @@ where
 							),
 						};
 
-						let rpc_middleware =
-							RpcServiceBuilder::new().option_layer(middleware_layer.clone());
+						let rpc_middleware = RpcServiceBuilder::new()
+							.rpc_logger(1024)
+							.option_layer(middleware_layer.clone());
 						let mut svc = service_builder
 							.set_rpc_middleware(rpc_middleware)
 							.build(methods, stop_handle);

substrate/client/rpc-spec-v2/Cargo.toml

@@ -28,7 +28,6 @@ sp-rpc = { workspace = true, default-features = true }
 sp-blockchain = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }
 sc-client-api = { workspace = true, default-features = true }
-sc-utils = { workspace = true, default-features = true }
 sc-rpc = { workspace = true, default-features = true }
 codec = { workspace = true, default-features = true }
 thiserror = { workspace = true }
@@ -56,6 +55,8 @@ sp-externalities = { workspace = true, default-features = true }
 sp-maybe-compressed-blob = { workspace = true, default-features = true }
 sc-block-builder = { workspace = true, default-features = true }
 sc-service = { features = ["test-helpers"], workspace = true, default-features = true }
+sc-rpc = { workspace = true, default-features = true, features = ["test-helpers"] }
 assert_matches = { workspace = true }
 pretty_assertions = { workspace = true }
 sc-transaction-pool = { workspace = true, default-features = true }
+sc-utils = { workspace = true, default-features = true }

substrate/client/rpc-spec-v2/src/archive/archive.rs

@@ -275,6 +275,7 @@ where
 			self.storage_max_descendant_responses,
 			self.storage_max_queried_items,
 		);
+
 		Ok(storage_client.handle_query(hash, items, child_trie))
 	}
 }