From 13413b7f417e3b87b2801dd068c72e95b4e966e7 Mon Sep 17 00:00:00 2001
From: Ermal Kaleci <ermalkaleci@gmail.com>
Date: Tue, 8 Jun 2021 00:16:28 +0200
Subject: [PATCH] Bencher update (#507)
* update weight template
* ignore resources consumed by benched submethods
* * refactor bencher / no need for build script
* add wasm_builder for building benches into wasm
* colorized output
* update docs
* remove cfg(test)
* clippy
* clippy
* wasm handle panic
* generate bench tests
* update docs
* fix features
* update the way benches are defined
* fix reset reduntant meter
* fix repeat-reads/writes
* cleanup
---
bencher/Cargo.toml | 22 +
bencher/src/bench_runner.rs | 28 +-
bencher/src/build_wasm/mod.rs | 64 ++
bencher/src/build_wasm/prerequisites.rs | 301 ++++++++
bencher/src/build_wasm/wasm_project.rs | 709 ++++++++++++++++++
bencher/src/colorize.rs | 39 +
bencher/src/handler.rs | 30 +-
bencher/src/lib.rs | 176 ++++-
bencher/src/macros.rs | 182 ++---
bencher/src/redundant_meter.rs | 111 +++
weight-gen/src/template.hbs | 4 +-
weight-meter/Cargo.toml | 6 +
weight-meter/src/lib.rs | 2 -
.../weight-meter-procedural/Cargo.toml | 3 +-
.../weight-meter-procedural/src/lib.rs | 17 +
15 files changed, 1554 insertions(+), 140 deletions(-)
create mode 100644 bencher/src/build_wasm/mod.rs
create mode 100644 bencher/src/build_wasm/prerequisites.rs
create mode 100644 bencher/src/build_wasm/wasm_project.rs
create mode 100644 bencher/src/colorize.rs
create mode 100644 bencher/src/redundant_meter.rs
diff --git a/bencher/Cargo.toml b/bencher/Cargo.toml
index d64711b..0f95818 100644
--- a/bencher/Cargo.toml
+++ b/bencher/Cargo.toml
@@ -8,6 +8,15 @@ authors = ["Laminar Developers <hello@laminar.one>"]
edition = "2018"
[dependencies]
+paste = "1.0"
+build-helper = { version = "0.1.1", optional = true }
+cargo_metadata = { version = "0.13.1", optional = true }
+tempfile = { version = "3.1.0", optional = true }
+toml = { version = "0.5.4", optional = true }
+walkdir = { version = "2.3.1", optional = true }
+ansi_term = { version = "0.12.1", optional = true }
+wasm-gc-api = { version = "0.1.11", optional = true }
+rand = {version = "0.8.3", optional = true }
linregress = { version = "0.4.0", optional = true }
serde = { version = "1.0.119", optional = true, features = ['derive'] }
serde_json = {version = "1.0.64", optional = true }
@@ -20,11 +29,20 @@ sp-state-machine = { git = "https://github.com/paritytech/substrate", branch = "
sc-executor = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false, features = ["wasmtime"], optional = true }
sc-executor-common = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", optional = true }
sc-client-db = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false, features = ["with-kvdb-rocksdb"], optional = true }
+sp-maybe-compressed-blob = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false, optional = true }
frame-benchmarking = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false }
[features]
default = ["std"]
std = [
+ "build-helper",
+ "cargo_metadata",
+ "tempfile",
+ "toml",
+ "walkdir",
+ "ansi_term",
+ "wasm-gc-api",
+ "rand",
"linregress",
"serde/std",
"serde_json/std",
@@ -37,5 +55,9 @@ std = [
"sc-executor/std",
"sc-executor-common",
"sc-client-db",
+ "sp-maybe-compressed-blob",
"frame-benchmarking/std",
]
+bench = [
+ "sp-io/disable_panic_handler"
+]
diff --git a/bencher/src/bench_runner.rs b/bencher/src/bench_runner.rs
index 1dc7fff..a667100 100644
--- a/bencher/src/bench_runner.rs
+++ b/bencher/src/bench_runner.rs
@@ -1,22 +1,18 @@
-use frame_benchmarking::{
- benchmarking,
- frame_support::sp_runtime::traits::{Block, NumberFor},
-};
-use sc_client_db::BenchmarkingState;
+use frame_benchmarking::frame_support::sp_runtime::traits::{Block, NumberFor};
use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutionMethod, WasmExecutor};
use sc_executor_common::runtime_blob::RuntimeBlob;
-use sp_io::SubstrateHostFunctions;
use sp_state_machine::{Ext, OverlayedChanges, StorageTransactionCache};
/// Run benches
-pub fn run<B: Block>(wasm_code: Vec<u8>) -> Vec<u8> {
+pub fn run<B: Block>(wasm_code: Vec<u8>) -> std::result::Result<Vec<u8>, String> {
let mut overlay = OverlayedChanges::default();
let mut cache = StorageTransactionCache::default();
- let state = BenchmarkingState::<B>::new(Default::default(), Default::default(), false).unwrap();
+ let state = sc_client_db::BenchmarkingState::<B>::new(Default::default(), Default::default(), false).unwrap();
let mut ext = Ext::<_, NumberFor<B>, _>::new(&mut overlay, &mut cache, &state, None, None);
- let mut host_functions = benchmarking::HostFunctions::host_functions();
- host_functions.append(&mut SubstrateHostFunctions::host_functions());
+ let mut host_functions = sp_io::SubstrateHostFunctions::host_functions();
+ host_functions.append(&mut frame_benchmarking::benchmarking::HostFunctions::host_functions());
+ host_functions.append(&mut super::bencher::HostFunctions::host_functions());
let executor = WasmExecutor::new(
WasmExecutionMethod::Compiled,
@@ -26,13 +22,7 @@ pub fn run<B: Block>(wasm_code: Vec<u8>) -> Vec<u8> {
None,
);
- executor
- .uncached_call(
- RuntimeBlob::uncompress_if_needed(&wasm_code[..]).unwrap(),
- &mut ext,
- true,
- "run_benches",
- &[],
- )
- .unwrap()
+ let blob = RuntimeBlob::uncompress_if_needed(&wasm_code[..]).unwrap();
+
+ executor.uncached_call(blob, &mut ext, true, "run_benches", &[])
}
diff --git a/bencher/src/build_wasm/mod.rs b/bencher/src/build_wasm/mod.rs
new file mode 100644
index 0000000..eca9348
--- /dev/null
+++ b/bencher/src/build_wasm/mod.rs
@@ -0,0 +1,64 @@
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+
+pub mod prerequisites;
+pub mod wasm_project;
+
+/// Environment variable to disable color output of the wasm build.
+const WASM_BUILD_NO_COLOR: &str = "WASM_BUILD_NO_COLOR";
+
+/// Returns `true` when color output is enabled.
+pub fn color_output_enabled() -> bool {
+ std::env::var(WASM_BUILD_NO_COLOR).is_err()
+}
+
+pub fn build() -> std::io::Result<Vec<u8>> {
+ let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap();
+ let pkg_name = std::env::var("CARGO_PKG_NAME").unwrap();
+
+ let random = thread_rng()
+ .sample_iter(&Alphanumeric)
+ .take(16)
+ .map(char::from)
+ .collect::<String>();
+
+ let mut out_dir = std::path::PathBuf::from(manifest_dir);
+ out_dir.push(format!("target/release/build/{}-{}/out", pkg_name, random));
+
+ std::env::set_var("OUT_DIR", out_dir.display().to_string());
+
+ let mut project_cargo_toml = std::env::current_dir()?;
+ project_cargo_toml.push("Cargo.toml");
+
+ let default_rustflags = "-Clink-arg=--export=__heap_base -C link-arg=--import-memory";
+ let cargo_cmd = match prerequisites::check() {
+ Ok(cmd) => cmd,
+ Err(err_msg) => {
+ eprintln!("{}", err_msg);
+ std::process::exit(1);
+ }
+ };
+
+ let (wasm_binary, bloaty) = wasm_project::create_and_compile(
+ &project_cargo_toml,
+ &default_rustflags,
+ cargo_cmd,
+ vec!["bench".to_string()],
+ None,
+ );
+
+ let (wasm_binary, _wasm_binary_bloaty) = if let Some(wasm_binary) = wasm_binary {
+ (
+ wasm_binary.wasm_binary_path_escaped(),
+ bloaty.wasm_binary_bloaty_path_escaped(),
+ )
+ } else {
+ (
+ bloaty.wasm_binary_bloaty_path_escaped(),
+ bloaty.wasm_binary_bloaty_path_escaped(),
+ )
+ };
+
+ let bytes = std::fs::read(wasm_binary)?;
+
+ Ok(bytes.to_vec())
+}
diff --git a/bencher/src/build_wasm/prerequisites.rs b/bencher/src/build_wasm/prerequisites.rs
new file mode 100644
index 0000000..fc09c4d
--- /dev/null
+++ b/bencher/src/build_wasm/prerequisites.rs
@@ -0,0 +1,301 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use crate::colorize::{color_output_enabled, red_bold, yellow_bold};
+use std::{
+ env, fs,
+ io::BufRead,
+ path::{Path, PathBuf},
+ process::Command,
+};
+
+/// Environment variable to set the toolchain used to compile the wasm binary.
+pub const WASM_BUILD_TOOLCHAIN: &str = "WASM_BUILD_TOOLCHAIN";
+
+/// Write to the given `file` if the `content` is different.
+pub fn write_file_if_changed(file: impl AsRef<Path>, content: impl AsRef<str>) {
+ if fs::read_to_string(file.as_ref()).ok().as_deref() != Some(content.as_ref()) {
+ fs::write(file.as_ref(), content.as_ref())
+ .unwrap_or_else(|_| panic!("Writing `{}` can not fail!", file.as_ref().display()));
+ }
+}
+
+/// Copy `src` to `dst` if the `dst` does not exist or is different.
+pub fn copy_file_if_changed(src: PathBuf, dst: PathBuf) {
+ let src_file = fs::read_to_string(&src).ok();
+ let dst_file = fs::read_to_string(&dst).ok();
+
+ if src_file != dst_file {
+ fs::copy(&src, &dst)
+ .unwrap_or_else(|_| panic!("Copying `{}` to `{}` can not fail; qed", src.display(), dst.display()));
+ }
+}
+
+/// Get a cargo command that compiles with nightly
+fn get_nightly_cargo() -> CargoCommand {
+ let env_cargo = CargoCommand::new(&env::var("CARGO").expect("`CARGO` env variable is always set by cargo"));
+ let default_cargo = CargoCommand::new("cargo");
+ let rustup_run_nightly = CargoCommand::new_with_args("rustup", &["run", "nightly", "cargo"]);
+ let wasm_toolchain = env::var(WASM_BUILD_TOOLCHAIN).ok();
+
+ // First check if the user requested a specific toolchain
+ if let Some(cmd) = wasm_toolchain.and_then(|t| get_rustup_nightly(Some(t))) {
+ cmd
+ } else if env_cargo.is_nightly() {
+ env_cargo
+ } else if default_cargo.is_nightly() {
+ default_cargo
+ } else if rustup_run_nightly.is_nightly() {
+ rustup_run_nightly
+ } else {
+ // If no command before provided us with a nightly compiler, we try to search
+ // one with rustup. If that fails as well, we return the default cargo and let
+ // the prequisities check fail.
+ get_rustup_nightly(None).unwrap_or(default_cargo)
+ }
+}
+
+/// Get a nightly from rustup. If `selected` is `Some(_)`, a `CargoCommand`
+/// using the given nightly is returned.
+fn get_rustup_nightly(selected: Option<String>) -> Option<CargoCommand> {
+ let host = format!("-{}", env::var("HOST").expect("`HOST` is always set by cargo"));
+
+ let version = match selected {
+ Some(selected) => selected,
+ None => {
+ let output = Command::new("rustup")
+ .args(&["toolchain", "list"])
+ .output()
+ .ok()?
+ .stdout;
+ let lines = output.as_slice().lines();
+
+ let mut latest_nightly = None;
+ for line in lines.filter_map(|l| l.ok()) {
+ if line.starts_with("nightly-") && line.ends_with(&host) {
+ // Rustup prints them sorted
+ latest_nightly = Some(line.clone());
+ }
+ }
+
+ latest_nightly?.trim_end_matches(&host).into()
+ }
+ };
+
+ Some(CargoCommand::new_with_args("rustup", &["run", &version, "cargo"]))
+}
+
+/// Wraps a specific command which represents a cargo invocation.
+#[derive(Debug)]
+pub struct CargoCommand {
+ program: String,
+ args: Vec<String>,
+}
+
+impl CargoCommand {
+ fn new(program: &str) -> Self {
+ CargoCommand {
+ program: program.into(),
+ args: Vec::new(),
+ }
+ }
+
+ fn new_with_args(program: &str, args: &[&str]) -> Self {
+ CargoCommand {
+ program: program.into(),
+ args: args.iter().map(ToString::to_string).collect(),
+ }
+ }
+
+ pub fn command(&self) -> Command {
+ let mut cmd = Command::new(&self.program);
+ cmd.args(&self.args);
+ cmd
+ }
+
+ /// Check if the supplied cargo command is a nightly version
+ fn is_nightly(&self) -> bool {
+ // `RUSTC_BOOTSTRAP` tells a stable compiler to behave like a nightly. So, when
+ // this env variable is set, we can assume that whatever rust compiler we have,
+ // it is a nightly compiler. For "more" information, see:
+ // https://github.com/rust-lang/rust/blob/fa0f7d0080d8e7e9eb20aa9cbf8013f96c81287f/src/libsyntax/feature_gate/check.rs#L891
+ env::var("RUSTC_BOOTSTRAP").is_ok()
+ || self
+ .command()
+ .arg("--version")
+ .output()
+ .map_err(|_| ())
+ .and_then(|o| String::from_utf8(o.stdout).map_err(|_| ()))
+ .unwrap_or_default()
+ .contains("-nightly")
+ }
+}
+
+/// Wraps a [`CargoCommand`] and the version of `rustc` the cargo command uses.
+pub struct CargoCommandVersioned {
+ command: CargoCommand,
+ version: String,
+}
+
+impl CargoCommandVersioned {
+ fn new(command: CargoCommand, version: String) -> Self {
+ Self { command, version }
+ }
+
+ /// Returns the `rustc` version.
+ pub fn rustc_version(&self) -> &str {
+ &self.version
+ }
+}
+
+impl std::ops::Deref for CargoCommandVersioned {
+ type Target = CargoCommand;
+
+ fn deref(&self) -> &CargoCommand {
+ &self.command
+ }
+}
+
+use tempfile::tempdir;
+
+/// Checks that all prerequisites are installed.
+///
+/// Returns the versioned cargo command on success.
+pub fn check() -> Result<CargoCommandVersioned, String> {
+ let cargo_command = get_nightly_cargo();
+
+ if !cargo_command.is_nightly() {
+ return Err(red_bold("Rust nightly not installed, please install it!"));
+ }
+
+ check_wasm_toolchain_installed(cargo_command)
+}
+
+/// Create the project that will be used to check that the wasm toolchain is
+/// installed and to extract the rustc version.
+fn create_check_toolchain_project(project_dir: &Path) {
+ let lib_rs_file = project_dir.join("src/lib.rs");
+ let main_rs_file = project_dir.join("src/main.rs");
+ let build_rs_file = project_dir.join("build.rs");
+ let manifest_path = project_dir.join("Cargo.toml");
+
+ write_file_if_changed(
+ &manifest_path,
+ r#"
+ [package]
+ name = "wasm-test"
+ version = "1.0.0"
+ edition = "2018"
+ build = "build.rs"
+
+ [lib]
+ name = "wasm_test"
+ crate-type = ["cdylib"]
+
+ [workspace]
+ "#,
+ );
+ write_file_if_changed(lib_rs_file, "pub fn test() {}");
+
+ // We want to know the rustc version of the rustc that is being used by our
+ // cargo command. The cargo command is determined by some *very* complex
+ // algorithm to find the cargo command that supports nightly.
+ // The best solution would be if there is a `cargo rustc --version` command,
+ // which sadly doesn't exists. So, the only available way of getting the rustc
+ // version is to build a project and capture the rustc version in this build
+ // process. This `build.rs` is exactly doing this. It gets the rustc version by
+ // calling `rustc --version` and exposing it in the `RUSTC_VERSION` environment
+ // variable.
+ write_file_if_changed(
+ build_rs_file,
+ r#"
+ fn main() {
+ let rustc_cmd = std::env::var("RUSTC").ok().unwrap_or_else(|| "rustc".into());
+
+ let rustc_version = std::process::Command::new(rustc_cmd)
+ .arg("--version")
+ .output()
+ .ok()
+ .and_then(|o| String::from_utf8(o.stdout).ok());
+
+ println!(
+ "cargo:rustc-env=RUSTC_VERSION={}",
+ rustc_version.unwrap_or_else(|| "unknown rustc version".into()),
+ );
+ }
+ "#,
+ );
+ // Just prints the `RURSTC_VERSION` environment variable that is being created
+ // by the `build.rs` script.
+ write_file_if_changed(
+ main_rs_file,
+ r#"
+ fn main() {
+ println!("{}", env!("RUSTC_VERSION"));
+ }
+ "#,
+ );
+}
+
+fn check_wasm_toolchain_installed(cargo_command: CargoCommand) -> Result<CargoCommandVersioned, String> {
+ let temp = tempdir().expect("Creating temp dir does not fail; qed");
+ fs::create_dir_all(temp.path().join("src")).expect("Creating src dir does not fail; qed");
+ create_check_toolchain_project(temp.path());
+
+ let err_msg = red_bold("Rust WASM toolchain not installed, please install it!");
+ let manifest_path = temp.path().join("Cargo.toml").display().to_string();
+
+ let mut build_cmd = cargo_command.command();
+ build_cmd.args(&[
+ "build",
+ "--target=wasm32-unknown-unknown",
+ "--manifest-path",
+ &manifest_path,
+ ]);
+
+ if color_output_enabled() {
+ build_cmd.arg("--color=always");
+ }
+
+ let mut run_cmd = cargo_command.command();
+ run_cmd.args(&["run", "--manifest-path", &manifest_path]);
+
+ build_cmd.output().map_err(|_| err_msg.clone()).and_then(|s| {
+ if s.status.success() {
+ let version = run_cmd.output().ok().and_then(|o| String::from_utf8(o.stdout).ok());
+ Ok(CargoCommandVersioned::new(
+ cargo_command,
+ version.unwrap_or_else(|| "unknown rustc version".into()),
+ ))
+ } else {
+ match String::from_utf8(s.stderr) {
+ Ok(ref err) if err.contains("linker `rust-lld` not found") => {
+ Err(red_bold("`rust-lld` not found, please install it!"))
+ }
+ Ok(ref err) => Err(format!(
+ "{}\n\n{}\n{}\n{}{}\n",
+ err_msg,
+ yellow_bold("Further error information:"),
+ yellow_bold(&"-".repeat(60)),
+ err,
+ yellow_bold(&"-".repeat(60)),
+ )),
+ Err(_) => Err(err_msg),
+ }
+ }
+ })
+}
diff --git a/bencher/src/build_wasm/wasm_project.rs b/bencher/src/build_wasm/wasm_project.rs
new file mode 100644
index 0000000..c924fb0
--- /dev/null
+++ b/bencher/src/build_wasm/wasm_project.rs
@@ -0,0 +1,709 @@
+// This file is part of Substrate.
+
+// Copyright (C) 2019-2021 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![allow(clippy::option_map_unit_fn)]
+
+use super::{
+ color_output_enabled,
+ prerequisites::{copy_file_if_changed, write_file_if_changed, CargoCommandVersioned},
+};
+
+use std::{
+ borrow::ToOwned,
+ collections::HashSet,
+ env, fs,
+ hash::{Hash, Hasher},
+ ops::Deref,
+ path::{Path, PathBuf},
+ process,
+};
+
+use toml::value::Table;
+
+use build_helper::rerun_if_changed;
+
+use cargo_metadata::{Metadata, MetadataCommand};
+
+/// Environment variable that tells us to skip building the wasm binary.
+const SKIP_BUILD_ENV: &str = "SKIP_WASM_BUILD";
+
+/// Environment variable to force a certain build type when building the wasm
+/// binary. Expects "debug" or "release" as value.
+///
+/// By default the WASM binary uses the same build type as the main cargo build.
+const WASM_BUILD_TYPE_ENV: &str = "WASM_BUILD_TYPE";
+
+/// Environment variable to extend the `RUSTFLAGS` variable given to the wasm
+/// build.
+const WASM_BUILD_RUSTFLAGS_ENV: &str = "WASM_BUILD_RUSTFLAGS";
+
+/// Environment variable to set the target directory to copy the final wasm
+/// binary.
+///
+/// The directory needs to be an absolute path.
+const WASM_TARGET_DIRECTORY: &str = "WASM_TARGET_DIRECTORY";
+
+/// Colorize an info message.
+///
+/// Returns the colorized message.
+fn colorize_info_message(message: &str) -> String {
+ if color_output_enabled() {
+ ansi_term::Color::Yellow.bold().paint(message).to_string()
+ } else {
+ message.into()
+ }
+}
+
+/// Holds the path to the bloaty WASM binary.
+pub struct WasmBinaryBloaty(PathBuf);
+
+impl WasmBinaryBloaty {
+ /// Returns the escaped path to the bloaty wasm binary.
+ pub fn wasm_binary_bloaty_path_escaped(&self) -> String {
+ self.0.display().to_string().escape_default().to_string()
+ }
+}
+
+/// Holds the path to the WASM binary.
+pub struct WasmBinary(PathBuf);
+
+impl WasmBinary {
+ /// Returns the path to the wasm binary.
+ pub fn wasm_binary_path(&self) -> &Path {
+ &self.0
+ }
+
+ /// Returns the escaped path to the wasm binary.
+ pub fn wasm_binary_path_escaped(&self) -> String {
+ self.0.display().to_string().escape_default().to_string()
+ }
+}
+
+fn crate_metadata(cargo_manifest: &Path) -> Metadata {
+ let mut cargo_lock = cargo_manifest.to_path_buf();
+ cargo_lock.set_file_name("Cargo.lock");
+
+ let cargo_lock_existed = cargo_lock.exists();
+
+ let crate_metadata = MetadataCommand::new()
+ .manifest_path(cargo_manifest)
+ .exec()
+ .expect("`cargo metadata` can not fail on project `Cargo.toml`; qed");
+
+ // If the `Cargo.lock` didn't exist, we need to remove it after
+ // calling `cargo metadata`. This is required to ensure that we don't change
+ // the build directory outside of the `target` folder. Commands like
+ // `cargo publish` require this.
+ if !cargo_lock_existed {
+ let _ = fs::remove_file(&cargo_lock);
+ }
+
+ crate_metadata
+}
+
+/// Creates the WASM project, compiles the WASM binary and compacts the WASM
+/// binary.
+///
+/// # Returns
+///
+/// The path to the compact WASM binary and the bloaty WASM binary.
+pub fn create_and_compile(
+ project_cargo_toml: &Path,
+ default_rustflags: &str,
+ cargo_cmd: CargoCommandVersioned,
+ features_to_enable: Vec<String>,
+ wasm_binary_name: Option<String>,
+) -> (Option<WasmBinary>, WasmBinaryBloaty) {
+ let wasm_workspace_root = get_wasm_workspace_root();
+ let wasm_workspace = wasm_workspace_root.join("wbuild");
+
+ let crate_metadata = crate_metadata(project_cargo_toml);
+
+ let project = create_project(
+ project_cargo_toml,
+ &wasm_workspace,
+ &crate_metadata,
+ crate_metadata.workspace_root.as_ref(),
+ features_to_enable,
+ );
+
+ build_project(&project, default_rustflags, cargo_cmd);
+ let (wasm_binary, wasm_binary_compressed, bloaty) =
+ compact_wasm_file(&project, project_cargo_toml, wasm_binary_name);
+
+ wasm_binary
+ .as_ref()
+ .map(|wasm_binary| copy_wasm_to_target_directory(project_cargo_toml, wasm_binary));
+
+ wasm_binary_compressed
+ .as_ref()
+ .map(|wasm_binary_compressed| copy_wasm_to_target_directory(project_cargo_toml, wasm_binary_compressed));
+
+ generate_rerun_if_changed_instructions(project_cargo_toml, &project, &wasm_workspace);
+
+ (wasm_binary_compressed.or(wasm_binary), bloaty)
+}
+
+/// Find the `Cargo.lock` relative to the `OUT_DIR` environment variable.
+///
+/// If the `Cargo.lock` cannot be found, we emit a warning and return `None`.
+fn find_cargo_lock(cargo_manifest: &Path) -> Option<PathBuf> {
+ fn find_impl(mut path: PathBuf) -> Option<PathBuf> {
+ loop {
+ if path.join("Cargo.lock").exists() {
+ return Some(path.join("Cargo.lock"));
+ }
+
+ if !path.pop() {
+ return None;
+ }
+ }
+ }
+
+ if let Some(path) = find_impl(build_helper::out_dir()) {
+ return Some(path);
+ }
+
+ if let Some(path) = find_impl(cargo_manifest.to_path_buf()) {
+ return Some(path);
+ }
+
+ build_helper::warning!(
+ "Could not find `Cargo.lock` for `{}`, while searching from `{}`.",
+ cargo_manifest.display(),
+ build_helper::out_dir().display()
+ );
+
+ None
+}
+
+/// Extract the crate name from the given `Cargo.toml`.
+fn get_crate_name(cargo_manifest: &Path) -> String {
+ let cargo_toml: Table =
+ toml::from_str(&fs::read_to_string(cargo_manifest).expect("File exists as checked before; qed"))
+ .expect("Cargo manifest is a valid toml file; qed");
+
+ let package = cargo_toml
+ .get("package")
+ .and_then(|t| t.as_table())
+ .expect("`package` key exists in valid `Cargo.toml`; qed");
+
+ package
+ .get("name")
+ .and_then(|p| p.as_str())
+ .map(ToOwned::to_owned)
+ .expect("Package name exists; qed")
+}
+
+/// Returns the name for the wasm binary.
+fn get_wasm_binary_name(cargo_manifest: &Path) -> String {
+ get_crate_name(cargo_manifest).replace('-', "_")
+}
+
+/// Returns the root path of the wasm workspace.
+fn get_wasm_workspace_root() -> PathBuf {
+ let mut out_dir = build_helper::out_dir();
+
+ loop {
+ match out_dir.parent() {
+ Some(parent) if out_dir.ends_with("build") => return parent.to_path_buf(),
+ _ => {
+ if !out_dir.pop() {
+ break;
+ }
+ }
+ }
+ }
+
+ panic!("Could not find target dir in: {}", build_helper::out_dir().display())
+}
+
+fn create_project_cargo_toml(
+ wasm_workspace: &Path,
+ workspace_root_path: &Path,
+ crate_name: &str,
+ crate_path: &Path,
+ wasm_binary: &str,
+ enabled_features: impl Iterator<Item = String>,
+) {
+ let mut workspace_toml: Table = toml::from_str(
+ &fs::read_to_string(workspace_root_path.join("Cargo.toml")).expect("Workspace root `Cargo.toml` exists; qed"),
+ )
+ .expect("Workspace root `Cargo.toml` is a valid toml file; qed");
+
+ let mut wasm_workspace_toml = Table::new();
+
+ // Add `profile` with release and dev
+ let mut release_profile = Table::new();
+ release_profile.insert("panic".into(), "abort".into());
+ release_profile.insert("lto".into(), true.into());
+
+ let mut dev_profile = Table::new();
+ dev_profile.insert("panic".into(), "abort".into());
+
+ let mut profile = Table::new();
+ profile.insert("release".into(), release_profile.into());
+ profile.insert("dev".into(), dev_profile.into());
+
+ wasm_workspace_toml.insert("profile".into(), profile.into());
+
+ // Add patch section from the project root `Cargo.toml`
+ if let Some(mut patch) = workspace_toml.remove("patch").and_then(|p| p.try_into::<Table>().ok()) {
+ // Iterate over all patches and make the patch path absolute from the workspace
+ // root path.
+ patch
+ .iter_mut()
+ .filter_map(|p| {
+ p.1.as_table_mut()
+ .map(|t| t.iter_mut().filter_map(|t| t.1.as_table_mut()))
+ })
+ .flatten()
+ .for_each(|p| {
+ p.iter_mut().filter(|(k, _)| k == &"path").for_each(|(_, v)| {
+ if let Some(path) = v.as_str().map(PathBuf::from) {
+ if path.is_relative() {
+ *v = workspace_root_path.join(path).display().to_string().into();
+ }
+ }
+ })
+ });
+
+ wasm_workspace_toml.insert("patch".into(), patch.into());
+ }
+
+ let mut package = Table::new();
+ package.insert("name".into(), format!("{}-wasm", crate_name).into());
+ package.insert("version".into(), "1.0.0".into());
+ package.insert("edition".into(), "2018".into());
+ package.insert("resolver".into(), "2".into());
+
+ wasm_workspace_toml.insert("package".into(), package.into());
+
+ let mut lib = Table::new();
+ lib.insert("name".into(), wasm_binary.into());
+ lib.insert("crate-type".into(), vec!["cdylib".to_string()].into());
+
+ wasm_workspace_toml.insert("lib".into(), lib.into());
+
+ let mut dependencies = Table::new();
+
+ let mut wasm_project = Table::new();
+ wasm_project.insert("package".into(), crate_name.into());
+ wasm_project.insert("path".into(), crate_path.display().to_string().into());
+ wasm_project.insert("default-features".into(), false.into());
+ wasm_project.insert("features".into(), enabled_features.collect::<Vec<_>>().into());
+
+ dependencies.insert("wasm-project".into(), wasm_project.into());
+
+ wasm_workspace_toml.insert("dependencies".into(), dependencies.into());
+
+ wasm_workspace_toml.insert("workspace".into(), Table::new().into());
+
+ write_file_if_changed(
+ wasm_workspace.join("Cargo.toml"),
+ toml::to_string_pretty(&wasm_workspace_toml).expect("Wasm workspace toml is valid; qed"),
+ );
+}
+
+/// Find a package by the given `manifest_path` in the metadata.
+///
+/// Panics if the package could not be found.
+fn find_package_by_manifest_path<'a>(
+ manifest_path: &Path,
+ crate_metadata: &'a cargo_metadata::Metadata,
+) -> &'a cargo_metadata::Package {
+ crate_metadata
+ .packages
+ .iter()
+ .find(|p| p.manifest_path == manifest_path)
+ .expect("Wasm project exists in its own metadata; qed")
+}
+
+/// Get a list of enabled features for the project.
+fn project_enabled_features(cargo_manifest: &Path, crate_metadata: &cargo_metadata::Metadata) -> Vec<String> {
+ let package = find_package_by_manifest_path(cargo_manifest, crate_metadata);
+
+ let mut enabled_features = package
+ .features
+ .keys()
+ .filter(|f| {
+ let mut feature_env = f.replace("-", "_");
+ feature_env.make_ascii_uppercase();
+
+ // We don't want to enable the `std`/`default` feature for the wasm build and
+ // we need to check if the feature is enabled by checking the env variable.
+ *f != "std"
+ && *f != "default"
+ && env::var(format!("CARGO_FEATURE_{}", feature_env))
+ .map(|v| v == "1")
+ .unwrap_or_default()
+ })
+ .cloned()
+ .collect::<Vec<_>>();
+
+ enabled_features.sort();
+ enabled_features
+}
+
+/// Returns if the project has the `runtime-wasm` feature
+fn has_runtime_wasm_feature_declared(cargo_manifest: &Path, crate_metadata: &cargo_metadata::Metadata) -> bool {
+ let package = find_package_by_manifest_path(cargo_manifest, crate_metadata);
+
+ package.features.keys().any(|k| k == "runtime-wasm")
+}
+
+/// Create the project used to build the wasm binary.
+///
+/// # Returns
+///
+/// The path to the created wasm project.
+fn create_project(
+ project_cargo_toml: &Path,
+ wasm_workspace: &Path,
+ crate_metadata: &Metadata,
+ workspace_root_path: &Path,
+ features_to_enable: Vec<String>,
+) -> PathBuf {
+ let crate_name = get_crate_name(project_cargo_toml);
+ let crate_path = project_cargo_toml.parent().expect("Parent path exists; qed");
+ let wasm_binary = get_wasm_binary_name(project_cargo_toml);
+ let wasm_project_folder = wasm_workspace.join(&crate_name);
+
+ fs::create_dir_all(wasm_project_folder.join("src")).expect("Wasm project dir create can not fail; qed");
+
+ let mut enabled_features = project_enabled_features(&project_cargo_toml, &crate_metadata);
+
+ if has_runtime_wasm_feature_declared(project_cargo_toml, crate_metadata) {
+ enabled_features.push("runtime-wasm".into());
+ }
+
+ let mut enabled_features = enabled_features.into_iter().collect::<HashSet<_>>();
+ enabled_features.extend(features_to_enable.into_iter());
+
+ create_project_cargo_toml(
+ &wasm_project_folder,
+ workspace_root_path,
+ &crate_name,
+ &crate_path,
+ &wasm_binary,
+ enabled_features.into_iter(),
+ );
+
+ write_file_if_changed(
+ wasm_project_folder.join("src/lib.rs"),
+ "#![no_std] pub use wasm_project::*;",
+ );
+
+ if let Some(crate_lock_file) = find_cargo_lock(project_cargo_toml) {
+ // Use the `Cargo.lock` of the main project.
+ copy_file_if_changed(crate_lock_file, wasm_project_folder.join("Cargo.lock"));
+ }
+
+ wasm_project_folder
+}
+
+/// Returns if the project should be built as a release.
+fn is_release_build() -> bool {
+ if let Ok(var) = env::var(WASM_BUILD_TYPE_ENV) {
+ match var.as_str() {
+ "release" => true,
+ "debug" => false,
+ var => panic!(
+ "Unexpected value for `{}` env variable: {}\nOne of the following are expected: `debug` or `release`.",
+ WASM_BUILD_TYPE_ENV, var,
+ ),
+ }
+ } else {
+ true
+ }
+}
+
+/// Build the project to create the WASM binary.
+fn build_project(project: &Path, default_rustflags: &str, cargo_cmd: CargoCommandVersioned) {
+ let manifest_path = project.join("Cargo.toml");
+ let mut build_cmd = cargo_cmd.command();
+
+ let rustflags = format!(
+ "-C link-arg=--export-table {} {}",
+ default_rustflags,
+ env::var(WASM_BUILD_RUSTFLAGS_ENV).unwrap_or_default(),
+ );
+
+ build_cmd
+ .args(&["rustc", "--target=wasm32-unknown-unknown"])
+ .arg(format!("--manifest-path={}", manifest_path.display()))
+ .env("RUSTFLAGS", rustflags)
+ // Unset the `CARGO_TARGET_DIR` to prevent a cargo deadlock (cargo locks a target dir exclusive).
+ // The runner project is created in `CARGO_TARGET_DIR` and executing it will create a sub target
+ // directory inside of `CARGO_TARGET_DIR`.
+ .env_remove("CARGO_TARGET_DIR")
+ // We don't want to call ourselves recursively
+ .env(SKIP_BUILD_ENV, "");
+
+ if color_output_enabled() {
+ build_cmd.arg("--color=always");
+ }
+
+ if is_release_build() {
+ build_cmd.arg("--release");
+ };
+
+ println!(
+ "{}",
+ colorize_info_message("Information that should be included in a bug report.")
+ );
+ println!("{} {:?}", colorize_info_message("Executing build command:"), build_cmd);
+ println!(
+ "{} {}",
+ colorize_info_message("Using rustc version:"),
+ cargo_cmd.rustc_version()
+ );
+
+ match build_cmd.status().map(|s| s.success()) {
+ Ok(true) => {}
+ // Use `process.exit(1)` to have a clean error output.
+ _ => process::exit(1),
+ }
+}
+
+/// Compact the WASM binary using `wasm-gc` and compress it using zstd.
+fn compact_wasm_file(
+ project: &Path,
+ cargo_manifest: &Path,
+ wasm_binary_name: Option<String>,
+) -> (Option<WasmBinary>, Option<WasmBinary>, WasmBinaryBloaty) {
+ let is_release_build = is_release_build();
+ let target = if is_release_build { "release" } else { "debug" };
+ let default_wasm_binary_name = get_wasm_binary_name(cargo_manifest);
+ let wasm_file = project
+ .join("target/wasm32-unknown-unknown")
+ .join(target)
+ .join(format!("{}.wasm", default_wasm_binary_name));
+
+ let wasm_compact_file = if is_release_build {
+ let wasm_compact_file = project.join(format!(
+ "{}.compact.wasm",
+ wasm_binary_name
+ .clone()
+ .unwrap_or_else(|| default_wasm_binary_name.clone()),
+ ));
+ wasm_gc::garbage_collect_file(&wasm_file, &wasm_compact_file)
+ .expect("Failed to compact generated WASM binary.");
+ Some(WasmBinary(wasm_compact_file))
+ } else {
+ None
+ };
+
+ let wasm_compact_compressed_file = wasm_compact_file.as_ref().and_then(|compact_binary| {
+ let file_name = wasm_binary_name
+ .clone()
+ .unwrap_or_else(|| default_wasm_binary_name.clone());
+
+ let wasm_compact_compressed_file = project.join(format!("{}.compact.compressed.wasm", file_name,));
+
+ if compress_wasm(&compact_binary.0, &wasm_compact_compressed_file) {
+ Some(WasmBinary(wasm_compact_compressed_file))
+ } else {
+ None
+ }
+ });
+
+ let bloaty_file_name = if let Some(name) = wasm_binary_name {
+ format!("{}.wasm", name)
+ } else {
+ format!("{}.wasm", default_wasm_binary_name)
+ };
+
+ let bloaty_file = project.join(bloaty_file_name);
+ fs::copy(wasm_file, &bloaty_file).expect("Copying the bloaty file to the project dir.");
+
+ (
+ wasm_compact_file,
+ wasm_compact_compressed_file,
+ WasmBinaryBloaty(bloaty_file),
+ )
+}
+
+fn compress_wasm(wasm_binary_path: &Path, compressed_binary_out_path: &Path) -> bool {
+ use sp_maybe_compressed_blob::CODE_BLOB_BOMB_LIMIT;
+
+ let data = fs::read(wasm_binary_path).expect("Failed to read WASM binary");
+ if let Some(compressed) = sp_maybe_compressed_blob::compress(&data, CODE_BLOB_BOMB_LIMIT) {
+ fs::write(compressed_binary_out_path, &compressed[..]).expect("Failed to write WASM binary");
+
+ true
+ } else {
+ println!(
+ "cargo:warning=Writing uncompressed wasm. Exceeded maximum size {}",
+ CODE_BLOB_BOMB_LIMIT,
+ );
+
+ false
+ }
+}
+
+/// Custom wrapper for a [`cargo_metadata::Package`] to store it in
+/// a `HashSet`.
+#[derive(Debug)]
+struct DeduplicatePackage<'a> {
+ package: &'a cargo_metadata::Package,
+ identifier: String,
+}
+
+impl<'a> From<&'a cargo_metadata::Package> for DeduplicatePackage<'a> {
+ fn from(package: &'a cargo_metadata::Package) -> Self {
+ Self {
+ package,
+ identifier: format!("{}{}{:?}", package.name, package.version, package.source),
+ }
+ }
+}
+
+impl<'a> Hash for DeduplicatePackage<'a> {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.identifier.hash(state);
+ }
+}
+
+impl<'a> PartialEq for DeduplicatePackage<'a> {
+ fn eq(&self, other: &Self) -> bool {
+ self.identifier == other.identifier
+ }
+}
+
+impl<'a> Eq for DeduplicatePackage<'a> {}
+
+impl<'a> Deref for DeduplicatePackage<'a> {
+ type Target = cargo_metadata::Package;
+
+ fn deref(&self) -> &Self::Target {
+ self.package
+ }
+}
+
+/// Generate the `rerun-if-changed` instructions for cargo to make sure that the
+/// WASM binary is rebuilt when needed.
+fn generate_rerun_if_changed_instructions(cargo_manifest: &Path, project_folder: &Path, wasm_workspace: &Path) {
+ // Rerun `build.rs` if the `Cargo.lock` changes
+ if let Some(cargo_lock) = find_cargo_lock(cargo_manifest) {
+ rerun_if_changed(cargo_lock);
+ }
+
+ let metadata = MetadataCommand::new()
+ .manifest_path(project_folder.join("Cargo.toml"))
+ .exec()
+ .expect("`cargo metadata` can not fail!");
+
+ let package = metadata
+ .packages
+ .iter()
+ .find(|p| p.manifest_path == cargo_manifest)
+ .expect("The crate package is contained in its own metadata; qed");
+
+ // Start with the dependencies of the crate we want to compile for wasm.
+ let mut dependencies = package.dependencies.iter().collect::<Vec<_>>();
+
+ // Collect all packages by follow the dependencies of all packages we find.
+ let mut packages = HashSet::new();
+ packages.insert(DeduplicatePackage::from(package));
+
+ while let Some(dependency) = dependencies.pop() {
+ let path_or_git_dep = dependency
+ .source
+ .as_ref()
+ .map(|s| s.starts_with("git+"))
+ .unwrap_or(true);
+
+ let package = metadata
+ .packages
+ .iter()
+ .filter(|p| !p.manifest_path.starts_with(wasm_workspace))
+ .find(|p| {
+ // Check that the name matches and that the version matches or this is
+ // a git or path dep. A git or path dependency can only occur once, so we don't
+ // need to check the version.
+ (path_or_git_dep || dependency.req.matches(&p.version)) && dependency.name == p.name
+ });
+
+ if let Some(package) = package {
+ if packages.insert(DeduplicatePackage::from(package)) {
+ dependencies.extend(package.dependencies.iter());
+ }
+ }
+ }
+
+ // Make sure that if any file/folder of a dependency change, we need to rerun
+ // the `build.rs`
+ packages.iter().for_each(package_rerun_if_changed);
+
+ // Register our env variables
+ println!("cargo:rerun-if-env-changed={}", SKIP_BUILD_ENV);
+ println!("cargo:rerun-if-env-changed={}", WASM_BUILD_TYPE_ENV);
+ println!("cargo:rerun-if-env-changed={}", WASM_BUILD_RUSTFLAGS_ENV);
+ println!("cargo:rerun-if-env-changed={}", WASM_TARGET_DIRECTORY);
+ println!(
+ "cargo:rerun-if-env-changed={}",
+ super::prerequisites::WASM_BUILD_TOOLCHAIN
+ );
+}
+
+/// Track files and paths related to the given package to rerun `build.rs` on
+/// any relevant change.
+fn package_rerun_if_changed(package: &DeduplicatePackage) {
+ let mut manifest_path = package.manifest_path.clone();
+ if manifest_path.ends_with("Cargo.toml") {
+ manifest_path.pop();
+ }
+
+ walkdir::WalkDir::new(&manifest_path)
+ .into_iter()
+ .filter_entry(|p| {
+ // Ignore this entry if it is a directory that contains a `Cargo.toml` that is
+ // not the `Cargo.toml` related to the current package. This is done to ignore
+ // sub-crates of a crate. If such a sub-crate is a dependency, it will be
+ // processed independently anyway.
+ p.path() == manifest_path || !p.path().is_dir() || !p.path().join("Cargo.toml").exists()
+ })
+ .filter_map(|p| p.ok().map(|p| p.into_path()))
+ .filter(|p| p.is_dir() || p.extension().map(|e| e == "rs" || e == "toml").unwrap_or_default())
+ .for_each(rerun_if_changed);
+}
+
+/// Copy the WASM binary to the target directory set in `WASM_TARGET_DIRECTORY`
+/// environment variable. If the variable is not set, this is a no-op.
+fn copy_wasm_to_target_directory(cargo_manifest: &Path, wasm_binary: &WasmBinary) {
+ let target_dir = match env::var(WASM_TARGET_DIRECTORY) {
+ Ok(path) => PathBuf::from(path),
+ Err(_) => return,
+ };
+
+ if !target_dir.is_absolute() {
+ panic!(
+ "Environment variable `{}` with `{}` is not an absolute path!",
+ WASM_TARGET_DIRECTORY,
+ target_dir.display(),
+ );
+ }
+
+ fs::create_dir_all(&target_dir).expect("Creates `WASM_TARGET_DIRECTORY`.");
+
+ fs::copy(
+ wasm_binary.wasm_binary_path(),
+ target_dir.join(format!("{}.wasm", get_wasm_binary_name(cargo_manifest))),
+ )
+ .expect("Copies WASM binary to `WASM_TARGET_DIRECTORY`.");
+}
diff --git a/bencher/src/colorize.rs b/bencher/src/colorize.rs
new file mode 100644
index 0000000..98cf737
--- /dev/null
+++ b/bencher/src/colorize.rs
@@ -0,0 +1,39 @@
+/// Environment variable to disable color output of the wasm build.
+const WASM_BUILD_NO_COLOR: &str = "WASM_BUILD_NO_COLOR";
+
+/// Returns `true` when color output is enabled.
+pub fn color_output_enabled() -> bool {
+ std::env::var(WASM_BUILD_NO_COLOR).is_err()
+}
+
+pub fn red_bold(message: &str) -> String {
+ if color_output_enabled() {
+ ansi_term::Color::Red.bold().paint(message).to_string()
+ } else {
+ message.into()
+ }
+}
+
+pub fn yellow_bold(message: &str) -> String {
+ if color_output_enabled() {
+ ansi_term::Color::Yellow.bold().paint(message).to_string()
+ } else {
+ message.into()
+ }
+}
+
+pub fn cyan(message: &str) -> String {
+ if crate::build_wasm::color_output_enabled() {
+ ansi_term::Color::Cyan.paint(message).to_string()
+ } else {
+ message.into()
+ }
+}
+
+pub fn green_bold(message: &str) -> String {
+ if crate::build_wasm::color_output_enabled() {
+ ansi_term::Color::Green.bold().paint(message).to_string()
+ } else {
+ message.into()
+ }
+}
diff --git a/bencher/src/handler.rs b/bencher/src/handler.rs
index 8533d2d..a855037 100644
--- a/bencher/src/handler.rs
+++ b/bencher/src/handler.rs
@@ -1,27 +1,35 @@
-use crate::BenchResult;
+use crate::{
+ colorize::{cyan, green_bold},
+ BenchResult,
+};
use codec::Decode;
use linregress::{FormulaRegressionBuilder, RegressionDataBuilder};
use serde::{Deserialize, Serialize};
use std::io::Write;
+use std::time::Duration;
#[derive(Serialize, Deserialize, Default, Debug, Clone)]
struct BenchData {
pub name: String,
pub base_weight: u64,
pub base_reads: u32,
+ pub base_repeat_reads: u32,
pub base_writes: u32,
+ pub base_repeat_writes: u32,
}
/// Handle bench results
pub fn handle(output: Vec<u8>) {
+ println!();
+
+ let pkg_name = std::env::var("CARGO_PKG_NAME").unwrap_or_default().replace("-", "_");
+
let results = <Vec<BenchResult> as Decode>::decode(&mut &output[..]).unwrap();
let data: Vec<BenchData> = results
.into_iter()
.map(|result| {
let name = String::from_utf8_lossy(&result.method).to_string();
- eprintln!("{:#?}", result);
-
let y: Vec<f64> = result.elapses.into_iter().map(|x| x as f64).collect();
let x: Vec<f64> = (0..y.len()).into_iter().map(|x| x as f64).collect();
let data = vec![("Y", y), ("X", x)];
@@ -34,15 +42,31 @@ pub fn handle(output: Vec<u8>) {
.fit()
.unwrap();
+ println!(
+ "{} {:<60} {:>20} {:<20} {:<20}",
+ green_bold("Bench"),
+ cyan(&format!("{}::{}", pkg_name, name)),
+ green_bold(&format!(
+ "{:?}",
+ Duration::from_nanos(model.parameters.intercept_value as u64)
+ )),
+ format!("reads: {}", green_bold(&result.reads.to_string())),
+ format!("writes: {}", green_bold(&result.writes.to_string()))
+ );
+
BenchData {
name,
base_weight: model.parameters.intercept_value as u64 * 1_000,
base_reads: result.reads,
+ base_repeat_reads: result.repeat_reads,
base_writes: result.writes,
+ base_repeat_writes: result.repeat_writes,
}
})
.collect();
+ println!();
+
if let Ok(json) = serde_json::to_string(&data) {
let stdout = ::std::io::stdout();
let mut handle = stdout.lock();
diff --git a/bencher/src/lib.rs b/bencher/src/lib.rs
index 2ef25f7..746f4e6 100644
--- a/bencher/src/lib.rs
+++ b/bencher/src/lib.rs
@@ -3,12 +3,29 @@
#[doc(hidden)]
pub extern crate frame_benchmarking;
#[doc(hidden)]
+pub extern crate paste;
+#[doc(hidden)]
pub extern crate sp_core;
#[doc(hidden)]
+pub extern crate sp_io;
+#[doc(hidden)]
pub extern crate sp_std;
+mod macros;
+
+#[cfg(feature = "std")]
+pub mod bench_runner;
+#[cfg(feature = "std")]
+pub mod build_wasm;
+#[cfg(feature = "std")]
+mod colorize;
+#[cfg(feature = "std")]
+pub mod handler;
+#[cfg(feature = "std")]
+mod redundant_meter;
+
use codec::{Decode, Encode};
-use sp_std::prelude::Vec;
+use sp_std::prelude::{Box, Vec};
#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
pub struct BenchResult {
@@ -20,9 +37,158 @@ pub struct BenchResult {
pub repeat_writes: u32,
}
-mod macros;
+pub struct Bencher {
+ pub name: Vec<u8>,
+ pub results: Vec<BenchResult>,
+ pub prepare: Box<dyn Fn()>,
+ pub bench: Box<dyn Fn()>,
+ pub verify: Box<dyn Fn()>,
+}
+
+impl Default for Bencher {
+ fn default() -> Self {
+ Bencher {
+ name: Vec::new(),
+ results: Vec::new(),
+ prepare: Box::new(|| {}),
+ bench: Box::new(|| {}),
+ verify: Box::new(|| {}),
+ }
+ }
+}
+
+impl Bencher {
+ /// Reset name and blocks
+ pub fn reset(&mut self) {
+ self.name = Vec::new();
+ self.prepare = Box::new(|| {});
+ self.bench = Box::new(|| {});
+ self.verify = Box::new(|| {});
+ }
+
+ /// Set bench name
+ pub fn name(&mut self, name: &str) -> &mut Self {
+ self.name = name.as_bytes().to_vec();
+ self
+ }
+
+ /// Set prepare block
+ pub fn prepare(&mut self, prepare: impl Fn() + 'static) -> &mut Self {
+ self.prepare = Box::new(prepare);
+ self
+ }
+
+ /// Set verify block
+ pub fn verify(&mut self, verify: impl Fn() + 'static) -> &mut Self {
+ self.verify = Box::new(verify);
+ self
+ }
+
+ /// Set bench block
+ pub fn bench(&mut self, bench: impl Fn() + 'static) -> &mut Self {
+ self.bench = Box::new(bench);
+ self
+ }
+
+ /// Run benchmark for tests
+ #[cfg(feature = "std")]
+ pub fn run(&mut self) {
+ // Execute prepare block
+ (self.prepare)();
+ // Execute bench block
+ (self.bench)();
+ // Execute verify block
+ (self.verify)();
+ }
+
+ /// Run benchmark
+ #[cfg(not(feature = "std"))]
+ pub fn run(&mut self) {
+ assert!(self.name.len() > 0, "bench name not defined");
+ // Warm up the DB
+ frame_benchmarking::benchmarking::commit_db();
+ frame_benchmarking::benchmarking::wipe_db();
+
+ let mut result = BenchResult {
+ method: self.name.clone(),
+ ..Default::default()
+ };
+
+ for _ in 0..50 {
+ // Execute prepare block
+ (self.prepare)();
+
+ frame_benchmarking::benchmarking::commit_db();
+ frame_benchmarking::benchmarking::reset_read_write_count();
+ bencher::reset();
+
+ let start_time = frame_benchmarking::benchmarking::current_time();
+ // Execute bench block
+ (self.bench)();
+ let end_time = frame_benchmarking::benchmarking::current_time();
+ frame_benchmarking::benchmarking::commit_db();
+
+ let (elapsed, reads, repeat_reads, writes, repeat_writes) =
+ bencher::finalized_results(end_time - start_time);
+
+ // Execute verify block
+ (self.verify)();
+
+ // Reset the DB
+ frame_benchmarking::benchmarking::wipe_db();
+
+ result.elapses.push(elapsed);
+
+ result.reads = sp_std::cmp::max(result.reads, reads);
+ result.repeat_reads = sp_std::cmp::max(result.repeat_reads, repeat_reads);
+ result.writes = sp_std::cmp::max(result.writes, writes);
+ result.repeat_writes = sp_std::cmp::max(result.repeat_writes, repeat_writes);
+ }
+ self.results.push(result);
+ }
+}
#[cfg(feature = "std")]
-pub mod bench_runner;
-#[cfg(feature = "std")]
-pub mod handler;
+thread_local! {
+ static REDUNDANT_METER: std::cell::RefCell<redundant_meter::RedundantMeter> = std::cell::RefCell::new(redundant_meter::RedundantMeter::default());
+}
+
+#[sp_runtime_interface::runtime_interface]
+pub trait Bencher {
+ fn panic(str: Vec<u8>) {
+ let msg = String::from_utf8_lossy(&str);
+ eprintln!("{}", colorize::red_bold(&msg));
+ std::process::exit(-1);
+ }
+
+ fn entering_method() -> Vec<u8> {
+ REDUNDANT_METER.with(|x| x.borrow_mut().entering_method())
+ }
+
+ fn leaving_method(identifier: Vec<u8>) {
+ REDUNDANT_METER.with(|x| {
+ x.borrow_mut().leaving_method(identifier);
+ });
+ }
+
+ fn finalized_results(elapsed: u128) -> (u128, u32, u32, u32, u32) {
+ let (reads, repeat_reads, writes, repeat_writes) = frame_benchmarking::benchmarking::read_write_count();
+
+ let (redundant_elapsed, redundant_reads, redundant_repeat_reads, redundant_writes, redundant_repeat_writes) =
+ REDUNDANT_METER.with(|x| x.borrow_mut().take_results());
+
+ let elapsed = elapsed - redundant_elapsed;
+ let reads = reads - redundant_reads;
+ let repeat_reads = repeat_reads - redundant_repeat_reads;
+ let writes = writes - redundant_writes;
+ let repeat_writes = repeat_writes - redundant_repeat_writes;
+
+ (elapsed, reads, repeat_reads, writes, repeat_writes)
+ }
+
+ fn reset() {
+ REDUNDANT_METER.with(|x| {
+ x.borrow_mut().reset();
+ });
+ }
+}
diff --git a/bencher/src/macros.rs b/bencher/src/macros.rs
index f654813..e8b1831 100644
--- a/bencher/src/macros.rs
+++ b/bencher/src/macros.rs
@@ -1,160 +1,126 @@
/// Run benches in WASM environment.
///
-/// Configure your module to build the mock runtime into wasm code.
-/// Create a `build.rs` like you do with your runtime.
-/// ```.ignore
-/// use substrate_wasm_builder::WasmBuilder;
-/// fn main() {
-/// WasmBuilder::new()
-/// .with_current_project()
-/// .export_heap_base()
-/// .import_memory()
-/// .build()
-/// }
-/// ```
-///
-/// Update mock runtime to be build into wasm code.
-/// ```.ignore
-/// #![cfg_attr(not(feature = "std"), no_std)]
-///
-/// #[cfg(feature = "std")]
-/// include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs"));
-///
-/// #[cfg(feature = "std")]
-/// pub fn wasm_binary_unwrap() -> &'static [u8] { WASM_BINARY.unwrap() }
-/// ..
-/// ```
-///
-/// Create a file `bench_runner.rs` with following code:
-/// ```.ignore
-/// orml_bencher::run_benches!(my_module::benches);
-/// ```
-///
/// Update Cargo.toml by adding:
/// ```toml
/// ..
/// [package]
-/// name = "my-module"
+/// name = "your-module"
/// ..
-/// build = 'build.rs'
-///
-/// [build-dependencies]
-/// substrate-wasm-builder = '4.0.0'
-///
/// [[bench]]
-/// name = 'benches'
+/// name = 'your-module-benches'
/// harness = false
-/// path = 'bench_runner.rs'
+/// path = 'src/benches.rs'
/// required-features = ['bench']
///
/// [features]
-/// bench = []
+/// bench = [
+/// 'orml-bencher/bench'
+/// 'orml-weight-meter/bench'
+/// ]
/// ..
/// ```
-///
-/// Run bench with features bench: `cargo bench --features=bench`
-#[cfg(feature = "std")]
-#[macro_export]
-macro_rules! run_benches {
- ($benches:path) => {
- use $benches::{wasm_binary_unwrap, Block};
- pub fn main() {
- let output = $crate::bench_runner::run::<Block>(wasm_binary_unwrap().to_vec());
- $crate::handler::handle(output);
- }
- };
-}
-
+///
/// Define benches
///
/// Create a file `src/benches.rs`:
/// ```.ignore
-/// #![cfg_attr(not(feature = "std"), no_std)]
/// #![allow(dead_code)]
///
-/// #[cfg(feature = "std")] // Re-export for bench_runner
-/// pub use crate::mock::{Block, wasm_binary_unwrap};
-///
-/// use crate::mock::YourModule;
+/// use orml_bencher::{Bencher, bench};
+/// use your_module::mock::{Block, YourModule};
///
/// fn foo(b: &mut Bencher) {
-/// b.bench("foo", || {
+/// b.prepare(|| {
+/// // optional. prepare block, runs before bench
+/// })
+/// .bench(|| {
+/// // foo must have macro `[orml_weight_meter::weight(..)]`
/// YourModule::foo();
+/// })
+/// .verify(|| {
+/// // optional. verify block, runs before bench
/// });
/// }
///
/// fn bar(b: &mut Bencher) {
-/// b.bench("bar", || {
+/// // optional. method name is used by default i.e: `bar`
+/// b.name("bench_name")
+/// .bench(|| {
+/// // bar must have macro `[orml_weight_meter::weight(..)]`
/// YourModule::bar();
/// });
/// }
///
-/// orml_bencher::bench!(foo, bar);
+/// bench!(Block, foo, bar); // Tests are generated automatically
/// ```
/// Update `src/lib.rs`:
/// ```.ignore
/// #[cfg(any(feature = "bench", test))]
/// pub mod mock; /* mock runtime needs to be compiled into wasm */
-/// #[cfg(feature = "bench")]
+/// #[cfg(any(feature = "bench", test))]
/// pub mod benches;
+///
+/// extern crate self as your_module;
/// ```
+///
+/// Run benchmarking: `cargo bench --features=bench`
#[macro_export]
macro_rules! bench {
(
+ $block:tt,
$($method:path),+
) => {
- use $crate::BenchResult;
- use $crate::sp_std::{cmp::max, prelude::Vec};
- use $crate::frame_benchmarking::{benchmarking, BenchmarkResults};
-
- #[derive(Default, Clone, PartialEq, Debug)]
- struct Bencher {
- pub results: Vec<BenchResult>,
- }
-
- impl Bencher {
- pub fn bench<F: Fn() -> ()>(&mut self, name: &str, block: F) {
- // Warm up the DB
- benchmarking::commit_db();
- benchmarking::wipe_db();
-
- let mut result = BenchResult {
- method: name.as_bytes().to_vec(),
- ..Default::default()
- };
-
- for _ in 0..50 {
- benchmarking::commit_db();
- benchmarking::reset_read_write_count();
-
- let start_time = benchmarking::current_time();
- block();
- let end_time = benchmarking::current_time();
- let elasped = end_time - start_time;
- result.elapses.push(elasped);
-
- benchmarking::commit_db();
- let (reads, repeat_reads, writes, repeat_writes) = benchmarking::read_write_count();
-
- result.reads = max(result.reads, reads);
- result.repeat_reads = max(result.repeat_reads, repeat_reads);
- result.writes = max(result.writes, writes);
- result.repeat_writes = max(result.repeat_writes, repeat_writes);
-
- benchmarking::wipe_db();
- }
- self.results.push(result);
- }
- }
-
+ #[cfg(feature = "bench")]
$crate::sp_core::wasm_export_functions! {
- fn run_benches() -> Vec<BenchResult> {
- let mut bencher = Bencher::default();
+ fn run_benches() -> $crate::sp_std::vec::Vec<$crate::BenchResult> {
+ let mut bencher = $crate::Bencher::default();
$(
+ bencher.reset();
$method(&mut bencher);
+ if bencher.name.len() == 0 {
+ // use method name as default bench name
+ bencher.name(stringify!($method));
+ }
+ bencher.run();
)+
bencher.results
}
}
+
+ #[cfg(all(feature = "bench", not(feature = "std")))]
+ #[panic_handler]
+ #[no_mangle]
+ fn panic_handler(info: &::core::panic::PanicInfo) -> ! {
+ unsafe {
+ let message = $crate::sp_std::alloc::format!("{}", info);
+ $crate::bencher::panic(message.as_bytes().to_vec());
+ core::arch::wasm32::unreachable();
+ }
+ }
+
+ #[cfg(all(feature = "std", feature = "bench"))]
+ pub fn main() -> std::io::Result<()> {
+ let wasm = $crate::build_wasm::build()?;
+ match $crate::bench_runner::run::<$block>(wasm) {
+ Ok(output) => { $crate::handler::handle(output); }
+ Err(e) => { eprintln!("{:?}", e); }
+ };
+ Ok(())
+ }
+
+ // Tests
+ $(
+ $crate::paste::item! {
+ #[test]
+ fn [<test_ $method>] () {
+ $crate::sp_io::TestExternalities::new_empty().execute_with(|| {
+ let mut bencher = $crate::Bencher::default();
+ $method(&mut bencher);
+ bencher.run();
+ });
+ }
+ }
+ )+
+
}
}
diff --git a/bencher/src/redundant_meter.rs b/bencher/src/redundant_meter.rs
new file mode 100644
index 0000000..64d11ce
--- /dev/null
+++ b/bencher/src/redundant_meter.rs
@@ -0,0 +1,111 @@
+use codec::{Decode, Encode};
+use rand::{distributions::Alphanumeric, thread_rng, Rng};
+
+#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)]
+struct RedundantResult {
+ identifier: Vec<u8>,
+ timestamp: u128,
+ reads: u32,
+ repeat_reads: u32,
+ writes: u32,
+ repeat_writes: u32,
+}
+
+/// RedundantMeter is used to measure resources been used by methods that
+/// already been benchmarked and have `[orml_weight_meter::weight(..)] macro
+/// defined. First method with that macro will be skipped and after that every
+/// method with macro defined will be measured as redundant result.
+#[derive(Default)]
+pub struct RedundantMeter {
+ started: bool,
+ results: Vec<RedundantResult>,
+ current: Option<RedundantResult>,
+}
+
+impl RedundantMeter {
+ /// Entering method with `[orml_weight_meter::weight(..)]` macro
+ pub fn entering_method(&mut self) -> Vec<u8> {
+ if !self.started {
+ self.started = true;
+ return Vec::new();
+ }
+
+ if self.current.is_some() {
+ return Vec::new();
+ }
+
+ let timestamp = frame_benchmarking::benchmarking::current_time();
+ frame_benchmarking::benchmarking::commit_db();
+ let (reads, repeat_reads, writes, repeat_writes) = frame_benchmarking::benchmarking::read_write_count();
+
+ let identifier: Vec<u8> = thread_rng()
+ .sample_iter(&Alphanumeric)
+ .take(10)
+ .map(char::from)
+ .collect::<String>()
+ .encode();
+
+ self.current = Some(RedundantResult {
+ identifier: identifier.to_owned(),
+ timestamp,
+ reads,
+ repeat_reads,
+ writes,
+ repeat_writes,
+ });
+
+ identifier
+ }
+
+ /// Leaving method with `[orml_weight_meter::weight(..)]` macro
+ pub fn leaving_method(&mut self, identifier: Vec<u8>) {
+ if let Some(current) = &self.current {
+ if current.identifier.eq(&identifier) {
+ frame_benchmarking::benchmarking::commit_db();
+ let (reads, repeat_reads, writes, repeat_writes) = frame_benchmarking::benchmarking::read_write_count();
+ let timestamp = frame_benchmarking::benchmarking::current_time();
+
+ self.results.push(RedundantResult {
+ identifier,
+ timestamp: timestamp - current.timestamp,
+ reads: reads - current.reads,
+ repeat_reads: repeat_reads - current.repeat_reads,
+ writes: writes - current.writes,
+ repeat_writes: repeat_writes - current.repeat_writes,
+ });
+
+ // reset current
+ self.current = None;
+ }
+ }
+ }
+
+ /// Take bench results and reset for next measurement
+ pub fn take_results(&mut self) -> (u128, u32, u32, u32, u32) {
+ assert!(self.current == None, "benchmark in progress");
+
+ let mut elapsed = 0u128;
+ let mut reads = 0u32;
+ let mut repeat_reads = 0u32;
+ let mut writes = 0u32;
+ let mut repeat_writes = 0u32;
+
+ self.results.iter().for_each(|x| {
+ elapsed += x.timestamp;
+ reads += x.reads;
+ repeat_reads += x.repeat_reads;
+ writes += x.writes;
+ repeat_writes += x.repeat_writes;
+ });
+
+ self.reset();
+
+ (elapsed, reads, repeat_reads, writes, repeat_writes)
+ }
+
+ pub fn reset(&mut self) {
+ self.started = false;
+ self.results = Vec::new();
+ self.current = None;
+ }
+}
diff --git a/weight-gen/src/template.hbs b/weight-gen/src/template.hbs
index 470b00c..3ad2b78 100644
--- a/weight-gen/src/template.hbs
+++ b/weight-gen/src/template.hbs
@@ -12,10 +12,10 @@ impl<T: frame_system::Config> ModuleWeights<T> {
{{~#each benchmarks as |benchmark|}}
pub fn {{benchmark.name~}} () -> Weight {
({{underscore benchmark.base_weight}} as Weight)
- {{~#if (ne benchmark.base_reads "0")}}
+ {{~#if (ne benchmark.base_reads 0)}}
.saturating_add(T::DbWeight::get().reads({{benchmark.base_reads}} as Weight))
{{~/if}}
- {{~#if (ne benchmark.base_writes "0")}}
+ {{~#if (ne benchmark.base_writes 0)}}
.saturating_add(T::DbWeight::get().writes({{benchmark.base_writes}} as Weight))
{{~/if}}
}
diff --git a/weight-meter/Cargo.toml b/weight-meter/Cargo.toml
index 781e12f..339d4a8 100644
--- a/weight-meter/Cargo.toml
+++ b/weight-meter/Cargo.toml
@@ -12,6 +12,7 @@ targets = ["x86_64-unknown-linux-gnu"]
spin = "0.7.1"
frame-support = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.4", default-features = false }
weight-meter-procedural = { path = "weight-meter-procedural", default-features = false }
+orml-bencher = { path = "../bencher", default-features = false, optional = true }
[dev-dependencies]
serde = { version = "1.0.124" }
@@ -29,4 +30,9 @@ default = ["std"]
std = [
"frame-support/std",
"weight-meter-procedural/std",
+ "orml-bencher/std",
+]
+bench = [
+ "weight-meter-procedural/bench",
+ "orml-bencher",
]
diff --git a/weight-meter/src/lib.rs b/weight-meter/src/lib.rs
index cfdc45d..84d20cd 100644
--- a/weight-meter/src/lib.rs
+++ b/weight-meter/src/lib.rs
@@ -57,8 +57,6 @@ struct Meter {
mod meter_no_std;
mod meter_std;
-// For use in mock file
-#[cfg(test)]
extern crate self as orml_weight_meter;
#[cfg(test)]
diff --git a/weight-meter/weight-meter-procedural/Cargo.toml b/weight-meter/weight-meter-procedural/Cargo.toml
index 05554ee..7447cb6 100644
--- a/weight-meter/weight-meter-procedural/Cargo.toml
+++ b/weight-meter/weight-meter-procedural/Cargo.toml
@@ -15,4 +15,5 @@ syn = { version = "1.0.58", features = ["full"] }
[features]
default = ["std"]
-std = []
\ No newline at end of file
+std = []
+bench = []
diff --git a/weight-meter/weight-meter-procedural/src/lib.rs b/weight-meter/weight-meter-procedural/src/lib.rs
index 87f5f75..953d7e6 100644
--- a/weight-meter/weight-meter-procedural/src/lib.rs
+++ b/weight-meter/weight-meter-procedural/src/lib.rs
@@ -17,6 +17,7 @@ pub fn start(_attr: TokenStream, item: TokenStream) -> TokenStream {
.into()
}
+#[cfg(not(feature = "bench"))]
#[proc_macro_attribute]
pub fn weight(attr: TokenStream, item: TokenStream) -> TokenStream {
let weight: syn::Expr = syn::parse(attr).unwrap();
@@ -30,3 +31,19 @@ pub fn weight(attr: TokenStream, item: TokenStream) -> TokenStream {
})
.into()
}
+
+#[cfg(feature = "bench")]
+#[proc_macro_attribute]
+pub fn weight(_attr: TokenStream, item: TokenStream) -> TokenStream {
+ let ItemFn { attrs, sig, block, .. } = syn::parse(item).unwrap();
+ (quote! {
+ #(#attrs)*
+ pub #sig {
+ let identifier: ::sp_std::vec::Vec<u8> = ::orml_bencher::bencher::entering_method();
+ let result = #block;
+ ::orml_bencher::bencher::leaving_method(identifier);
+ result
+ }
+ })
+ .into()
+}
--
GitLab