diff --git a/packer/Cargo.lock b/packer/Cargo.lock index 25cee9a5e..726e04781 100644 --- a/packer/Cargo.lock +++ b/packer/Cargo.lock @@ -121,12 +121,65 @@ dependencies = [ "vec_map", ] +[[package]] +name = "console" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea0f3e2e8d7dba335e913b97f9e1992c86c4399d54f8be1d31c8727d0652064" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "regex", + "terminal_size", + "termios", + "unicode-width", + "winapi", + "winapi-util", +] + [[package]] name = "constant_time_eq" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c695eeca1e7173472a32221542ae469b3e9aac3a4fc81f7696bcad82029493db" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.7.2" @@ -181,6 +234,18 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" +[[package]] +name = "either" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3" + +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "fake-simd" version = "0.1.2" @@ -206,6 +271,8 @@ dependencies = [ "anyhow", "clap", "data-encoding", + "indicatif", + "rayon", "serde", "serde_json", "serde_yaml", @@ -250,6 +317,18 @@ dependencies = [ "libc", ] +[[package]] +name = "indicatif" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a68371cf417889c9d7f98235b7102ea7c54fc59bcbd22f3dea785be9d27e40" +dependencies = [ + "console", + "lazy_static", + "number_prefix", + "regex", +] + [[package]] name = "itoa" version = "0.4.5" @@ -274,6 +353,37 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memoffset" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4fc2c02a7e374099d4ee95a193111f72d2110197fe200272371758f6c3643d8" +dependencies = [ + "autocfg", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" + [[package]] name = "opaque-debug" version = "0.2.3" @@ -326,6 +436,30 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" +[[package]] +name = "rayon" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db6ce3297f9c85e16621bb8cca38a06779ffc31bb8184e1be4bed2be4678a098" +dependencies = [ + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08a89b46efaf957e52b18062fb2f4660f8b8a4dde1807ca002690868ef2c85a9" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "lazy_static", + "num_cpus", +] + [[package]] name = "rdrand" version = "0.4.0" @@ -352,6 +486,21 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "regex" +version = "1.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6020f034922e3194c711b82a627453881bc4682166cabb07134a10c26ba7692" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.6.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fe5bd57d1d7414c6b5ed48563a2c855d995ff777729dcd91c369ec7fea395ae" + [[package]] name = "remove_dir_all" version = "0.5.2" @@ -379,6 +528,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + [[package]] name = "serde" version = "1.0.106" @@ -482,6 +637,25 @@ dependencies = [ "remove_dir_all", ] +[[package]] +name = "terminal_size" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8038f95fc7a6f351163f4b964af631bd26c9e828f7db085f2a84aca56f70d13b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "termios" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0fcee7b24a25675de40d5bb4de6e41b0df07bc9856295e7e2b3a3600c400c2" +dependencies = [ + "libc", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -537,6 +711,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/packer/Cargo.toml b/packer/Cargo.toml index 592dcc0ad..475f8c2bb 100644 --- a/packer/Cargo.toml +++ b/packer/Cargo.toml @@ -14,6 +14,7 @@ anyhow = "1.0.28" sha2 = "0.8.1" data-encoding = "2.2.0" serde_json = "1.0.52" +rayon = "1.3.0" [dev-dependencies] tempdir = "0.3.7" diff --git a/packer/src/main.rs b/packer/src/main.rs index ab69e2b37..542e59fa0 100644 --- a/packer/src/main.rs +++ b/packer/src/main.rs @@ -10,6 +10,7 @@ mod types; use anyhow::{bail, Context, Result}; use clap::value_t_or_exit; +use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use std::collections::BTreeMap; use std::fs::File; use std::io::{stdout, BufReader, Read, Write}; @@ -42,7 +43,7 @@ fn pack( .get(platform) .ok_or_else(|| error::Error::MissingPlatformDefinition(platform.clone()))?; packtype_paths - .into_iter() + .into_par_iter() .map(|(pack_type, pack_files)| { print!( "Packing for platform {:?} type {:?} ... ", @@ -193,7 +194,7 @@ fn gen_manifest_files( archive_paths: &[(PackType, path::PathBuf)], ) -> Result> { let res = archive_paths - .into_iter() + .into_par_iter() .map(|(pack_type, path)| { let reader = BufReader::new(File::open(path)?); let hash = sha256_digest(reader)?;