diff --git a/packer/Cargo.lock b/packer/Cargo.lock index 60c47e233..25cee9a5e 100644 --- a/packer/Cargo.lock +++ b/packer/Cargo.lock @@ -67,6 +67,39 @@ dependencies = [ "constant_time_eq", ] +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + [[package]] name = "cfg-if" version = "0.1.10" @@ -105,6 +138,21 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "data-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11c0346158a19b3627234e15596f5e465c360fcdb97d817bcb255e0510f5a788" + +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + [[package]] name = "dirs" version = "2.0.2" @@ -133,6 +181,12 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "filetime" version = "0.2.9" @@ -151,10 +205,29 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", + "data-encoding", "serde", + "serde_json", "serde_yaml", + "sha2", "shellexpand", "tar", + "tempdir", +] + +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + +[[package]] +name = "generic-array" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +dependencies = [ + "typenum", ] [[package]] @@ -177,6 +250,12 @@ dependencies = [ "libc", ] +[[package]] +name = "itoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" + [[package]] name = "lazy_static" version = "1.4.0" @@ -195,6 +274,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "proc-macro2" version = "1.0.10" @@ -213,6 +298,43 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "redox_syscall" version = "0.1.56" @@ -230,6 +352,15 @@ dependencies = [ "rust-argon2", ] +[[package]] +name = "remove_dir_all" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e" +dependencies = [ + "winapi", +] + [[package]] name = "rust-argon2" version = "0.7.0" @@ -242,6 +373,12 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "ryu" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1" + [[package]] name = "serde" version = "1.0.106" @@ -262,6 +399,17 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd" +dependencies = [ + "itoa", + "ryu", + "serde", +] + [[package]] name = "serde_yaml" version = "0.8.11" @@ -274,6 +422,18 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha2" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + [[package]] name = "shellexpand" version = "2.0.0" @@ -312,6 +472,16 @@ dependencies = [ "xattr", ] +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand", + "remove_dir_all", +] + [[package]] name = "textwrap" version = "0.11.0" @@ -321,6 +491,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "typenum" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" + [[package]] name = "unicode-width" version = "0.1.7" diff --git a/packer/Cargo.toml b/packer/Cargo.toml index 1c5dda65d..592dcc0ad 100644 --- a/packer/Cargo.toml +++ b/packer/Cargo.toml @@ -11,3 +11,9 @@ tar = "0.4.26" serde = { version = "1.0.106", features = ["derive"] } serde_yaml = "0.8.11" anyhow = "1.0.28" +sha2 = "0.8.1" +data-encoding = "2.2.0" +serde_json = "1.0.52" + +[dev-dependencies] +tempdir = "0.3.7" diff --git a/packer/src/main.rs b/packer/src/main.rs index 91a3c4474..bd6df7a56 100644 --- a/packer/src/main.rs +++ b/packer/src/main.rs @@ -12,6 +12,7 @@ use anyhow::{bail, Result}; use clap::value_t_or_exit; use std::collections::BTreeMap; use std::fs::File; +use std::io::{stdout, BufReader, Read, Write}; use std::path; use types::{PackType, Platform}; @@ -22,31 +23,44 @@ type PackListPlatform = BTreeMap>; #[derive(Debug, serde::Deserialize)] struct PackList(pub BTreeMap); +#[derive(Debug, serde::Serialize)] +struct HashSum(String); + +#[derive(Debug, serde::Serialize)] +struct PackManifest { + files: BTreeMap, +} + fn pack( platform: &Platform, dist_dir: &std::path::PathBuf, pack_list: &PackList, output_directory: &std::path::PathBuf, -) -> Result<()> { +) -> Result> { let packtype_paths = pack_list .0 .get(platform) .ok_or_else(|| error::Error::MissingPlatformDefinition(platform.clone()))?; - for (pack_type, pack_files) in packtype_paths { - print!( - "Packing for platform {:?} type {:?} ...", - platform, pack_type - ); - let output_path = path::Path::new(output_directory).join(format!("{}.tar", pack_type)); - let mut tar = tar::Builder::new(File::create(output_path)?); - // MacOS uses symlinks for bundling multiple framework versions and pointing - // to the "Current" one. - tar.follow_symlinks(false); - pack_platform(platform, dist_dir, pack_files, pack_type, &mut tar)?; - tar.finish()?; - println!(" done."); - } - Ok(()) + packtype_paths + .iter() + .try_fold(vec![], |mut acc, (pack_type, pack_files)| { + print!( + "Packing for platform {:?} type {:?} ...", + platform, pack_type + ); + let _ = stdout().flush(); + let output_path = path::Path::new(output_directory).join(format!("{}.tar", pack_type)); + let mut tar = tar::Builder::new(File::create(&output_path)?); + // MacOS uses symlinks for bundling multiple framework versions and pointing + // to the "Current" one. + tar.follow_symlinks(false); + pack_platform(platform, dist_dir, pack_files, pack_type, &mut tar)?; + tar.finish()?; + println!(" done."); + + acc.push((*pack_type, output_path)); + Ok(acc) + }) } fn pack_platform( @@ -82,7 +96,17 @@ fn pack_platform( Ok(()) } -fn main() -> Result<(), error::Error> { +fn sha256_digest(mut reader: R) -> Result { + use sha2::{Digest, Sha256}; + + let mut sha256 = Sha256::new(); + std::io::copy(&mut reader, &mut sha256)?; + let hash = sha256.result(); + + Ok(HashSum(data_encoding::HEXLOWER.encode(&hash))) +} + +fn main() -> Result<(), anyhow::Error> { // Ensure to define all env vars used here in the BUCK env, too. let args = clap::App::new(env!("CARGO_PKG_NAME")) .version(env!("CARGO_PKG_VERSION")) @@ -117,17 +141,55 @@ fn main() -> Result<(), error::Error> { .unwrap_or(DEFAULT_PACKLIST.to_string()); let pack_list: PackList = serde_yaml::from_str(&pack_list_str).expect("Failed to deserialize YAML packlist."); - pack( - &platform, - &dist_dir, - &pack_list, - &path::PathBuf::from(args.value_of("output").expect("argument has default")), - ) - .unwrap(); + let output_directory = + &path::PathBuf::from(args.value_of("output").expect("argument has default")); + let archive_paths = pack(&platform, &dist_dir, &pack_list, output_directory)?; + manifest(&archive_paths, &output_directory)?; Ok(()) } +fn manifest( + archive_paths: &[(PackType, path::PathBuf)], + output_directory: &path::PathBuf, +) -> Result { + print!("Generating manifest ..."); + let _ = stdout().flush(); + // TODO: This could easily be parallelised. + let archive_manifest = gen_manifest(&archive_paths)?; + println!(" done."); + write_manifest(&output_directory, &archive_manifest) +} + +fn write_manifest( + output_directory: &path::PathBuf, + archive_manifest: &PackManifest, +) -> Result { + let path = path::PathBuf::from(output_directory).join("manifest.json"); + let mut file = File::create(&path)?; + file.write_all(&serde_json::to_string_pretty(archive_manifest)?.as_bytes())?; + Ok(path) +} + +fn gen_manifest(archive_paths: &[(PackType, path::PathBuf)]) -> Result { + Ok(PackManifest { + files: gen_manifest_files(archive_paths)?, + }) +} + +fn gen_manifest_files( + archive_paths: &[(PackType, path::PathBuf)], +) -> Result> { + archive_paths.into_iter().try_fold( + BTreeMap::new(), + |mut acc: BTreeMap, (pack_type, path)| { + let reader = BufReader::new(File::open(path)?); + acc.insert(*pack_type, sha256_digest(reader)?); + Ok(acc) + }, + ) +} + #[cfg(test)] mod test { use super::*; @@ -138,4 +200,21 @@ mod test { serde_yaml::from_str(DEFAULT_PACKLIST).expect("Default packlist doesn't deserialize"); assert_eq!(res.0.len(), 3); } + + #[test] + fn test_manifest() -> anyhow::Result<()> { + let tmp_dir = tempdir::TempDir::new("manifest_test")?; + let artifact_path = path::PathBuf::from(tmp_dir.path()).join("core.tar"); + let mut artifact = File::create(&artifact_path)?; + artifact.write_all("Hello World.".as_bytes())?; + + let archive_paths = &[(PackType::Core, artifact_path)]; + let path = manifest(archive_paths, &tmp_dir.path().to_path_buf())?; + + let manifest_content = std::fs::read_to_string(&path)?; + + assert_eq!(manifest_content, "{\n \"files\": {\n \"core\": \"f4bb1975bf1f81f76ce824f7536c1e101a8060a632a52289d530a6f600d52c92\"\n }\n}"); + + Ok(()) + } } diff --git a/packer/src/types.rs b/packer/src/types.rs index 3ca4b85db..5dda84a72 100644 --- a/packer/src/types.rs +++ b/packer/src/types.rs @@ -18,7 +18,17 @@ arg_enum! { } } -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + serde::Deserialize, + serde::Serialize +)] #[serde(rename_all = "lowercase")] pub enum PackType { Frameworks,