Hashing and Manifest

Summary: Hashes the artifacts and adds the information to a manifest. This will then be used to upload to a blob store, and save the information so it can be retrieved again later. Not sure if I want to stick with SHA256 or if there's something more efficient like Cityhash, given that we don't really care about cryptographic properties here.

Reviewed By: jknoxville

Differential Revision: D21380369

fbshipit-source-id: e9c5cd56d94f3083ae5ed6396673d00cbf98ce39

packer/Cargo.lock

@@ -67,6 +67,39 @@ dependencies = [
  "constant_time_eq",
 ]
 
+[[package]]
+name = "block-buffer"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
+dependencies = [
+ "block-padding",
+ "byte-tools",
+ "byteorder",
+ "generic-array",
+]
+
+[[package]]
+name = "block-padding"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
+dependencies = [
+ "byte-tools",
+]
+
+[[package]]
+name = "byte-tools"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7"
+
+[[package]]
+name = "byteorder"
+version = "1.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -105,6 +138,21 @@ dependencies = [
  "lazy_static",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "11c0346158a19b3627234e15596f5e465c360fcdb97d817bcb255e0510f5a788"
+
+[[package]]
+name = "digest"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5"
+dependencies = [
+ "generic-array",
+]
+
 [[package]]
 name = "dirs"
 version = "2.0.2"
@@ -133,6 +181,12 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3"
 
+[[package]]
+name = "fake-simd"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed"
+
 [[package]]
 name = "filetime"
 version = "0.2.9"
@@ -151,10 +205,29 @@ version = "0.1.0"
 dependencies = [
  "anyhow",
  "clap",
+ "data-encoding",
  "serde",
+ "serde_json",
  "serde_yaml",
+ "sha2",
  "shellexpand",
  "tar",
+ "tempdir",
+]
+
+[[package]]
+name = "fuchsia-cprng"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
+
+[[package]]
+name = "generic-array"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec"
+dependencies = [
+ "typenum",
 ]
 
 [[package]]
@@ -177,6 +250,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "itoa"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
+
 [[package]]
 name = "lazy_static"
 version = "1.4.0"
@@ -195,6 +274,12 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 
+[[package]]
+name = "opaque-debug"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.10"
@@ -213,6 +298,43 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "rand"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293"
+dependencies = [
+ "fuchsia-cprng",
+ "libc",
+ "rand_core 0.3.1",
+ "rdrand",
+ "winapi",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"
+dependencies = [
+ "rand_core 0.4.2",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"
+
+[[package]]
+name = "rdrand"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2"
+dependencies = [
+ "rand_core 0.3.1",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.1.56"
@@ -230,6 +352,15 @@ dependencies = [
  "rust-argon2",
 ]
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a83fa3702a688b9359eccba92d153ac33fd2e8462f9e0e3fdf155239ea7792e"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "rust-argon2"
 version = "0.7.0"
@@ -242,6 +373,12 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "ryu"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed3d612bc64430efeb3f7ee6ef26d590dce0c43249217bddc62112540c7941e1"
+
 [[package]]
 name = "serde"
 version = "1.0.106"
@@ -262,6 +399,17 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "serde_json"
+version = "1.0.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7894c8ed05b7a3a279aeb79025fdec1d3158080b75b98a08faf2806bb799edd"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
 [[package]]
 name = "serde_yaml"
 version = "0.8.11"
@@ -274,6 +422,18 @@ dependencies = [
  "yaml-rust",
 ]
 
+[[package]]
+name = "sha2"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27044adfd2e1f077f649f59deb9490d3941d674002f7d062870a60ebe9bd47a0"
+dependencies = [
+ "block-buffer",
+ "digest",
+ "fake-simd",
+ "opaque-debug",
+]
+
 [[package]]
 name = "shellexpand"
 version = "2.0.0"
@@ -312,6 +472,16 @@ dependencies = [
  "xattr",
 ]
 
+[[package]]
+name = "tempdir"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8"
+dependencies = [
+ "rand",
+ "remove_dir_all",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.11.0"
@@ -321,6 +491,12 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "typenum"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33"
+
 [[package]]
 name = "unicode-width"
 version = "0.1.7"

packer/Cargo.toml

@@ -11,3 +11,9 @@ tar = "0.4.26"
 serde = { version = "1.0.106", features = ["derive"] }
 serde_yaml = "0.8.11"
 anyhow = "1.0.28"
+sha2 = "0.8.1"
+data-encoding = "2.2.0"
+serde_json = "1.0.52"
+
+[dev-dependencies]
+tempdir = "0.3.7"

packer/src/main.rs

@@ -12,6 +12,7 @@ use anyhow::{bail, Result};
 use clap::value_t_or_exit;
 use std::collections::BTreeMap;
 use std::fs::File;
+use std::io::{stdout, BufReader, Read, Write};
 use std::path;
 use types::{PackType, Platform};
 
@@ -22,31 +23,44 @@ type PackListPlatform = BTreeMap<PackType, Vec<path::PathBuf>>;
 #[derive(Debug, serde::Deserialize)]
 struct PackList(pub BTreeMap<Platform, PackListPlatform>);
 
+#[derive(Debug, serde::Serialize)]
+struct HashSum(String);
+
+#[derive(Debug, serde::Serialize)]
+struct PackManifest {
+    files: BTreeMap<PackType, HashSum>,
+}
+
 fn pack(
     platform: &Platform,
     dist_dir: &std::path::PathBuf,
     pack_list: &PackList,
     output_directory: &std::path::PathBuf,
-) -> Result<()> {
+) -> Result<Vec<(PackType, path::PathBuf)>> {
     let packtype_paths = pack_list
         .0
         .get(platform)
         .ok_or_else(|| error::Error::MissingPlatformDefinition(platform.clone()))?;
-    for (pack_type, pack_files) in packtype_paths {
+    packtype_paths
-        print!(
+        .iter()
-            "Packing for platform {:?} type {:?} ...",
+        .try_fold(vec![], |mut acc, (pack_type, pack_files)| {
-            platform, pack_type
+            print!(
-        );
+                "Packing for platform {:?} type {:?} ...",
-        let output_path = path::Path::new(output_directory).join(format!("{}.tar", pack_type));
+                platform, pack_type
-        let mut tar = tar::Builder::new(File::create(output_path)?);
+            );
-        // MacOS uses symlinks for bundling multiple framework versions and pointing
+            let _ = stdout().flush();
-        // to the "Current" one.
+            let output_path = path::Path::new(output_directory).join(format!("{}.tar", pack_type));
-        tar.follow_symlinks(false);
+            let mut tar = tar::Builder::new(File::create(&output_path)?);
-        pack_platform(platform, dist_dir, pack_files, pack_type, &mut tar)?;
+            // MacOS uses symlinks for bundling multiple framework versions and pointing
-        tar.finish()?;
+            // to the "Current" one.
-        println!(" done.");
+            tar.follow_symlinks(false);
-    }
+            pack_platform(platform, dist_dir, pack_files, pack_type, &mut tar)?;
-    Ok(())
+            tar.finish()?;
+            println!(" done.");
+
+            acc.push((*pack_type, output_path));
+            Ok(acc)
+        })
 }
 
 fn pack_platform(
@@ -82,7 +96,17 @@ fn pack_platform(
     Ok(())
 }
 
-fn main() -> Result<(), error::Error> {
+fn sha256_digest<R: Read>(mut reader: R) -> Result<HashSum> {
+    use sha2::{Digest, Sha256};
+
+    let mut sha256 = Sha256::new();
+    std::io::copy(&mut reader, &mut sha256)?;
+    let hash = sha256.result();
+
+    Ok(HashSum(data_encoding::HEXLOWER.encode(&hash)))
+}
+
+fn main() -> Result<(), anyhow::Error> {
     // Ensure to define all env vars used here in the BUCK env, too.
     let args = clap::App::new(env!("CARGO_PKG_NAME"))
         .version(env!("CARGO_PKG_VERSION"))
@@ -117,17 +141,55 @@ fn main() -> Result<(), error::Error> {
         .unwrap_or(DEFAULT_PACKLIST.to_string());
     let pack_list: PackList =
         serde_yaml::from_str(&pack_list_str).expect("Failed to deserialize YAML packlist.");
-    pack(
+    let output_directory =
-        &platform,
+        &path::PathBuf::from(args.value_of("output").expect("argument has default"));
-        &dist_dir,
+    let archive_paths = pack(&platform, &dist_dir, &pack_list, output_directory)?;
-        &pack_list,
+    manifest(&archive_paths, &output_directory)?;
-        &path::PathBuf::from(args.value_of("output").expect("argument has default")),
-    )
-    .unwrap();
 
     Ok(())
 }
 
+fn manifest(
+    archive_paths: &[(PackType, path::PathBuf)],
+    output_directory: &path::PathBuf,
+) -> Result<path::PathBuf> {
+    print!("Generating manifest ...");
+    let _ = stdout().flush();
+    // TODO: This could easily be parallelised.
+    let archive_manifest = gen_manifest(&archive_paths)?;
+    println!(" done.");
+    write_manifest(&output_directory, &archive_manifest)
+}
+
+fn write_manifest(
+    output_directory: &path::PathBuf,
+    archive_manifest: &PackManifest,
+) -> Result<path::PathBuf> {
+    let path = path::PathBuf::from(output_directory).join("manifest.json");
+    let mut file = File::create(&path)?;
+    file.write_all(&serde_json::to_string_pretty(archive_manifest)?.as_bytes())?;
+    Ok(path)
+}
+
+fn gen_manifest(archive_paths: &[(PackType, path::PathBuf)]) -> Result<PackManifest> {
+    Ok(PackManifest {
+        files: gen_manifest_files(archive_paths)?,
+    })
+}
+
+fn gen_manifest_files(
+    archive_paths: &[(PackType, path::PathBuf)],
+) -> Result<BTreeMap<PackType, HashSum>> {
+    archive_paths.into_iter().try_fold(
+        BTreeMap::new(),
+        |mut acc: BTreeMap<PackType, HashSum>, (pack_type, path)| {
+            let reader = BufReader::new(File::open(path)?);
+            acc.insert(*pack_type, sha256_digest(reader)?);
+            Ok(acc)
+        },
+    )
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -138,4 +200,21 @@ mod test {
             serde_yaml::from_str(DEFAULT_PACKLIST).expect("Default packlist doesn't deserialize");
         assert_eq!(res.0.len(), 3);
     }
+
+    #[test]
+    fn test_manifest() -> anyhow::Result<()> {
+        let tmp_dir = tempdir::TempDir::new("manifest_test")?;
+        let artifact_path = path::PathBuf::from(tmp_dir.path()).join("core.tar");
+        let mut artifact = File::create(&artifact_path)?;
+        artifact.write_all("Hello World.".as_bytes())?;
+
+        let archive_paths = &[(PackType::Core, artifact_path)];
+        let path = manifest(archive_paths, &tmp_dir.path().to_path_buf())?;
+
+        let manifest_content = std::fs::read_to_string(&path)?;
+
+        assert_eq!(manifest_content, "{\n  \"files\": {\n    \"core\": \"f4bb1975bf1f81f76ce824f7536c1e101a8060a632a52289d530a6f600d52c92\"\n  }\n}");
+
+        Ok(())
+    }
 }

packer/src/types.rs

@@ -18,7 +18,17 @@ arg_enum! {
     }
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)]
+#[derive(
+    Debug,
+    Clone,
+    Copy,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    serde::Deserialize,
+    serde::Serialize
+)]
 #[serde(rename_all = "lowercase")]
 pub enum PackType {
     Frameworks,