Implement compression

Summary:
Compresses artifacts with LZMA by default. `--no-compression` can be used
to skip the quite time-intensive task to just quickly check if the archives
still add up to a full distribution.

Reviewed By: jknoxville

Differential Revision: D21456052

fbshipit-source-id: dfa4686f3fe7ce342b62dab4b5203bc0aad37d9a

packer/Cargo.lock

@@ -100,6 +100,12 @@ version = "1.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
 
+[[package]]
+name = "cc"
+version = "1.0.52"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d"
+
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -280,6 +286,7 @@ dependencies = [
  "shellexpand",
  "tar",
  "tempdir",
+ "xz2",
 ]
 
 [[package]]
@@ -353,6 +360,17 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83"
 
+[[package]]
+name = "lzma-sys"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e48818fd597d46155132bbbb9505d6d1b3d360b4ee25cfa91c406f8a90fe91"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "maybe-uninit"
 version = "2.0.0"
@@ -390,6 +408,12 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.10"
@@ -735,6 +759,15 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "xz2"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "yaml-rust"
 version = "0.4.3"

packer/Cargo.toml

@@ -16,6 +16,7 @@ data-encoding = "^2.2.0"
 serde_json = "^1.0.52"
 rayon = "^1.3.0"
 indicatif = "^0.14.0"
+xz2 = "^0.1.6"
 
 [dev-dependencies]
 tempdir = "^0.3.7"

packer/src/main.rs

@@ -13,7 +13,7 @@ use clap::value_t_or_exit;
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 use std::collections::BTreeMap;
 use std::fs::File;
-use std::io::{BufReader, Read, Write};
+use std::io::{BufReader, BufWriter, Read, Write};
 use std::path;
 use types::{PackType, Platform};
 
@@ -146,6 +146,9 @@ fn main() -> Result<(), anyhow::Error> {
         .arg(clap::Arg::from_usage(
             "-p, --packlist=packlist.yaml 'Custom list of files to pack.'",
         ))
+        .arg(clap::Arg::from_usage(
+            "--no-compression 'Skip compressing the archives (for debugging)'",
+        ))
         .arg(
             clap::Arg::from_usage("[PLATFORM] 'Platform to build for'")
                 .case_insensitive(true)
@@ -158,6 +161,7 @@ fn main() -> Result<(), anyhow::Error> {
     let dist_dir = path::PathBuf::from(
         shellexpand::tilde(args.value_of("dist").expect("argument has default")).to_string(),
     );
+    let compress = !args.is_present("no-compression");
     let pack_list_str = args
         .value_of("packlist")
         .map(|f| std::fs::read_to_string(f).expect(&format!("Failed to open packfile {}.", f)))
@@ -173,11 +177,55 @@ fn main() -> Result<(), anyhow::Error> {
         )
     })?;
     let archive_paths = pack(&platform, &dist_dir, &pack_list, output_directory)?;
-    manifest(&archive_paths, &output_directory)?;
+    let compressed_archive_paths = if compress {
+        compress_paths(&archive_paths)?
+    } else {
+        archive_paths
+    };
+    manifest(&compressed_archive_paths, &output_directory)?;
 
     Ok(())
 }
 
+/// Takes a list of archive paths, compresses them with LZMA and returns
+/// the updated paths.
+/// TODO: Remove compressed artifacts.
+fn compress_paths(
+    archive_paths: &[(PackType, path::PathBuf)],
+) -> Result<Vec<(PackType, path::PathBuf)>> {
+    let pb = default_progress_bar(archive_paths.len() as u64 - 1);
+    pb.set_prefix(&format!(
+        "{:width$}",
+        "Compressing archives",
+        width = PROGRESS_PREFIX_LEN
+    ));
+    let res = archive_paths
+        .into_par_iter()
+        .map(|(pack_type, path)| {
+            let input_file = File::open(&path).with_context(|| {
+                format!("Failed to open archive '{}'.", &path.to_string_lossy())
+            })?;
+            let mut reader = BufReader::new(input_file);
+            let mut output_path = path::PathBuf::from(path);
+            output_path.set_extension("tar.xz");
+            let output_file: File = File::create(&output_path).with_context(|| {
+                format!(
+                    "Failed opening compressed archive '{}' for writing.",
+                    &output_path.to_string_lossy()
+                )
+            })?;
+            let writer = BufWriter::new(output_file);
+            let mut encoder = xz2::write::XzEncoder::new(writer, 9);
+            std::io::copy(&mut reader, &mut encoder)?;
+            pb.inc(1);
+            Ok((*pack_type, output_path))
+        })
+        .collect::<Result<Vec<(PackType, path::PathBuf)>>>()?;
+    pb.finish();
+
+    Ok(res)
+}
+
 fn manifest(
     archive_paths: &[(PackType, path::PathBuf)],
     output_directory: &path::PathBuf,