diff --git a/packer/Cargo.lock b/packer/Cargo.lock index 726e04781..374628f63 100644 --- a/packer/Cargo.lock +++ b/packer/Cargo.lock @@ -100,6 +100,12 @@ version = "1.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +[[package]] +name = "cc" +version = "1.0.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d87b23d6a92cd03af510a5ade527033f6aa6fa92161e2d5863a907d4c5e31d" + [[package]] name = "cfg-if" version = "0.1.10" @@ -280,6 +286,7 @@ dependencies = [ "shellexpand", "tar", "tempdir", + "xz2", ] [[package]] @@ -353,6 +360,17 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" +[[package]] +name = "lzma-sys" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e48818fd597d46155132bbbb9505d6d1b3d360b4ee25cfa91c406f8a90fe91" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "maybe-uninit" version = "2.0.0" @@ -390,6 +408,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +[[package]] +name = "pkg-config" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05da548ad6865900e60eaba7f589cc0783590a92e940c26953ff81ddbab2d677" + [[package]] name = "proc-macro2" version = "1.0.10" @@ -735,6 +759,15 @@ dependencies = [ "libc", ] +[[package]] +name = "xz2" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c179869f34fc7c01830d3ce7ea2086bc3a07e0d35289b667d0a8bf910258926c" +dependencies = [ + "lzma-sys", +] + [[package]] name = "yaml-rust" version = "0.4.3" diff --git a/packer/Cargo.toml b/packer/Cargo.toml index 94a9f4eaf..059bf2f32 100644 --- a/packer/Cargo.toml +++ b/packer/Cargo.toml @@ -16,6 +16,7 @@ data-encoding = "^2.2.0" serde_json = "^1.0.52" rayon = "^1.3.0" indicatif = "^0.14.0" +xz2 = "^0.1.6" [dev-dependencies] tempdir = "^0.3.7" diff --git a/packer/src/main.rs b/packer/src/main.rs index 101f46b75..a69526ebb 100644 --- a/packer/src/main.rs +++ b/packer/src/main.rs @@ -13,7 +13,7 @@ use clap::value_t_or_exit; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; use std::collections::BTreeMap; use std::fs::File; -use std::io::{BufReader, Read, Write}; +use std::io::{BufReader, BufWriter, Read, Write}; use std::path; use types::{PackType, Platform}; @@ -146,6 +146,9 @@ fn main() -> Result<(), anyhow::Error> { .arg(clap::Arg::from_usage( "-p, --packlist=packlist.yaml 'Custom list of files to pack.'", )) + .arg(clap::Arg::from_usage( + "--no-compression 'Skip compressing the archives (for debugging)'", + )) .arg( clap::Arg::from_usage("[PLATFORM] 'Platform to build for'") .case_insensitive(true) @@ -158,6 +161,7 @@ fn main() -> Result<(), anyhow::Error> { let dist_dir = path::PathBuf::from( shellexpand::tilde(args.value_of("dist").expect("argument has default")).to_string(), ); + let compress = !args.is_present("no-compression"); let pack_list_str = args .value_of("packlist") .map(|f| std::fs::read_to_string(f).expect(&format!("Failed to open packfile {}.", f))) @@ -173,11 +177,55 @@ fn main() -> Result<(), anyhow::Error> { ) })?; let archive_paths = pack(&platform, &dist_dir, &pack_list, output_directory)?; - manifest(&archive_paths, &output_directory)?; + let compressed_archive_paths = if compress { + compress_paths(&archive_paths)? + } else { + archive_paths + }; + manifest(&compressed_archive_paths, &output_directory)?; Ok(()) } +/// Takes a list of archive paths, compresses them with LZMA and returns +/// the updated paths. +/// TODO: Remove compressed artifacts. +fn compress_paths( + archive_paths: &[(PackType, path::PathBuf)], +) -> Result> { + let pb = default_progress_bar(archive_paths.len() as u64 - 1); + pb.set_prefix(&format!( + "{:width$}", + "Compressing archives", + width = PROGRESS_PREFIX_LEN + )); + let res = archive_paths + .into_par_iter() + .map(|(pack_type, path)| { + let input_file = File::open(&path).with_context(|| { + format!("Failed to open archive '{}'.", &path.to_string_lossy()) + })?; + let mut reader = BufReader::new(input_file); + let mut output_path = path::PathBuf::from(path); + output_path.set_extension("tar.xz"); + let output_file: File = File::create(&output_path).with_context(|| { + format!( + "Failed opening compressed archive '{}' for writing.", + &output_path.to_string_lossy() + ) + })?; + let writer = BufWriter::new(output_file); + let mut encoder = xz2::write::XzEncoder::new(writer, 9); + std::io::copy(&mut reader, &mut encoder)?; + pb.inc(1); + Ok((*pack_type, output_path)) + }) + .collect::>>()?; + pb.finish(); + + Ok(res) +} + fn manifest( archive_paths: &[(PackType, path::PathBuf)], output_directory: &path::PathBuf,