From 155a8c6952aaf41cfa099736ab9bbebd82c04b35 Mon Sep 17 00:00:00 2001 From: Pascal Hartig Date: Fri, 1 May 2020 03:59:12 -0700 Subject: [PATCH] Initial packer project Summary: This is a tool (right now more a script, really) for splitting a Flipper distribution into smaller chunks that can be individually cached, hashed and downloaded. It's incomplete but can already bundle up Mac apps into smaller chunks that can be merged back again into one app. It takes a config file that specifies how to slice the inputs up and comes with a default copy bundled in. That's in line with the general "design" where it tries to make the right assumptions, for instance defaulting to the normal dist output directory. Reviewed By: jknoxville Differential Revision: D21302551 fbshipit-source-id: b22670ff4825ce526dd0f20320f87080c058cd93 --- packer/.gitignore | 1 + packer/Cargo.lock | 379 +++++++++++++++++++++++++++++++++++++++ packer/Cargo.toml | 12 ++ packer/README.md | 64 +++++++ packer/src/error.rs | 40 +++++ packer/src/main.rs | 158 ++++++++++++++++ packer/src/packlist.yaml | 16 ++ packer/src/types.rs | 25 +++ 8 files changed, 695 insertions(+) create mode 100644 packer/.gitignore create mode 100644 packer/Cargo.lock create mode 100644 packer/Cargo.toml create mode 100644 packer/README.md create mode 100644 packer/src/error.rs create mode 100644 packer/src/main.rs create mode 100644 packer/src/packlist.yaml create mode 100644 packer/src/types.rs diff --git a/packer/.gitignore b/packer/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/packer/.gitignore @@ -0,0 +1 @@ +/target diff --git a/packer/Cargo.lock b/packer/Cargo.lock new file mode 100644 index 000000000..de164378b --- /dev/null +++ b/packer/Cargo.lock @@ -0,0 +1,379 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if", + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" +dependencies = [ + "cfg-if", + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dtoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" + +[[package]] +name = "filetime" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f59efc38004c988e4201d11d263b8171f49a2e7ec0bdbb71773433f271504a5e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "flipper-packer" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "serde_yaml", + "shellexpand", + "tar", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61565ff7aaace3525556587bd2dc31d4a07071957be715e63ce7b1eccf51a8f4" +dependencies = [ + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" + +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" + +[[package]] +name = "proc-macro2" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "redox_users" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "rust-argon2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" +dependencies = [ + "base64", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "serde" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + +[[package]] +name = "shellexpand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2b22262a9aaf9464d356f656fea420634f78c881c5eebd5ef5e66d8b9bc603" +dependencies = [ + "dirs", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +dependencies = [ + "filetime", + "libc", + "redox_syscall", + "xattr", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +dependencies = [ + "linked-hash-map", +] diff --git a/packer/Cargo.toml b/packer/Cargo.toml new file mode 100644 index 000000000..51480dfe8 --- /dev/null +++ b/packer/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "flipper-packer" +version = "0.1.0" +authors = ["Facebook, Inc."] +edition = "2018" + +[dependencies] +clap = "2.33.0" +shellexpand = "2.0.0" +tar = "0.4.26" +serde = { version = "1.0.106", features = ["derive"] } +serde_yaml = "0.8.11" diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 000000000..9e133939e --- /dev/null +++ b/packer/README.md @@ -0,0 +1,64 @@ +# Flipper Packer + +*Work in progress and incomplete* + +A tool for splitting a Flipper build into smaller artifacts that can be distributed and cached separately. + +**N.B. Buck build files are not included in the open-source export of this tool.** + +## Building + +With cargo: +``` +$ cargo build +``` + +With Buck: +``` +$ buck build :packer +``` + +## Testing + +With cargo: +``` +$ cargo test +``` + +With Buck: +``` +$ buck test :packer +``` + +## Usage + +See help page: + +``` +$ cargo run -- --help +flipper-packer 0.1.0 +Facebook, Inc. + + +USAGE: + flipper-packer [FLAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -p, --packlist Custom list of files to pack. + -V, --version Prints version information + +OPTIONS: + -d, --dist Flipper dist directory to read from. [default: ~/fbsource/xplat/sonar/dist] + -o, --output Directory to write output files to. [default: .] + +ARGS: + Platform to build for [possible values: Mac, Linux, Windows] + +``` + +Buck: + +``` +$ buck run :packer +``` diff --git a/packer/src/error.rs b/packer/src/error.rs new file mode 100644 index 000000000..7e3f0e53f --- /dev/null +++ b/packer/src/error.rs @@ -0,0 +1,40 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use crate::types::{PackType, Platform}; +use std::fmt; +use std::io; +use std::path::PathBuf; + +#[derive(Debug)] +pub enum Error { + IOError(io::Error), + MissingPackFile(Platform, PackType, PathBuf), +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Error::IOError(e) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Error::*; + + match self { + IOError(e) => write!(f, "IO Error: {}", e), + Error::MissingPackFile(platform, pack_type, path) => write!( + f, + "Couldn't open file to pack for platform {:?} and type {:?}: {}", + platform, + pack_type, + path.to_string_lossy() + ), + } + } +} diff --git a/packer/src/main.rs b/packer/src/main.rs new file mode 100644 index 000000000..be14ec588 --- /dev/null +++ b/packer/src/main.rs @@ -0,0 +1,158 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +mod error; +mod types; + +use clap::value_t_or_exit; +use std::collections::BTreeMap; +use std::fs::File; +use std::path; +use types::{PackType, Platform}; + +const DEFAULT_PACKLIST: &str = include_str!("packlist.yaml"); + +type PackListPlatform = BTreeMap>; + +#[derive(Debug, serde::Deserialize)] +struct PackList(pub BTreeMap); + +fn pack( + platform: &Platform, + dist_dir: &std::path::PathBuf, + pack_list: &PackList, + output_directory: &std::path::PathBuf, +) -> Result<(), error::Error> { + let mut frameworks_path = output_directory.clone(); + frameworks_path.push("frameworks.tar"); + let mut frameworks_tar = tar::Builder::new(File::create(frameworks_path)?); + // MacOS uses symlinks for bundling multiple framework versions and pointing + // to the "Current" one. + frameworks_tar.follow_symlinks(false); + pack_platform( + platform, + dist_dir, + pack_list, + &PackType::Frameworks, + &mut frameworks_tar, + )?; + frameworks_tar.finish()?; + + // TODO: Instead of hard-coding the two types, just iterate over the packlist. + let mut core_path = output_directory.clone(); + core_path.push("core.tar"); + let mut core_tar = tar::Builder::new(File::create(core_path)?); + pack_platform( + platform, + dist_dir, + pack_list, + &PackType::Core, + &mut core_tar, + )?; + + eprintln!("Written."); + Ok(()) +} + +fn pack_platform( + platform: &Platform, + dist_dir: &std::path::PathBuf, + pack_list: &PackList, + pack_type: &PackType, + tar_builder: &mut tar::Builder, +) -> Result<(), error::Error> { + let pack_files = pack_list + .0 + .get(platform) + .and_then(|f| f.get(pack_type)) + .expect(&format!( + "Missing packlist definition for platform {:?} and pack type {:?}.", + platform, pack_type + )); + let base_dir = match platform { + Platform::Mac => path::Path::new(dist_dir).join("mac"), + // TODO: Verify this. + Platform::Linux => path::Path::new(dist_dir).join("linux-unpacked"), + Platform::Windows => path::Path::new(dist_dir).join("win-unpacked"), + }; + + for f in pack_files { + let full_path = path::Path::new(&base_dir).join(f); + if !full_path.exists() { + return Err(error::Error::MissingPackFile( + platform.clone(), + pack_type.clone(), + full_path, + )); + } + if full_path.is_file() { + tar_builder.append_path_with_name(full_path, f)?; + } else if full_path.is_dir() { + tar_builder.append_dir_all(f, full_path)?; + } + } + + Ok(()) +} + +fn main() -> Result<(), error::Error> { + // Ensure to define all env vars used here in the BUCK env, too. + let args = clap::App::new(env!("CARGO_PKG_NAME")) + .version(env!("CARGO_PKG_VERSION")) + .author(env!("CARGO_PKG_AUTHORS")) + .about("Split the Flipper distribution into smaller, cacheable artifacts") + .arg( + clap::Arg::from_usage("-o, --output [DIRECTORY] 'Directory to write output files to.'") + .default_value("."), + ) + .arg( + clap::Arg::from_usage("-d, --dist [DIRECTORY] 'Flipper dist directory to read from.'") + .default_value("~/fbsource/xplat/sonar/dist"), + ) + .arg(clap::Arg::from_usage( + "-p, --packlist=packlist.yaml 'Custom list of files to pack.'", + )) + .arg( + clap::Arg::from_usage("[PLATFORM] 'Platform to build for'") + .case_insensitive(true) + .required(true) + .possible_values(&Platform::variants()), + ) + .get_matches(); + + let platform = value_t_or_exit!(args.value_of("PLATFORM"), Platform); + let dist_dir = path::PathBuf::from( + shellexpand::tilde(args.value_of("dist").expect("argument has default")).to_string(), + ); + let pack_list_str = args + .value_of("packlist") + .map(|f| std::fs::read_to_string(f).expect(&format!("Failed to open packfile {}.", f))) + .unwrap_or(DEFAULT_PACKLIST.to_string()); + let pack_list: PackList = + serde_yaml::from_str(&pack_list_str).expect("Failed to deserialize YAML packlist."); + pack( + &platform, + &dist_dir, + &pack_list, + &path::PathBuf::from(args.value_of("output").expect("argument has default")), + ) + .unwrap(); + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_included_packlist_parses() { + let res: PackList = + serde_yaml::from_str(DEFAULT_PACKLIST).expect("Default packlist doesn't deserialize"); + assert_eq!(res.0.len(), 3); + } +} diff --git a/packer/src/packlist.yaml b/packer/src/packlist.yaml new file mode 100644 index 000000000..26df7a40d --- /dev/null +++ b/packer/src/packlist.yaml @@ -0,0 +1,16 @@ +mac: + frameworks: + - Flipper.app/Contents/Frameworks/ + - Flipper.app/Contents/Info.plist + - Flipper.app/Contents/MacOS + - Flipper.app/Contents/PkgInfo + core: + - Flipper.app/Contents/Resources + +linux: + frameworks: [] + core: [] + +windows: + frameworks: [] + core: [] \ No newline at end of file diff --git a/packer/src/types.rs b/packer/src/types.rs new file mode 100644 index 000000000..386c7a226 --- /dev/null +++ b/packer/src/types.rs @@ -0,0 +1,25 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use clap::arg_enum; + +arg_enum! { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] + #[serde(rename_all = "lowercase")] + pub enum Platform { + Mac, + Linux, + Windows + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PackType { + Frameworks, + Core, +}