diff --git a/packer/.gitignore b/packer/.gitignore new file mode 100644 index 000000000..ea8c4bf7f --- /dev/null +++ b/packer/.gitignore @@ -0,0 +1 @@ +/target diff --git a/packer/Cargo.lock b/packer/Cargo.lock new file mode 100644 index 000000000..de164378b --- /dev/null +++ b/packer/Cargo.lock @@ -0,0 +1,379 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" + +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clap" +version = "2.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if", + "lazy_static", +] + +[[package]] +name = "dirs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" +dependencies = [ + "cfg-if", + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afa0b23de8fd801745c471deffa6e12d248f962c9fd4b4c33787b055599bde7b" +dependencies = [ + "cfg-if", + "libc", + "redox_users", + "winapi", +] + +[[package]] +name = "dtoa" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4358a9e11b9a09cf52383b451b49a169e8d797b68aa02301ff586d70d9661ea3" + +[[package]] +name = "filetime" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f59efc38004c988e4201d11d263b8171f49a2e7ec0bdbb71773433f271504a5e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "winapi", +] + +[[package]] +name = "flipper-packer" +version = "0.1.0" +dependencies = [ + "clap", + "serde", + "serde_yaml", + "shellexpand", + "tar", +] + +[[package]] +name = "getrandom" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61565ff7aaace3525556587bd2dc31d4a07071957be715e63ce7b1eccf51a8f4" +dependencies = [ + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" + +[[package]] +name = "linked-hash-map" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae91b68aebc4ddb91978b11a1b02ddd8602a05ec19002801c5666000e05e0f83" + +[[package]] +name = "proc-macro2" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bdc6c187c65bca4260c9011c9e3132efe4909da44726bad24cf7572ae338d7f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.1.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2439c63f3f6139d1b57529d16bc3b8bb855230c8efcc5d3a896c8bea7c3b1e84" + +[[package]] +name = "redox_users" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09b23093265f8d200fa7b4c2c76297f47e681c655f6f1285a8780d6a022f7431" +dependencies = [ + "getrandom", + "redox_syscall", + "rust-argon2", +] + +[[package]] +name = "rust-argon2" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bc8af4bda8e1ff4932523b94d3dd20ee30a87232323eda55903ffd71d2fb017" +dependencies = [ + "base64", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "serde" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36df6ac6412072f67cf767ebbde4133a5b2e88e76dc6187fa7104cd16f783399" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e549e3abf4fb8621bd1609f11dfc9f5e50320802273b12f3811a67e6716ea6c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_yaml" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "691b17f19fc1ec9d94ec0b5864859290dff279dbd7b03f017afda54eb36c3c35" +dependencies = [ + "dtoa", + "linked-hash-map", + "serde", + "yaml-rust", +] + +[[package]] +name = "shellexpand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2b22262a9aaf9464d356f656fea420634f78c881c5eebd5ef5e66d8b9bc603" +dependencies = [ + "dirs", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "410a7488c0a728c7ceb4ad59b9567eb4053d02e8cc7f5c0e0eeeb39518369213" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tar" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3196bfbffbba3e57481b6ea32249fbaf590396a52505a2615adbb79d9d826d3" +dependencies = [ + "filetime", + "libc", + "redox_syscall", + "xattr", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-width" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" + +[[package]] +name = "unicode-xid" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" + +[[package]] +name = "vec_map" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c78687fb1a80548ae3250346c3db86a80a7cdd77bda190189f2d0a0987c81a" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8093091eeb260906a183e6ae1abdba2ef5ef2257a21801128899c3fc699229c6" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "xattr" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "244c3741f4240ef46274860397c7c74e50eb23624996930e484c16679633a54c" +dependencies = [ + "libc", +] + +[[package]] +name = "yaml-rust" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65923dd1784f44da1d2c3dbbc5e822045628c590ba72123e1c73d3c230c4434d" +dependencies = [ + "linked-hash-map", +] diff --git a/packer/Cargo.toml b/packer/Cargo.toml new file mode 100644 index 000000000..51480dfe8 --- /dev/null +++ b/packer/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "flipper-packer" +version = "0.1.0" +authors = ["Facebook, Inc."] +edition = "2018" + +[dependencies] +clap = "2.33.0" +shellexpand = "2.0.0" +tar = "0.4.26" +serde = { version = "1.0.106", features = ["derive"] } +serde_yaml = "0.8.11" diff --git a/packer/README.md b/packer/README.md new file mode 100644 index 000000000..9e133939e --- /dev/null +++ b/packer/README.md @@ -0,0 +1,64 @@ +# Flipper Packer + +*Work in progress and incomplete* + +A tool for splitting a Flipper build into smaller artifacts that can be distributed and cached separately. + +**N.B. Buck build files are not included in the open-source export of this tool.** + +## Building + +With cargo: +``` +$ cargo build +``` + +With Buck: +``` +$ buck build :packer +``` + +## Testing + +With cargo: +``` +$ cargo test +``` + +With Buck: +``` +$ buck test :packer +``` + +## Usage + +See help page: + +``` +$ cargo run -- --help +flipper-packer 0.1.0 +Facebook, Inc. + + +USAGE: + flipper-packer [FLAGS] [OPTIONS] + +FLAGS: + -h, --help Prints help information + -p, --packlist Custom list of files to pack. + -V, --version Prints version information + +OPTIONS: + -d, --dist Flipper dist directory to read from. [default: ~/fbsource/xplat/sonar/dist] + -o, --output Directory to write output files to. [default: .] + +ARGS: + Platform to build for [possible values: Mac, Linux, Windows] + +``` + +Buck: + +``` +$ buck run :packer +``` diff --git a/packer/src/error.rs b/packer/src/error.rs new file mode 100644 index 000000000..7e3f0e53f --- /dev/null +++ b/packer/src/error.rs @@ -0,0 +1,40 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use crate::types::{PackType, Platform}; +use std::fmt; +use std::io; +use std::path::PathBuf; + +#[derive(Debug)] +pub enum Error { + IOError(io::Error), + MissingPackFile(Platform, PackType, PathBuf), +} + +impl From for Error { + fn from(e: io::Error) -> Self { + Error::IOError(e) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use Error::*; + + match self { + IOError(e) => write!(f, "IO Error: {}", e), + Error::MissingPackFile(platform, pack_type, path) => write!( + f, + "Couldn't open file to pack for platform {:?} and type {:?}: {}", + platform, + pack_type, + path.to_string_lossy() + ), + } + } +} diff --git a/packer/src/main.rs b/packer/src/main.rs new file mode 100644 index 000000000..be14ec588 --- /dev/null +++ b/packer/src/main.rs @@ -0,0 +1,158 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +mod error; +mod types; + +use clap::value_t_or_exit; +use std::collections::BTreeMap; +use std::fs::File; +use std::path; +use types::{PackType, Platform}; + +const DEFAULT_PACKLIST: &str = include_str!("packlist.yaml"); + +type PackListPlatform = BTreeMap>; + +#[derive(Debug, serde::Deserialize)] +struct PackList(pub BTreeMap); + +fn pack( + platform: &Platform, + dist_dir: &std::path::PathBuf, + pack_list: &PackList, + output_directory: &std::path::PathBuf, +) -> Result<(), error::Error> { + let mut frameworks_path = output_directory.clone(); + frameworks_path.push("frameworks.tar"); + let mut frameworks_tar = tar::Builder::new(File::create(frameworks_path)?); + // MacOS uses symlinks for bundling multiple framework versions and pointing + // to the "Current" one. + frameworks_tar.follow_symlinks(false); + pack_platform( + platform, + dist_dir, + pack_list, + &PackType::Frameworks, + &mut frameworks_tar, + )?; + frameworks_tar.finish()?; + + // TODO: Instead of hard-coding the two types, just iterate over the packlist. + let mut core_path = output_directory.clone(); + core_path.push("core.tar"); + let mut core_tar = tar::Builder::new(File::create(core_path)?); + pack_platform( + platform, + dist_dir, + pack_list, + &PackType::Core, + &mut core_tar, + )?; + + eprintln!("Written."); + Ok(()) +} + +fn pack_platform( + platform: &Platform, + dist_dir: &std::path::PathBuf, + pack_list: &PackList, + pack_type: &PackType, + tar_builder: &mut tar::Builder, +) -> Result<(), error::Error> { + let pack_files = pack_list + .0 + .get(platform) + .and_then(|f| f.get(pack_type)) + .expect(&format!( + "Missing packlist definition for platform {:?} and pack type {:?}.", + platform, pack_type + )); + let base_dir = match platform { + Platform::Mac => path::Path::new(dist_dir).join("mac"), + // TODO: Verify this. + Platform::Linux => path::Path::new(dist_dir).join("linux-unpacked"), + Platform::Windows => path::Path::new(dist_dir).join("win-unpacked"), + }; + + for f in pack_files { + let full_path = path::Path::new(&base_dir).join(f); + if !full_path.exists() { + return Err(error::Error::MissingPackFile( + platform.clone(), + pack_type.clone(), + full_path, + )); + } + if full_path.is_file() { + tar_builder.append_path_with_name(full_path, f)?; + } else if full_path.is_dir() { + tar_builder.append_dir_all(f, full_path)?; + } + } + + Ok(()) +} + +fn main() -> Result<(), error::Error> { + // Ensure to define all env vars used here in the BUCK env, too. + let args = clap::App::new(env!("CARGO_PKG_NAME")) + .version(env!("CARGO_PKG_VERSION")) + .author(env!("CARGO_PKG_AUTHORS")) + .about("Split the Flipper distribution into smaller, cacheable artifacts") + .arg( + clap::Arg::from_usage("-o, --output [DIRECTORY] 'Directory to write output files to.'") + .default_value("."), + ) + .arg( + clap::Arg::from_usage("-d, --dist [DIRECTORY] 'Flipper dist directory to read from.'") + .default_value("~/fbsource/xplat/sonar/dist"), + ) + .arg(clap::Arg::from_usage( + "-p, --packlist=packlist.yaml 'Custom list of files to pack.'", + )) + .arg( + clap::Arg::from_usage("[PLATFORM] 'Platform to build for'") + .case_insensitive(true) + .required(true) + .possible_values(&Platform::variants()), + ) + .get_matches(); + + let platform = value_t_or_exit!(args.value_of("PLATFORM"), Platform); + let dist_dir = path::PathBuf::from( + shellexpand::tilde(args.value_of("dist").expect("argument has default")).to_string(), + ); + let pack_list_str = args + .value_of("packlist") + .map(|f| std::fs::read_to_string(f).expect(&format!("Failed to open packfile {}.", f))) + .unwrap_or(DEFAULT_PACKLIST.to_string()); + let pack_list: PackList = + serde_yaml::from_str(&pack_list_str).expect("Failed to deserialize YAML packlist."); + pack( + &platform, + &dist_dir, + &pack_list, + &path::PathBuf::from(args.value_of("output").expect("argument has default")), + ) + .unwrap(); + + Ok(()) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_included_packlist_parses() { + let res: PackList = + serde_yaml::from_str(DEFAULT_PACKLIST).expect("Default packlist doesn't deserialize"); + assert_eq!(res.0.len(), 3); + } +} diff --git a/packer/src/packlist.yaml b/packer/src/packlist.yaml new file mode 100644 index 000000000..26df7a40d --- /dev/null +++ b/packer/src/packlist.yaml @@ -0,0 +1,16 @@ +mac: + frameworks: + - Flipper.app/Contents/Frameworks/ + - Flipper.app/Contents/Info.plist + - Flipper.app/Contents/MacOS + - Flipper.app/Contents/PkgInfo + core: + - Flipper.app/Contents/Resources + +linux: + frameworks: [] + core: [] + +windows: + frameworks: [] + core: [] \ No newline at end of file diff --git a/packer/src/types.rs b/packer/src/types.rs new file mode 100644 index 000000000..386c7a226 --- /dev/null +++ b/packer/src/types.rs @@ -0,0 +1,25 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use clap::arg_enum; + +arg_enum! { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] + #[serde(rename_all = "lowercase")] + pub enum Platform { + Mac, + Linux, + Windows + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum PackType { + Frameworks, + Core, +}