diff --git a/packer/Cargo.lock b/packer/Cargo.lock index 760fee11d..dca5041b0 100644 --- a/packer/Cargo.lock +++ b/packer/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.11.0" @@ -49,6 +58,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bstr" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d" +dependencies = [ + "memchr", +] + [[package]] name = "cc" version = "1.0.67" @@ -215,6 +233,7 @@ dependencies = [ "anyhow", "clap", "data-encoding", + "ignore", "indicatif", "rayon", "serde", @@ -227,6 +246,12 @@ dependencies = [ "xz2", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "fuchsia-cprng" version = "0.1.1" @@ -254,6 +279,19 @@ dependencies = [ "wasi", ] +[[package]] +name = "globset" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c152169ef1e421390738366d2f796655fec62621dabbd0fd476f905934061e4a" +dependencies = [ + "aho-corasick", + "bstr", + "fnv", + "log", + "regex", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -263,6 +301,24 @@ dependencies = [ "libc", ] +[[package]] +name = "ignore" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b287fb45c60bb826a0dc68ff08742b9d88a2fea13d6e0c286b3172065aaf878c" +dependencies = [ + "crossbeam-utils", + "globset", + "lazy_static", + "log", + "memchr", + "regex", + "same-file", + "thread_local", + "walkdir", + "winapi-util", +] + [[package]] name = "indicatif" version = "0.15.0" @@ -299,6 +355,15 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + [[package]] name = "lzma-sys" version = "0.1.17" @@ -310,6 +375,12 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "memchr" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" + [[package]] name = "memoffset" version = "0.6.1" @@ -335,6 +406,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a" +[[package]] +name = "once_cell" +version = "1.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" + [[package]] name = "opaque-debug" version = "0.3.0" @@ -452,6 +529,8 @@ version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", ] @@ -476,6 +555,15 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -604,6 +692,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.13.0" @@ -634,6 +731,17 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "walkdir" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -656,6 +764,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/packer/Cargo.toml b/packer/Cargo.toml index e1ffa6306..59859970b 100644 --- a/packer/Cargo.toml +++ b/packer/Cargo.toml @@ -22,6 +22,7 @@ serde_json = "1.0.64" rayon = "1.5.0" indicatif = "0.15.0" xz2 = "0.1.6" +ignore = "^0.4.17" [dev-dependencies] tempdir = "0.3.7" diff --git a/packer/src/main.rs b/packer/src/main.rs index 7e971bc59..5604d08a5 100644 --- a/packer/src/main.rs +++ b/packer/src/main.rs @@ -26,7 +26,7 @@ const DEFAULT_PACKLIST: &str = include_str!("packlist.yaml"); // This is to ensure that all progress bar prefixes are aligned. const PROGRESS_PREFIX_LEN: usize = 24; -type PackListPlatform = BTreeMap>; +type PackListPlatform = BTreeMap>; #[derive(Debug, serde::Deserialize)] struct PackListSpec { @@ -91,7 +91,11 @@ fn pack( // MacOS uses symlinks for bundling multiple framework versions and pointing // to the "Current" one. tar.follow_symlinks(false); - pack_platform(platform, &base_dir, pack_files, pack_type, &mut tar)?; + let pack_platform_fn = match packlist_spec.mode { + types::PackMode::Exact => pack_platform_exact, + types::PackMode::Glob => pack_platform_glob, + }; + pack_platform_fn(platform, &base_dir, pack_files, pack_type, &mut tar)?; pb.inc(1); tar.finish()?; pb.inc(1); @@ -112,10 +116,52 @@ fn platform_base_dir(dist_dir: &path::Path, platform: Platform) -> path::PathBuf } } -fn pack_platform>( +fn pack_platform_glob( platform: Platform, base_dir: &path::Path, - pack_files: &[P], + pack_files: &[String], + pack_type: PackType, + tar_builder: &mut tar::Builder, +) -> Result<()> { + let mut ov = ignore::overrides::OverrideBuilder::new(base_dir); + for f in pack_files { + ov.add(f.as_ref())?; + } + + let walker = ignore::WalkBuilder::new(base_dir) + // Otherwise we'll include .ignore files, etc. + .standard_filters(false) + // This is a big restriction. We won't traverse directories at all. + // If we did, we could no longer make use of tar's `append_dir_all` + // method which does a lot of clever stuff about symlinks, etc. + .max_depth(Some(1)) + .overrides(ov.build()?) + .build(); + + // The first entry is the root dir, which we don't care about. + for f in walker.into_iter().skip(1) { + let dir_entry = f?; + let path = dir_entry.file_name(); + let full_path = path::Path::new(&base_dir).join(&path); + if !full_path.exists() { + bail!(error::Error::MissingPackFile( + platform, pack_type, full_path, + )); + } + if full_path.is_file() { + tar_builder.append_path_with_name(full_path, &path)?; + } else if full_path.is_dir() { + tar_builder.append_dir_all(path, full_path)?; + } + } + + Ok(()) +} + +fn pack_platform_exact( + platform: Platform, + base_dir: &path::Path, + pack_files: &[String], pack_type: PackType, tar_builder: &mut tar::Builder, ) -> Result<()> { diff --git a/packer/src/packlist.yaml b/packer/src/packlist.yaml index 70f0c3043..8aa2a3cee 100644 --- a/packer/src/packlist.yaml +++ b/packer/src/packlist.yaml @@ -10,47 +10,19 @@ mac: - Flipper.app/Contents/Info.plist linux: - mode: exact + mode: glob files: frameworks: - - chrome-sandbox - - chrome_100_percent.pak - - chrome_200_percent.pak - - flipper - - icudtl.dat - - libEGL.so - - libffmpeg.so - - libGLESv2.so - - libvk_swiftshader.so - - libvulkan.so.1 - - locales - - resources.pak - - snapshot_blob.bin - - swiftshader - - v8_context_snapshot.bin - - vk_swiftshader_icd.json + - '!resources/' core: + - '!*' - resources windows: - mode: exact + mode: glob files: frameworks: - - chrome_100_percent.pak - - chrome_200_percent.pak - - d3dcompiler_47.dll - - ffmpeg.dll - - Flipper.exe - - icudtl.dat - - libEGL.dll - - libGLESv2.dll - - locales - - resources.pak - - snapshot_blob.bin - - swiftshader - - v8_context_snapshot.bin - - vk_swiftshader.dll - - vk_swiftshader_icd.json - - vulkan-1.dll + - '!resources/' core: + - '!*' - resources