Implement glob packing mode

Summary:
Best seen by looking at the packfile. This makes upgrades and changes
to the file format a lot more robust.

Reviewed By: priteshrnandgaonkar

Differential Revision: D27194152

fbshipit-source-id: a722adb3a59d73b7530287f7abd84ed865a35fa7

packer/Cargo.lock

@@ -2,6 +2,15 @@
 # It is not intended for manual editing.
 version = 3
 
+[[package]]
+name = "aho-corasick"
+version = "0.7.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "ansi_term"
 version = "0.11.0"
@@ -49,6 +58,15 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "bstr"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.67"
@@ -215,6 +233,7 @@ dependencies = [
  "anyhow",
  "clap",
  "data-encoding",
+ "ignore",
  "indicatif",
  "rayon",
  "serde",
@@ -227,6 +246,12 @@ dependencies = [
  "xz2",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "fuchsia-cprng"
 version = "0.1.1"
@@ -254,6 +279,19 @@ dependencies = [
  "wasi",
 ]
 
+[[package]]
+name = "globset"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c152169ef1e421390738366d2f796655fec62621dabbd0fd476f905934061e4a"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
 [[package]]
 name = "hermit-abi"
 version = "0.1.18"
@@ -263,6 +301,24 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "ignore"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b287fb45c60bb826a0dc68ff08742b9d88a2fea13d6e0c286b3172065aaf878c"
+dependencies = [
+ "crossbeam-utils",
+ "globset",
+ "lazy_static",
+ "log",
+ "memchr",
+ "regex",
+ "same-file",
+ "thread_local",
+ "walkdir",
+ "winapi-util",
+]
+
 [[package]]
 name = "indicatif"
 version = "0.15.0"
@@ -299,6 +355,15 @@ version = "0.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
 
+[[package]]
+name = "log"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
+dependencies = [
+ "cfg-if",
+]
+
 [[package]]
 name = "lzma-sys"
 version = "0.1.17"
@@ -310,6 +375,12 @@ dependencies = [
  "pkg-config",
 ]
 
+[[package]]
+name = "memchr"
+version = "2.3.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
+
 [[package]]
 name = "memoffset"
 version = "0.6.1"
@@ -335,6 +406,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 
+[[package]]
+name = "once_cell"
+version = "1.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
+
 [[package]]
 name = "opaque-debug"
 version = "0.3.0"
@@ -452,6 +529,8 @@ version = "1.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
 dependencies = [
+ "aho-corasick",
+ "memchr",
  "regex-syntax",
 ]
 
@@ -476,6 +555,15 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "same-file"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
+dependencies = [
+ "winapi-util",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -604,6 +692,15 @@ dependencies = [
  "unicode-width",
 ]
 
+[[package]]
+name = "thread_local"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "typenum"
 version = "1.13.0"
@@ -634,6 +731,17 @@ version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
 
+[[package]]
+name = "walkdir"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
+dependencies = [
+ "same-file",
+ "winapi",
+ "winapi-util",
+]
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"
@@ -656,6 +764,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
 
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "winapi-x86_64-pc-windows-gnu"
 version = "0.4.0"

packer/Cargo.toml

@@ -22,6 +22,7 @@ serde_json = "1.0.64"
 rayon = "1.5.0"
 indicatif = "0.15.0"
 xz2 = "0.1.6"
+ignore = "^0.4.17"
 
 [dev-dependencies]
 tempdir = "0.3.7"

packer/src/main.rs

@@ -26,7 +26,7 @@ const DEFAULT_PACKLIST: &str = include_str!("packlist.yaml");
 // This is to ensure that all progress bar prefixes are aligned.
 const PROGRESS_PREFIX_LEN: usize = 24;
 
-type PackListPlatform = BTreeMap<PackType, Vec<path::PathBuf>>;
+type PackListPlatform = BTreeMap<PackType, Vec<String>>;
 
 #[derive(Debug, serde::Deserialize)]
 struct PackListSpec {
@@ -91,7 +91,11 @@ fn pack(
             // MacOS uses symlinks for bundling multiple framework versions and pointing
             // to the "Current" one.
             tar.follow_symlinks(false);
-            pack_platform(platform, &base_dir, pack_files, pack_type, &mut tar)?;
+            let pack_platform_fn = match packlist_spec.mode {
+                types::PackMode::Exact => pack_platform_exact,
+                types::PackMode::Glob => pack_platform_glob,
+            };
+            pack_platform_fn(platform, &base_dir, pack_files, pack_type, &mut tar)?;
             pb.inc(1);
             tar.finish()?;
             pb.inc(1);
@@ -112,10 +116,52 @@ fn platform_base_dir(dist_dir: &path::Path, platform: Platform) -> path::PathBuf
     }
 }
 
-fn pack_platform<P: AsRef<path::Path>>(
+fn pack_platform_glob(
     platform: Platform,
     base_dir: &path::Path,
-    pack_files: &[P],
+    pack_files: &[String],
+    pack_type: PackType,
+    tar_builder: &mut tar::Builder<File>,
+) -> Result<()> {
+    let mut ov = ignore::overrides::OverrideBuilder::new(base_dir);
+    for f in pack_files {
+        ov.add(f.as_ref())?;
+    }
+
+    let walker = ignore::WalkBuilder::new(base_dir)
+        // Otherwise we'll include .ignore files, etc.
+        .standard_filters(false)
+        // This is a big restriction. We won't traverse directories at all.
+        // If we did, we could no longer make use of tar's `append_dir_all`
+        // method which does a lot of clever stuff about symlinks, etc.
+        .max_depth(Some(1))
+        .overrides(ov.build()?)
+        .build();
+
+    // The first entry is the root dir, which we don't care about.
+    for f in walker.into_iter().skip(1) {
+        let dir_entry = f?;
+        let path = dir_entry.file_name();
+        let full_path = path::Path::new(&base_dir).join(&path);
+        if !full_path.exists() {
+            bail!(error::Error::MissingPackFile(
+                platform, pack_type, full_path,
+            ));
+        }
+        if full_path.is_file() {
+            tar_builder.append_path_with_name(full_path, &path)?;
+        } else if full_path.is_dir() {
+            tar_builder.append_dir_all(path, full_path)?;
+        }
+    }
+
+    Ok(())
+}
+
+fn pack_platform_exact(
+    platform: Platform,
+    base_dir: &path::Path,
+    pack_files: &[String],
     pack_type: PackType,
     tar_builder: &mut tar::Builder<File>,
 ) -> Result<()> {

packer/src/packlist.yaml

@@ -10,47 +10,19 @@ mac:
       - Flipper.app/Contents/Info.plist
 
 linux:
-  mode: exact
+  mode: glob
   files:
     frameworks:
-      - chrome-sandbox
+      - '!resources/'
-      - chrome_100_percent.pak
-      - chrome_200_percent.pak
-      - flipper
-      - icudtl.dat
-      - libEGL.so
-      - libffmpeg.so
-      - libGLESv2.so
-      - libvk_swiftshader.so
-      - libvulkan.so.1
-      - locales
-      - resources.pak
-      - snapshot_blob.bin
-      - swiftshader
-      - v8_context_snapshot.bin
-      - vk_swiftshader_icd.json
     core:
+      - '!*'
       - resources
 
 windows:
-  mode: exact
+  mode: glob
   files:
     frameworks:
-      - chrome_100_percent.pak
+      - '!resources/'
-      - chrome_200_percent.pak
-      - d3dcompiler_47.dll
-      - ffmpeg.dll
-      - Flipper.exe
-      - icudtl.dat
-      - libEGL.dll
-      - libGLESv2.dll
-      - locales
-      - resources.pak
-      - snapshot_blob.bin
-      - swiftshader
-      - v8_context_snapshot.bin
-      - vk_swiftshader.dll
-      - vk_swiftshader_icd.json
-      - vulkan-1.dll
     core:
+      - '!*'
       - resources