jens/flipper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement glob packing mode

Browse Source 
Summary:
Best seen by looking at the packfile. This makes upgrades and changes
to the file format a lot more robust.

Reviewed By: priteshrnandgaonkar

Differential Revision: D27194152

fbshipit-source-id: a722adb3a59d73b7530287f7abd84ed865a35fa7
This commit is contained in:
Pascal Hartig
2021-03-23 13:26:31 -07:00
committed by Facebook GitHub Bot
parent ca55e68c89
commit fa97809178
4 changed files with 174 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

117
packer/Cargo.lock generated
View File

@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5"
dependencies = [
"memchr",
]
[[package]]
name = "ansi_term"
version = "0.11.0"
@@ -49,6 +58,15 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bstr"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a40b47ad93e1a5404e6c18dec46b628214fee441c70f4ab5d6942142cc268a3d"
dependencies = [
"memchr",
]
[[package]]
name = "cc"
version = "1.0.67"
@@ -215,6 +233,7 @@ dependencies = [
"anyhow",
"clap",
"data-encoding",
"ignore",
"indicatif",
"rayon",
"serde",
@@ -227,6 +246,12 @@ dependencies = [
"xz2",
]
[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "fuchsia-cprng"
version = "0.1.1"
@@ -254,6 +279,19 @@ dependencies = [
"wasi",
]
[[package]]
name = "globset"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c152169ef1e421390738366d2f796655fec62621dabbd0fd476f905934061e4a"
dependencies = [
"aho-corasick",
"bstr",
"fnv",
"log",
"regex",
]
[[package]]
name = "hermit-abi"
version = "0.1.18"
@@ -263,6 +301,24 @@ dependencies = [
"libc",
]
[[package]]
name = "ignore"
version = "0.4.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b287fb45c60bb826a0dc68ff08742b9d88a2fea13d6e0c286b3172065aaf878c"
dependencies = [
"crossbeam-utils",
"globset",
"lazy_static",
"log",
"memchr",
"regex",
"same-file",
"thread_local",
"walkdir",
"winapi-util",
]
[[package]]
name = "indicatif"
version = "0.15.0"
@@ -299,6 +355,15 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3"
[[package]]
name = "log"
version = "0.4.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710"
dependencies = [
"cfg-if",
]
[[package]]
name = "lzma-sys"
version = "0.1.17"
@@ -310,6 +375,12 @@ dependencies = [
"pkg-config",
]
[[package]]
name = "memchr"
version = "2.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
name = "memoffset"
version = "0.6.1"
@@ -335,6 +406,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
[[package]]
name = "once_cell"
version = "1.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3"
[[package]]
name = "opaque-debug"
version = "0.3.0"
@@ -452,6 +529,8 @@ version = "1.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax",
]
@@ -476,6 +555,15 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
@@ -604,6 +692,15 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "thread_local"
version = "1.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd"
dependencies = [
"once_cell",
]
[[package]]
name = "typenum"
version = "1.13.0"
@@ -634,6 +731,17 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "walkdir"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "777182bc735b6424e1a57516d35ed72cb8019d85c8c9bf536dccb3445c1a2f7d"
dependencies = [
"same-file",
"winapi",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
@@ -656,6 +764,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [
"winapi",
]
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"

1
packer/Cargo.toml
View File

@@ -22,6 +22,7 @@ serde_json = "1.0.64"
rayon = "1.5.0"
indicatif = "0.15.0"
xz2 = "0.1.6"
ignore = "^0.4.17"
[dev-dependencies]
tempdir = "0.3.7"

54
packer/src/main.rs
View File

@@ -26,7 +26,7 @@ const DEFAULT_PACKLIST: &str = include_str!("packlist.yaml");
// This is to ensure that all progress bar prefixes are aligned.
const PROGRESS_PREFIX_LEN: usize = 24;
type PackListPlatform = BTreeMap<PackType, Vec<path::PathBuf>>;
type PackListPlatform = BTreeMap<PackType, Vec<String>>;
#[derive(Debug, serde::Deserialize)]
struct PackListSpec {
@@ -91,7 +91,11 @@ fn pack(
// MacOS uses symlinks for bundling multiple framework versions and pointing
// to the "Current" one.
tar.follow_symlinks(false);
pack_platform(platform, &base_dir, pack_files, pack_type, &mut tar)?;
let pack_platform_fn = match packlist_spec.mode {
types::PackMode::Exact => pack_platform_exact,
types::PackMode::Glob => pack_platform_glob,
};
pack_platform_fn(platform, &base_dir, pack_files, pack_type, &mut tar)?;
pb.inc(1);
tar.finish()?;
pb.inc(1);
@@ -112,10 +116,52 @@ fn platform_base_dir(dist_dir: &path::Path, platform: Platform) -> path::PathBuf
}
}
fn pack_platform<P: AsRef<path::Path>>(
fn pack_platform_glob(
platform: Platform,
base_dir: &path::Path,
pack_files: &[P],
pack_files: &[String],
pack_type: PackType,
tar_builder: &mut tar::Builder<File>,
) -> Result<()> {
let mut ov = ignore::overrides::OverrideBuilder::new(base_dir);
for f in pack_files {
ov.add(f.as_ref())?;
}
let walker = ignore::WalkBuilder::new(base_dir)
// Otherwise we'll include .ignore files, etc.
.standard_filters(false)
// This is a big restriction. We won't traverse directories at all.
// If we did, we could no longer make use of tar's `append_dir_all`
// method which does a lot of clever stuff about symlinks, etc.
.max_depth(Some(1))
.overrides(ov.build()?)
.build();
// The first entry is the root dir, which we don't care about.
for f in walker.into_iter().skip(1) {
let dir_entry = f?;
let path = dir_entry.file_name();
let full_path = path::Path::new(&base_dir).join(&path);
if !full_path.exists() {
bail!(error::Error::MissingPackFile(
platform, pack_type, full_path,
));
}
if full_path.is_file() {
tar_builder.append_path_with_name(full_path, &path)?;
} else if full_path.is_dir() {
tar_builder.append_dir_all(path, full_path)?;
}
}
Ok(())
}
fn pack_platform_exact(
platform: Platform,
base_dir: &path::Path,
pack_files: &[String],
pack_type: PackType,
tar_builder: &mut tar::Builder<File>,
) -> Result<()> {

40
packer/src/packlist.yaml
View File

@@ -10,47 +10,19 @@ mac:
- Flipper.app/Contents/Info.plist
linux:
mode: exact
mode: glob
files:
frameworks:
- chrome-sandbox
- chrome_100_percent.pak
- chrome_200_percent.pak
- flipper
- icudtl.dat
- libEGL.so
- libffmpeg.so
- libGLESv2.so
- libvk_swiftshader.so
- libvulkan.so.1
- locales
- resources.pak
- snapshot_blob.bin
- swiftshader
- v8_context_snapshot.bin
- vk_swiftshader_icd.json
- '!resources/'
core:
- '!*'
- resources
windows:
mode: exact
mode: glob
files:
frameworks:
- chrome_100_percent.pak
- chrome_200_percent.pak
- d3dcompiler_47.dll
- ffmpeg.dll
- Flipper.exe
- icudtl.dat
- libEGL.dll
- libGLESv2.dll
- locales
- resources.pak
- snapshot_blob.bin
- swiftshader
- v8_context_snapshot.bin
- vk_swiftshader.dll
- vk_swiftshader_icd.json
- vulkan-1.dll
- '!resources/'
core:
- '!*'
- resources