Stable babel transformation cache key

Summary:
This diff makes babel transformation cache key stable. With the previous approach it was changed on each CI build so it was not possible to persist cache between builds. Now it is computed from the transformation package content after each build. Because of that it is equal for every CI build while Babel transformations unchanged and so we could use same cache on different Sandcastle agents.

In addition to that, it makes it possible to specify directory for Metro cache so we can save/restore it in CI builds.

Reviewed By: mweststrate

Differential Revision: D26877989

fbshipit-source-id: 7cb04a177f86e61986585e5a74d9c7396ddddc18

desktop/babel-transformer/package.json

@@ -10,8 +10,8 @@
   "bugs": "https://github.com/facebook/flipper/issues",
   "dependencies": {
     "@babel/core": "^7.13.8",
-    "@babel/parser": "^7.13.9",
     "@babel/generator": "^7.13.9",
+    "@babel/parser": "^7.13.9",
     "@babel/plugin-proposal-class-properties": "^7.13.0",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8",
     "@babel/plugin-proposal-object-rest-spread": "^7.13.8",
@@ -40,7 +40,7 @@
   },
   "scripts": {
     "reset": "rimraf lib *.tsbuildinfo",
-    "build": "tsc -b",
+    "build": "tsc -b && cd .. && ./ts-node ./scripts/compute-package-checksum.ts -d ./babel-transformer -o ./lib/checksum.txt",
     "prepack": "yarn reset && yarn build"
   },
   "files": [

desktop/babel-transformer/src/get-cache-key.ts

@@ -18,28 +18,22 @@ import {default as flipperEnv} from './flipper-env';
 import fs from 'fs-extra';
 import path from 'path';
 
-let baseHash = '';
+let selfChecksum: string | undefined;
-const tsbuildinfoPath = path.resolve(__dirname, '..', 'tsconfig.tsbuildinfo');
+function getSelfChecksum() {
-const packageJsonPath = path.resolve(__dirname, '..', 'package.json');
+  if (!selfChecksum) {
-if (fs.pathExistsSync(tsbuildinfoPath)) {
+    selfChecksum = fs
-  /**
+      .readFileSync(path.resolve(__dirname, '..', 'lib', 'checksum.txt'))
-   * tsconfig.tsbuildinfo is changed each time TS incremental build detects changes and rebuilds the package,
+      .toString();
-   * so we can use its modification date as cache key to invalidate the cache each time when babel transformations changed.
+  }
-   */
+  return selfChecksum;
-  baseHash = fs.lstatSync(tsbuildinfoPath).ctime.toUTCString();
-} else if (fs.pathExistsSync(packageJsonPath)) {
-  /**
-   * tsconfig.tsbuildinfo will not exist in case if the package is installed from npm rather than built locally.
-   * In such case we should use version of npm package as hash key to invalidate the cache after updates.
-   */
-  baseHash = fs.readJsonSync(packageJsonPath).version;
 }
 
 export default function getCacheKey() {
-  return [
+  const key = [
-    baseHash,
+    getSelfChecksum(),
     ...Object.entries(flipperEnv)
       .sort(([name1, _value1], [name2, _value2]) => name1.localeCompare(name2))
       .map(([name, value]) => `${name}=${value}`),
   ].join('|');
+  return key;
 }

desktop/package.json

@@ -218,11 +218,10 @@
   "scripts": {
     "build": "cross-env NODE_ENV=production ./ts-node scripts/build-release.ts $@",
     "build-plugin": "./ts-node scripts/build-plugin.ts",
-    "build:babel-transformer": "cd babel-transformer && yarn build",
     "build:dev": "cross-env NODE_ENV=development ./ts-node scripts/build-release.ts $@",
     "build:eslint": "cd eslint-plugin-flipper && yarn build",
-    "build:pkg": "cd pkg && yarn build",
     "build:themes": "lessc --js themes/light.less static/themes/light.css && lessc --js themes/dark.less static/themes/dark.css",
+    "build:tsc": "tsc -b tsc-root/tsconfig.json && ./ts-node ./scripts/compute-package-checksum.ts -d ./babel-transformer -o ./lib/checksum.txt",
     "bump-versions": "./ts-node scripts/bump-versions.ts",
     "dev-server": "cross-env NODE_ENV=development ./ts-node scripts/start-dev-server.ts",
     "everything": "yarn reset && yarn install && yarn lint && yarn test && yarn test-electron && yarn build --mac --mac-dmg --win --linux --linux-deb && yarn start",
@@ -232,12 +231,12 @@
     "lint:tsc": "tsc --noemit",
     "list-plugins": "./ts-node scripts/list-plugins.ts",
     "open-dist": "open ../dist/mac/Flipper.app --args --launcher=false --inspect=9229",
-    "postinstall": "patch-package && ./ts-node scripts/yarn-install-fb-plugins.ts && yarn build:pkg && yarn build:eslint && ./ts-node scripts/generate-plugin-entry-points.ts && yarn build:themes",
+    "postinstall": "patch-package && ./ts-node scripts/yarn-install-fb-plugins.ts && yarn build:tsc && ./ts-node scripts/generate-plugin-entry-points.ts && yarn build:themes",
-    "prebuild": "yarn build:pkg && yarn rm-dist && yarn build:themes",
+    "prebuild": "yarn build:tsc && yarn rm-dist && yarn build:themes",
-    "predev-server": "yarn build:pkg",
+    "predev-server": "yarn build:tsc",
     "preinstall": "node scripts/prepare-watchman-config.js && yarn config set ignore-engines",
     "prelint:eslint": "yarn build:eslint",
-    "pretest": "yarn build:pkg",
+    "pretest": "yarn build:tsc",
     "publish-packages": "./ts-node scripts/publish-packages.ts",
     "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-modules",
     "resolve-plugin-dir": "./ts-node scripts/resolve-plugin-dir.ts",
@@ -252,9 +251,9 @@
     "start:no-embedded-plugins": "yarn start --no-embedded-plugins",
     "test": "jest",
     "test-e2e": "cd e2e && yarn test",
-    "test-electron": "yarn build:pkg && jest --testMatch=\"**/**.electron\\.(js|jsx|ts|tsx)\" --testEnvironment=@jest-runner/electron/environment --runner=@jest-runner/electron",
+    "test-electron": "yarn build:tsc && jest --testMatch=\"**/**.electron\\.(js|jsx|ts|tsx)\" --testEnvironment=@jest-runner/electron/environment --runner=@jest-runner/electron",
-    "test-with-device": "yarn build:pkg && USE_ELECTRON_STUBS=1 jest --testMatch=\"**/**.device\\.(js|jsx|ts|tsx)\" --detectOpenHandles",
+    "test-with-device": "yarn build:tsc && USE_ELECTRON_STUBS=1 jest --testMatch=\"**/**.device\\.(js|jsx|ts|tsx)\" --detectOpenHandles",
-    "test:debug": "yarn build:pkg && node --inspect node_modules/.bin/jest --runInBand",
+    "test:debug": "yarn build:tsc && node --inspect node_modules/.bin/jest --runInBand",
     "watch": "jest --watch"
   },
   "version": "0.79.1",

desktop/pkg-lib/package.json

@@ -13,6 +13,7 @@
     "flipper-plugin-lib": "0.0.0",
     "fs-extra": "^9.0.1",
     "metro": "^0.65.2",
+    "metro-cache": "^0.65.2",
     "metro-minify-terser": "^0.65.2",
     "npm-packlist": "^2.1.4"
   },

desktop/pkg-lib/src/computePackageChecksum.ts

@@ -14,16 +14,41 @@ import fs from 'fs-extra';
 
 export default async function computePackageChecksum(
   dir: string,
+  checksumFilePath?: string,
 ): Promise<string> {
+  const fullChecksumFilePath = checksumFilePath
+    ? path.resolve(dir, checksumFilePath)
+    : undefined;
+  if (fullChecksumFilePath) {
+    // This block is an optimisation to not recompute checksum if nothing changed.
+    // tsbuildinfo file is changed every time when typescript compiler re-compiles anything,
+    // so we could compare its modification date with checksum modification date to
+    // decide whether we need to re-compute checksum or we can just use already computed one.
+    const tsBuildInfoPath = path.join(dir, 'tsconfig.tsbuildinfo');
+    try {
+      const [tsBuildInfoStat, checksumStat] = await Promise.all([
+        fs.stat(tsBuildInfoPath),
+        fs.stat(fullChecksumFilePath),
+      ]);
+      if (checksumStat.mtime > tsBuildInfoStat.mtime) {
+        return (await fs.readFile(fullChecksumFilePath)).toString();
+      }
+    } catch {}
+  }
   const hash = crypto.createHash('sha1');
   hash.setEncoding('hex');
   const files = (await packlist({path: dir})).sort();
   for (const file of files) {
-    // add hash of relative file path
-    hash.write(process.platform === 'win32' ? file.replace(/\\/g, '/') : file);
-
     const filePath = path.resolve(dir, file);
 
+    if (filePath === fullChecksumFilePath) {
+      // If there is already existing checksum file, we need to ignore it as it is not a part of package content.
+      continue;
+    }
+
+    // add hash of relative normalized file path
+    hash.write(process.platform === 'win32' ? file.replace(/\\/g, '/') : file);
+
     if (file === 'package.json') {
       // add hash of package.json with version set to "0.0.0" to avoid changing hash when only version changed
       const packageJson = await fs.readJson(filePath);
@@ -46,5 +71,9 @@ export default async function computePackageChecksum(
     }
   }
   hash.end();
-  return hash.read();
+  const checksum = hash.read();
+  if (fullChecksumFilePath) {
+    await fs.writeFile(fullChecksumFilePath, checksum);
+  }
+  return checksum;
 }

desktop/pkg-lib/src/runBuild.ts

@@ -12,6 +12,8 @@ import getWatchFolders from './getWatchFolders';
 import path from 'path';
 import fs from 'fs-extra';
 import {getInstalledPluginDetails} from 'flipper-plugin-lib';
+import {FileStore} from 'metro-cache';
+import os from 'os';
 
 let metroDir: string | undefined;
 const metroDirPromise = getMetroDir().then((dir) => (metroDir = dir));
@@ -77,11 +79,18 @@ export default async function bundlePlugin(pluginDir: string, dev: boolean) {
       sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs', 'cjs'],
       blacklistRE: /\.native\.js$/,
     },
+    cacheStores: [
+      new FileStore({
+        root:
+          process.env.FLIPPER_METRO_CACHE ??
+          path.join(os.tmpdir(), 'metro-cache'),
+      }),
+    ],
   });
   await Metro.runBuild(config, {
     dev,
     minify: !dev,
-    resetCache: !dev,
+    resetCache: false,
     sourceMap: dev,
     sourceMapUrl,
     entry,

desktop/scripts/compute-package-checksum.ts

@@ -0,0 +1,41 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {computePackageChecksum} from 'flipper-pkg-lib';
+import yargs from 'yargs';
+
+const argv = yargs
+  .usage('yarn compute-package-checksum [args]')
+  .version(false)
+  .options({
+    dir: {
+      description: 'Package root directory',
+      type: 'string',
+      demandOption: true,
+      alias: 'd',
+    },
+    out: {
+      description:
+        'File relative to the package root directory where the computed checksum should be saved, e.g. "lib/checksum.txt"',
+      type: 'string',
+      alias: 'o',
+    },
+  })
+  .help()
+  .strict()
+  .parse(process.argv.slice(1));
+
+computePackageChecksum(argv.dir, argv.out)
+  .then(() => {
+    process.exit(0);
+  })
+  .catch((err: any) => {
+    console.error(err);
+    process.exit(1);
+  });

desktop/tsc-root/tsconfig.json

@@ -0,0 +1,25 @@
+{
+    "references": [
+        {
+            "path": "../babel-transformer"
+        },
+        {
+            "path": "../plugin-lib"
+        },
+        {
+            "path": "../pkg"
+        },
+        {
+            "path": "../pkg-lib"
+        },
+        {
+            "path": "../doctor"
+        },
+        {
+            "path": "../eslint-plugin-flipper"
+        },
+        {
+            "path": "../test-utils"
+        }
+    ]
+}

desktop/types/metro-cache.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+declare module 'metro-cache';

desktop/yarn.lock

@@ -8741,7 +8741,7 @@ metro-cache-key@0.65.2:
   resolved "https://registry.yarnpkg.com/metro-cache-key/-/metro-cache-key-0.65.2.tgz#b841ceacda1c97a5616ad99e728a3bf31c6390df"
   integrity sha512-2Re5w1kxIpdBFYavrAOrAu0pIj/gj2DUQGcCbDiYWpDw0ibyvOakmIbMZRH/J6c1KXJIGzb4YZpisKeAOsNODQ==
 
-metro-cache@0.65.2:
+metro-cache@0.65.2, metro-cache@^0.65.2:
   version "0.65.2"
   resolved "https://registry.yarnpkg.com/metro-cache/-/metro-cache-0.65.2.tgz#8922c8d4d882014e745aa01c9d1c64aeb3b08306"
   integrity sha512-+f7A57qlF2IChrFsHsYN4Lhr7nwb3lsS+Z/M1833Bhj25VBhbe3dMqt8+rOWncgkOY3aHtH7PphRABy5yfDQ3g==