Enable Metro caching

Summary:
Enabling Metro cache for dev mode. For release builds we reset the cache.

Cache is used for faster compilation in dev mode for both main and renderer bundles, as well as for plugins.

Currently we have few side effects based on env vars, so cache is invalidated when they are changed. Also the cache is invalidated when transformations are changed (changed code, bumped dependency etc). Also added a script to reset the cache if something is going wrong.

Reviewed By: mweststrate

Differential Revision: D20691464

fbshipit-source-id: 478947d438bd3090f052dbfa6ad5c649523ecacb

desktop/babel-transformer/package.json

@@ -12,8 +12,6 @@
     "@babel/core": "^7.9.0",
     "@babel/generator": "^7.9.0",
     "@babel/parser": "^7.9.0",
-    "@babel/types": "^7.9.0",
-    "@babel/traverse": "^7.9.0",
     "@babel/plugin-proposal-class-properties": "^7.8.3",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
     "@babel/plugin-proposal-object-rest-spread": "^7.9.0",
@@ -23,6 +21,8 @@
     "@babel/plugin-transform-typescript": "^7.9.0",
     "@babel/preset-env": "^7.9.0",
     "@babel/preset-react": "^7.9.1",
+    "@babel/traverse": "^7.9.0",
+    "@babel/types": "^7.9.0",
     "@types/fs-extra": "^8.1.0",
     "@types/node": "^13.7.5",
     "fs-extra": "^8.1.0",
@@ -33,13 +33,14 @@
     "@types/jest": "25.1.4",
     "jest": "^25.1.0",
     "prettier": "^2.0.0",
+    "rimraf": "^3.0.2",
     "ts-jest": "^25.2.1",
     "ts-node": "^8",
     "typescript": "^3.7.2"
   },
   "scripts": {
     "build": "tsc -b",
-    "prepack": "rm -rf lib && tsc -b",
+    "prepack": "rimraf lib *.tsbuildinfo && tsc -b",
     "prepublishOnly": "yarn test",
     "test": "jest --config jestconfig.json"
   },

desktop/babel-transformer/src/fb-stubs.ts

@@ -18,7 +18,7 @@ const requireFromFolder = (folder: string, path: string) =>
   new RegExp(folder + '/[A-Za-z0-9.-_]+(.js)?$', 'g').test(path);
 
 module.exports = () => ({
-  name: 'replace-dynamic-requires',
+  name: 'replace-fb-stubs',
   visitor: {
     CallExpression(path: NodePath<CallExpression>, state: any) {
       if (

desktop/babel-transformer/src/flipper-env.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+/**
+ * There are some env vars which affect transformations, so the Metro/Babel cache should be invalidated when at least one of them changed.
+ * They are used in get-cache-key.ts for cache key generation.
+ */
+type FlipperEnvVars = {
+  FLIPPER_HEADLESS?: string;
+  FLIPPER_FB?: string;
+  FLIPPER_TEST_RUNNER?: string;
+  FLIPPER_ELECTRON_VERSION?: string;
+  NODE_ENV?: string;
+};
+
+const flipperEnv = new Proxy(
+  {
+    FLIPPER_HEADLESS: undefined,
+    FLIPPER_FB: undefined,
+    FLIPPER_TEST_RUNNER: undefined,
+    FLIPPER_ELECTRON_VERSION: undefined,
+    NODE_ENV: undefined,
+  } as FlipperEnvVars,
+  {
+    get: function (obj, prop) {
+      if (typeof prop === 'string') {
+        return process.env[prop];
+      } else {
+        return (obj as any)[prop];
+      }
+    },
+    set: function () {
+      throw new Error('flipperEnv is read-only');
+    },
+  },
+);
+
+export default flipperEnv;

desktop/babel-transformer/src/get-cache-key.ts

@@ -7,9 +7,39 @@
  * @format
  */
 
-// Disable caching of babel transforms all together. We haven't found a good
-// way to cache our transforms, as they rely on side effects like env vars or
-// the existence of folders in the file system.
-export default function getCacheKey() {
-  return Math.random().toString(36);
+/**
+ * There are some env vars which affect transformations, so the Metro/Babel cache should be invalidated when at least one of them changed.
+ *
+ * If any issues found with such approach, we can fallback to the implementation which always invalidates caches, but also makes bundling significantly slower:
+ * export default function getCacheKey() { return Math.random().toString(36); }
+ */
+
+import {default as flipperEnv} from './flipper-env';
+import fs from 'fs-extra';
+import path from 'path';
+
+let baseHash = '';
+const tsbuildinfoPath = path.resolve(__dirname, '..', 'tsconfig.tsbuildinfo');
+const packageJsonPath = path.resolve(__dirname, '..', 'package.json');
+if (fs.pathExistsSync(tsbuildinfoPath)) {
+  /**
+   * tsconfig.tsbuildinfo is changed each time TS incremental build detects changes and rebuilds the package,
+   * so we can use its modification date as cache key to invalidate the cache each time when babel transformations changed.
+   */
+  baseHash = fs.lstatSync(tsbuildinfoPath).ctime.toUTCString();
+} else if (fs.pathExistsSync(packageJsonPath)) {
+  /**
+   * tsconfig.tsbuildinfo will not exist in case if the package is installed from npm rather than built locally.
+   * In such case we should use version of npm package as hash key to invalidate the cache after updates.
+   */
+  baseHash = fs.readJsonSync(packageJsonPath).version;
+}
+
+export default function getCacheKey() {
+  return [
+    baseHash,
+    ...Object.entries(flipperEnv)
+      .sort(([name1, _value1], [name2, _value2]) => name1.localeCompare(name2))
+      .map(([name, value]) => `${name}=${value}`),
+  ].join('|');
 }

desktop/babel-transformer/src/transform-app.ts

@@ -9,6 +9,7 @@
 
 import {default as doTransform} from './transform';
 import {default as getCacheKey} from './get-cache-key';
+import {default as flipperEnv} from './flipper-env';
 
 module.exports = {
   transform,
@@ -26,10 +27,10 @@ function transform({
 }) {
   const presets = [require('@babel/preset-react')];
   const plugins = [];
-  if (process.env.FLIPPER_FB) {
+  if (flipperEnv.FLIPPER_FB) {
     plugins.push(require('./fb-stubs'));
   }
-  if (process.env.BUILD_HEADLESS) {
+  if (flipperEnv.FLIPPER_HEADLESS) {
     plugins.push(require('./electron-stubs'));
   }
   plugins.push(require('./electron-requires'));

desktop/babel-transformer/src/transform-jest.ts

@@ -9,6 +9,7 @@
 
 import {default as doTransform} from './transform';
 import {default as getCacheKey} from './get-cache-key';
+import {default as flipperEnv} from './flipper-env';
 
 module.exports = {
   transform,
@@ -26,10 +27,10 @@ function transform({
 }) {
   const presets = [require('@babel/preset-react')];
   const plugins = [];
-  if (process.env.FLIPPER_FB) {
+  if (flipperEnv.FLIPPER_FB) {
     plugins.push(require('./fb-stubs'));
   }
-  if (process.env.BUILD_HEADLESS) {
+  if (flipperEnv.FLIPPER_HEADLESS) {
     plugins.push(require('./electron-stubs'));
   }
   plugins.push(require('./import-react'));

desktop/babel-transformer/src/transform-main.ts

@@ -9,6 +9,7 @@
 
 import {default as doTransform} from './transform';
 import {default as getCacheKey} from './get-cache-key';
+import {default as flipperEnv} from './flipper-env';
 
 module.exports = {
   transform,
@@ -27,11 +28,11 @@ function transform({
   const presets = [
     [
       require('@babel/preset-env'),
-      {targets: {electron: process.env.FLIPPER_ELECTRON_VERSION}},
+      {targets: {electron: flipperEnv.FLIPPER_ELECTRON_VERSION}},
     ],
   ];
   const plugins = [];
-  if (process.env.FLIPPER_FB) {
+  if (flipperEnv.FLIPPER_FB) {
     plugins.push(require('./fb-stubs'));
   }
   plugins.push(require('./electron-requires-main'));

desktop/babel-transformer/src/transform-plugin.ts

@@ -8,6 +8,7 @@
  */
 
 import {default as doTransform} from './transform';
+import {default as flipperEnv} from './flipper-env';
 
 export default function transform({
   filename,
@@ -24,10 +25,10 @@ export default function transform({
 }) {
   presets = presets ?? [require('@babel/preset-react')];
   plugins = plugins ?? [];
-  if (process.env.FLIPPER_FB) {
+  if (flipperEnv.FLIPPER_FB) {
     plugins.push(require('./fb-stubs'));
   }
-  if (process.env.BUILD_HEADLESS) {
+  if (flipperEnv.FLIPPER_HEADLESS) {
     plugins.push(require('./electron-stubs'));
   }
   plugins.push(require('./electron-requires'));

desktop/babel-transformer/src/transform.ts

@@ -10,6 +10,7 @@
 import {default as generate} from '@babel/generator';
 import {parse} from '@babel/parser';
 import {transformFromAstSync} from '@babel/core';
+import {default as flipperEnv} from './flipper-env';
 
 export default function transform({
   filename,
@@ -77,7 +78,7 @@ export default function transform({
     plugins,
     presets,
     sourceMaps: true,
-    retainLines: !!options.isTestRunner,
+    retainLines: !!flipperEnv.FLIPPER_TEST_RUNNER,
   });
   if (!transformed) {
     throw new Error('Failed to transform');
@@ -88,7 +89,7 @@ export default function transform({
       filename,
       sourceFileName: filename,
       sourceMaps: true,
-      retainLines: !!options.isTestRunner,
+      retainLines: !!flipperEnv.FLIPPER_TEST_RUNNER,
     },
     src,
   );

desktop/headless/index.tsx

@@ -30,6 +30,8 @@ import {getStringFromErrorLike} from '../app/src/utils/index';
 import AndroidDevice from '../app/src/devices/AndroidDevice';
 import {Store} from 'flipper';
 
+process.env.FLIPPER_HEADLESS = 'true';
+
 type Action = {exit: boolean; result?: string};
 
 type UserArguments = {

desktop/package.json

@@ -184,8 +184,11 @@
     "rm-modules": "rimraf **/node_modules node_modules",
     "rm-temp": "rimraf $TMPDIR/jest* $TMPDIR/react-native-packager*",
     "rm-bundle": "rimraf static/main.bundle.* **/lib **/*.tsbuildinfo",
-    "reset": "yarn rm-dist && yarn rm-temp && yarn cache clean && yarn rm-bundle && yarn rm-modules",
-    "dev-server": "yarn build:babel-transformer && cross-env NODE_ENV=development TS_NODE_FILES=true node --require ts-node/register scripts/start-dev-server.ts",
+    "rm-watches": "watchman watch-del-all",
+    "rm-metro-cache": "rimraf $TMPDIR/metro-cache*",
+    "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-watches && yarn rm-modules",
+    "predev-server": "yarn build:babel-transformer",
+    "dev-server": "cross-env NODE_ENV=development TS_NODE_FILES=true node --require ts-node/register scripts/start-dev-server.ts",
     "start": "yarn dev-server --inspect=9229",
     "start:break": "yarn dev-server --inspect-brk=9229",
     "start:no-embedded-plugins": "yarn start --no-embedded-plugins",

desktop/scripts/build-headless.ts

@@ -98,7 +98,7 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
   // developement iteration by not including any plugins.
   const skipPlugins = process.argv.indexOf('--no-plugins') > -1;
 
-  process.env.BUILD_HEADLESS = 'true';
+  process.env.FLIPPER_HEADLESS = 'true';
   const buildDir = await buildFolder();
   // eslint-disable-next-line no-console
   console.log('Created build directory', buildDir);

desktop/scripts/build-release.ts

@@ -179,7 +179,7 @@ function downloadIcons(buildFolder: string) {
   // eslint-disable-next-line no-console
   console.log('Created build directory', dir);
 
-  await compileMain({dev: false});
+  await compileMain();
   await copyStaticFolder(dir);
   await downloadIcons(dir);
   if (!process.argv.includes('--no-embedded-plugins')) {

desktop/scripts/build-utils.ts

@@ -24,6 +24,8 @@ import {
   babelTransformationsDir,
 } from './paths';
 
+const dev = process.env.NODE_ENV !== 'production';
+
 async function mostRecentlyChanged(
   dir: string,
   ignores: string[],
@@ -89,9 +91,9 @@ async function compile(
       },
     },
     {
-      dev: false,
+      dev,
       minify: false,
-      resetCache: true,
+      resetCache: !dev,
       sourceMap: true,
       entry,
       out: path.join(buildFolder, 'bundle.js'),
@@ -145,7 +147,7 @@ export async function compileRenderer(buildFolder: string) {
   }
 }
 
-export async function compileMain({dev}: {dev: boolean}) {
+export async function compileMain() {
   const out = path.join(staticDir, 'main.bundle.js');
   process.env.FLIPPER_ELECTRON_VERSION = require('electron/package.json').version;
   // check if main needs to be compiled
@@ -182,7 +184,7 @@ export async function compileMain({dev}: {dev: boolean}) {
       dev,
       minify: false,
       sourceMap: true,
-      resetCache: true,
+      resetCache: !dev,
     });
     console.log('✅  Compiled main bundle.');
   } catch (err) {

desktop/scripts/jest-transform.js

@@ -15,6 +15,8 @@ if (isFB && process.env.FLIPPER_FB === undefined) {
   process.env.FLIPPER_FB = 'true';
 }
 
+process.env.FLIPPER_TEST_RUNNER = 'true';
+
 module.exports = {
   process(src, filename, config, options) {
     return transform({

desktop/scripts/start-dev-server.ts

@@ -263,7 +263,7 @@ function outputScreen(socket?: socketIo.Server) {
   const socket = await addWebsocket(server);
   await startMetroServer(app);
   outputScreen(socket);
-  await compileMain({dev: true});
+  await compileMain();
   shutdownElectron = launchElectron({
     devServerURL: `http://localhost:${port}`,
     bundleURL: `http://localhost:${port}/src/init.bundle`,

desktop/static/compilePlugins.ts

@@ -33,8 +33,6 @@ export type CompileOptions = {
   recompileOnChanges: boolean;
 };
 
-type DynamicCompileOptions = CompileOptions & {force: boolean};
-
 export type PluginManifest = {
   version: string;
   name: string;
@@ -69,10 +67,7 @@ export default async function (
   const compilations = pMap(
     Object.values(plugins),
     (plugin) => {
-      const dynamicOptions: DynamicCompileOptions = Object.assign(options, {
-        force: false,
-      });
-      return compilePlugin(plugin, pluginCache, dynamicOptions);
+      return compilePlugin(plugin, pluginCache, options);
     },
     {concurrency: 4},
   );
@@ -244,11 +239,12 @@ async function mostRecentlyChanged(dir: string) {
 async function compilePlugin(
   pluginInfo: PluginInfo,
   pluginCache: string,
-  options: DynamicCompileOptions,
+  {force, failSilently}: CompileOptions,
 ): Promise<CompiledPluginInfo | null> {
   const {rootDir, manifest, entry, name} = pluginInfo;
   const bundleMain = manifest.bundleMain ?? path.join('dist', 'index.js');
   const bundlePath = path.join(rootDir, bundleMain);
+  const dev = process.env.NODE_ENV !== 'production';
   if (await fs.pathExists(bundlePath)) {
     // eslint-disable-next-line no-console
     const out = path.join(rootDir, bundleMain);
@@ -262,7 +258,7 @@ async function compilePlugin(
     const result = Object.assign({}, pluginInfo.manifest, {out});
     const rootDirCtime = await mostRecentlyChanged(rootDir);
     if (
-      !options.force &&
+      !force &&
       (await fs.pathExists(out)) &&
       rootDirCtime < (await fs.lstat(out)).ctime
     ) {
@@ -296,13 +292,14 @@ async function compilePlugin(
           {
             entry: entry.replace(rootDir, '.'),
             out,
-            dev: false,
+            dev,
             sourceMap: true,
             minify: false,
+            resetCache: !dev,
           },
         );
       } catch (e) {
-        if (options.failSilently) {
+        if (failSilently) {
           console.error(
             `❌  Plugin ${name} is ignored, because it could not be compiled.`,
           );

desktop/yarn.lock

@@ -9918,11 +9918,6 @@ rimraf@^3.0.0, rimraf@^3.0.2:
   dependencies:
     glob "^7.1.3"
 
-rimraf@~2.2.6:
-  version "2.2.8"
-  resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582"
-  integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI=
-
 rn-host-detect@^1.1.5:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/rn-host-detect/-/rn-host-detect-1.2.0.tgz#8b0396fc05631ec60c1cb8789e5070cdb04d0da0"
@@ -10865,12 +10860,11 @@ temp@0.8.3, temp@0.9.0:
     rimraf "~2.6.2"
 
 temp@^0.8.1:
-  version "0.8.3"
-  resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.3.tgz#e0c6bc4d26b903124410e4fed81103014dfc1f59"
-  integrity sha1-4Ma8TSa5AxJEEOT+2BEDAU38H1k=
+  version "0.8.4"
+  resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.4.tgz#8c97a33a4770072e0a05f919396c7665a7dd59f2"
+  integrity sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==
   dependencies:
-    os-tmpdir "^1.0.0"
-    rimraf "~2.2.6"
+    rimraf "~2.6.2"
 
 term-size@^2.1.0:
   version "2.2.0"