From da7449c20b2a04038c439986c5226386f184df50 Mon Sep 17 00:00:00 2001 From: Anton Nikolaev Date: Mon, 30 Mar 2020 09:23:09 -0700 Subject: [PATCH] Enable Metro caching Summary: Enabling Metro cache for dev mode. For release builds we reset the cache. Cache is used for faster compilation in dev mode for both main and renderer bundles, as well as for plugins. Currently we have few side effects based on env vars, so cache is invalidated when they are changed. Also the cache is invalidated when transformations are changed (changed code, bumped dependency etc). Also added a script to reset the cache if something is going wrong. Reviewed By: mweststrate Differential Revision: D20691464 fbshipit-source-id: 478947d438bd3090f052dbfa6ad5c649523ecacb --- desktop/babel-transformer/package.json | 7 +-- desktop/babel-transformer/src/fb-stubs.ts | 2 +- desktop/babel-transformer/src/flipper-env.ts | 44 +++++++++++++++++++ .../babel-transformer/src/get-cache-key.ts | 40 ++++++++++++++--- .../babel-transformer/src/transform-app.ts | 5 ++- .../babel-transformer/src/transform-jest.ts | 5 ++- .../babel-transformer/src/transform-main.ts | 5 ++- .../babel-transformer/src/transform-plugin.ts | 5 ++- desktop/babel-transformer/src/transform.ts | 5 ++- desktop/headless/index.tsx | 2 + desktop/package.json | 7 ++- desktop/scripts/build-headless.ts | 2 +- desktop/scripts/build-release.ts | 2 +- desktop/scripts/build-utils.ts | 10 +++-- desktop/scripts/jest-transform.js | 2 + desktop/scripts/start-dev-server.ts | 2 +- desktop/static/compilePlugins.ts | 17 +++---- desktop/yarn.lock | 14 ++---- 18 files changed, 128 insertions(+), 48 deletions(-) create mode 100644 desktop/babel-transformer/src/flipper-env.ts diff --git a/desktop/babel-transformer/package.json b/desktop/babel-transformer/package.json index f63fe1395..02403066a 100644 --- a/desktop/babel-transformer/package.json +++ b/desktop/babel-transformer/package.json @@ -12,8 +12,6 @@ "@babel/core": "^7.9.0", "@babel/generator": "^7.9.0", "@babel/parser": "^7.9.0", - "@babel/types": "^7.9.0", - "@babel/traverse": "^7.9.0", "@babel/plugin-proposal-class-properties": "^7.8.3", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3", "@babel/plugin-proposal-object-rest-spread": "^7.9.0", @@ -23,6 +21,8 @@ "@babel/plugin-transform-typescript": "^7.9.0", "@babel/preset-env": "^7.9.0", "@babel/preset-react": "^7.9.1", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", "@types/fs-extra": "^8.1.0", "@types/node": "^13.7.5", "fs-extra": "^8.1.0", @@ -33,13 +33,14 @@ "@types/jest": "25.1.4", "jest": "^25.1.0", "prettier": "^2.0.0", + "rimraf": "^3.0.2", "ts-jest": "^25.2.1", "ts-node": "^8", "typescript": "^3.7.2" }, "scripts": { "build": "tsc -b", - "prepack": "rm -rf lib && tsc -b", + "prepack": "rimraf lib *.tsbuildinfo && tsc -b", "prepublishOnly": "yarn test", "test": "jest --config jestconfig.json" }, diff --git a/desktop/babel-transformer/src/fb-stubs.ts b/desktop/babel-transformer/src/fb-stubs.ts index 8f7e33fc3..525e63116 100644 --- a/desktop/babel-transformer/src/fb-stubs.ts +++ b/desktop/babel-transformer/src/fb-stubs.ts @@ -18,7 +18,7 @@ const requireFromFolder = (folder: string, path: string) => new RegExp(folder + '/[A-Za-z0-9.-_]+(.js)?$', 'g').test(path); module.exports = () => ({ - name: 'replace-dynamic-requires', + name: 'replace-fb-stubs', visitor: { CallExpression(path: NodePath, state: any) { if ( diff --git a/desktop/babel-transformer/src/flipper-env.ts b/desktop/babel-transformer/src/flipper-env.ts new file mode 100644 index 000000000..df5d7046e --- /dev/null +++ b/desktop/babel-transformer/src/flipper-env.ts @@ -0,0 +1,44 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +/** + * There are some env vars which affect transformations, so the Metro/Babel cache should be invalidated when at least one of them changed. + * They are used in get-cache-key.ts for cache key generation. + */ +type FlipperEnvVars = { + FLIPPER_HEADLESS?: string; + FLIPPER_FB?: string; + FLIPPER_TEST_RUNNER?: string; + FLIPPER_ELECTRON_VERSION?: string; + NODE_ENV?: string; +}; + +const flipperEnv = new Proxy( + { + FLIPPER_HEADLESS: undefined, + FLIPPER_FB: undefined, + FLIPPER_TEST_RUNNER: undefined, + FLIPPER_ELECTRON_VERSION: undefined, + NODE_ENV: undefined, + } as FlipperEnvVars, + { + get: function (obj, prop) { + if (typeof prop === 'string') { + return process.env[prop]; + } else { + return (obj as any)[prop]; + } + }, + set: function () { + throw new Error('flipperEnv is read-only'); + }, + }, +); + +export default flipperEnv; diff --git a/desktop/babel-transformer/src/get-cache-key.ts b/desktop/babel-transformer/src/get-cache-key.ts index 656d26842..4f19551ce 100644 --- a/desktop/babel-transformer/src/get-cache-key.ts +++ b/desktop/babel-transformer/src/get-cache-key.ts @@ -7,9 +7,39 @@ * @format */ -// Disable caching of babel transforms all together. We haven't found a good -// way to cache our transforms, as they rely on side effects like env vars or -// the existence of folders in the file system. -export default function getCacheKey() { - return Math.random().toString(36); +/** + * There are some env vars which affect transformations, so the Metro/Babel cache should be invalidated when at least one of them changed. + * + * If any issues found with such approach, we can fallback to the implementation which always invalidates caches, but also makes bundling significantly slower: + * export default function getCacheKey() { return Math.random().toString(36); } + */ + +import {default as flipperEnv} from './flipper-env'; +import fs from 'fs-extra'; +import path from 'path'; + +let baseHash = ''; +const tsbuildinfoPath = path.resolve(__dirname, '..', 'tsconfig.tsbuildinfo'); +const packageJsonPath = path.resolve(__dirname, '..', 'package.json'); +if (fs.pathExistsSync(tsbuildinfoPath)) { + /** + * tsconfig.tsbuildinfo is changed each time TS incremental build detects changes and rebuilds the package, + * so we can use its modification date as cache key to invalidate the cache each time when babel transformations changed. + */ + baseHash = fs.lstatSync(tsbuildinfoPath).ctime.toUTCString(); +} else if (fs.pathExistsSync(packageJsonPath)) { + /** + * tsconfig.tsbuildinfo will not exist in case if the package is installed from npm rather than built locally. + * In such case we should use version of npm package as hash key to invalidate the cache after updates. + */ + baseHash = fs.readJsonSync(packageJsonPath).version; +} + +export default function getCacheKey() { + return [ + baseHash, + ...Object.entries(flipperEnv) + .sort(([name1, _value1], [name2, _value2]) => name1.localeCompare(name2)) + .map(([name, value]) => `${name}=${value}`), + ].join('|'); } diff --git a/desktop/babel-transformer/src/transform-app.ts b/desktop/babel-transformer/src/transform-app.ts index ce38b341a..9eec2edcc 100644 --- a/desktop/babel-transformer/src/transform-app.ts +++ b/desktop/babel-transformer/src/transform-app.ts @@ -9,6 +9,7 @@ import {default as doTransform} from './transform'; import {default as getCacheKey} from './get-cache-key'; +import {default as flipperEnv} from './flipper-env'; module.exports = { transform, @@ -26,10 +27,10 @@ function transform({ }) { const presets = [require('@babel/preset-react')]; const plugins = []; - if (process.env.FLIPPER_FB) { + if (flipperEnv.FLIPPER_FB) { plugins.push(require('./fb-stubs')); } - if (process.env.BUILD_HEADLESS) { + if (flipperEnv.FLIPPER_HEADLESS) { plugins.push(require('./electron-stubs')); } plugins.push(require('./electron-requires')); diff --git a/desktop/babel-transformer/src/transform-jest.ts b/desktop/babel-transformer/src/transform-jest.ts index 827377c01..c71950608 100644 --- a/desktop/babel-transformer/src/transform-jest.ts +++ b/desktop/babel-transformer/src/transform-jest.ts @@ -9,6 +9,7 @@ import {default as doTransform} from './transform'; import {default as getCacheKey} from './get-cache-key'; +import {default as flipperEnv} from './flipper-env'; module.exports = { transform, @@ -26,10 +27,10 @@ function transform({ }) { const presets = [require('@babel/preset-react')]; const plugins = []; - if (process.env.FLIPPER_FB) { + if (flipperEnv.FLIPPER_FB) { plugins.push(require('./fb-stubs')); } - if (process.env.BUILD_HEADLESS) { + if (flipperEnv.FLIPPER_HEADLESS) { plugins.push(require('./electron-stubs')); } plugins.push(require('./import-react')); diff --git a/desktop/babel-transformer/src/transform-main.ts b/desktop/babel-transformer/src/transform-main.ts index 0787d290f..471084a73 100644 --- a/desktop/babel-transformer/src/transform-main.ts +++ b/desktop/babel-transformer/src/transform-main.ts @@ -9,6 +9,7 @@ import {default as doTransform} from './transform'; import {default as getCacheKey} from './get-cache-key'; +import {default as flipperEnv} from './flipper-env'; module.exports = { transform, @@ -27,11 +28,11 @@ function transform({ const presets = [ [ require('@babel/preset-env'), - {targets: {electron: process.env.FLIPPER_ELECTRON_VERSION}}, + {targets: {electron: flipperEnv.FLIPPER_ELECTRON_VERSION}}, ], ]; const plugins = []; - if (process.env.FLIPPER_FB) { + if (flipperEnv.FLIPPER_FB) { plugins.push(require('./fb-stubs')); } plugins.push(require('./electron-requires-main')); diff --git a/desktop/babel-transformer/src/transform-plugin.ts b/desktop/babel-transformer/src/transform-plugin.ts index 425c34a63..ccebdfed7 100644 --- a/desktop/babel-transformer/src/transform-plugin.ts +++ b/desktop/babel-transformer/src/transform-plugin.ts @@ -8,6 +8,7 @@ */ import {default as doTransform} from './transform'; +import {default as flipperEnv} from './flipper-env'; export default function transform({ filename, @@ -24,10 +25,10 @@ export default function transform({ }) { presets = presets ?? [require('@babel/preset-react')]; plugins = plugins ?? []; - if (process.env.FLIPPER_FB) { + if (flipperEnv.FLIPPER_FB) { plugins.push(require('./fb-stubs')); } - if (process.env.BUILD_HEADLESS) { + if (flipperEnv.FLIPPER_HEADLESS) { plugins.push(require('./electron-stubs')); } plugins.push(require('./electron-requires')); diff --git a/desktop/babel-transformer/src/transform.ts b/desktop/babel-transformer/src/transform.ts index 87fa33ea1..0e17c962c 100644 --- a/desktop/babel-transformer/src/transform.ts +++ b/desktop/babel-transformer/src/transform.ts @@ -10,6 +10,7 @@ import {default as generate} from '@babel/generator'; import {parse} from '@babel/parser'; import {transformFromAstSync} from '@babel/core'; +import {default as flipperEnv} from './flipper-env'; export default function transform({ filename, @@ -77,7 +78,7 @@ export default function transform({ plugins, presets, sourceMaps: true, - retainLines: !!options.isTestRunner, + retainLines: !!flipperEnv.FLIPPER_TEST_RUNNER, }); if (!transformed) { throw new Error('Failed to transform'); @@ -88,7 +89,7 @@ export default function transform({ filename, sourceFileName: filename, sourceMaps: true, - retainLines: !!options.isTestRunner, + retainLines: !!flipperEnv.FLIPPER_TEST_RUNNER, }, src, ); diff --git a/desktop/headless/index.tsx b/desktop/headless/index.tsx index 37a7f6be6..b6dc307e2 100644 --- a/desktop/headless/index.tsx +++ b/desktop/headless/index.tsx @@ -30,6 +30,8 @@ import {getStringFromErrorLike} from '../app/src/utils/index'; import AndroidDevice from '../app/src/devices/AndroidDevice'; import {Store} from 'flipper'; +process.env.FLIPPER_HEADLESS = 'true'; + type Action = {exit: boolean; result?: string}; type UserArguments = { diff --git a/desktop/package.json b/desktop/package.json index 8b595425a..7ebf33e9c 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -184,8 +184,11 @@ "rm-modules": "rimraf **/node_modules node_modules", "rm-temp": "rimraf $TMPDIR/jest* $TMPDIR/react-native-packager*", "rm-bundle": "rimraf static/main.bundle.* **/lib **/*.tsbuildinfo", - "reset": "yarn rm-dist && yarn rm-temp && yarn cache clean && yarn rm-bundle && yarn rm-modules", - "dev-server": "yarn build:babel-transformer && cross-env NODE_ENV=development TS_NODE_FILES=true node --require ts-node/register scripts/start-dev-server.ts", + "rm-watches": "watchman watch-del-all", + "rm-metro-cache": "rimraf $TMPDIR/metro-cache*", + "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-watches && yarn rm-modules", + "predev-server": "yarn build:babel-transformer", + "dev-server": "cross-env NODE_ENV=development TS_NODE_FILES=true node --require ts-node/register scripts/start-dev-server.ts", "start": "yarn dev-server --inspect=9229", "start:break": "yarn dev-server --inspect-brk=9229", "start:no-embedded-plugins": "yarn start --no-embedded-plugins", diff --git a/desktop/scripts/build-headless.ts b/desktop/scripts/build-headless.ts index 6efa68811..ef3fe2a5d 100644 --- a/desktop/scripts/build-headless.ts +++ b/desktop/scripts/build-headless.ts @@ -98,7 +98,7 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) { // developement iteration by not including any plugins. const skipPlugins = process.argv.indexOf('--no-plugins') > -1; - process.env.BUILD_HEADLESS = 'true'; + process.env.FLIPPER_HEADLESS = 'true'; const buildDir = await buildFolder(); // eslint-disable-next-line no-console console.log('Created build directory', buildDir); diff --git a/desktop/scripts/build-release.ts b/desktop/scripts/build-release.ts index 9ffebfb5a..1bd330850 100755 --- a/desktop/scripts/build-release.ts +++ b/desktop/scripts/build-release.ts @@ -179,7 +179,7 @@ function downloadIcons(buildFolder: string) { // eslint-disable-next-line no-console console.log('Created build directory', dir); - await compileMain({dev: false}); + await compileMain(); await copyStaticFolder(dir); await downloadIcons(dir); if (!process.argv.includes('--no-embedded-plugins')) { diff --git a/desktop/scripts/build-utils.ts b/desktop/scripts/build-utils.ts index e373c7290..db6c9dcf3 100644 --- a/desktop/scripts/build-utils.ts +++ b/desktop/scripts/build-utils.ts @@ -24,6 +24,8 @@ import { babelTransformationsDir, } from './paths'; +const dev = process.env.NODE_ENV !== 'production'; + async function mostRecentlyChanged( dir: string, ignores: string[], @@ -89,9 +91,9 @@ async function compile( }, }, { - dev: false, + dev, minify: false, - resetCache: true, + resetCache: !dev, sourceMap: true, entry, out: path.join(buildFolder, 'bundle.js'), @@ -145,7 +147,7 @@ export async function compileRenderer(buildFolder: string) { } } -export async function compileMain({dev}: {dev: boolean}) { +export async function compileMain() { const out = path.join(staticDir, 'main.bundle.js'); process.env.FLIPPER_ELECTRON_VERSION = require('electron/package.json').version; // check if main needs to be compiled @@ -182,7 +184,7 @@ export async function compileMain({dev}: {dev: boolean}) { dev, minify: false, sourceMap: true, - resetCache: true, + resetCache: !dev, }); console.log('✅ Compiled main bundle.'); } catch (err) { diff --git a/desktop/scripts/jest-transform.js b/desktop/scripts/jest-transform.js index 4d038d5ab..d1b94b8b4 100644 --- a/desktop/scripts/jest-transform.js +++ b/desktop/scripts/jest-transform.js @@ -15,6 +15,8 @@ if (isFB && process.env.FLIPPER_FB === undefined) { process.env.FLIPPER_FB = 'true'; } +process.env.FLIPPER_TEST_RUNNER = 'true'; + module.exports = { process(src, filename, config, options) { return transform({ diff --git a/desktop/scripts/start-dev-server.ts b/desktop/scripts/start-dev-server.ts index cca63bfd3..74c8856a6 100644 --- a/desktop/scripts/start-dev-server.ts +++ b/desktop/scripts/start-dev-server.ts @@ -263,7 +263,7 @@ function outputScreen(socket?: socketIo.Server) { const socket = await addWebsocket(server); await startMetroServer(app); outputScreen(socket); - await compileMain({dev: true}); + await compileMain(); shutdownElectron = launchElectron({ devServerURL: `http://localhost:${port}`, bundleURL: `http://localhost:${port}/src/init.bundle`, diff --git a/desktop/static/compilePlugins.ts b/desktop/static/compilePlugins.ts index e17fd66a9..16ed90c05 100644 --- a/desktop/static/compilePlugins.ts +++ b/desktop/static/compilePlugins.ts @@ -33,8 +33,6 @@ export type CompileOptions = { recompileOnChanges: boolean; }; -type DynamicCompileOptions = CompileOptions & {force: boolean}; - export type PluginManifest = { version: string; name: string; @@ -69,10 +67,7 @@ export default async function ( const compilations = pMap( Object.values(plugins), (plugin) => { - const dynamicOptions: DynamicCompileOptions = Object.assign(options, { - force: false, - }); - return compilePlugin(plugin, pluginCache, dynamicOptions); + return compilePlugin(plugin, pluginCache, options); }, {concurrency: 4}, ); @@ -244,11 +239,12 @@ async function mostRecentlyChanged(dir: string) { async function compilePlugin( pluginInfo: PluginInfo, pluginCache: string, - options: DynamicCompileOptions, + {force, failSilently}: CompileOptions, ): Promise { const {rootDir, manifest, entry, name} = pluginInfo; const bundleMain = manifest.bundleMain ?? path.join('dist', 'index.js'); const bundlePath = path.join(rootDir, bundleMain); + const dev = process.env.NODE_ENV !== 'production'; if (await fs.pathExists(bundlePath)) { // eslint-disable-next-line no-console const out = path.join(rootDir, bundleMain); @@ -262,7 +258,7 @@ async function compilePlugin( const result = Object.assign({}, pluginInfo.manifest, {out}); const rootDirCtime = await mostRecentlyChanged(rootDir); if ( - !options.force && + !force && (await fs.pathExists(out)) && rootDirCtime < (await fs.lstat(out)).ctime ) { @@ -296,13 +292,14 @@ async function compilePlugin( { entry: entry.replace(rootDir, '.'), out, - dev: false, + dev, sourceMap: true, minify: false, + resetCache: !dev, }, ); } catch (e) { - if (options.failSilently) { + if (failSilently) { console.error( `❌ Plugin ${name} is ignored, because it could not be compiled.`, ); diff --git a/desktop/yarn.lock b/desktop/yarn.lock index 60856fc0b..6a5b96b12 100644 --- a/desktop/yarn.lock +++ b/desktop/yarn.lock @@ -9918,11 +9918,6 @@ rimraf@^3.0.0, rimraf@^3.0.2: dependencies: glob "^7.1.3" -rimraf@~2.2.6: - version "2.2.8" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.2.8.tgz#e439be2aaee327321952730f99a8929e4fc50582" - integrity sha1-5Dm+Kq7jJzIZUnMPmaiSnk/FBYI= - rn-host-detect@^1.1.5: version "1.2.0" resolved "https://registry.yarnpkg.com/rn-host-detect/-/rn-host-detect-1.2.0.tgz#8b0396fc05631ec60c1cb8789e5070cdb04d0da0" @@ -10865,12 +10860,11 @@ temp@0.8.3, temp@0.9.0: rimraf "~2.6.2" temp@^0.8.1: - version "0.8.3" - resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.3.tgz#e0c6bc4d26b903124410e4fed81103014dfc1f59" - integrity sha1-4Ma8TSa5AxJEEOT+2BEDAU38H1k= + version "0.8.4" + resolved "https://registry.yarnpkg.com/temp/-/temp-0.8.4.tgz#8c97a33a4770072e0a05f919396c7665a7dd59f2" + integrity sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg== dependencies: - os-tmpdir "^1.0.0" - rimraf "~2.2.6" + rimraf "~2.6.2" term-size@^2.1.0: version "2.2.0"