From c065760d150da6291e0d2f4aa69d4ff4a9a14a39 Mon Sep 17 00:00:00 2001 From: Anton Nikolaev Date: Wed, 10 Mar 2021 08:06:19 -0800 Subject: [PATCH] Stable babel transformation cache key Summary: This diff makes babel transformation cache key stable. With the previous approach it was changed on each CI build so it was not possible to persist cache between builds. Now it is computed from the transformation package content after each build. Because of that it is equal for every CI build while Babel transformations unchanged and so we could use same cache on different Sandcastle agents. In addition to that, it makes it possible to specify directory for Metro cache so we can save/restore it in CI builds. Reviewed By: mweststrate Differential Revision: D26877989 fbshipit-source-id: 7cb04a177f86e61986585e5a74d9c7396ddddc18 --- desktop/babel-transformer/package.json | 4 +- .../babel-transformer/src/get-cache-key.ts | 28 +++++-------- desktop/package.json | 17 ++++---- desktop/pkg-lib/package.json | 1 + desktop/pkg-lib/src/computePackageChecksum.ts | 37 +++++++++++++++-- desktop/pkg-lib/src/runBuild.ts | 11 ++++- desktop/scripts/compute-package-checksum.ts | 41 +++++++++++++++++++ desktop/tsc-root/tsconfig.json | 25 +++++++++++ desktop/types/metro-cache.d.ts | 10 +++++ desktop/yarn.lock | 2 +- 10 files changed, 142 insertions(+), 34 deletions(-) create mode 100644 desktop/scripts/compute-package-checksum.ts create mode 100644 desktop/tsc-root/tsconfig.json create mode 100644 desktop/types/metro-cache.d.ts diff --git a/desktop/babel-transformer/package.json b/desktop/babel-transformer/package.json index 47f7eb560..fdbc6f4b5 100644 --- a/desktop/babel-transformer/package.json +++ b/desktop/babel-transformer/package.json @@ -10,8 +10,8 @@ "bugs": "https://github.com/facebook/flipper/issues", "dependencies": { "@babel/core": "^7.13.8", - "@babel/parser": "^7.13.9", "@babel/generator": "^7.13.9", + "@babel/parser": "^7.13.9", "@babel/plugin-proposal-class-properties": "^7.13.0", "@babel/plugin-proposal-nullish-coalescing-operator": "^7.13.8", "@babel/plugin-proposal-object-rest-spread": "^7.13.8", @@ -40,7 +40,7 @@ }, "scripts": { "reset": "rimraf lib *.tsbuildinfo", - "build": "tsc -b", + "build": "tsc -b && cd .. && ./ts-node ./scripts/compute-package-checksum.ts -d ./babel-transformer -o ./lib/checksum.txt", "prepack": "yarn reset && yarn build" }, "files": [ diff --git a/desktop/babel-transformer/src/get-cache-key.ts b/desktop/babel-transformer/src/get-cache-key.ts index 4f19551ce..63366b534 100644 --- a/desktop/babel-transformer/src/get-cache-key.ts +++ b/desktop/babel-transformer/src/get-cache-key.ts @@ -18,28 +18,22 @@ import {default as flipperEnv} from './flipper-env'; import fs from 'fs-extra'; import path from 'path'; -let baseHash = ''; -const tsbuildinfoPath = path.resolve(__dirname, '..', 'tsconfig.tsbuildinfo'); -const packageJsonPath = path.resolve(__dirname, '..', 'package.json'); -if (fs.pathExistsSync(tsbuildinfoPath)) { - /** - * tsconfig.tsbuildinfo is changed each time TS incremental build detects changes and rebuilds the package, - * so we can use its modification date as cache key to invalidate the cache each time when babel transformations changed. - */ - baseHash = fs.lstatSync(tsbuildinfoPath).ctime.toUTCString(); -} else if (fs.pathExistsSync(packageJsonPath)) { - /** - * tsconfig.tsbuildinfo will not exist in case if the package is installed from npm rather than built locally. - * In such case we should use version of npm package as hash key to invalidate the cache after updates. - */ - baseHash = fs.readJsonSync(packageJsonPath).version; +let selfChecksum: string | undefined; +function getSelfChecksum() { + if (!selfChecksum) { + selfChecksum = fs + .readFileSync(path.resolve(__dirname, '..', 'lib', 'checksum.txt')) + .toString(); + } + return selfChecksum; } export default function getCacheKey() { - return [ - baseHash, + const key = [ + getSelfChecksum(), ...Object.entries(flipperEnv) .sort(([name1, _value1], [name2, _value2]) => name1.localeCompare(name2)) .map(([name, value]) => `${name}=${value}`), ].join('|'); + return key; } diff --git a/desktop/package.json b/desktop/package.json index 0cd84cb7a..b92050137 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -218,11 +218,10 @@ "scripts": { "build": "cross-env NODE_ENV=production ./ts-node scripts/build-release.ts $@", "build-plugin": "./ts-node scripts/build-plugin.ts", - "build:babel-transformer": "cd babel-transformer && yarn build", "build:dev": "cross-env NODE_ENV=development ./ts-node scripts/build-release.ts $@", "build:eslint": "cd eslint-plugin-flipper && yarn build", - "build:pkg": "cd pkg && yarn build", "build:themes": "lessc --js themes/light.less static/themes/light.css && lessc --js themes/dark.less static/themes/dark.css", + "build:tsc": "tsc -b tsc-root/tsconfig.json && ./ts-node ./scripts/compute-package-checksum.ts -d ./babel-transformer -o ./lib/checksum.txt", "bump-versions": "./ts-node scripts/bump-versions.ts", "dev-server": "cross-env NODE_ENV=development ./ts-node scripts/start-dev-server.ts", "everything": "yarn reset && yarn install && yarn lint && yarn test && yarn test-electron && yarn build --mac --mac-dmg --win --linux --linux-deb && yarn start", @@ -232,12 +231,12 @@ "lint:tsc": "tsc --noemit", "list-plugins": "./ts-node scripts/list-plugins.ts", "open-dist": "open ../dist/mac/Flipper.app --args --launcher=false --inspect=9229", - "postinstall": "patch-package && ./ts-node scripts/yarn-install-fb-plugins.ts && yarn build:pkg && yarn build:eslint && ./ts-node scripts/generate-plugin-entry-points.ts && yarn build:themes", - "prebuild": "yarn build:pkg && yarn rm-dist && yarn build:themes", - "predev-server": "yarn build:pkg", + "postinstall": "patch-package && ./ts-node scripts/yarn-install-fb-plugins.ts && yarn build:tsc && ./ts-node scripts/generate-plugin-entry-points.ts && yarn build:themes", + "prebuild": "yarn build:tsc && yarn rm-dist && yarn build:themes", + "predev-server": "yarn build:tsc", "preinstall": "node scripts/prepare-watchman-config.js && yarn config set ignore-engines", "prelint:eslint": "yarn build:eslint", - "pretest": "yarn build:pkg", + "pretest": "yarn build:tsc", "publish-packages": "./ts-node scripts/publish-packages.ts", "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-modules", "resolve-plugin-dir": "./ts-node scripts/resolve-plugin-dir.ts", @@ -252,9 +251,9 @@ "start:no-embedded-plugins": "yarn start --no-embedded-plugins", "test": "jest", "test-e2e": "cd e2e && yarn test", - "test-electron": "yarn build:pkg && jest --testMatch=\"**/**.electron\\.(js|jsx|ts|tsx)\" --testEnvironment=@jest-runner/electron/environment --runner=@jest-runner/electron", - "test-with-device": "yarn build:pkg && USE_ELECTRON_STUBS=1 jest --testMatch=\"**/**.device\\.(js|jsx|ts|tsx)\" --detectOpenHandles", - "test:debug": "yarn build:pkg && node --inspect node_modules/.bin/jest --runInBand", + "test-electron": "yarn build:tsc && jest --testMatch=\"**/**.electron\\.(js|jsx|ts|tsx)\" --testEnvironment=@jest-runner/electron/environment --runner=@jest-runner/electron", + "test-with-device": "yarn build:tsc && USE_ELECTRON_STUBS=1 jest --testMatch=\"**/**.device\\.(js|jsx|ts|tsx)\" --detectOpenHandles", + "test:debug": "yarn build:tsc && node --inspect node_modules/.bin/jest --runInBand", "watch": "jest --watch" }, "version": "0.79.1", diff --git a/desktop/pkg-lib/package.json b/desktop/pkg-lib/package.json index 26c3ab477..90dd1f27e 100644 --- a/desktop/pkg-lib/package.json +++ b/desktop/pkg-lib/package.json @@ -13,6 +13,7 @@ "flipper-plugin-lib": "0.0.0", "fs-extra": "^9.0.1", "metro": "^0.65.2", + "metro-cache": "^0.65.2", "metro-minify-terser": "^0.65.2", "npm-packlist": "^2.1.4" }, diff --git a/desktop/pkg-lib/src/computePackageChecksum.ts b/desktop/pkg-lib/src/computePackageChecksum.ts index ff811898c..b6183bac6 100644 --- a/desktop/pkg-lib/src/computePackageChecksum.ts +++ b/desktop/pkg-lib/src/computePackageChecksum.ts @@ -14,16 +14,41 @@ import fs from 'fs-extra'; export default async function computePackageChecksum( dir: string, + checksumFilePath?: string, ): Promise { + const fullChecksumFilePath = checksumFilePath + ? path.resolve(dir, checksumFilePath) + : undefined; + if (fullChecksumFilePath) { + // This block is an optimisation to not recompute checksum if nothing changed. + // tsbuildinfo file is changed every time when typescript compiler re-compiles anything, + // so we could compare its modification date with checksum modification date to + // decide whether we need to re-compute checksum or we can just use already computed one. + const tsBuildInfoPath = path.join(dir, 'tsconfig.tsbuildinfo'); + try { + const [tsBuildInfoStat, checksumStat] = await Promise.all([ + fs.stat(tsBuildInfoPath), + fs.stat(fullChecksumFilePath), + ]); + if (checksumStat.mtime > tsBuildInfoStat.mtime) { + return (await fs.readFile(fullChecksumFilePath)).toString(); + } + } catch {} + } const hash = crypto.createHash('sha1'); hash.setEncoding('hex'); const files = (await packlist({path: dir})).sort(); for (const file of files) { - // add hash of relative file path - hash.write(process.platform === 'win32' ? file.replace(/\\/g, '/') : file); - const filePath = path.resolve(dir, file); + if (filePath === fullChecksumFilePath) { + // If there is already existing checksum file, we need to ignore it as it is not a part of package content. + continue; + } + + // add hash of relative normalized file path + hash.write(process.platform === 'win32' ? file.replace(/\\/g, '/') : file); + if (file === 'package.json') { // add hash of package.json with version set to "0.0.0" to avoid changing hash when only version changed const packageJson = await fs.readJson(filePath); @@ -46,5 +71,9 @@ export default async function computePackageChecksum( } } hash.end(); - return hash.read(); + const checksum = hash.read(); + if (fullChecksumFilePath) { + await fs.writeFile(fullChecksumFilePath, checksum); + } + return checksum; } diff --git a/desktop/pkg-lib/src/runBuild.ts b/desktop/pkg-lib/src/runBuild.ts index c6f45ed25..413dfc701 100644 --- a/desktop/pkg-lib/src/runBuild.ts +++ b/desktop/pkg-lib/src/runBuild.ts @@ -12,6 +12,8 @@ import getWatchFolders from './getWatchFolders'; import path from 'path'; import fs from 'fs-extra'; import {getInstalledPluginDetails} from 'flipper-plugin-lib'; +import {FileStore} from 'metro-cache'; +import os from 'os'; let metroDir: string | undefined; const metroDirPromise = getMetroDir().then((dir) => (metroDir = dir)); @@ -77,11 +79,18 @@ export default async function bundlePlugin(pluginDir: string, dev: boolean) { sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs', 'cjs'], blacklistRE: /\.native\.js$/, }, + cacheStores: [ + new FileStore({ + root: + process.env.FLIPPER_METRO_CACHE ?? + path.join(os.tmpdir(), 'metro-cache'), + }), + ], }); await Metro.runBuild(config, { dev, minify: !dev, - resetCache: !dev, + resetCache: false, sourceMap: dev, sourceMapUrl, entry, diff --git a/desktop/scripts/compute-package-checksum.ts b/desktop/scripts/compute-package-checksum.ts new file mode 100644 index 000000000..bf70de48d --- /dev/null +++ b/desktop/scripts/compute-package-checksum.ts @@ -0,0 +1,41 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {computePackageChecksum} from 'flipper-pkg-lib'; +import yargs from 'yargs'; + +const argv = yargs + .usage('yarn compute-package-checksum [args]') + .version(false) + .options({ + dir: { + description: 'Package root directory', + type: 'string', + demandOption: true, + alias: 'd', + }, + out: { + description: + 'File relative to the package root directory where the computed checksum should be saved, e.g. "lib/checksum.txt"', + type: 'string', + alias: 'o', + }, + }) + .help() + .strict() + .parse(process.argv.slice(1)); + +computePackageChecksum(argv.dir, argv.out) + .then(() => { + process.exit(0); + }) + .catch((err: any) => { + console.error(err); + process.exit(1); + }); diff --git a/desktop/tsc-root/tsconfig.json b/desktop/tsc-root/tsconfig.json new file mode 100644 index 000000000..57aa1165e --- /dev/null +++ b/desktop/tsc-root/tsconfig.json @@ -0,0 +1,25 @@ +{ + "references": [ + { + "path": "../babel-transformer" + }, + { + "path": "../plugin-lib" + }, + { + "path": "../pkg" + }, + { + "path": "../pkg-lib" + }, + { + "path": "../doctor" + }, + { + "path": "../eslint-plugin-flipper" + }, + { + "path": "../test-utils" + } + ] +} diff --git a/desktop/types/metro-cache.d.ts b/desktop/types/metro-cache.d.ts new file mode 100644 index 000000000..899856442 --- /dev/null +++ b/desktop/types/metro-cache.d.ts @@ -0,0 +1,10 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +declare module 'metro-cache'; diff --git a/desktop/yarn.lock b/desktop/yarn.lock index 90b50bc28..2c4002f12 100644 --- a/desktop/yarn.lock +++ b/desktop/yarn.lock @@ -8741,7 +8741,7 @@ metro-cache-key@0.65.2: resolved "https://registry.yarnpkg.com/metro-cache-key/-/metro-cache-key-0.65.2.tgz#b841ceacda1c97a5616ad99e728a3bf31c6390df" integrity sha512-2Re5w1kxIpdBFYavrAOrAu0pIj/gj2DUQGcCbDiYWpDw0ibyvOakmIbMZRH/J6c1KXJIGzb4YZpisKeAOsNODQ== -metro-cache@0.65.2: +metro-cache@0.65.2, metro-cache@^0.65.2: version "0.65.2" resolved "https://registry.yarnpkg.com/metro-cache/-/metro-cache-0.65.2.tgz#8922c8d4d882014e745aa01c9d1c64aeb3b08306" integrity sha512-+f7A57qlF2IChrFsHsYN4Lhr7nwb3lsS+Z/M1833Bhj25VBhbe3dMqt8+rOWncgkOY3aHtH7PphRABy5yfDQ3g==