Re-use babel transformations

Summary:
SORRY FOR BIG DIFF, but it's really hard to split it as all these changes are cross-dependent and should be made at once:
1. Moved transformations to separate package "flipper-babel-transformer" and linked it using yarn workspaces to "static" and "pkg" packages where they are re-used. Removed double copies of transformations we had before int these two packages.
2. Converted transformations to typescript
3. Refactored transformations to avoid relying on file system paths for customisation (FB stubs and Electron stubs for headless build)
4. As babel transformations must be built before other builds - enabled incremental build for them and changed scripts to invoke the transformations build before other build scripts
5. As we need to deploy all the dependencies including the fresh "flipper-babel-transformer" as a part of "static" - implemented script which copies package with all the dependencies taking in account yarn workspaces (hoisting and symlinks)

Reviewed By: passy, mweststrate

Differential Revision: D20690662

fbshipit-source-id: 38a275b60d3c91e01ec21d1dbd72d03c05cfac0b

desktop/scripts/build-headless.ts

@@ -7,7 +7,7 @@
  * @format
  */
 
-import fs from 'fs';
+import fs from 'fs-extra';
 import path from 'path';
 import lineReplace from 'line-replace';
 import yazl from 'yazl';
@@ -19,6 +19,8 @@ import {
   getVersionNumber,
   genMercurialRevision,
 } from './build-utils';
+import isFB from './isFB';
+import {distDir} from './paths';
 
 const PLUGINS_FOLDER_NAME = 'plugins';
 
@@ -68,6 +70,9 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
 }
 
 (async () => {
+  if (isFB) {
+    process.env.FLIPPER_FB = 'true';
+  }
   const targets: {mac?: string; linux?: string; win?: string} = {};
   let platformPostfix: string = '';
   if (process.argv.indexOf('--mac') > -1) {
@@ -95,7 +100,6 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
 
   process.env.BUILD_HEADLESS = 'true';
   const buildDir = await buildFolder();
-  const distDir = path.join(__dirname, '..', '..', 'dist');
   // eslint-disable-next-line no-console
   console.log('Created build directory', buildDir);
   await compileHeadless(buildDir);
@@ -114,6 +118,7 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
     Object.values(targets).join(','),
     '--debug',
   ]);
+  await fs.ensureDir(distDir);
   await createZip(buildDir, distDir, Object.keys(targets));
   // eslint-disable-next-line no-console
   console.log('✨  Done');

desktop/scripts/build-release.ts

@@ -22,14 +22,13 @@ import {
 } from './build-utils';
 import fetch from 'node-fetch';
 import {getIcons, buildLocalIconPath, getIconURL} from '../app/src/utils/icons';
+import isFB from './isFB';
+import copyPackageWithDependencies from './copy-package-with-dependencies';
+import {staticDir, distDir} from './paths';
 
-function generateManifest(versionNumber: string) {
-  const filePath = path.join(__dirname, '..', '..', 'dist');
-  if (!fs.existsSync(filePath)) {
-    fs.mkdirSync(filePath);
-  }
-  fs.writeFileSync(
-    path.join(__dirname, '..', '..', 'dist', 'manifest.json'),
+async function generateManifest(versionNumber: string) {
+  await fs.writeFile(
+    path.join(distDir, 'manifest.json'),
     JSON.stringify({
       package: 'com.facebook.sonar',
       version_name: versionNumber,
@@ -37,7 +36,7 @@ function generateManifest(versionNumber: string) {
   );
 }
 
-function modifyPackageManifest(
+async function modifyPackageManifest(
   buildFolder: string,
   versionNumber: string,
   hgRevision: string | null,
@@ -56,7 +55,7 @@ function modifyPackageManifest(
   if (hgRevision != null) {
     manifest.revision = hgRevision;
   }
-  fs.writeFileSync(
+  await fs.writeFile(
     path.join(buildFolder, 'package.json'),
     JSON.stringify(manifest, null, '  '),
   );
@@ -74,7 +73,7 @@ async function buildDist(buildFolder: string) {
     }
     postBuildCallbacks.push(() =>
       spawn('zip', ['-qyr9', '../Flipper-mac.zip', 'Flipper.app'], {
-        cwd: path.join(__dirname, '..', '..', 'dist', 'mac'),
+        cwd: path.join(distDir, 'mac'),
         encoding: 'utf-8',
       }),
     );
@@ -107,8 +106,8 @@ async function buildDist(buildFolder: string) {
       config: {
         appId: `com.facebook.sonar`,
         directories: {
-          buildResources: path.join(__dirname, '..', 'static'),
-          output: path.join(__dirname, '..', '..', 'dist'),
+          buildResources: buildFolder,
+          output: distDir,
         },
         electronDownload: electronDownloadOptions,
         npmRebuild: false,
@@ -122,10 +121,8 @@ async function buildDist(buildFolder: string) {
   }
 }
 
-function copyStaticFolder(buildFolder: string) {
-  fs.copySync(path.join(__dirname, '..', 'static'), buildFolder, {
-    dereference: true,
-  });
+async function copyStaticFolder(buildFolder: string) {
+  await copyPackageWithDependencies(staticDir, buildFolder);
 }
 
 function downloadIcons(buildFolder: string) {
@@ -175,18 +172,25 @@ function downloadIcons(buildFolder: string) {
 }
 
 (async () => {
+  if (isFB) {
+    process.env.FLIPPER_FB = 'true';
+  }
   const dir = await buildFolder();
   // eslint-disable-next-line no-console
   console.log('Created build directory', dir);
+
   await compileMain({dev: false});
-  copyStaticFolder(dir);
+  await copyStaticFolder(dir);
   await downloadIcons(dir);
-  await compileDefaultPlugins(path.join(dir, 'defaultPlugins'));
+  if (!process.argv.includes('--no-embedded-plugins')) {
+    await compileDefaultPlugins(path.join(dir, 'defaultPlugins'));
+  }
   await compileRenderer(dir);
   const versionNumber = getVersionNumber();
   const hgRevision = await genMercurialRevision();
-  modifyPackageManifest(dir, versionNumber, hgRevision);
-  generateManifest(versionNumber);
+  await modifyPackageManifest(dir, versionNumber, hgRevision);
+  await fs.ensureDir(distDir);
+  await generateManifest(versionNumber);
   await buildDist(dir);
   // eslint-disable-next-line no-console
   console.log('✨  Done');

desktop/scripts/build-utils.ts

@@ -16,7 +16,13 @@ import fs from 'fs-extra';
 import {spawn} from 'promisify-child-process';
 import recursiveReaddir from 'recursive-readdir';
 import {default as getWatchFolders} from '../static/get-watch-folders';
-import {appDir, staticDir, pluginsDir, headlessDir} from './paths';
+import {
+  appDir,
+  staticDir,
+  pluginsDir,
+  headlessDir,
+  babelTransformationsDir,
+} from './paths';
 
 async function mostRecentlyChanged(
   dir: string,
@@ -72,7 +78,10 @@ async function compile(
       watchFolders,
       serializer: {},
       transformer: {
-        babelTransformerPath: path.join(staticDir, 'transforms', 'index.js'),
+        babelTransformerPath: path.join(
+          babelTransformationsDir,
+          'transform-app',
+        ),
       },
       resolver: {
         resolverMainFields: ['flipper:source', 'module', 'main'],
@@ -138,6 +147,7 @@ export async function compileRenderer(buildFolder: string) {
 
 export async function compileMain({dev}: {dev: boolean}) {
   const out = path.join(staticDir, 'main.bundle.js');
+  process.env.FLIPPER_ELECTRON_VERSION = require('electron/package.json').version;
   // check if main needs to be compiled
   if (await fs.pathExists(out)) {
     const staticDirCtime = await mostRecentlyChanged(staticDir, ['*.bundle.*']);
@@ -147,14 +157,17 @@ export async function compileMain({dev}: {dev: boolean}) {
       return;
     }
   }
-  console.log(`⚙️  Compiling main bundle... ${staticDir}`);
+  console.log('⚙️  Compiling main bundle...');
   try {
     const config = Object.assign({}, await Metro.loadConfig(), {
       reporter: {update: () => {}},
       projectRoot: staticDir,
       watchFolders: await getWatchFolders(staticDir),
       transformer: {
-        babelTransformerPath: path.join(staticDir, 'transforms', 'index.js'),
+        babelTransformerPath: path.join(
+          babelTransformationsDir,
+          'transform-main',
+        ),
       },
       resolver: {
         sourceExts: ['tsx', 'ts', 'js'],

desktop/scripts/copy-package-with-dependencies.ts

@@ -0,0 +1,110 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import fs from 'fs-extra';
+import path from 'path';
+
+/**
+ * This function copies package into the specified target dir with all its dependencies:
+ * 1) Both direct and transitive dependencies are copied.
+ * 2) Symlinks are dereferenced and copied to the target dir as normal dirs.
+ * 3) Hoisting is supported, so the function scans node_modules up the file tree until dependency is resolved.
+ * 4) All the dependencies keep their scopes, e.g. dependency from <packageDir>/node_modules/package1/node_modules/package2
+ *    is copied to <targetDir>/node_modules/package1/node_modules/package2.
+ * 5) Prints informative error and fails fast if a dependency is not resolved.
+ */
+export default async function copyPackageWithDependencies(
+  packageDir: string,
+  targetDir: string,
+) {
+  await fs.remove(targetDir);
+  await copyPackageWithDependenciesRecursive(packageDir, targetDir, targetDir);
+}
+
+async function copyPackageWithDependenciesRecursive(
+  packageDir: string,
+  targetDir: string,
+  rootTargetDir: string,
+) {
+  if (await fs.pathExists(targetDir)) {
+    return;
+  }
+  await fs.mkdirp(targetDir);
+  if ((await fs.stat(packageDir)).isSymbolicLink()) {
+    packageDir = await fs.readlink(packageDir);
+  }
+  await fs.copy(packageDir, targetDir, {
+    dereference: true,
+    recursive: true,
+    filter: (src) => !src.startsWith(path.join(packageDir, 'node_modules')),
+  });
+  const pkg = await fs.readJson(path.join(packageDir, 'package.json'));
+  const dependencies = (pkg.dependencies ?? {}) as {[key: string]: string};
+  let unresolvedCount = Object.keys(dependencies).length;
+  let curPackageDir = packageDir;
+  let curTargetDir = targetDir;
+  while (unresolvedCount > 0) {
+    const curPackageModulesDir = path.join(curPackageDir, 'node_modules');
+    if (await fs.pathExists(curPackageModulesDir)) {
+      for (const moduleName of Object.keys(dependencies)) {
+        const curModulePath = path.join(
+          curPackageModulesDir,
+          ...moduleName.split('/'),
+        );
+        const targetModulePath = path.join(
+          curTargetDir,
+          'node_modules',
+          ...moduleName.split('/'),
+        );
+        if (await fs.pathExists(curModulePath)) {
+          await copyPackageWithDependenciesRecursive(
+            curModulePath,
+            targetModulePath,
+            rootTargetDir,
+          );
+          delete dependencies[moduleName];
+          unresolvedCount--;
+        }
+      }
+    }
+
+    const parentPackageDir = getParentPackageDir(curPackageDir);
+    if (
+      !parentPackageDir ||
+      parentPackageDir === '' ||
+      parentPackageDir === curPackageDir
+    ) {
+      break;
+    }
+    curPackageDir = parentPackageDir;
+
+    curTargetDir = getParentPackageDir(curTargetDir);
+    if (!curTargetDir || curTargetDir.length < rootTargetDir.length) {
+      curTargetDir = rootTargetDir;
+    }
+  }
+
+  if (unresolvedCount > 0) {
+    for (const unresolvedDependency of Object.keys(dependencies)) {
+      console.error(`Cannot resolve ${unresolvedDependency} in ${packageDir}`);
+    }
+    process.exit(1);
+  }
+}
+
+function getParentPackageDir(packageDir: string) {
+  packageDir = path.dirname(packageDir);
+  while (
+    path.basename(packageDir) === 'node_modules' ||
+    path.basename(packageDir).startsWith('@')
+  ) {
+    packageDir = path.dirname(packageDir);
+  }
+  return packageDir;
+}

desktop/scripts/isFB.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const isFB: boolean;
+export default isFB;

desktop/scripts/isFB.js

@@ -0,0 +1,16 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ * @ts-check
+ */
+
+const fs = require('fs-extra');
+const path = require('path');
+
+const isFB = fs.pathExistsSync(path.resolve(__dirname, '..', 'static', 'fb'));
+
+module.exports = isFB;

desktop/scripts/jest-setup.js

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+global.fetch = require('jest-fetch-mock');
+
+require('immer').enableMapSet();

desktop/scripts/jest-transform.js

@@ -0,0 +1,27 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+// eslint-disable-next-line import/no-unresolved
+const {transform} = require('../babel-transformer/lib/transform-jest');
+const isFB = require('./isFB');
+
+if (isFB && process.env.FLIPPER_FB === undefined) {
+  process.env.FLIPPER_FB = 'true';
+}
+
+module.exports = {
+  process(src, filename, config, options) {
+    return transform({
+      src,
+      filename,
+      config,
+      options: {...options, isTestRunner: true},
+    });
+  },
+};

desktop/scripts/paths.ts

@@ -14,3 +14,9 @@ export const appDir = path.join(rootDir, 'app');
 export const staticDir = path.join(rootDir, 'static');
 export const pluginsDir = path.join(rootDir, 'plugins');
 export const headlessDir = path.join(rootDir, 'headless');
+export const distDir = path.resolve(rootDir, '..', 'dist');
+export const babelTransformationsDir = path.resolve(
+  rootDir,
+  'babel-transformer',
+  'src',
+);

desktop/scripts/start-dev-server.ts

@@ -8,7 +8,7 @@
  */
 
 const electronBinary: string = require('electron') as any;
-import codeFrame from 'babel-code-frame';
+import codeFrame from '@babel/code-frame';
 import socketIo from 'socket.io';
 import express, {Express} from 'express';
 import detect from 'detect-port';
@@ -23,7 +23,8 @@ import Watchman from '../static/watchman';
 import Metro from 'metro';
 import MetroResolver from 'metro-resolver';
 import {default as getWatchFolders} from '../static/get-watch-folders';
-import {staticDir, pluginsDir, appDir} from './paths';
+import {staticDir, pluginsDir, appDir, babelTransformationsDir} from './paths';
+import isFB from './isFB';
 
 const ansiToHtmlConverter = new AnsiToHtmlConverter();
 
@@ -40,6 +41,9 @@ function launchElectron({
   bundleURL: string;
   electronURL: string;
 }) {
+  if (process.argv.includes('--no-embedded-plugins')) {
+    process.env.FLIPPER_NO_EMBEDDED_PLUGINS = 'true';
+  }
   const args = [
     path.join(staticDir, 'index.js'),
     '--remote-debugging-port=9222',
@@ -88,7 +92,7 @@ async function startMetroServer(app: Express) {
     projectRoot: appDir,
     watchFolders,
     transformer: {
-      babelTransformerPath: path.join(staticDir, 'transforms', 'index.js'),
+      babelTransformerPath: path.join(babelTransformationsDir, 'transform-app'),
     },
     resolver: {
       resolverMainFields: ['flipper:source', 'module', 'main'],
@@ -251,6 +255,9 @@ function outputScreen(socket?: socketIo.Server) {
 }
 
 (async () => {
+  if (isFB && process.env.FLIPPER_FB === undefined) {
+    process.env.FLIPPER_FB = 'true';
+  }
   const port = await detect(DEFAULT_PORT);
   const {app, server} = await startAssetServer(port);
   const socket = await addWebsocket(server);