Include default plugins into app bundle (#998)

Summary:
Pull Request resolved: https://github.com/facebook/flipper/pull/998

After this diff all the default plugins (which are distributed with Flipper) will be included into the main app bundle instead of bundling each of them separately and then loading from file system. This is done by auto-generating plugins index in build-time and importing it from Flipper app bundle, so Metro can follow these imports and bundle all the plugins to the app bundle.
This provides several benefits:
1) reduced Flipper bundle size (~10% reduction of zipped Flipper archive), because Metro bundles each of re-used dependencies only once instead of bundling them for each plugin where such dependency used.
2) Faster Flipper startup because of reduced bundle and the fact that we don't need to load each plugin bundle from disk - just need to load the single bundle where everything is already included.
3) Metro dev server for plugins works in the same way as for Flipper app itself, e.g. simple refresh automatically recompiles bundled plugins too if there are changes. This also potentially should allow us to enable "fast refresh" for quicker iterations while developing plugins.
4) Faster build ("yarn build --mac" is 2 times faster on my machine after this change)

Potential downsides:
1) Currently all the plugins are identically loaded from disk. After this change some of plugins will be bundled, and some of them (third-party) will be loaded from disk.
2) In future when it will be possible to publish new versions of default plugins separately, installing new version of such plugin (e.g. with some urgent fix) will mean the "default" pre-built version will still be bundled (we cannot "unbundle" it :)), but we'll skip it and instead load new version from disk.

Changelog: Internals: include default plugins into the main bundle instead producing separate bundles for them.

Reviewed By: passy

Differential Revision: D20864002

fbshipit-source-id: 2968f3b786cdd1767d6223996090143d03894b92

desktop/scripts/build-headless.ts

@@ -15,15 +15,13 @@ const {exec: createBinary} = require('pkg');
 import {
   buildFolder,
   compileHeadless,
-  compileDefaultPlugins,
   getVersionNumber,
   genMercurialRevision,
+  generatePluginEntryPoints,
 } from './build-utils';
 import isFB from './isFB';
 import {distDir} from './paths';
 
-const PLUGINS_FOLDER_NAME = 'plugins';
-
 function preludeBundle(
   dir: string,
   versionNumber: string,
@@ -52,15 +50,6 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
       zip.addFile(path.join(buildDir, binary), binary);
     });
 
-    // add plugins
-    const pluginDir = path.join(buildDir, PLUGINS_FOLDER_NAME);
-    fs.readdirSync(pluginDir).forEach((file) => {
-      zip.addFile(
-        path.join(pluginDir, file),
-        path.join(PLUGINS_FOLDER_NAME, file),
-      );
-    });
-
     // write zip file
     zip.outputStream
       .pipe(fs.createWriteStream(path.join(distDir, 'Flipper-headless.zip')))
@@ -94,22 +83,15 @@ async function createZip(buildDir: string, distDir: string, targets: string[]) {
     // platformPostfix is automatically added by pkg
     platformPostfix = '';
   }
-  // Compiling all plugins takes a long time. Use this flag for quicker
-  // developement iteration by not including any plugins.
-  const skipPlugins = process.argv.indexOf('--no-plugins') > -1;
-
   process.env.FLIPPER_HEADLESS = 'true';
   const buildDir = await buildFolder();
   // eslint-disable-next-line no-console
   console.log('Created build directory', buildDir);
+  await generatePluginEntryPoints();
   await compileHeadless(buildDir);
   const versionNumber = getVersionNumber();
   const buildRevision = await genMercurialRevision();
   await preludeBundle(buildDir, versionNumber, buildRevision);
-  await compileDefaultPlugins(
-    path.join(buildDir, PLUGINS_FOLDER_NAME),
-    skipPlugins,
-  );
   await createBinary([
     path.join(buildDir, 'bundle.js'),
     '--output',

desktop/scripts/build-release.ts

@@ -16,9 +16,9 @@ import {
   compileRenderer,
   compileMain,
   die,
-  compileDefaultPlugins,
   getVersionNumber,
   genMercurialRevision,
+  generatePluginEntryPoints,
 } from './build-utils';
 import fetch from 'node-fetch';
 import {getIcons, buildLocalIconPath, getIconURL} from '../app/src/utils/icons';
@@ -126,7 +126,9 @@ async function buildDist(buildFolder: string) {
 }
 
 async function copyStaticFolder(buildFolder: string) {
+  console.log(`⚙️  Copying static package with dependencies...`);
   await copyPackageWithDependencies(staticDir, buildFolder);
+  console.log('✅  Copied static package with dependencies.');
 }
 
 function downloadIcons(buildFolder: string) {
@@ -184,11 +186,9 @@ function downloadIcons(buildFolder: string) {
   console.log('Created build directory', dir);
 
   await compileMain();
+  await generatePluginEntryPoints();
   await copyStaticFolder(dir);
   await downloadIcons(dir);
-  if (!process.argv.includes('--no-embedded-plugins')) {
-    await compileDefaultPlugins(path.join(dir, 'defaultPlugins'));
-  }
   await compileRenderer(dir);
   const versionNumber = getVersionNumber();
   const hgRevision = await genMercurialRevision();

desktop/scripts/build-utils.ts

@@ -8,64 +8,53 @@
  */
 
 import Metro from 'metro';
-import compilePlugins from '../static/compilePlugins';
-import util from 'util';
 import tmp from 'tmp';
 import path from 'path';
 import fs from 'fs-extra';
 import {spawn} from 'promisify-child-process';
-import recursiveReaddir from 'recursive-readdir';
 import getWatchFolders from '../static/getWatchFolders';
 import getAppWatchFolders from './get-app-watch-folders';
+import getPlugins from '../static/getPlugins';
 import {
   appDir,
   staticDir,
-  pluginsDir,
+  defaultPluginsIndexDir,
   headlessDir,
   babelTransformationsDir,
 } from './paths';
+import getPluginFolders from '../static/getPluginFolders';
 
 const dev = process.env.NODE_ENV !== 'production';
 
-async function mostRecentlyChanged(
-  dir: string,
-  ignores: string[],
-): Promise<Date> {
-  const files = await util.promisify<string, string[], string[]>(
-    recursiveReaddir,
-  )(dir, ignores);
-  return files
-    .map((f) => fs.lstatSync(f).ctime)
-    .reduce((a, b) => (a > b ? a : b), new Date(0));
-}
-
 export function die(err: Error) {
   console.error(err.stack);
   process.exit(1);
 }
 
-export function compileDefaultPlugins(
-  defaultPluginDir: string,
-  skipAll: boolean = false,
-) {
-  return compilePlugins(
-    null,
-    skipAll ? [] : [pluginsDir, path.join(pluginsDir, 'fb')],
-    defaultPluginDir,
-    {force: true, failSilently: false, recompileOnChanges: false},
-  )
-    .then((defaultPlugins) =>
-      fs.writeFileSync(
-        path.join(defaultPluginDir, 'index.json'),
-        JSON.stringify(
-          defaultPlugins.map(({entry, rootDir, out, ...plugin}) => ({
-            ...plugin,
-            out: path.parse(out).base,
-          })),
-        ),
-      ),
-    )
-    .catch(die);
+export async function generatePluginEntryPoints() {
+  console.log('⚙️  Generating plugin entry points...');
+  const pluginEntryPoints = await getPlugins();
+  if (await fs.pathExists(defaultPluginsIndexDir)) {
+    await fs.remove(defaultPluginsIndexDir);
+  }
+  await fs.mkdirp(defaultPluginsIndexDir);
+  await fs.writeJSON(
+    path.join(defaultPluginsIndexDir, 'index.json'),
+    pluginEntryPoints.map((plugin) => plugin.manifest),
+  );
+  const pluginRequres = pluginEntryPoints
+    .map((x) => `  '${x.name}': require('${x.name}')`)
+    .join(',\n');
+  const generatedIndex = `
+  // THIS FILE IS AUTO-GENERATED by function "generatePluginEntryPoints" in "build-utils.ts".
+  export default {\n${pluginRequres}\n} as any
+  `;
+  await fs.ensureDir(path.join(appDir, 'src', 'defaultPlugins'));
+  await fs.writeFile(
+    path.join(appDir, 'src', 'defaultPlugins', 'index.tsx'),
+    generatedIndex,
+  );
+  console.log('✅  Generated plugin entry points.');
 }
 
 async function compile(
@@ -89,6 +78,7 @@ async function compile(
       resolver: {
         resolverMainFields: ['flipper:source', 'module', 'main'],
         blacklistRE: /\.native\.js$/,
+        sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs'],
       },
     },
     {
@@ -108,6 +98,7 @@ export async function compileHeadless(buildFolder: string) {
     headlessDir,
     ...(await getWatchFolders(staticDir)),
     ...(await getAppWatchFolders()),
+    ...(await getPluginFolders()),
   ]
     .filter((value, index, self) => self.indexOf(value) === index)
     .filter(fs.pathExistsSync);
@@ -126,7 +117,10 @@ export async function compileHeadless(buildFolder: string) {
 
 export async function compileRenderer(buildFolder: string) {
   console.log(`⚙️  Compiling renderer bundle...`);
-  const watchFolders = await getAppWatchFolders();
+  const watchFolders = [
+    ...(await getAppWatchFolders()),
+    ...(await getPluginFolders()),
+  ];
   try {
     await compile(
       buildFolder,
@@ -143,15 +137,6 @@ export async function compileRenderer(buildFolder: string) {
 export async function compileMain() {
   const out = path.join(staticDir, 'main.bundle.js');
   process.env.FLIPPER_ELECTRON_VERSION = require('electron/package.json').version;
-  // check if main needs to be compiled
-  if (await fs.pathExists(out)) {
-    const staticDirCtime = await mostRecentlyChanged(staticDir, ['*.bundle.*']);
-    const bundleCtime = (await fs.lstat(out)).ctime;
-    if (staticDirCtime < bundleCtime) {
-      console.log(`🥫  Using cached version of main bundle...`);
-      return;
-    }
-  }
   console.log('⚙️  Compiling main bundle...');
   try {
     const config = Object.assign({}, await Metro.loadConfig(), {

desktop/scripts/generate-plugin-entry-points.ts

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {generatePluginEntryPoints} from './build-utils';
+
+generatePluginEntryPoints().catch((err) => {
+  console.error(err);
+  process.exit(1);
+});

desktop/scripts/paths.ts

@@ -12,6 +12,7 @@ import path from 'path';
 export const rootDir = path.resolve(__dirname, '..');
 export const appDir = path.join(rootDir, 'app');
 export const staticDir = path.join(rootDir, 'static');
+export const defaultPluginsIndexDir = path.join(staticDir, 'defaultPlugins');
 export const pluginsDir = path.join(rootDir, 'plugins');
 export const headlessDir = path.join(rootDir, 'headless');
 export const distDir = path.resolve(rootDir, '..', 'dist');

desktop/scripts/start-dev-server.ts

@@ -18,13 +18,16 @@ import chalk from 'chalk';
 import http from 'http';
 import path from 'path';
 import fs from 'fs-extra';
-import {compileMain} from './build-utils';
+import {compileMain, generatePluginEntryPoints} from './build-utils';
 import Watchman from '../static/watchman';
 import Metro from 'metro';
 import MetroResolver from 'metro-resolver';
-import getAppWatchFolders from './get-app-watch-folders';
 import {staticDir, appDir, babelTransformationsDir} from './paths';
 import isFB from './isFB';
+import getAppWatchFolders from './get-app-watch-folders';
+import getPlugins from '../static/getPlugins';
+import getPluginFolders from '../static/getPluginFolders';
+import startWatchPlugins from '../static/startWatchPlugins';
 
 const ansiToHtmlConverter = new AnsiToHtmlConverter();
 
@@ -81,7 +84,9 @@ function launchElectron({
 }
 
 async function startMetroServer(app: Express) {
-  const watchFolders = await getAppWatchFolders();
+  const watchFolders = (await getAppWatchFolders()).concat(
+    await getPluginFolders(),
+  );
   const metroBundlerServer = await Metro.runMetro({
     projectRoot: appDir,
     watchFolders,
@@ -101,6 +106,7 @@ async function startMetroServer(app: Express) {
           platform,
         );
       },
+      sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs'],
     },
     watch: true,
   });
@@ -170,7 +176,7 @@ async function addWebsocket(server: http.Server) {
     }
   });
 
-  // refresh the app on changes to the src folder
+  // refresh the app on changes
   // this can be removed once metroServer notifies us about file changes
   try {
     const watchman = new Watchman(path.resolve(__dirname, '..'));
@@ -188,6 +194,10 @@ async function addWebsocket(server: http.Server) {
         ),
       ),
     );
+    const plugins = await getPlugins();
+    await startWatchPlugins(plugins, () => {
+      io.emit('refresh');
+    });
   } catch (err) {
     console.error(
       'Failed to start watching for changes using Watchman, continue without hot reloading',
@@ -258,6 +268,7 @@ function outputScreen(socket?: socketIo.Server) {
   await startMetroServer(app);
   outputScreen(socket);
   await compileMain();
+  await generatePluginEntryPoints();
   shutdownElectron = launchElectron({
     devServerURL: `http://localhost:${port}`,
     bundleURL: `http://localhost:${port}/src/init.bundle`,