Make loading of bundled plugins working and robust

Summary:
Bundled plugins so far didn't load because the defaultPlugins was not generated yet. Fixed follow up errors that resulted from node modules not being available in the browser, and made the process more robust so that one plugin that fails to initialise doesn't kill all bundled plugins from being loaded.

Cleaned up the server scripts, and reused the BUILTINS list that is already maintained in the babel transformer.

Report errors during the build process if modules are referred that are stubbed out (later on we could maybe error on that)

Reviewed By: aigoncharov

Differential Revision: D33020243

fbshipit-source-id: 3ce13b0049664b5fb19c1f45f0b33c1d7fdbea4c

desktop/.gitignore

@@ -5,6 +5,7 @@ node_modules/
 /static/themes/
 /static/defaultPlugins/
 /app/src/defaultPlugins/index.tsx
+/flipper-ui-browser/src/defaultPlugins/index.tsx
 /coverage
 .env
 tsc-error.log

desktop/babel-transformer/src/electron-requires.ts

@@ -84,3 +84,6 @@ module.exports = () => ({
     },
   },
 });
+
+// used by startWebServerDev to know which modules to stub
+module.exports.BUILTINS = BUILTINS;

desktop/flipper-server/src/startWebServerDev.tsx

@@ -19,6 +19,11 @@ import pFilter from 'p-filter';
 // provided by Metro
 // eslint-disable-next-line
 import MetroResolver from 'metro-resolver';
+import {homedir} from 'os';
+
+// This file is heavily inspired by scripts/start-dev-server.ts!
+// part of that is done by start-flipper-server (compiling "main"),
+// the other part ("renderer") here.
 
 const uiSourceDirs = [
   'flipper-ui-browser',
@@ -27,81 +32,28 @@ const uiSourceDirs = [
   'flipper-common',
 ];
 
-const stubModules = new Set<string>([
+// copied from plugin-lib/src/pluginPaths
-  // 'fs',
+export async function getPluginSourceFolders(): Promise<string[]> {
-  // 'path',
+  const pluginFolders: string[] = [];
-  // 'crypto',
+  if (process.env.FLIPPER_NO_DEFAULT_PLUGINS) {
-  // 'process',
+    console.log(
-  // 'os',
+      '🥫  Skipping default plugins because "--no-default-plugins" flag provided',
-  // 'util',
+    );
-  // 'child_process',
+    return pluginFolders;
-  // 'assert',
+  }
-  // 'adbkit', // TODO: factor out!
+  const flipperConfigPath = path.join(homedir(), '.flipper', 'config.json');
-  // 'zlib',
+  if (await fs.pathExists(flipperConfigPath)) {
-  // 'events',
+    const config = await fs.readJson(flipperConfigPath);
-  // 'fs-extra',
+    if (config.pluginPaths) {
-  // 'archiver',
+      pluginFolders.push(...config.pluginPaths);
-  // 'graceful-fs',
+    }
-  // 'stream',
+  }
-  // 'url',
+  pluginFolders.push(path.resolve(__dirname, '..', '..', 'plugins', 'public'));
-  // 'node-fetch',
+  pluginFolders.push(path.resolve(__dirname, '..', '..', 'plugins', 'fb'));
-  // 'net',
+  return pFilter(pluginFolders, (p) => fs.pathExists(p));
-  // 'vm',
-  // 'debug',
-  // 'lockfile',
-  // 'constants',
-  // 'https',
-  // 'plugin-lib', // TODO: we only want the types?
-  // 'flipper-plugin-lib',
-  // 'tar',
-  // 'minipass',
-  // 'live-plugin-manager',
-  // 'decompress-tar',
-  // 'readable-stream',
-  // 'archiver-utils',
-  // 'metro',
-  // 'decompress',
-  // 'temp',
-  // 'tmp',
-  // 'promisify-child-process',
-  // 'jsdom',
-  // 'extract-zip',
-  // 'yauzl',
-  // 'fd-slicer',
-  // 'envinfo',
-  // 'bser',
-  // 'fb-watchman',
-  // TODO fix me
-]);
-
-// This file is heavily inspired by scripts/start-dev-server.ts!
-export async function startWebServerDev(
-  app: Express,
-  server: http.Server,
-  socket: socketio.Server,
-  rootDir: string,
-) {
-  //   await prepareDefaultPlugins(
-  //     process.env.FLIPPER_RELEASE_CHANNEL === 'insiders',
-  //   );
-  //   await ensurePluginFoldersWatchable();
-  await startMetroServer(app, server, socket, rootDir);
-  //   await compileMain();
-  //   if (dotenv && dotenv.parsed) {
-  //     console.log('✅  Loaded env vars from .env file: ', dotenv.parsed);
-  //   }
-  //   shutdownElectron = launchElectron(port);
-
-  // Refresh the app on changes.
-  // When Fast Refresh enabled, reloads are performed by HMRClient, so don't need to watch manually here.
-  //   if (!process.env.FLIPPER_FAST_REFRESH) {
-  //     await startWatchChanges(io);
-  //   }
-
-  console.log('DEV webserver started.');
 }
 
-async function startMetroServer(
+export async function startWebServerDev(
   app: Express,
   server: http.Server,
   socket: socketio.Server,
@@ -112,13 +64,26 @@ async function startMetroServer(
     'babel-transformer',
     'lib', // Note: required pre-compiled!
   );
-  const watchFolders = await dedupeFolders(
+
-    (
+  const electronRequires = path.join(
+    babelTransformationsDir,
+    'electron-requires.js',
+  );
+  const stubModules = new Set<string>(
+    global.electronRequire(electronRequires).BUILTINS,
+  );
+  if (!stubModules.size) {
+    throw new Error('Failed to load list of Node builtins');
+  }
+
+  const watchFolders = await dedupeFolders([
+    ...(
       await Promise.all(
         uiSourceDirs.map((dir) => getWatchFolders(path.resolve(rootDir, dir))),
       )
     ).flat(),
-  );
+    ...(await getPluginSourceFolders()),
+  ]);
 
   const baseConfig = await Metro.loadConfig();
   const config = Object.assign({}, baseConfig, {
@@ -137,8 +102,16 @@ async function startMetroServer(
       blacklistRE: [/\.native\.js$/],
       sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs', 'cjs'],
       resolveRequest(context: any, moduleName: string, ...rest: any[]) {
+        // flipper is special cased, for plugins that we bundle,
+        // we want to resolve `impoSrt from 'flipper'` to 'flipper-ui-core', which
+        // defines all the deprecated exports
+        if (moduleName === 'flipper') {
+          return MetroResolver.resolve(context, 'flipper-ui-core', ...rest);
+        }
         if (stubModules.has(moduleName)) {
-          // console.warn("Found reference to ", moduleName)
+          console.warn(
+            `Found a reference to built-in module '${moduleName}', which will be stubbed out. Referer: ${context.originModulePath}`,
+          );
           return {
             type: 'empty',
           };
@@ -154,8 +127,6 @@ async function startMetroServer(
       },
     },
     watch: true,
-    // only needed when medling with babel transforms
-    // cacheVersion: Math.random(), // only cache for current run
   });
   const connectMiddleware = await Metro.createConnectMiddleware(config);
   app.use(connectMiddleware.middleware);
@@ -165,6 +136,8 @@ async function startMetroServer(
     socket.local.emit('hasErrors', err.toString());
     next();
   });
+
+  console.log('DEV webserver started.');
 }
 
 async function dedupeFolders(paths: string[]): Promise<string[]> {

desktop/flipper-ui-browser/src/defaultPlugins/__mocks__/index.tsx

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+export default {} as any;

desktop/flipper-ui-browser/src/initializeRenderHost.tsx

@@ -76,9 +76,7 @@ export function initializeRenderHost(
 }
 
 function getDefaultPluginsIndex() {
-  // TODO:
-  return {};
   // eslint-disable-next-line import/no-unresolved
-  // const index = require('../defaultPlugins');
+  const index = require('./defaultPlugins');
-  // return index.default || index;
+  return index.default || index;
 }

desktop/flipper-ui-core/src/dispatcher/plugins.tsx

@@ -293,6 +293,11 @@ const requirePluginInternal = async (
   let plugin = pluginDetails.isBundled
     ? defaultPluginsIndex[pluginDetails.name]
     : await getRenderHostInstance().requirePlugin(pluginDetails.entry);
+  if (!plugin) {
+    throw new Error(
+      `Failed to obtain plugin source for: ${pluginDetails.name}`,
+    );
+  }
   if (isSandyPlugin(pluginDetails)) {
     // Sandy plugin
     return new _SandyPluginDefinition(pluginDetails, plugin);

desktop/scripts/build-utils.ts

@@ -30,6 +30,7 @@ import {
   babelTransformationsDir,
   serverDir,
   rootDir,
+  browserUiDir,
 } from './paths';
 
 const {version} = require('../package.json');
@@ -121,11 +122,25 @@ async function generateDefaultPluginEntryPoints(
     bundledPlugins,
   );
   const pluginRequres = bundledPlugins
-    .map((x) => `  '${x.name}': require('${x.name}')`)
+    .map(
+      (x) =>
+        `  '${x.name}': tryRequire('${x.name}', () =>  require('${x.name}'))`,
+    )
     .join(',\n');
   const generatedIndex = `
   /* eslint-disable */
   // THIS FILE IS AUTO-GENERATED by function "generateDefaultPluginEntryPoints" in "build-utils.ts".
+
+  // This function exists to make sure that if one require fails in its module initialisation, not everything fails
+  function tryRequire(module: string, fn: () => any): any {
+    try {
+      return fn();
+    } catch (e) {
+      console.error(\`Could not require ${module}: \`, e)
+      return {};
+    }
+  }
+
   export default {\n${pluginRequres}\n} as any
   `;
   await fs.ensureDir(path.join(appDir, 'src', 'defaultPlugins'));
@@ -133,6 +148,11 @@ async function generateDefaultPluginEntryPoints(
     path.join(appDir, 'src', 'defaultPlugins', 'index.tsx'),
     generatedIndex,
   );
+  await fs.ensureDir(path.join(browserUiDir, 'src', 'defaultPlugins'));
+  await fs.writeFile(
+    path.join(browserUiDir, 'src', 'defaultPlugins', 'index.tsx'),
+    generatedIndex,
+  );
   console.log('✅  Generated bundled plugin entry points.');
 }
 

desktop/scripts/paths.ts

@@ -11,6 +11,7 @@ import path from 'path';
 
 export const rootDir = path.resolve(__dirname, '..');
 export const appDir = path.join(rootDir, 'app');
+export const browserUiDir = path.join(rootDir, 'flipper-ui-browser');
 export const staticDir = path.join(rootDir, 'static');
 export const serverDir = path.join(rootDir, 'flipper-server');
 export const defaultPluginsDir = path.join(staticDir, 'defaultPlugins');

desktop/scripts/start-flipper-server.ts

@@ -7,15 +7,17 @@
  * @format
  */
 
+const dotenv = require('dotenv').config();
 import child from 'child_process';
 import chalk from 'chalk';
 import path from 'path';
-import {compileServerMain} from './build-utils';
+import {compileServerMain, prepareDefaultPlugins} from './build-utils';
 import Watchman from './watchman';
 import {serverDir} from './paths';
 import isFB from './isFB';
 import yargs from 'yargs';
 import open from 'open';
+import ensurePluginFoldersWatchable from './ensurePluginFoldersWatchable';
 
 const argv = yargs
   .usage('yarn start [args]')
@@ -191,10 +193,20 @@ async function startWatchChanges() {
 }
 
 (async () => {
+  if (dotenv && dotenv.parsed) {
+    console.log('✅  Loaded env vars from .env file: ', dotenv.parsed);
+  }
+  await prepareDefaultPlugins(
+    process.env.FLIPPER_RELEASE_CHANNEL === 'insiders',
+  );
+
+  // build?
   if (argv['build']) {
     await compileServerMain();
   } else {
+    // watch
     await startWatchChanges();
+    await ensurePluginFoldersWatchable();
     // builds and starts
     await restartServer();