Fix generation of bundled.json, make source maps work in prod builds

Summary:
This diff fixes several issues around loading plugin, such as:

* make suresource maps work in the flipper-server production build
* make sure default plugins are no symlinked, which wouldn't work anywhere else but on the the system where it was build
* support release channel param for flipper-server

Bundled flipper-server is now 42MB (with icons (see later diffs) and plugins
```
ll flipper-server-v0.0.0.tgz
-rw-r--r--  1 mweststrate  staff    42M 23 Dec 15:29 flipper-server-v0.0.0.tgz
```

Reviewed By: nikoant

Differential Revision: D33294677

fbshipit-source-id: 63538dc8127f883fee6a3608673ad11ce239b350

desktop/flipper-server-core/src/FlipperServerImpl.tsx

@@ -81,6 +81,9 @@ export class FlipperServerImpl implements FlipperServer {
     keytarModule?: KeytarModule,
   ) {
     setFlipperServerConfig(config);
+    console.log(
+      'Loaded flipper config, paths: ' + JSON.stringify(config.paths, null, 2),
+    );
     const server = (this.server = new ServerController(this));
     this.android = new AndroidDeviceManager(this);
     this.ios = new IOSDeviceManager(this);

desktop/flipper-server-core/src/plugins/loadDynamicPlugins.tsx

@@ -45,6 +45,11 @@ export async function loadDynamicPlugins(): Promise<InstalledPluginDetails[]> {
       )
     ).map((p: any) => p.name) as string[],
   );
+  console.log(
+    `✅  Detected ${bundledPlugins.size} bundled plugins: ${Array.from(
+      bundledPlugins,
+    ).join(', ')}.`,
+  );
   const [installedPlugins, unfilteredSourcePlugins] = await Promise.all([
     process.env.FLIPPER_NO_PLUGIN_MARKETPLACE
       ? Promise.resolve([])
@@ -60,23 +65,23 @@ export async function loadDynamicPlugins(): Promise<InstalledPluginDetails[]> {
   const defaultPlugins = await getAllInstalledPluginsInDir(defaultPluginsDir);
   if (defaultPlugins.length > 0) {
     console.log(
-      `✅  Loaded ${defaultPlugins.length} default plugins: ${defaultPlugins
+      `✅  Loaded ${defaultPlugins.length} default plugins:\n${defaultPlugins
-        .map((x) => x.title)
+        .map((x) => `${x.title}@${x.version}`)
-        .join(', ')}.`,
+        .join('\n')}.`,
     );
   }
   if (installedPlugins.length > 0) {
     console.log(
-      `✅  Loaded ${installedPlugins.length} installed plugins: ${Array.from(
+      `✅  Loaded ${installedPlugins.length} installed plugins:\n${Array.from(
-        new Set(installedPlugins.map((x) => x.title)),
+        new Set(installedPlugins.map((x) => `${x.title}@${x.version}`)),
-      ).join(', ')}.`,
+      ).join('\n')}.`,
     );
   }
   if (sourcePlugins.length > 0) {
     console.log(
-      `✅  Loaded ${sourcePlugins.length} source plugins: ${sourcePlugins
+      `✅  Loaded ${sourcePlugins.length} source plugins:\n${sourcePlugins
-        .map((x) => x.title)
+        .map((x) => `${x.title} - ${x.dir}`)
-        .join(', ')}.`,
+        .join('\n')}.`,
     );
   }
   return [...defaultPlugins, ...installedPlugins, ...sourcePlugins];

desktop/flipper-server/src/startWebServerDev.tsx

@@ -17,7 +17,7 @@ import pFilter from 'p-filter';
 import {homedir} from 'os';
 
 // This file is heavily inspired by scripts/start-dev-server.ts!
-// part of that is done by start-flipper-server (compiling "main"),
+// part of that is done by start-flipper-server-dev (compiling "main"),
 // the other part ("renderer") here.
 
 const uiSourceDirs = [

desktop/flipper-ui-browser/src/initializeRenderHost.tsx

@@ -59,10 +59,18 @@ export function initializeRenderHost(
     },
     flipperServer,
     async requirePlugin(path) {
-      // TODO: use `await import(path)`?
+      let source = await flipperServer.exec('plugin-source', path);
-      const source = await flipperServer.exec('plugin-source', path);
+      // append source url (to make sure a file entry shows up in the debugger)
-      // eslint-disable-next-line no-new-func
+      source += `\n//# sourceURL=file://${path}`;
-      const cjsLoader = new Function('module', source);
+      // and source map url (to get source code if available)
+      source += `\n//# sourceMappingURL=file://${path.replace(/.js$/, '.map')}`;
+
+      // Plugins are compiled as typical CJS modules, referring to the global
+      // 'module', which we'll make available by loading the source into a closure that captures 'module'.
+      // Note that we use 'eval', and not 'new Function', because the latter will cause the source maps
+      // to be off by two lines (as the function declaration uses two lines in the generated source)
+      // eslint-disable-next-line no-eval
+      const cjsLoader = eval('(module) => {' + source + '\n}');
       const theModule = {exports: {}};
       cjsLoader(theModule);
       return theModule.exports;

desktop/package.json

@@ -128,7 +128,7 @@
     "predev-server": "yarn build:tsc",
     "dev-server": "cross-env NODE_ENV=development ./ts-node scripts/start-dev-server.ts",
     "preflipper-server": "yarn build:tsc",
-    "flipper-server": "cross-env NODE_ENV=development ./ts-node scripts/start-flipper-server.ts",
+    "flipper-server": "cross-env NODE_ENV=development ./ts-node scripts/start-flipper-server-dev.ts",
     "fix": "eslint . --fix --ext .js,.ts,.tsx",
     "lint": "yarn lint:eslint && yarn lint:tsc && yarn tsc-plugins",
     "lint:eslint": "eslint . --ext .js,.ts,.tsx",

desktop/scripts/build-flipper-server-release.ts

@@ -15,10 +15,18 @@ import {
   launchServer,
   prepareDefaultPlugins,
 } from './build-utils';
-import {serverStaticDir, staticDir} from './paths';
+import {
+  defaultPluginsDir,
+  serverDefaultPluginsDir,
+  serverStaticDir,
+  staticDir,
+} from './paths';
 import isFB from './isFB';
 import yargs from 'yargs';
-import {copy, mkdir, remove} from 'fs-extra';
+import fs from 'fs-extra';
+import copyPackageWithDependencies, {
+  copyPackageWithDependenciesRecursive,
+} from './copy-package-with-dependencies';
 
 const argv = yargs
   .usage('yarn build-flipper-server [args]')
@@ -51,6 +59,11 @@ const argv = yargs
         'Load only specified plugins and skip loading rest. This is useful when you are developing only one or few plugins. Plugins to load can be specified as a comma-separated list with either plugin id or name used as identifier, e.g. "--enabled-plugins network,inspector". The flag is not provided by default which means that all plugins loaded.',
       type: 'array',
     },
+    channel: {
+      description: 'Release channel for the build',
+      choices: ['stable', 'insiders'],
+      default: 'stable',
+    },
   })
   .version('DEV')
   .help()
@@ -60,7 +73,8 @@ if (isFB) {
   process.env.FLIPPER_FB = 'true';
 }
 
-// Don't bundle any plugins into the UI
+process.env.FLIPPER_RELEASE_CHANNEL = argv.channel;
+
 process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true';
 
 // Don't rebuild default plugins, mostly to speed up testing
@@ -93,10 +107,10 @@ if (argv['enabled-plugins'] !== undefined) {
   }
 
   // clear and re-create static dir
-  await remove(serverStaticDir);
+  await fs.remove(serverStaticDir);
-  await mkdir(serverStaticDir);
+  await fs.mkdir(serverStaticDir);
 
-  await prepareDefaultPlugins(false);
+  await prepareDefaultPlugins(argv.channel === 'insiders');
 
   await compileServerMain(false);
   await buildBrowserBundle(false);
@@ -111,11 +125,32 @@ if (argv['enabled-plugins'] !== undefined) {
 });
 
 async function copyStaticResources() {
+  console.log(`⚙️  Copying default plugins...`);
+  await fs.mkdirp(serverDefaultPluginsDir);
+  const plugins = await fs.readdir(defaultPluginsDir);
+  for (const plugin of plugins) {
+    let source = path.join(defaultPluginsDir, plugin);
+    // static/defaultPlugins will symlink, resolve those first
+    while ((await fs.lstat(source)).isSymbolicLink()) {
+      source = await fs.readlink(source);
+    }
+    const target = path.join(serverDefaultPluginsDir, plugin);
+    if ((await fs.stat(source)).isDirectory()) {
+      // for plugins, only copy package.json & dist, to keep impact minimal
+      await fs.copy(
+        path.join(source, 'package.json'),
+        path.join(target, 'package.json'),
+      );
+      await fs.copy(path.join(source, 'dist'), path.join(target, 'dist'));
+    } else {
+      await fs.copy(source, target);
+    }
+  }
+
   console.log(`⚙️  Copying static resources...`);
 
   // static folder, without the things that are only for Electron
   const thingsToCopy = [
-    'defaultPlugins',
     'facebook',
     'icons',
     'native-modules',
@@ -134,7 +169,7 @@ async function copyStaticResources() {
 
   await Promise.all(
     thingsToCopy.map((e) =>
-      copy(path.join(staticDir, e), path.join(serverStaticDir, e)),
+      fs.copy(path.join(staticDir, e), path.join(serverStaticDir, e)),
     ),
   );
   console.log('✅  Copied static resources.');

desktop/scripts/build-release.ts

@@ -31,7 +31,7 @@ import {
 import fetch from '@adobe/node-fetch-retry';
 import isFB from './isFB';
 import copyPackageWithDependencies from './copy-package-with-dependencies';
-import {staticDir, distDir} from './paths';
+import {staticDir, distDir, defaultPluginsDir} from './paths';
 import yargs from 'yargs';
 import {WinPackager} from 'app-builder-lib/out/winPackager';
 // eslint-disable-next-line node/no-extraneous-import

desktop/scripts/build-utils.ts

@@ -420,9 +420,6 @@ export async function compileServerMain(dev: boolean) {
     resetCache: !dev,
   });
   console.log('✅  Compiled server bundle.');
-  if (!dev) {
-    stripSourceMapComment(out);
-  }
 }
 
 // TODO: needed?
@@ -503,9 +500,6 @@ export async function buildBrowserBundle(dev: boolean) {
     inlineSourceMap: false,
   });
   console.log('✅  Compiled browser bundle.');
-  if (!dev) {
-    stripSourceMapComment(out);
-  }
 }
 
 async function dedupeFolders(paths: string[]): Promise<string[]> {

desktop/scripts/paths.ts

@@ -16,6 +16,10 @@ export const staticDir = path.join(rootDir, 'static');
 export const serverDir = path.join(rootDir, 'flipper-server');
 export const serverStaticDir = path.join(serverDir, 'static'); // for pre-bundled server, static resources are copied here
 export const defaultPluginsDir = path.join(staticDir, 'defaultPlugins');
+export const serverDefaultPluginsDir = path.join(
+  serverStaticDir,
+  'defaultPlugins',
+);
 export const pluginsDir = path.join(rootDir, 'plugins');
 export const fbPluginsDir = path.join(pluginsDir, 'fb');
 export const publicPluginsDir = path.join(pluginsDir, 'public');

desktop/scripts/start-flipper-server-dev.ts

@@ -65,6 +65,11 @@ const argv = yargs
       type: 'boolean',
       default: true,
     },
+    channel: {
+      description: 'Release channel for the build',
+      choices: ['stable', 'insiders'],
+      default: 'stable',
+    },
   })
   .version('DEV')
   .help()
@@ -74,6 +79,8 @@ if (isFB) {
   process.env.FLIPPER_FB = 'true';
 }
 
+process.env.FLIPPER_RELEASE_CHANNEL = argv.channel;
+
 if (argv['default-plugins'] === true) {
   delete process.env.FLIPPER_NO_DEFAULT_PLUGINS;
 } else if (argv['default-plugins'] === false) {