Remove plugin compilation and loading from startup hot path

Summary:
- Removed compilation on startup which is not required anymore after switching to plugin spec v2.
- Removed from Node process ("static" package) all the dependencies which are not required anymore:  e.g. metro, babel etc.
- Plugin loading code from node process moved to browser process and made asyncronous.

Some expected benefits after these changes:
1) Reduced size of Flipper bundle (~4.5MB reduction for lzma package in my tests) as well as startup time. It's hard to say the exact startup time difference as it is very machine-dependent, and on my machine it was already fast ~1500ms (vs 5500ms for p95) and decreased by just 100ms. But I think we should definitely see some improvements on "launch time" analytics graph for p95/p99.
2) Plugin loading is async now and happens when UI is already shown, so perceptive startup time should be also better now.
3) All plugin loading code is now consolidated in "app/dispatcher/plugins.tsx" instead of being splitted between Node and Browser processes as before. So it will be easier to debug plugin loading.
4) Now it is possible to apply updates of plugins by simple refresh of browser window instead of full Electron process restart as before.
5) 60% less code in Node process. This is good because it is harder to debug changes in Node process than in Browser process, especially taking in account differences between dev/release builds. Because of this Node process often ended up broken after changes. Hopefully now it will be more stable.

Changelog: changed the way of plugin loading, and removed obsolete dependencies, which should reduce bundle size and startup time.

Reviewed By: passy

Differential Revision: D23682756

fbshipit-source-id: 8445c877234b41c73853cebe585e2fdb1638b2c9

desktop/scripts/build-utils.ts

@@ -14,7 +14,7 @@ import fs from 'fs-extra';
 import {spawn} from 'promisify-child-process';
 import {getWatchFolders} from 'flipper-pkg-lib';
 import getAppWatchFolders from './get-app-watch-folders';
-import {getSourcePlugins} from '../static/getPlugins';
+import {getSourcePlugins, getPluginSourceFolders} from 'flipper-plugin-lib';
 import {
   appDir,
   staticDir,
@@ -22,7 +22,6 @@ import {
   headlessDir,
   babelTransformationsDir,
 } from './paths';
-import {getPluginSourceFolders} from '../static/getPluginFolders';
 
 const dev = process.env.NODE_ENV !== 'production';
 

desktop/scripts/ensurePluginFoldersWatchable.ts

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {getPluginSourceFolders} from 'flipper-plugin-lib';
+import fs from 'fs-extra';
+
+const watchmanconfigName = '.watchmanconfig';
+
+import path from 'path';
+
+export default async function ensurePluginFoldersWatchable() {
+  const pluginFolders = await getPluginSourceFolders();
+  for (const pluginFolder of pluginFolders) {
+    if (!(await hasParentWithWatchmanConfig(pluginFolder))) {
+      // If no watchman config found in the plugins folder or any its parent, we need to create it.
+      // Otherwise we won't be able to listen for plugin changes.
+      await fs.writeJson(path.join(pluginFolder, watchmanconfigName), {});
+    }
+  }
+}
+
+async function hasParentWithWatchmanConfig(dir: string): Promise<boolean> {
+  if (await fs.pathExists(path.join(dir, watchmanconfigName))) {
+    return true;
+  } else {
+    const parent = path.dirname(dir);
+    if (parent && parent != '' && parent !== dir) {
+      return await hasParentWithWatchmanConfig(parent);
+    }
+  }
+  return false;
+}

desktop/scripts/start-dev-server.ts

@@ -20,16 +20,15 @@ import path from 'path';
 import fs from 'fs-extra';
 import {hostname} from 'os';
 import {compileMain, generatePluginEntryPoints} from './build-utils';
-import Watchman from '../static/watchman';
+import Watchman from './watchman';
 import Metro from 'metro';
 import MetroResolver from 'metro-resolver';
 import {staticDir, appDir, babelTransformationsDir} from './paths';
 import isFB from './isFB';
 import getAppWatchFolders from './get-app-watch-folders';
-import {getSourcePlugins} from '../static/getPlugins';
-import {getPluginSourceFolders} from '../static/getPluginFolders';
-import startWatchPlugins from '../static/startWatchPlugins';
-import ensurePluginFoldersWatchable from '../static/ensurePluginFoldersWatchable';
+import {getPluginSourceFolders} from 'flipper-plugin-lib';
+import ensurePluginFoldersWatchable from './ensurePluginFoldersWatchable';
+import startWatchPlugins from './startWatchPlugins';
 
 const ansiToHtmlConverter = new AnsiToHtmlConverter();
 
@@ -211,7 +210,7 @@ async function startWatchChanges(io: socketIo.Server) {
     const watchman = new Watchman(path.resolve(__dirname, '..'));
     await watchman.initialize();
     await Promise.all(
-      ['app', 'pkg', 'doctor', 'flipper-plugin'].map((dir) =>
+      ['app', 'pkg', 'doctor', 'plugin-lib', 'flipper-plugin'].map((dir) =>
         watchman.startWatchFiles(
           dir,
           () => {
@@ -223,8 +222,7 @@ async function startWatchChanges(io: socketIo.Server) {
         ),
       ),
     );
-    const plugins = await getSourcePlugins();
-    await startWatchPlugins(plugins, () => {
+    await startWatchPlugins(() => {
       io.emit('refresh');
     });
   } catch (err) {

desktop/scripts/startWatchPlugins.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import Watchman from './watchman';
+import {getPluginSourceFolders} from 'flipper-plugin-lib';
+
+export default async function startWatchPlugins(
+  onChanged: () => void | Promise<void>,
+) {
+  // eslint-disable-next-line no-console
+  console.log('🕵️‍  Watching for plugin changes');
+
+  let delayedCompilation: NodeJS.Timeout | undefined;
+  const kCompilationDelayMillis = 1000;
+  const onPluginChangeDetected = () => {
+    if (!delayedCompilation) {
+      delayedCompilation = setTimeout(() => {
+        delayedCompilation = undefined;
+        // eslint-disable-next-line no-console
+        console.log(`🕵️‍  Detected plugin change`);
+        onChanged();
+      }, kCompilationDelayMillis);
+    }
+  };
+  try {
+    await startWatchingPluginsUsingWatchman(onPluginChangeDetected);
+  } catch (err) {
+    console.error(
+      'Failed to start watching plugin files using Watchman, continue without hot reloading',
+      err,
+    );
+  }
+}
+
+async function startWatchingPluginsUsingWatchman(onChange: () => void) {
+  const pluginFolders = await getPluginSourceFolders();
+  await Promise.all(
+    pluginFolders.map(async (pluginFolder) => {
+      const watchman = new Watchman(pluginFolder);
+      await watchman.initialize();
+      await watchman.startWatchFiles('.', () => onChange(), {
+        excludes: ['**/__tests__/**/*', '**/node_modules/**/*', '**/.*'],
+      });
+    }),
+  );
+}

desktop/scripts/watchman.ts

@@ -0,0 +1,118 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {Client} from 'fb-watchman';
+import {v4 as uuid} from 'uuid';
+import path from 'path';
+
+export default class Watchman {
+  constructor(private rootDir: string) {}
+
+  private client?: Client;
+  private watch?: any;
+  private relativeRoot?: string;
+
+  async initialize(): Promise<void> {
+    if (this.client) {
+      return;
+    }
+    this.client = new Client();
+    this.client.setMaxListeners(250);
+    await new Promise((resolve, reject) => {
+      const onError = (err: Error) => {
+        this.client!.removeAllListeners('error');
+        reject(err);
+        this.client!.end();
+        delete this.client;
+      };
+      this.client!.once('error', onError);
+      this.client!.capabilityCheck(
+        {optional: [], required: ['relative_root']},
+        (error) => {
+          if (error) {
+            onError(error);
+            return;
+          }
+          this.client!.command(
+            ['watch-project', this.rootDir],
+            (error, resp) => {
+              if (error) {
+                onError(error);
+                return;
+              }
+              if ('warning' in resp) {
+                console.warn(resp.warning);
+              }
+              this.watch = resp.watch;
+              this.relativeRoot = resp.relative_path;
+              resolve();
+            },
+          );
+        },
+      );
+    });
+  }
+
+  async startWatchFiles(
+    relativeDir: string,
+    handler: (resp: any) => void,
+    options: {excludes: string[]},
+  ): Promise<void> {
+    if (!this.watch) {
+      throw new Error(
+        'Watchman is not initialized, please call "initialize" function and wait for the returned promise completion before calling "startWatchFiles".',
+      );
+    }
+    options = Object.assign({excludes: []}, options);
+    return new Promise((resolve, reject) => {
+      this.client!.command(['clock', this.watch], (error, resp) => {
+        if (error) {
+          return reject(error);
+        }
+
+        try {
+          const {clock} = resp;
+
+          const sub = {
+            expression: [
+              'allof',
+              ['not', ['type', 'd']],
+              ...options!.excludes.map((e) => [
+                'not',
+                ['match', e, 'wholename'],
+              ]),
+            ],
+            fields: ['name'],
+            since: clock,
+            relative_root: this.relativeRoot
+              ? path.join(this.relativeRoot, relativeDir)
+              : relativeDir,
+          };
+
+          const id = uuid();
+
+          this.client!.command(['subscribe', this.watch, id, sub], (error) => {
+            if (error) {
+              return reject(error);
+            }
+            this.client!.on('subscription', (resp) => {
+              if (resp.subscription !== id || !resp.files) {
+                return;
+              }
+              handler(resp);
+            });
+            resolve();
+          });
+        } catch (err) {
+          reject(err);
+        }
+      });
+    });
+  }
+}