Option for "yarn start" and "yarn build" scripts to pre-install default plugin packages instead of bundling them

Summary:
Sorry for long diff! I can try to split it if necessary, but many changes here are 1-1 replacements / renames.

**Preambule**
Currently we bundle default plugins into the Flipper main bundle. This helps us to reduce bundle size, because of plugin dependencies re-use. E.g. if multiple plugins use "lodash" when they are bundled together, only one copy of "lodash" added. When they are bundled separately, the same dependency might be added to each of them. However as we're not going to include most of plugins into Flipper distributive anymore and going to rely on Marketplace instead, this bundling doesn't provide significant size benefits anymore. In addition to that, bundling makes it impossible to differentiate whether thrown errors are originated from Flipper core or one of its plugins.

Why don't we remove plugin bundling at all? Because for "dev mode" it actually quite useful. It makes dev build start much faster and also enables using of Fast Refresh for plugin development (fast refresh won't work for plugins loaded from disk).

**Changes**
This diff introduces new option "no-bundled-plugins" for "yarn start" and "yarn build" commands. For now, by default, we will continue bundling default plugins into the Flipper main bundle, but if this option provided then we will build each default plugin separately and include their packages into the Flipper distributive as "pre-installed" to be able to load them from disk even without access to Marketplace.

For "yarn start", we're adding symlinks to plugin folders in "static/defaultPlugins" and then they are loaded by Flipper. For "yarn build" we are dereferencing these symlinks to include physical files of plugins into folder "defaultPlugins" of the produced distributive. Folder "defaultPlugins" is excluded from asar, because loading of plugins from asar archive might introduce some unexpected issues depending on their implementation.

Reviewed By: mweststrate

Differential Revision: D28431838

fbshipit-source-id: f7757e9f5ba9183ed918d70252de3ce0e823177d

desktop/app/src/dispatcher/plugins.tsx

@@ -27,16 +27,15 @@ import {
 import GK from '../fb-stubs/GK';
 import {FlipperBasePlugin} from '../plugin';
 import {setupMenuBar} from '../MenuBar';
+import fs from 'fs-extra';
 import path from 'path';
 import {default as config} from '../utils/processConfig';
-import isProduction from '../utils/isProduction';
 import {notNull} from '../utils/typeUtils';
 import {sideEffect} from '../utils/sideEffect';
-import semver from 'semver';
 import {
   ActivatablePluginDetails,
   BundledPluginDetails,
-  PluginDetails,
+  ConcretePluginDetails,
 } from 'flipper-plugin-lib';
 import {tryCatchReportPluginFailures, reportUsage} from '../utils/metrics';
 import * as FlipperPluginSDK from 'flipper-plugin';
@@ -53,7 +52,8 @@ import * as crc32 from 'crc32';
 import getDefaultPluginsIndex from '../utils/getDefaultPluginsIndex';
 import {isDevicePluginDefinition} from '../utils/pluginUtils';
 import isPluginCompatible from '../utils/isPluginCompatible';
-
+import isPluginVersionMoreRecent from '../utils/isPluginVersionMoreRecent';
+import {getStaticPath} from '../utils/pathUtils';
 let defaultPluginsIndex: any = null;
 
 export default async (store: Store, logger: Logger) => {
@@ -78,37 +78,23 @@ export default async (store: Store, logger: Logger) => {
 
   defaultPluginsIndex = getDefaultPluginsIndex();
 
-  const marketplacePlugins = store.getState().plugins.marketplacePlugins;
-  store.dispatch(
-    registerMarketplacePlugins(
-      selectCompatibleMarketplaceVersions(marketplacePlugins),
-    ),
+  const marketplacePlugins = selectCompatibleMarketplaceVersions(
+    store.getState().plugins.marketplacePlugins,
   );
+  store.dispatch(registerMarketplacePlugins(marketplacePlugins));
 
   const uninstalledPluginNames =
     store.getState().plugins.uninstalledPluginNames;
 
-  const bundledPlugins = getBundledPlugins();
+  const bundledPlugins = await getBundledPlugins();
 
   const allLocalVersions = [
-    ...getBundledPlugins(),
+    ...bundledPlugins,
     ...(await getDynamicPlugins()),
   ].filter((p) => !uninstalledPluginNames.has(p.name));
 
-  const loadedVersionsMap: Map<string, ActivatablePluginDetails> = new Map();
-  for (const localVersion of allLocalVersions) {
-    if (isPluginCompatible(localVersion)) {
-      const loadedVersion = loadedVersionsMap.get(localVersion.id);
-      if (
-        !loadedVersion ||
-        semver.gt(localVersion.version, loadedVersion.version)
-      ) {
-        loadedVersionsMap.set(localVersion.id, localVersion);
-      }
-    }
-  }
-
-  const loadedPlugins = Array.from(loadedVersionsMap.values());
+  const loadedPlugins =
+    getLatestCompatibleVersionOfEachPlugin(allLocalVersions);
 
   const initialPlugins: PluginDefinition[] = loadedPlugins
     .map(reportVersion)
@@ -150,40 +136,33 @@ function reportVersion(pluginDetails: ActivatablePluginDetails) {
   return pluginDetails;
 }
 
-export function filterNewestVersionOfEachPlugin<
-  T1 extends PluginDetails,
-  T2 extends PluginDetails,
->(bundledPlugins: T1[], dynamicPlugins: T2[]): (T1 | T2)[] {
-  const pluginByName: {[key: string]: T1 | T2} = {};
-  for (const plugin of bundledPlugins) {
-    pluginByName[plugin.name] = plugin;
-  }
-  for (const plugin of dynamicPlugins) {
-    if (
-      !pluginByName[plugin.name] ||
-      (!process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE &&
-        semver.gt(plugin.version, pluginByName[plugin.name].version, true))
-    ) {
-      pluginByName[plugin.name] = plugin;
+export function getLatestCompatibleVersionOfEachPlugin<
+  T extends ConcretePluginDetails,
+>(plugins: T[]): T[] {
+  const latestCompatibleVersions: Map<string, T> = new Map();
+  for (const plugin of plugins) {
+    if (isPluginCompatible(plugin)) {
+      const loadedVersion = latestCompatibleVersions.get(plugin.id);
+      if (!loadedVersion || isPluginVersionMoreRecent(plugin, loadedVersion)) {
+        latestCompatibleVersions.set(plugin.id, plugin);
+      }
     }
   }
-  return Object.values(pluginByName);
+  return Array.from(latestCompatibleVersions.values());
 }
 
-function getBundledPlugins(): Array<BundledPluginDetails> {
-  // DefaultPlugins that are included in the bundle.
-  // List of defaultPlugins is written at build time
-  const pluginPath =
-    process.env.BUNDLED_PLUGIN_PATH ||
-    (isProduction()
-      ? path.join(__dirname, 'defaultPlugins')
-      : './defaultPlugins/index.json');
-
+async function getBundledPlugins(): Promise<Array<BundledPluginDetails>> {
+  // defaultPlugins that are included in the Flipper distributive.
+  // List of default bundled plugins is written at build time to defaultPlugins/bundled.json.
+  const pluginPath = getStaticPath(
+    path.join('defaultPlugins', 'bundled.json'),
+    {asarUnpacked: true},
+  );
   let bundledPlugins: Array<BundledPluginDetails> = [];
   try {
-    bundledPlugins = global.electronRequire(pluginPath);
+    bundledPlugins = await fs.readJson(pluginPath);
   } catch (e) {
-    console.error('Failed to load bundled plugins', e);
+    console.error('Failed to load list of bundled plugins', e);
   }
 
   return bundledPlugins;