move plugin management from ui-core to server-core

Summary:
Follow up of D32665064, this diff moves all plugin management logic from flipper-ui to flipper-server. Things like downloading, installing, querying new plugins.

Loading plugins is handled separately in the next diff.

Reviewed By: nikoant

Differential Revision: D32666537

fbshipit-source-id: 9786b82987f00180bb26200e38735b334dc4d5c3

desktop/flipper-server-core/src/FlipperServerImpl.tsx

@@ -34,6 +34,7 @@ import {setFlipperServerConfig} from './FlipperServerConfig';
 import {saveSettings} from './utils/settings';
 import {saveLauncherSettings} from './utils/launcherSettings';
 import {KeytarManager} from './utils/keytar';
+import {PluginManager} from './plugins/PluginManager';
 
 /**
  * FlipperServer takes care of all incoming device & client connections.
@@ -53,6 +54,7 @@ export class FlipperServerImpl implements FlipperServer {
   android: AndroidDeviceManager;
   ios: IOSDeviceManager;
   keytarManager: KeytarManager;
+  pluginManager: PluginManager;
 
   constructor(
     public config: FlipperServerConfig,
@@ -64,6 +66,9 @@ export class FlipperServerImpl implements FlipperServer {
     this.android = new AndroidDeviceManager(this);
     this.ios = new IOSDeviceManager(this);
     this.keytarManager = new KeytarManager(keytarModule);
+    // TODO: given flipper-dump, it might make more sense to have the plugin command
+    // handled by moved to flipper-server & app, but let's keep things simple for now
+    this.pluginManager = new PluginManager();
 
     server.addListener('error', (err) => {
       this.emit('server-error', err);
@@ -122,6 +127,7 @@ export class FlipperServerImpl implements FlipperServer {
 
     try {
       await this.server.init();
+      await this.pluginManager.start();
       await this.startDeviceListeners();
       this.setServerState('started');
     } catch (e) {
@@ -247,6 +253,21 @@ export class FlipperServerImpl implements FlipperServer {
     'keychain-write': (service, password) =>
       this.keytarManager.writeKeychain(service, password),
     'keychain-unset': (service) => this.keytarManager.unsetKeychain(service),
+    'plugins-load-dynamic-plugins': () =>
+      this.pluginManager.loadDynamicPlugins(),
+    'plugins-get-bundled-plugins': () => this.pluginManager.getBundledPlugins(),
+    'plugins-get-installed-plugins': () =>
+      this.pluginManager.getInstalledPlugins(),
+    'plugins-remove-plugins': (plugins) =>
+      this.pluginManager.removePlugins(plugins),
+    'plugin-start-download': (details) =>
+      this.pluginManager.downloadPlugin(details),
+    'plugins-get-updatable-plugins': (query) =>
+      this.pluginManager.getUpdatablePlugins(query),
+    'plugins-install-from-file': (path) =>
+      this.pluginManager.installPluginFromFile(path),
+    'plugins-install-from-npm': (name) =>
+      this.pluginManager.installPluginFromNpm(name),
   };
 
   registerDevice(device: ServerDevice) {

desktop/flipper-server-core/src/fb-stubs/constants.tsx

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+export const isFBBuild: boolean = false;

desktop/flipper-server-core/src/plugins/PluginManager.tsx

@@ -0,0 +1,147 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import fs from 'fs-extra';
+import path from 'path';
+import tmp from 'tmp';
+import {promisify} from 'util';
+import {default as axios} from 'axios';
+import {
+  BundledPluginDetails,
+  DownloadablePluginDetails,
+  InstalledPluginDetails,
+} from 'flipper-common';
+import {getStaticPath} from '../utils/pathUtils';
+import {loadDynamicPlugins} from './loadDynamicPlugins';
+import {
+  cleanupOldInstalledPluginVersions,
+  getInstalledPluginDetails,
+  getInstalledPlugins,
+  getPluginVersionInstallationDir,
+  installPluginFromFile,
+  removePlugins,
+  getUpdatablePlugins,
+  getInstalledPlugin,
+  installPluginFromNpm,
+} from 'flipper-plugin-lib';
+
+const maxInstalledPluginVersionsToKeep = 2;
+
+// Adapter which forces node.js implementation for axios instead of browser implementation
+// used by default in Electron. Node.js implementation is better, because it
+// supports streams which can be used for direct downloading to disk.
+const axiosHttpAdapter = require('axios/lib/adapters/http'); // eslint-disable-line import/no-commonjs
+
+const getTempDirName = promisify(tmp.dir) as (
+  options?: tmp.DirOptions,
+) => Promise<string>;
+
+export class PluginManager {
+  async start() {
+    // This needn't happen immediately and is (light) I/O work.
+    (window.requestIdleCallback || setImmediate)(() => {
+      cleanupOldInstalledPluginVersions(maxInstalledPluginVersionsToKeep).catch(
+        (err) =>
+          console.error('Failed to clean up old installed plugins:', err),
+      );
+    });
+  }
+
+  loadDynamicPlugins = loadDynamicPlugins;
+  getInstalledPlugins = getInstalledPlugins;
+  removePlugins = removePlugins;
+  getUpdatablePlugins = getUpdatablePlugins;
+  getInstalledPlugin = getInstalledPlugin;
+  installPluginFromFile = installPluginFromFile;
+  installPluginFromNpm = installPluginFromNpm;
+
+  async getBundledPlugins(): Promise<Array<BundledPluginDetails>> {
+    if (process.env.NODE_ENV === 'test') {
+      return [];
+    }
+    // defaultPlugins that are included in the Flipper distributive.
+    // List of default bundled plugins is written at build time to defaultPlugins/bundled.json.
+    const pluginPath = getStaticPath(
+      path.join('defaultPlugins', 'bundled.json'),
+      {asarUnpacked: true},
+    );
+    let bundledPlugins: Array<BundledPluginDetails> = [];
+    try {
+      bundledPlugins = await fs.readJson(pluginPath);
+    } catch (e) {
+      console.error('Failed to load list of bundled plugins', e);
+    }
+    return bundledPlugins;
+  }
+
+  async downloadPlugin(
+    plugin: DownloadablePluginDetails,
+  ): Promise<InstalledPluginDetails> {
+    const {name, title, version, downloadUrl} = plugin;
+    const installationDir = getPluginVersionInstallationDir(name, version);
+    console.log(
+      `Downloading plugin "${title}" v${version} from "${downloadUrl}" to "${installationDir}".`,
+    );
+    const tmpDir = await getTempDirName();
+    const tmpFile = path.join(tmpDir, `${name}-${version}.tgz`);
+    try {
+      const cancelationSource = axios.CancelToken.source();
+      if (await fs.pathExists(installationDir)) {
+        console.log(
+          `Using existing files instead of downloading plugin "${title}" v${version} from "${downloadUrl}" to "${installationDir}"`,
+        );
+        return await getInstalledPluginDetails(installationDir);
+      } else {
+        await fs.ensureDir(tmpDir);
+        let percentCompleted = 0;
+        const response = await axios.get(plugin.downloadUrl, {
+          adapter: axiosHttpAdapter,
+          cancelToken: cancelationSource.token,
+          responseType: 'stream',
+          headers: {
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-Mode': 'navigate',
+          },
+          onDownloadProgress: async (progressEvent) => {
+            const newPercentCompleted = !progressEvent.total
+              ? 0
+              : Math.round((progressEvent.loaded * 100) / progressEvent.total);
+            if (newPercentCompleted - percentCompleted >= 20) {
+              percentCompleted = newPercentCompleted;
+              console.log(
+                `Downloading plugin "${title}" v${version} from "${downloadUrl}": ${percentCompleted}% completed (${progressEvent.loaded} from ${progressEvent.total})`,
+              );
+            }
+          },
+        });
+        if (response.headers['content-type'] !== 'application/octet-stream') {
+          throw new Error(
+            `It looks like you are not on VPN/Lighthouse. Unexpected content type received: ${response.headers['content-type']}.`,
+          );
+        }
+        const responseStream = response.data as fs.ReadStream;
+        const writeStream = responseStream.pipe(
+          fs.createWriteStream(tmpFile, {autoClose: true}),
+        );
+        await new Promise((resolve, reject) =>
+          writeStream.once('finish', resolve).once('error', reject),
+        );
+        return await installPluginFromFile(tmpFile);
+      }
+    } catch (error) {
+      console.error(
+        `Failed to download plugin "${title}" v${version} from "${downloadUrl}" to "${installationDir}".`,
+        error,
+      );
+      throw error;
+    } finally {
+      await fs.remove(tmpDir);
+    }
+  }
+}

desktop/flipper-server-core/src/plugins/loadDynamicPlugins.tsx

@@ -0,0 +1,83 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import path from 'path';
+import fs from 'fs-extra';
+import {
+  getSourcePlugins,
+  moveInstalledPluginsFromLegacyDir,
+  getAllInstalledPluginVersions,
+  getAllInstalledPluginsInDir,
+} from 'flipper-plugin-lib';
+import {InstalledPluginDetails} from 'flipper-common';
+import {getStaticPath} from '../utils/pathUtils';
+
+// Load "dynamic" plugins, e.g. those which are either pre-installed (default), installed or loaded from sources (for development).
+// This opposed to "bundled" plugins which are included into Flipper bundle.
+export async function loadDynamicPlugins(): Promise<InstalledPluginDetails[]> {
+  if (process.env.NODE_ENV === 'test') {
+    return [];
+  }
+  if (process.env.FLIPPER_FAST_REFRESH) {
+    console.log(
+      '❌  Skipping loading of dynamic plugins because Fast Refresh is enabled. Fast Refresh only works with bundled plugins.',
+    );
+    return [];
+  }
+  await moveInstalledPluginsFromLegacyDir().catch((ex) =>
+    console.error(
+      'Eror while migrating installed plugins from legacy folder',
+      ex,
+    ),
+  );
+  const bundledPlugins = new Set<string>(
+    (
+      await fs.readJson(
+        getStaticPath(path.join('defaultPlugins', 'bundled.json'), {
+          asarUnpacked: true,
+        }),
+      )
+    ).map((p: any) => p.name) as string[],
+  );
+  const [installedPlugins, unfilteredSourcePlugins] = await Promise.all([
+    process.env.FLIPPER_NO_PLUGIN_MARKETPLACE
+      ? Promise.resolve([])
+      : getAllInstalledPluginVersions(),
+    getSourcePlugins(),
+  ]);
+  const sourcePlugins = unfilteredSourcePlugins.filter(
+    (p) => !bundledPlugins.has(p.name),
+  );
+  const defaultPluginsDir = getStaticPath('defaultPlugins', {
+    asarUnpacked: true,
+  });
+  const defaultPlugins = await getAllInstalledPluginsInDir(defaultPluginsDir);
+  if (defaultPlugins.length > 0) {
+    console.log(
+      `✅  Loaded ${defaultPlugins.length} default plugins: ${defaultPlugins
+        .map((x) => x.title)
+        .join(', ')}.`,
+    );
+  }
+  if (installedPlugins.length > 0) {
+    console.log(
+      `✅  Loaded ${installedPlugins.length} installed plugins: ${Array.from(
+        new Set(installedPlugins.map((x) => x.title)),
+      ).join(', ')}.`,
+    );
+  }
+  if (sourcePlugins.length > 0) {
+    console.log(
+      `✅  Loaded ${sourcePlugins.length} source plugins: ${sourcePlugins
+        .map((x) => x.title)
+        .join(', ')}.`,
+    );
+  }
+  return [...defaultPlugins, ...installedPlugins, ...sourcePlugins];
+}

desktop/flipper-server-core/src/utils/pathUtils.tsx

@@ -0,0 +1,40 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+// We use sync access once per startup.
+/* eslint-disable node/no-sync */
+
+import path from 'path';
+import fs from 'fs';
+import {getFlipperServerConfig} from '../FlipperServerConfig';
+import {isFBBuild} from '../fb-stubs/constants';
+
+export function getStaticPath(
+  relativePath: string = '.',
+  {asarUnpacked}: {asarUnpacked: boolean} = {asarUnpacked: false},
+) {
+  const staticDir = getFlipperServerConfig().paths.staticPath;
+  const absolutePath = path.resolve(staticDir, relativePath);
+  // Unfortunately, path.resolve, fs.pathExists, fs.read etc do not automatically work with asarUnpacked files.
+  // All these functions still look for files in "app.asar" even if they are unpacked.
+  // Looks like automatic resolving for asarUnpacked files only work for "child_process" module.
+  // So we're using a hack here to actually look to "app.asar.unpacked" dir instead of app.asar package.
+  return asarUnpacked
+    ? absolutePath.replace('app.asar', 'app.asar.unpacked')
+    : absolutePath;
+}
+
+export function getChangelogPath() {
+  const changelogPath = getStaticPath(isFBBuild ? 'facebook' : '.');
+  if (fs.existsSync(changelogPath)) {
+    return changelogPath;
+  } else {
+    throw new Error('Changelog path path does not exist: ' + changelogPath);
+  }
+}