Move the code related to plugin loading / installation to "flipper-plugin-lib"

Summary:
Sorry for so long diff, but actually there are no functional changes, just refactoring to make further changes of Plugin Manager easier to understand.

I've de-coupled the code related to plugin management from UI code and moved it from PluginInstaller UI component (which will be replaced soon by new UI) to "flipper-plugin-lib".  So pretty much everything related to plugin discovery and installation now consolidated in this package.

Additionally, this refactoring enables re-using of plugin management code in "flipper-pkg", e.g. to create CLI command for plugin installation from NPM, e.g.: `flipper-pkg install flipper-plugin-reactotron`.

Reviewed By: passy

Differential Revision: D23679346

fbshipit-source-id: 82e7b9de9afa08c508c1b228c2038b4ba423571c

desktop/plugin-lib/src/__tests__/getUpdatablePlugins.node.ts

@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+jest.mock('../getInstalledPlugins');
+jest.mock('../getNpmHostedPlugins');
+
+import {getUpdatablePlugins} from '../getUpdatablePlugins';
+import {
+  getNpmHostedPlugins,
+  NpmPackageDescriptor,
+} from '../getNpmHostedPlugins';
+import type {InstalledPluginDetails} from '../getInstalledPlugins';
+import {getInstalledPlugins} from '../getInstalledPlugins';
+import {mocked} from 'ts-jest/utils';
+import type {Package} from 'npm-api';
+
+jest.mock('npm-api', () => {
+  return jest.fn().mockImplementation(() => {
+    return {
+      repo: jest.fn().mockImplementation((name: string) => {
+        let pkg: Package | undefined;
+        if (name === 'flipper-plugin-hello') {
+          pkg = {
+            $schema: 'https://fbflipper.com/schemas/plugin-package/v2.json',
+            name: 'flipper-plugin-hello',
+            title: 'Hello',
+            version: '0.1.0',
+            main: 'dist/bundle.js',
+            flipperBundlerEntry: 'src/index.js',
+            description: 'World?',
+          };
+        } else if (name === 'flipper-plugin-world') {
+          pkg = {
+            $schema: 'https://fbflipper.com/schemas/plugin-package/v2.json',
+            name: 'flipper-plugin-world',
+            title: 'World',
+            version: '0.3.0',
+            main: 'dist/bundle.js',
+            flipperBundlerEntry: 'src/index.js',
+            description: 'World?',
+          };
+        }
+        return {
+          package: jest.fn().mockImplementation(() => Promise.resolve(pkg)),
+        };
+      }),
+    };
+  });
+});
+
+const installedPlugins: InstalledPluginDetails[] = [
+  {
+    name: 'flipper-plugin-hello',
+    entry: './test/index.js',
+    version: '0.1.0',
+    specVersion: 2,
+    main: 'dist/bundle.js',
+    dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-sample1',
+    source: 'src/index.js',
+    id: 'Hello',
+    title: 'Hello',
+    description: 'World?',
+    isDefault: false,
+    installationStatus: 'installed',
+  },
+  {
+    name: 'flipper-plugin-world',
+    entry: './test/index.js',
+    version: '0.2.0',
+    specVersion: 2,
+    main: 'dist/bundle.js',
+    dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-sample2',
+    source: 'src/index.js',
+    id: 'World',
+    title: 'World',
+    description: 'Hello?',
+    isDefault: false,
+    installationStatus: 'pending',
+  },
+];
+
+const updates: NpmPackageDescriptor[] = [
+  {name: 'flipper-plugin-hello', version: '0.1.0'},
+  {name: 'flipper-plugin-world', version: '0.3.0'},
+];
+
+test('annotatePluginsWithUpdates', async () => {
+  const getInstalledPluginsMock = mocked(getInstalledPlugins);
+  getInstalledPluginsMock.mockReturnValue(Promise.resolve(installedPlugins));
+
+  const getNpmHostedPluginsMock = mocked(getNpmHostedPlugins);
+  getNpmHostedPluginsMock.mockReturnValue(Promise.resolve(updates));
+
+  const res = await getUpdatablePlugins();
+
+  expect(res.length).toBe(2);
+  expect({
+    name: res[0].name,
+    version: res[0].version,
+    updateStatus: res[0].updateStatus,
+  }).toMatchInlineSnapshot(`
+    Object {
+      "name": "flipper-plugin-hello",
+      "updateStatus": Object {
+        "kind": "up-to-date",
+      },
+      "version": "0.1.0",
+    }
+  `);
+
+  expect({
+    name: res[1].name,
+    version: res[1].version,
+    updateStatus: res[1].updateStatus,
+  }).toMatchInlineSnapshot(`
+    Object {
+      "name": "flipper-plugin-world",
+      "updateStatus": Object {
+        "kind": "update-available",
+        "version": "0.3.0",
+      },
+      "version": "0.3.0",
+    }
+  `);
+});

desktop/plugin-lib/src/getInstalledPlugins.ts

@@ -0,0 +1,104 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import fs from 'fs-extra';
+import path from 'path';
+import semver from 'semver';
+import {
+  pluginPendingInstallationDir,
+  pluginInstallationDir,
+} from './pluginPaths';
+import PluginDetails from './PluginDetails';
+import getPluginDetails from './getPluginDetails';
+import pmap from 'p-map';
+import {notNull} from './typeUtils';
+
+export type PluginInstallationStatus =
+  | 'not-installed'
+  | 'installed'
+  | 'pending';
+
+export type InstalledPluginDetails = PluginDetails & {
+  installationStatus: PluginInstallationStatus;
+};
+
+async function getFullyInstalledPlugins(): Promise<PluginDetails[]> {
+  const pluginDirExists = await fs.pathExists(pluginInstallationDir);
+  if (!pluginDirExists) {
+    return [];
+  }
+  const dirs = await fs.readdir(pluginInstallationDir);
+  const plugins = await pmap(dirs, async (dirName) => {
+    const pluginDir = path.join(pluginInstallationDir, dirName);
+    if (!(await fs.lstat(pluginDir)).isDirectory()) {
+      return undefined;
+    }
+    try {
+      return await getPluginDetails(pluginDir);
+    } catch (e) {
+      console.error(`Failed to load plugin from ${pluginDir}`, e);
+      return undefined;
+    }
+  });
+  return plugins.filter(notNull);
+}
+
+async function getPendingInstallationPlugins(): Promise<PluginDetails[]> {
+  const pluginDirExists = await fs.pathExists(pluginPendingInstallationDir);
+  if (!pluginDirExists) {
+    return [];
+  }
+  const dirs = await fs.readdir(pluginPendingInstallationDir);
+  const plugins = await pmap(dirs, async (dirName) => {
+    const versions = (
+      await fs.readdir(path.join(pluginPendingInstallationDir, dirName))
+    ).sort((v1, v2) => semver.compare(v2, v1, true));
+    if (versions.length === 0) {
+      return undefined;
+    }
+    const pluginDir = path.join(
+      pluginPendingInstallationDir,
+      dirName,
+      versions[0],
+    );
+    if (!(await fs.lstat(pluginDir)).isDirectory()) {
+      return undefined;
+    }
+    try {
+      return await getPluginDetails(pluginDir);
+    } catch (e) {
+      console.error(`Failed to load plugin from ${pluginDir}`, e);
+      return undefined;
+    }
+  });
+  return plugins.filter(notNull);
+}
+
+export async function getInstalledPlugins(): Promise<InstalledPluginDetails[]> {
+  const map = new Map<string, InstalledPluginDetails>(
+    (await getFullyInstalledPlugins()).map((p) => [
+      p.name,
+      {...p, installationStatus: 'installed'},
+    ]),
+  );
+  for (const p of await getPendingInstallationPlugins()) {
+    if (!map.get(p.name) || semver.gt(p.version, map.get(p.name)!.version)) {
+      map.set(p.name, {...p, installationStatus: 'pending'});
+    }
+  }
+  const allPlugins = [...map.values()].sort((p1, p2) =>
+    p1.installationStatus === 'installed' && p2.installationStatus === 'pending'
+      ? 1
+      : p1.installationStatus === 'pending' &&
+        p2.installationStatus === 'installed'
+      ? -1
+      : p1.name.localeCompare(p2.name),
+  );
+  return allPlugins;
+}

desktop/plugin-lib/src/getNpmHostedPlugins.ts

@@ -0,0 +1,46 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {default as algoliasearch, SearchIndex} from 'algoliasearch';
+
+const ALGOLIA_APPLICATION_ID = 'OFCNCOG2CU';
+const ALGOLIA_API_KEY = 'f54e21fa3a2a0160595bb058179bfb1e';
+
+function provideSearchIndex(): SearchIndex {
+  const client = algoliasearch(ALGOLIA_APPLICATION_ID, ALGOLIA_API_KEY);
+  return client.initIndex('npm-search');
+}
+
+export type NpmPackageDescriptor = {
+  name: string;
+  version: string;
+};
+
+export type NpmHostedPluginsSearchArgs = {
+  query?: string;
+};
+
+export async function getNpmHostedPlugins(
+  args: NpmHostedPluginsSearchArgs = {},
+): Promise<NpmPackageDescriptor[]> {
+  const index = provideSearchIndex();
+  args = Object.assign(
+    {
+      query: '',
+      filters: 'keywords:flipper-plugin',
+      hitsPerPage: 50,
+    },
+    args,
+  );
+  const {hits} = await index.search<NpmPackageDescriptor>(
+    args.query || '',
+    args,
+  );
+  return hits;
+}

desktop/plugin-lib/src/getUpdatablePlugins.ts

@@ -0,0 +1,120 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import PluginDetails from './PluginDetails';
+import {getInstalledPlugins} from './getInstalledPlugins';
+import semver from 'semver';
+import {getNpmHostedPlugins, NpmPackageDescriptor} from './getNpmHostedPlugins';
+import NpmApi from 'npm-api';
+import getPluginDetails from './getPluginDetails';
+import {getPluginInstallationDir} from './pluginInstaller';
+import pmap from 'p-map';
+import {notNull} from './typeUtils';
+
+export type UpdateResult =
+  | {kind: 'not-installed'; version: string}
+  | {kind: 'pending'}
+  | {kind: 'up-to-date'}
+  | {kind: 'error'; error: Error}
+  | {kind: 'update-available'; version: string};
+
+export type UpdatablePlugin = {
+  updateStatus: UpdateResult;
+};
+
+export type UpdatablePluginDetails = PluginDetails & UpdatablePlugin;
+
+export async function getUpdatablePlugins(): Promise<UpdatablePluginDetails[]> {
+  const npmApi = new NpmApi();
+  const installedPlugins = await getInstalledPlugins();
+  const npmHostedPlugins = new Map<string, NpmPackageDescriptor>(
+    (await getNpmHostedPlugins()).map((p) => [p.name, p]),
+  );
+  const annotatedInstalledPlugins = await pmap(
+    installedPlugins,
+    async (installedPlugin): Promise<UpdatablePluginDetails> => {
+      try {
+        const npmPackageDescriptor = npmHostedPlugins.get(installedPlugin.name);
+        if (npmPackageDescriptor) {
+          npmHostedPlugins.delete(installedPlugin.name);
+          if (
+            semver.lt(installedPlugin.version, npmPackageDescriptor.version)
+          ) {
+            const pkg = await npmApi.repo(npmPackageDescriptor.name).package();
+            const npmPluginDetails = await getPluginDetails(
+              getPluginInstallationDir(npmPackageDescriptor.name),
+              pkg,
+            );
+            return {
+              ...npmPluginDetails,
+              updateStatus: {
+                kind: 'update-available',
+                version: npmPluginDetails.version,
+              },
+            };
+          }
+        }
+        const updateStatus: UpdateResult =
+          installedPlugin.installationStatus === 'installed'
+            ? {kind: 'up-to-date'}
+            : {kind: 'pending'};
+        return {
+          ...installedPlugin,
+          updateStatus,
+        };
+      } catch (error) {
+        return {
+          ...installedPlugin,
+          updateStatus: {
+            kind: 'error',
+            error,
+          },
+        };
+      }
+    },
+    {
+      concurrency: 4,
+    },
+  );
+  const annotatedNotInstalledPlugins = await pmap(
+    npmHostedPlugins.values(),
+    async (notInstalledPlugin) => {
+      try {
+        const pkg = await npmApi.repo(notInstalledPlugin.name).package();
+        const npmPluginDetails = await getPluginDetails(
+          getPluginInstallationDir(notInstalledPlugin.name),
+          pkg,
+        );
+        return {
+          ...npmPluginDetails,
+          updateStatus: {
+            kind: 'not-installed',
+            version: npmPluginDetails.version,
+          },
+        } as UpdatablePluginDetails;
+      } catch (error) {
+        console.log(
+          `Failed to load details from npm for plugin ${notInstalledPlugin.name}`,
+        );
+        return null;
+      }
+    },
+    {
+      concurrency: 4,
+    },
+  );
+  return [
+    ...annotatedInstalledPlugins.sort((p1, p2) =>
+      p1.name.localeCompare(p2.name),
+    ),
+    ...annotatedNotInstalledPlugins
+      .filter(notNull)
+      .sort((p1, p2) => p1.name.localeCompare(p2.name)),
+  ];
+}

desktop/plugin-lib/src/index.ts

@@ -10,3 +10,5 @@
 export {default as PluginDetails} from './PluginDetails';
 export {default as getPluginDetails} from './getPluginDetails';
 export * from './pluginInstaller';
+export * from './getInstalledPlugins';
+export * from './getUpdatablePlugins';

desktop/plugin-lib/src/pluginInstaller.ts

@@ -23,8 +23,6 @@ import {
 } from './pluginPaths';
 import semver from 'semver';
 
-export type PluginMap = Map<string, PluginDetails>;
-
 const getTmpDir = promisify(tmp.dir) as () => Promise<string>;
 
 function providePluginManager(): PM {
@@ -51,7 +49,7 @@ function getPluginPendingInstallationsDir(name: string): string {
   );
 }
 
-function getPluginInstallationDir(name: string): string {
+export function getPluginInstallationDir(name: string): string {
   return path.join(
     pluginInstallationDir,
     replaceInvalidPathSegmentCharacters(name),
@@ -183,83 +181,11 @@ export async function installPluginFromFile(
   }
 }
 
-export async function getInstalledPlugins(): Promise<PluginMap> {
-  const pluginDirExists = await fs.pathExists(pluginInstallationDir);
-  if (!pluginDirExists) {
-    return new Map();
-  }
-  const dirs = await fs.readdir(pluginInstallationDir);
-  const plugins = await Promise.all<[string, PluginDetails]>(
-    dirs.map(
-      (dirName) =>
-        new Promise(async (resolve, reject) => {
-          const pluginDir = path.join(pluginInstallationDir, dirName);
-          if (!(await fs.lstat(pluginDir)).isDirectory()) {
-            return resolve(undefined);
-          }
-          try {
-            const details = await getPluginDetails(pluginDir);
-            resolve([details.name, details]);
-          } catch (e) {
-            reject(e);
-          }
-        }),
-    ),
-  );
-  return new Map(plugins.filter(Boolean));
-}
-
-export async function getPendingInstallationPlugins(): Promise<PluginMap> {
-  const pluginDirExists = await fs.pathExists(pluginPendingInstallationDir);
-  if (!pluginDirExists) {
-    return new Map();
-  }
-  const dirs = await fs.readdir(pluginPendingInstallationDir);
-  const plugins = await Promise.all<[string, PluginDetails]>(
-    dirs.map(
-      (dirName) =>
-        new Promise(async (resolve, reject) => {
-          const versions = (
-            await fs.readdir(path.join(pluginPendingInstallationDir, dirName))
-          ).sort((v1, v2) => semver.compare(v2, v1, true));
-          if (versions.length === 0) {
-            return resolve(undefined);
-          }
-          const pluginDir = path.join(
-            pluginPendingInstallationDir,
-            dirName,
-            versions[0],
-          );
-          if (!(await fs.lstat(pluginDir)).isDirectory()) {
-            return resolve(undefined);
-          }
-          try {
-            const details = await getPluginDetails(pluginDir);
-            resolve([details.name, details]);
-          } catch (e) {
-            reject(e);
-          }
-        }),
-    ),
-  );
-  return new Map(plugins.filter(Boolean));
-}
-
-export async function getPendingAndInstalledPlugins(): Promise<PluginMap> {
-  const plugins = await getInstalledPlugins();
-  for (const [name, details] of await getPendingInstallationPlugins()) {
-    if (
-      !plugins.get(name) ||
-      semver.gt(details.version, plugins.get(name)!.version)
-    ) {
-      plugins.set(name, details);
-    }
-  }
-  return plugins;
-}
-
 export async function removePlugin(name: string): Promise<void> {
-  await fs.remove(getPluginInstallationDir(name));
+  await Promise.all([
+    fs.remove(getPluginInstallationDir(name)),
+    fs.remove(getPluginPendingInstallationsDir(name)),
+  ]);
 }
 
 export async function finishPendingPluginInstallations() {

desktop/plugin-lib/src/typeUtils.ts

@@ -0,0 +1,15 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+// TODO T75614643: move to a separate lib for utils, e.g. flipper-utils
+// Typescript doesn't know Array.filter(Boolean) won't contain nulls.
+// So use Array.filter(notNull) instead.
+export function notNull<T>(x: T | null | undefined): x is T {
+  return x !== null && x !== undefined;
+}