From a4eb2a56d60d925b73e36fa9a98e827915e96a52 Mon Sep 17 00:00:00 2001 From: Anton Nikolaev Date: Tue, 18 May 2021 08:06:07 -0700 Subject: [PATCH] Option for "yarn start" and "yarn build" scripts to pre-install default plugin packages instead of bundling them Summary: Sorry for long diff! I can try to split it if necessary, but many changes here are 1-1 replacements / renames. **Preambule** Currently we bundle default plugins into the Flipper main bundle. This helps us to reduce bundle size, because of plugin dependencies re-use. E.g. if multiple plugins use "lodash" when they are bundled together, only one copy of "lodash" added. When they are bundled separately, the same dependency might be added to each of them. However as we're not going to include most of plugins into Flipper distributive anymore and going to rely on Marketplace instead, this bundling doesn't provide significant size benefits anymore. In addition to that, bundling makes it impossible to differentiate whether thrown errors are originated from Flipper core or one of its plugins. Why don't we remove plugin bundling at all? Because for "dev mode" it actually quite useful. It makes dev build start much faster and also enables using of Fast Refresh for plugin development (fast refresh won't work for plugins loaded from disk). **Changes** This diff introduces new option "no-bundled-plugins" for "yarn start" and "yarn build" commands. For now, by default, we will continue bundling default plugins into the Flipper main bundle, but if this option provided then we will build each default plugin separately and include their packages into the Flipper distributive as "pre-installed" to be able to load them from disk even without access to Marketplace. For "yarn start", we're adding symlinks to plugin folders in "static/defaultPlugins" and then they are loaded by Flipper. For "yarn build" we are dereferencing these symlinks to include physical files of plugins into folder "defaultPlugins" of the produced distributive. Folder "defaultPlugins" is excluded from asar, because loading of plugins from asar archive might introduce some unexpected issues depending on their implementation. Reviewed By: mweststrate Differential Revision: D28431838 fbshipit-source-id: f7757e9f5ba9183ed918d70252de3ce0e823177d --- desktop/.eslintignore | 2 +- desktop/.gitignore | 2 +- .../src/dispatcher/__tests__/plugins.node.tsx | 69 +++------------ desktop/app/src/dispatcher/iOSDevice.tsx | 10 ++- desktop/app/src/dispatcher/plugins.tsx | 83 +++++++----------- .../sandy-chrome/appinspect/PluginList.tsx | 2 +- desktop/app/src/utils/icons.ts | 2 +- desktop/app/src/utils/info.tsx | 2 +- .../src/utils/isPluginVersionMoreRecent.tsx | 47 ++++++++++ desktop/app/src/utils/loadDynamicPlugins.tsx | 37 +++++--- desktop/app/src/utils/pathUtils.tsx | 34 ++++---- desktop/app/src/utils/pluginUtils.tsx | 10 +-- desktop/package.json | 3 +- desktop/plugin-lib/src/getPluginDetails.ts | 14 +++ desktop/plugin-lib/src/getSourcePlugins.ts | 12 +-- desktop/plugin-lib/src/pluginInstaller.ts | 27 +++++- desktop/plugin-lib/src/pluginPaths.ts | 4 +- desktop/scripts/build-release.ts | 26 +++++- desktop/scripts/build-utils.ts | 86 ++++++++++++++----- .../scripts/generate-plugin-entry-points.ts | 4 +- desktop/scripts/start-dev-server.ts | 53 ++++++++---- desktop/scripts/tsc-plugins.ts | 3 +- desktop/scripts/workspaces.ts | 6 +- 23 files changed, 332 insertions(+), 206 deletions(-) create mode 100644 desktop/app/src/utils/isPluginVersionMoreRecent.tsx diff --git a/desktop/.eslintignore b/desktop/.eslintignore index 883a9a5f7..c0e9a2f2e 100644 --- a/desktop/.eslintignore +++ b/desktop/.eslintignore @@ -11,6 +11,6 @@ website/build react-native/ReactNativeFlipperExample scripts/generate-changelog.js static/index.js -static/defaultPlugins/index.json +static/defaultPlugins/* app/src/defaultPlugins/index.tsx generated diff --git a/desktop/.gitignore b/desktop/.gitignore index 6c3e8c204..7590a0c99 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -2,7 +2,7 @@ lib/ node_modules/ *.tsbuildinfo /static/themes/ -/static/defaultPlugins/index.json +/static/defaultPlugins/ /app/src/defaultPlugins/index.tsx /coverage .env diff --git a/desktop/app/src/dispatcher/__tests__/plugins.node.tsx b/desktop/app/src/dispatcher/__tests__/plugins.node.tsx index 6ce67656d..d92474b87 100644 --- a/desktop/app/src/dispatcher/__tests__/plugins.node.tsx +++ b/desktop/app/src/dispatcher/__tests__/plugins.node.tsx @@ -14,7 +14,7 @@ import dispatcher, { checkDisabled, checkGK, createRequirePluginFunction, - filterNewestVersionOfEachPlugin, + getLatestCompatibleVersionOfEachPlugin, } from '../plugins'; import {BundledPluginDetails, InstalledPluginDetails} from 'flipper-plugin-lib'; import path from 'path'; @@ -54,6 +54,7 @@ const sampleInstalledPluginDetails: InstalledPluginDetails = { const sampleBundledPluginDetails: BundledPluginDetails = { ...sampleInstalledPluginDetails, + id: 'SampleBundled', isBundled: true, }; @@ -169,11 +170,13 @@ test('newest version of each plugin is used', () => { const bundledPlugins: BundledPluginDetails[] = [ { ...sampleBundledPluginDetails, + id: 'TestPlugin1', name: 'flipper-plugin-test1', version: '0.1.0', }, { ...sampleBundledPluginDetails, + id: 'TestPlugin2', name: 'flipper-plugin-test2', version: '0.1.0-alpha.201', }, @@ -181,6 +184,7 @@ test('newest version of each plugin is used', () => { const installedPlugins: InstalledPluginDetails[] = [ { ...sampleInstalledPluginDetails, + id: 'TestPlugin2', name: 'flipper-plugin-test2', version: '0.1.0-alpha.21', dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-test2', @@ -188,19 +192,21 @@ test('newest version of each plugin is used', () => { }, { ...sampleInstalledPluginDetails, + id: 'TestPlugin1', name: 'flipper-plugin-test1', version: '0.10.0', dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-test1', entry: './test/index.js', }, ]; - const filteredPlugins = filterNewestVersionOfEachPlugin( - bundledPlugins, - installedPlugins, - ); + const filteredPlugins = getLatestCompatibleVersionOfEachPlugin([ + ...bundledPlugins, + ...installedPlugins, + ]); expect(filteredPlugins).toHaveLength(2); expect(filteredPlugins).toContainEqual({ ...sampleInstalledPluginDetails, + id: 'TestPlugin1', name: 'flipper-plugin-test1', version: '0.10.0', dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-test1', @@ -208,62 +214,12 @@ test('newest version of each plugin is used', () => { }); expect(filteredPlugins).toContainEqual({ ...sampleBundledPluginDetails, + id: 'TestPlugin2', name: 'flipper-plugin-test2', version: '0.1.0-alpha.201', }); }); -test('bundled versions are used when env var FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE is set even if newer versions are installed', () => { - process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE = 'true'; - try { - const bundledPlugins: BundledPluginDetails[] = [ - { - ...sampleBundledPluginDetails, - name: 'flipper-plugin-test1', - version: '0.1.0', - }, - { - ...sampleBundledPluginDetails, - name: 'flipper-plugin-test2', - version: '0.1.0-alpha.21', - }, - ]; - const installedPlugins: InstalledPluginDetails[] = [ - { - ...sampleInstalledPluginDetails, - name: 'flipper-plugin-test2', - version: '0.1.0-alpha.201', - dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-test2', - entry: './test/index.js', - }, - { - ...sampleInstalledPluginDetails, - name: 'flipper-plugin-test1', - version: '0.10.0', - dir: '/Users/mock/.flipper/thirdparty/flipper-plugin-test1', - entry: './test/index.js', - }, - ]; - const filteredPlugins = filterNewestVersionOfEachPlugin( - bundledPlugins, - installedPlugins, - ); - expect(filteredPlugins).toHaveLength(2); - expect(filteredPlugins).toContainEqual({ - ...sampleBundledPluginDetails, - name: 'flipper-plugin-test1', - version: '0.1.0', - }); - expect(filteredPlugins).toContainEqual({ - ...sampleBundledPluginDetails, - name: 'flipper-plugin-test2', - version: '0.1.0-alpha.21', - }); - } finally { - delete process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE; - } -}); - test('requirePlugin loads valid Sandy plugin', () => { const name = 'pluginID'; const requireFn = createRequirePluginFunction([], require); @@ -324,6 +280,7 @@ test('requirePlugin loads valid Sandy Device plugin', () => { const requireFn = createRequirePluginFunction([], require); const plugin = requireFn({ ...sampleInstalledPluginDetails, + pluginType: 'device', name, dir: path.join( __dirname, diff --git a/desktop/app/src/dispatcher/iOSDevice.tsx b/desktop/app/src/dispatcher/iOSDevice.tsx index f0d540184..7a9b13ad2 100644 --- a/desktop/app/src/dispatcher/iOSDevice.tsx +++ b/desktop/app/src/dispatcher/iOSDevice.tsx @@ -54,9 +54,13 @@ function isAvailable(simulator: iOSSimulatorDevice): boolean { ); } -const portforwardingClient = path.join( - getStaticPath(), - 'PortForwardingMacApp.app/Contents/MacOS/PortForwardingMacApp', +const portforwardingClient = getStaticPath( + path.join( + 'PortForwardingMacApp.app', + 'Contents', + 'MacOS', + 'PortForwardingMacApp', + ), ); function forwardPort(port: number, multiplexChannelPort: number) { diff --git a/desktop/app/src/dispatcher/plugins.tsx b/desktop/app/src/dispatcher/plugins.tsx index 582eaab4d..83aed4787 100644 --- a/desktop/app/src/dispatcher/plugins.tsx +++ b/desktop/app/src/dispatcher/plugins.tsx @@ -27,16 +27,15 @@ import { import GK from '../fb-stubs/GK'; import {FlipperBasePlugin} from '../plugin'; import {setupMenuBar} from '../MenuBar'; +import fs from 'fs-extra'; import path from 'path'; import {default as config} from '../utils/processConfig'; -import isProduction from '../utils/isProduction'; import {notNull} from '../utils/typeUtils'; import {sideEffect} from '../utils/sideEffect'; -import semver from 'semver'; import { ActivatablePluginDetails, BundledPluginDetails, - PluginDetails, + ConcretePluginDetails, } from 'flipper-plugin-lib'; import {tryCatchReportPluginFailures, reportUsage} from '../utils/metrics'; import * as FlipperPluginSDK from 'flipper-plugin'; @@ -53,7 +52,8 @@ import * as crc32 from 'crc32'; import getDefaultPluginsIndex from '../utils/getDefaultPluginsIndex'; import {isDevicePluginDefinition} from '../utils/pluginUtils'; import isPluginCompatible from '../utils/isPluginCompatible'; - +import isPluginVersionMoreRecent from '../utils/isPluginVersionMoreRecent'; +import {getStaticPath} from '../utils/pathUtils'; let defaultPluginsIndex: any = null; export default async (store: Store, logger: Logger) => { @@ -78,37 +78,23 @@ export default async (store: Store, logger: Logger) => { defaultPluginsIndex = getDefaultPluginsIndex(); - const marketplacePlugins = store.getState().plugins.marketplacePlugins; - store.dispatch( - registerMarketplacePlugins( - selectCompatibleMarketplaceVersions(marketplacePlugins), - ), + const marketplacePlugins = selectCompatibleMarketplaceVersions( + store.getState().plugins.marketplacePlugins, ); + store.dispatch(registerMarketplacePlugins(marketplacePlugins)); const uninstalledPluginNames = store.getState().plugins.uninstalledPluginNames; - const bundledPlugins = getBundledPlugins(); + const bundledPlugins = await getBundledPlugins(); const allLocalVersions = [ - ...getBundledPlugins(), + ...bundledPlugins, ...(await getDynamicPlugins()), ].filter((p) => !uninstalledPluginNames.has(p.name)); - const loadedVersionsMap: Map = new Map(); - for (const localVersion of allLocalVersions) { - if (isPluginCompatible(localVersion)) { - const loadedVersion = loadedVersionsMap.get(localVersion.id); - if ( - !loadedVersion || - semver.gt(localVersion.version, loadedVersion.version) - ) { - loadedVersionsMap.set(localVersion.id, localVersion); - } - } - } - - const loadedPlugins = Array.from(loadedVersionsMap.values()); + const loadedPlugins = + getLatestCompatibleVersionOfEachPlugin(allLocalVersions); const initialPlugins: PluginDefinition[] = loadedPlugins .map(reportVersion) @@ -150,40 +136,33 @@ function reportVersion(pluginDetails: ActivatablePluginDetails) { return pluginDetails; } -export function filterNewestVersionOfEachPlugin< - T1 extends PluginDetails, - T2 extends PluginDetails, ->(bundledPlugins: T1[], dynamicPlugins: T2[]): (T1 | T2)[] { - const pluginByName: {[key: string]: T1 | T2} = {}; - for (const plugin of bundledPlugins) { - pluginByName[plugin.name] = plugin; - } - for (const plugin of dynamicPlugins) { - if ( - !pluginByName[plugin.name] || - (!process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE && - semver.gt(plugin.version, pluginByName[plugin.name].version, true)) - ) { - pluginByName[plugin.name] = plugin; +export function getLatestCompatibleVersionOfEachPlugin< + T extends ConcretePluginDetails, +>(plugins: T[]): T[] { + const latestCompatibleVersions: Map = new Map(); + for (const plugin of plugins) { + if (isPluginCompatible(plugin)) { + const loadedVersion = latestCompatibleVersions.get(plugin.id); + if (!loadedVersion || isPluginVersionMoreRecent(plugin, loadedVersion)) { + latestCompatibleVersions.set(plugin.id, plugin); + } } } - return Object.values(pluginByName); + return Array.from(latestCompatibleVersions.values()); } -function getBundledPlugins(): Array { - // DefaultPlugins that are included in the bundle. - // List of defaultPlugins is written at build time - const pluginPath = - process.env.BUNDLED_PLUGIN_PATH || - (isProduction() - ? path.join(__dirname, 'defaultPlugins') - : './defaultPlugins/index.json'); - +async function getBundledPlugins(): Promise> { + // defaultPlugins that are included in the Flipper distributive. + // List of default bundled plugins is written at build time to defaultPlugins/bundled.json. + const pluginPath = getStaticPath( + path.join('defaultPlugins', 'bundled.json'), + {asarUnpacked: true}, + ); let bundledPlugins: Array = []; try { - bundledPlugins = global.electronRequire(pluginPath); + bundledPlugins = await fs.readJson(pluginPath); } catch (e) { - console.error('Failed to load bundled plugins', e); + console.error('Failed to load list of bundled plugins', e); } return bundledPlugins; diff --git a/desktop/app/src/sandy-chrome/appinspect/PluginList.tsx b/desktop/app/src/sandy-chrome/appinspect/PluginList.tsx index c2d41a266..34bb5a156 100644 --- a/desktop/app/src/sandy-chrome/appinspect/PluginList.tsx +++ b/desktop/app/src/sandy-chrome/appinspect/PluginList.tsx @@ -40,7 +40,7 @@ import { switchPlugin, uninstallPlugin, } from '../../reducers/pluginManager'; -import {BundledPluginDetails} from 'plugin-lib'; +import {BundledPluginDetails} from 'flipper-plugin-lib'; import {reportUsage} from '../../utils/metrics'; const {SubMenu} = Menu; diff --git a/desktop/app/src/utils/icons.ts b/desktop/app/src/utils/icons.ts index c9a7a40af..a6d888358 100644 --- a/desktop/app/src/utils/icons.ts +++ b/desktop/app/src/utils/icons.ts @@ -16,7 +16,7 @@ const AVAILABLE_SIZES = [8, 10, 12, 16, 18, 20, 24, 32]; const DENSITIES = [1, 1.5, 2, 3, 4]; function getIconsPath() { - return path.resolve(getStaticPath(), 'icons.json'); + return getStaticPath('icons.json'); } export type Icons = { diff --git a/desktop/app/src/utils/info.tsx b/desktop/app/src/utils/info.tsx index f8617f036..7420d45aa 100644 --- a/desktop/app/src/utils/info.tsx +++ b/desktop/app/src/utils/info.tsx @@ -47,7 +47,7 @@ export function getAppVersion(): string { (isTest() ? '0.0.0' : (isProduction() - ? fs.readJsonSync(path.join(getStaticPath(), 'package.json'), { + ? fs.readJsonSync(getStaticPath('package.json'), { throws: false, })?.version : require('../../package.json').version) ?? '0.0.0')); diff --git a/desktop/app/src/utils/isPluginVersionMoreRecent.tsx b/desktop/app/src/utils/isPluginVersionMoreRecent.tsx new file mode 100644 index 000000000..4cbeaf1c7 --- /dev/null +++ b/desktop/app/src/utils/isPluginVersionMoreRecent.tsx @@ -0,0 +1,47 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {ConcretePluginDetails} from 'flipper-plugin-lib'; +import semver from 'semver'; +import isPluginCompatible from './isPluginCompatible'; + +export function isPluginVersionMoreRecent( + versionDetails: ConcretePluginDetails, + otherVersionDetails: ConcretePluginDetails, +) { + const isPlugin1Compatible = isPluginCompatible(versionDetails); + const isPlugin2Compatible = isPluginCompatible(otherVersionDetails); + + // prefer compatible plugins + if (isPlugin1Compatible && !isPlugin2Compatible) return true; + if (!isPlugin1Compatible && isPlugin2Compatible) return false; + + // prefer plugins with greater version + if (semver.gt(versionDetails.version, otherVersionDetails.version)) { + return true; + } + if ( + semver.eq(versionDetails.version, otherVersionDetails.version) && + versionDetails.isBundled + ) { + // prefer bundled versions + return true; + } + if ( + semver.eq(versionDetails.version, otherVersionDetails.version) && + versionDetails.isActivatable && + !otherVersionDetails.isActivatable + ) { + // prefer locally available versions to the versions available remotely on marketplace + return true; + } + return false; +} + +export default isPluginVersionMoreRecent; diff --git a/desktop/app/src/utils/loadDynamicPlugins.tsx b/desktop/app/src/utils/loadDynamicPlugins.tsx index 264a4d575..e5f3d8869 100644 --- a/desktop/app/src/utils/loadDynamicPlugins.tsx +++ b/desktop/app/src/utils/loadDynamicPlugins.tsx @@ -14,11 +14,12 @@ import { moveInstalledPluginsFromLegacyDir, InstalledPluginDetails, getAllInstalledPluginVersions, + getAllInstalledPluginsInDir, } from 'flipper-plugin-lib'; import {getStaticPath} from '../utils/pathUtils'; -// Load "dynamic" plugins, e.g. those which are either installed or loaded from sources for development purposes. -// This opposed to "default" plugins which are included into Flipper bundle. +// Load "dynamic" plugins, e.g. those which are either pre-installed (default), installed or loaded from sources (for development). +// This opposed to "bundled" plugins which are included into Flipper bundle. export default async function loadDynamicPlugins(): Promise< InstalledPluginDetails[] > { @@ -34,26 +35,40 @@ export default async function loadDynamicPlugins(): Promise< ex, ), ); - const staticPath = getStaticPath(); - const defaultPlugins = new Set( + const bundledPlugins = new Set( ( - await fs.readJson(path.join(staticPath, 'defaultPlugins', 'index.json')) + await fs.readJson( + getStaticPath(path.join('defaultPlugins', 'bundled.json'), { + asarUnpacked: true, + }), + ) ).map((p: any) => p.name) as string[], ); const [installedPlugins, unfilteredSourcePlugins] = await Promise.all([ - process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE + process.env.FLIPPER_NO_PLUGIN_MARKETPLACE ? Promise.resolve([]) : getAllInstalledPluginVersions(), getSourcePlugins(), ]); const sourcePlugins = unfilteredSourcePlugins.filter( - (p) => !defaultPlugins.has(p.name), + (p) => !bundledPlugins.has(p.name), ); + const defaultPluginsDir = getStaticPath('defaultPlugins', { + asarUnpacked: true, + }); + const defaultPlugins = await getAllInstalledPluginsInDir(defaultPluginsDir); + if (defaultPlugins.length > 0) { + console.log( + `✅ Loaded ${defaultPlugins.length} default plugins: ${defaultPlugins + .map((x) => x.title) + .join(', ')}.`, + ); + } if (installedPlugins.length > 0) { console.log( - `✅ Loaded ${ - installedPlugins.length - } installed plugins: ${installedPlugins.map((x) => x.title).join(', ')}.`, + `✅ Loaded ${installedPlugins.length} installed plugins: ${Array.from( + new Set(installedPlugins.map((x) => x.title)), + ).join(', ')}.`, ); } if (sourcePlugins.length > 0) { @@ -63,5 +78,5 @@ export default async function loadDynamicPlugins(): Promise< .join(', ')}.`, ); } - return [...installedPlugins, ...sourcePlugins]; + return [...defaultPlugins, ...installedPlugins, ...sourcePlugins]; } diff --git a/desktop/app/src/utils/pathUtils.tsx b/desktop/app/src/utils/pathUtils.tsx index ea5a9aad9..072529eae 100644 --- a/desktop/app/src/utils/pathUtils.tsx +++ b/desktop/app/src/utils/pathUtils.tsx @@ -14,7 +14,7 @@ import config from '../fb-stubs/config'; let _staticPath = ''; -export function getStaticPath() { +function getStaticDir() { if (_staticPath) { return _staticPath; } @@ -31,31 +31,35 @@ export function getStaticPath() { return _staticPath; } +export function getStaticPath( + relativePath: string = '.', + {asarUnpacked}: {asarUnpacked: boolean} = {asarUnpacked: false}, +) { + const staticDir = getStaticDir(); + const absolutePath = path.resolve(staticDir, relativePath); + // Unfortunately, path.resolve, fs.pathExists, fs.read etc do not automatically work with asarUnpacked files. + // All these functions still look for files in "app.asar" even if they are unpacked. + // Looks like automatic resolving for asarUnpacked files only work for "child_process" module. + // So we're using a hack here to actually look to "app.asar.unpacked" dir instead of app.asar package. + return asarUnpacked + ? absolutePath.replace('app.asar', 'app.asar.unpacked') + : absolutePath; +} + let _appPath: string | undefined = undefined; export function getAppPath() { if (!_appPath) { - _appPath = path.join(getStaticPath(), '..'); + _appPath = getStaticPath('..'); } return _appPath; } export function getChangelogPath() { - const staticPath = getStaticPath(); - let changelogPath = ''; - - if (config.isFBBuild) { - changelogPath = path.resolve(staticPath, 'facebook'); - } else { - changelogPath = staticPath; - } - + const changelogPath = getStaticPath(config.isFBBuild ? 'facebook' : '.'); if (fs.existsSync(changelogPath)) { return changelogPath; - } - - if (!fs.existsSync(changelogPath)) { + } else { throw new Error('Changelog path path does not exist: ' + changelogPath); } - return changelogPath; } diff --git a/desktop/app/src/utils/pluginUtils.tsx b/desktop/app/src/utils/pluginUtils.tsx index 8edfa3743..056965e4b 100644 --- a/desktop/app/src/utils/pluginUtils.tsx +++ b/desktop/app/src/utils/pluginUtils.tsx @@ -25,7 +25,7 @@ import type { DownloadablePluginDetails, PluginDetails, } from 'flipper-plugin-lib'; -import {filterNewestVersionOfEachPlugin} from '../dispatcher/plugins'; +import {getLatestCompatibleVersionOfEachPlugin} from '../dispatcher/plugins'; export const defaultEnabledBackgroundPlugins = ['Navigation']; // The navigation plugin is enabled always, to make sure the navigation features works @@ -188,10 +188,10 @@ export function computePluginLists( enabledDevicePluginsState: Set, _pluginsChanged?: number, // this argument is purely used to invalidate the memoization cache ) { - const uninstalledMarketplacePlugins = filterNewestVersionOfEachPlugin( - [...plugins.bundledPlugins.values()], - plugins.marketplacePlugins, - ).filter((p) => !plugins.loadedPlugins.has(p.id)); + const uninstalledMarketplacePlugins = getLatestCompatibleVersionOfEachPlugin([ + ...plugins.bundledPlugins.values(), + ...plugins.marketplacePlugins, + ]).filter((p) => !plugins.loadedPlugins.has(p.id)); const devicePlugins: DevicePluginDefinition[] = [ ...plugins.devicePlugins.values(), ] diff --git a/desktop/package.json b/desktop/package.json index 077035d7e..403580b50 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -9,7 +9,8 @@ "artifactName": "Flipper-${os}.${ext}", "asar": true, "asarUnpack": [ - "PortForwardingMacApp.app/**/*" + "PortForwardingMacApp.app/**/*", + "defaultPlugins/**/*" ], "dmg": { "background": "dmgBackground.png", diff --git a/desktop/plugin-lib/src/getPluginDetails.ts b/desktop/plugin-lib/src/getPluginDetails.ts index ee1a2c053..7f4cf6fb4 100644 --- a/desktop/plugin-lib/src/getPluginDetails.ts +++ b/desktop/plugin-lib/src/getPluginDetails.ts @@ -16,6 +16,20 @@ import { } from './PluginDetails'; import {pluginCacheDir} from './pluginPaths'; +export function isPluginJson(packageJson: any): boolean { + return packageJson?.keywords?.includes('flipper-plugin'); +} + +export async function isPluginDir(dir: string): Promise { + const packageJsonPath = path.join(dir, 'package.json'); + const json = (await fs.pathExists(packageJsonPath)) + ? await fs.readJson(path.join(dir, 'package.json'), { + throws: false, + }) + : undefined; + return isPluginJson(json); +} + export function getPluginDetails(packageJson: any): PluginDetails { const specVersion = packageJson.$schema && diff --git a/desktop/plugin-lib/src/getSourcePlugins.ts b/desktop/plugin-lib/src/getSourcePlugins.ts index 9434185c8..b66540281 100644 --- a/desktop/plugin-lib/src/getSourcePlugins.ts +++ b/desktop/plugin-lib/src/getSourcePlugins.ts @@ -14,7 +14,7 @@ import {getPluginSourceFolders} from './pluginPaths'; import pmap from 'p-map'; import pfilter from 'p-filter'; import {satisfies} from 'semver'; -import {getInstalledPluginDetails} from './getPluginDetails'; +import {getInstalledPluginDetails, isPluginJson} from './getPluginDetails'; import {InstalledPluginDetails} from './PluginDetails'; const flipperVersion = require('../package.json').version; @@ -75,8 +75,8 @@ async function entryPointForPluginFolder( } catch (e) { console.error( `Could not load plugin from "${dir}", because package.json is invalid.`, + e, ); - console.error(e); return null; } }), @@ -84,10 +84,10 @@ async function entryPointForPluginFolder( .then((packages) => packages.filter(notNull)) .then((packages) => packages.filter(({manifest}) => !manifest.workspaces)) .then((packages) => - packages.filter(({manifest: {keywords, name}}) => { - if (!keywords || !keywords.includes('flipper-plugin')) { + packages.filter(({manifest}) => { + if (!isPluginJson(manifest)) { console.log( - `Skipping package "${name}" as its "keywords" field does not contain tag "flipper-plugin"`, + `Skipping package "${manifest.name}" as its "keywords" field does not contain tag "flipper-plugin"`, ); return false; } @@ -110,8 +110,8 @@ async function entryPointForPluginFolder( } catch (e) { console.error( `Could not load plugin from "${dir}", because package.json is invalid.`, + e, ); - console.error(e); return null; } }), diff --git a/desktop/plugin-lib/src/pluginInstaller.ts b/desktop/plugin-lib/src/pluginInstaller.ts index 80c4bf9eb..7e3c28103 100644 --- a/desktop/plugin-lib/src/pluginInstaller.ts +++ b/desktop/plugin-lib/src/pluginInstaller.ts @@ -16,7 +16,7 @@ import decompressTargz from 'decompress-targz'; import decompressUnzip from 'decompress-unzip'; import tmp from 'tmp'; import {InstalledPluginDetails} from './PluginDetails'; -import {getInstalledPluginDetails} from './getPluginDetails'; +import {getInstalledPluginDetails, isPluginDir} from './getPluginDetails'; import { getPluginVersionInstallationDir, getPluginDirNameFromPackageName, @@ -267,3 +267,28 @@ async function getInstalledPluginVersionDirs(): Promise { + const plugins: InstalledPluginDetails[] = []; + if (!((await fs.pathExists(dir)) && (await fs.stat(dir)).isDirectory())) { + console.log('defaultPlugins dir not found'); + return plugins; + } + const items = await fs.readdir(dir); + await pmap(items, async (item) => { + const fullPath = path.join(dir, item); + if (await isPluginDir(fullPath)) { + try { + plugins.push(await getInstalledPluginDetails(fullPath)); + } catch (err) { + console.error(`Failed to load plugin from ${fullPath}`); + } + } else if (recursive) { + plugins.push(...(await getAllInstalledPluginsInDir(fullPath, recursive))); + } + }); + return plugins; +} diff --git a/desktop/plugin-lib/src/pluginPaths.ts b/desktop/plugin-lib/src/pluginPaths.ts index 216a0064d..6e10c1c7f 100644 --- a/desktop/plugin-lib/src/pluginPaths.ts +++ b/desktop/plugin-lib/src/pluginPaths.ts @@ -28,9 +28,9 @@ export const pluginCacheDir = path.join(flipperDataDir, 'plugins'); export async function getPluginSourceFolders(): Promise { const pluginFolders: string[] = []; - if (process.env.FLIPPER_NO_EMBEDDED_PLUGINS) { + if (process.env.FLIPPER_NO_DEFAULT_PLUGINS) { console.log( - '🥫 Skipping embedded plugins because "--no-embedded-plugins" flag provided', + '🥫 Skipping default plugins because "--no-default-plugins" flag provided', ); return pluginFolders; } diff --git a/desktop/scripts/build-release.ts b/desktop/scripts/build-release.ts index 2c8b4d8cb..395f88627 100755 --- a/desktop/scripts/build-release.ts +++ b/desktop/scripts/build-release.ts @@ -25,7 +25,7 @@ import { die, getVersionNumber, genMercurialRevision, - generatePluginEntryPoints, + prepareDefaultPlugins, } from './build-utils'; import fetch from '@adobe/node-fetch-retry'; import {getIcons, buildLocalIconPath, getIconURL} from '../app/src/utils/icons'; @@ -78,6 +78,16 @@ const argv = yargs choices: ['stable', 'insiders'], default: 'stable', }, + 'bundled-plugins': { + describe: + 'Enables bundling of plugins into Flipper bundle. Env var FLIPPER_NO_BUNDLED_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".', + type: 'boolean', + }, + 'rebuild-plugins': { + describe: + 'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.', + type: 'boolean', + }, }) .help() .strict() @@ -102,6 +112,18 @@ if (isFB) { process.env.FLIPPER_RELEASE_CHANNEL = argv.channel; +if (argv['bundled-plugins'] === false) { + process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true'; +} else if (argv['bundled-plugins'] === true) { + delete process.env.FLIPPER_NO_BUNDLED_PLUGINS; +} + +if (argv['rebuild-plugins'] === false) { + process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true'; +} else if (argv['rebuild-plugins'] === true) { + delete process.env.FLIPPER_NO_REBUILD_PLUGINS; +} + async function generateManifest(versionNumber: string) { await fs.writeFile( path.join(distDir, 'manifest.json'), @@ -319,7 +341,7 @@ function downloadIcons(buildFolder: string) { console.log('Created build directory', dir); await compileMain(); - await generatePluginEntryPoints(argv.channel === 'insiders'); + await prepareDefaultPlugins(argv.channel === 'insiders'); await copyStaticFolder(dir); await downloadIcons(dir); await compileRenderer(dir); diff --git a/desktop/scripts/build-utils.ts b/desktop/scripts/build-utils.ts index cabe6c974..064480f14 100644 --- a/desktop/scripts/build-utils.ts +++ b/desktop/scripts/build-utils.ts @@ -12,12 +12,13 @@ import tmp from 'tmp'; import path from 'path'; import fs from 'fs-extra'; import {spawn} from 'promisify-child-process'; -import {getWatchFolders} from 'flipper-pkg-lib'; +import {getWatchFolders, runBuild} from 'flipper-pkg-lib'; import getAppWatchFolders from './get-app-watch-folders'; import { getSourcePlugins, getPluginSourceFolders, BundledPluginDetails, + InstalledPluginDetails, } from 'flipper-plugin-lib'; import { appDir, @@ -58,32 +59,42 @@ export function die(err: Error) { process.exit(1); } -export async function generatePluginEntryPoints( - isInsidersBuild: boolean = false, -) { +export async function prepareDefaultPlugins(isInsidersBuild: boolean = false) { console.log( - `⚙️ Generating plugin entry points (isInsidersBuild=${isInsidersBuild})...`, + `⚙️ Preparing default plugins (isInsidersBuild=${isInsidersBuild})...`, ); - const sourcePlugins = await getSourcePlugins(); - const bundledPlugins = sourcePlugins + await fs.emptyDir(defaultPluginsIndexDir); + const sourcePlugins = process.env.FLIPPER_NO_DEFAULT_PLUGINS + ? [] + : await getSourcePlugins(); + const defaultPlugins = sourcePlugins // we only include predefined set of plugins into insiders release - .filter((p) => !isInsidersBuild || hardcodedPlugins.has(p.id)) - .map( - (p) => - ({ - ...p, - isBundled: true, - version: p.version === '0.0.0' ? version : p.version, - flipperSDKVersion: - p.flipperSDKVersion === '0.0.0' ? version : p.flipperSDKVersion, - } as BundledPluginDetails), - ); - if (await fs.pathExists(defaultPluginsIndexDir)) { - await fs.remove(defaultPluginsIndexDir); + .filter((p) => !isInsidersBuild || hardcodedPlugins.has(p.id)); + if (isInsidersBuild || process.env.FLIPPER_NO_BUNDLED_PLUGINS) { + await buildDefaultPlugins(defaultPlugins); + await generateDefaultPluginEntryPoints([]); // calling it here just to generate empty indexes + } else { + await generateDefaultPluginEntryPoints(defaultPlugins); } - await fs.mkdirp(defaultPluginsIndexDir); +} + +async function generateDefaultPluginEntryPoints( + defaultPlugins: InstalledPluginDetails[], +) { + const bundledPlugins = defaultPlugins.map( + (p) => + ({ + ...p, + isBundled: true, + version: p.version === '0.0.0' ? version : p.version, + flipperSDKVersion: + p.flipperSDKVersion === '0.0.0' ? version : p.flipperSDKVersion, + dir: undefined, + entry: undefined, + } as BundledPluginDetails), + ); await fs.writeJSON( - path.join(defaultPluginsIndexDir, 'index.json'), + path.join(defaultPluginsIndexDir, 'bundled.json'), bundledPlugins, ); const pluginRequres = bundledPlugins @@ -91,7 +102,7 @@ export async function generatePluginEntryPoints( .join(',\n'); const generatedIndex = ` /* eslint-disable */ - // THIS FILE IS AUTO-GENERATED by function "generatePluginEntryPoints" in "build-utils.ts". + // THIS FILE IS AUTO-GENERATED by function "generateDefaultPluginEntryPoints" in "build-utils.ts". export default {\n${pluginRequres}\n} as any `; await fs.ensureDir(path.join(appDir, 'src', 'defaultPlugins')); @@ -102,6 +113,35 @@ export async function generatePluginEntryPoints( console.log('✅ Generated plugin entry points.'); } +async function buildDefaultPlugins(defaultPlugins: InstalledPluginDetails[]) { + if (process.env.FLIPPER_NO_REBUILD_PLUGINS) { + console.log( + `⚙️ Including ${ + defaultPlugins.length + } plugins into the default plugins list. Skipping rebuilding because "no-rebuild-plugins" option provided. List of default plugins: ${defaultPlugins + .map((p) => p.id) + .join(', ')}`, + ); + } + for (const plugin of defaultPlugins) { + try { + if (!process.env.FLIPPER_NO_REBUILD_PLUGINS) { + console.log( + `⚙️ Building plugin ${plugin.id} to include it into the default plugins list...`, + ); + await runBuild(plugin.dir, dev); + } + await fs.ensureSymlink( + plugin.dir, + path.join(defaultPluginsIndexDir, plugin.name), + 'junction', + ); + } catch (err) { + console.error(`✖ Failed to build plugin ${plugin.id}`, err); + } + } +} + const minifierConfig = { minifierPath: require.resolve('metro-minify-terser'), minifierConfig: { diff --git a/desktop/scripts/generate-plugin-entry-points.ts b/desktop/scripts/generate-plugin-entry-points.ts index 6da52acf7..eb3d4c0cf 100644 --- a/desktop/scripts/generate-plugin-entry-points.ts +++ b/desktop/scripts/generate-plugin-entry-points.ts @@ -7,9 +7,9 @@ * @format */ -import {generatePluginEntryPoints} from './build-utils'; +import {prepareDefaultPlugins} from './build-utils'; -generatePluginEntryPoints().catch((err) => { +prepareDefaultPlugins().catch((err) => { console.error(err); process.exit(1); }); diff --git a/desktop/scripts/start-dev-server.ts b/desktop/scripts/start-dev-server.ts index 711e7482b..18c2ac2e9 100644 --- a/desktop/scripts/start-dev-server.ts +++ b/desktop/scripts/start-dev-server.ts @@ -20,7 +20,7 @@ import http from 'http'; import path from 'path'; import fs from 'fs-extra'; import {hostname} from 'os'; -import {compileMain, generatePluginEntryPoints} from './build-utils'; +import {compileMain, prepareDefaultPlugins} from './build-utils'; import Watchman from './watchman'; import Metro from 'metro'; import {staticDir, babelTransformationsDir, rootDir} from './paths'; @@ -34,9 +34,19 @@ import yargs from 'yargs'; const argv = yargs .usage('yarn start [args]') .options({ - 'embedded-plugins': { + 'default-plugins': { describe: - 'Enables embedding of plugins into Flipper bundle. If it disabled then only installed plugins are loaded. The flag is enabled by default. Env var FLIPPER_NO_EMBEDDED_PLUGINS is equivalent to the command-line option "--no-embedded-plugins".', + 'Enables embedding of default plugins into Flipper package so they are always available. The flag is enabled by default. Env var FLIPPER_NO_DEFAULT_PLUGINS is equivalent to the command-line option "--no-default-plugins".', + type: 'boolean', + }, + 'bundled-plugins': { + describe: + 'Enables bundling of plugins into Flipper bundle. This is useful for debugging, because it makes Flipper dev mode loading faster and unblocks fast refresh. The flag is enabled by default. Env var FLIPPER_NO_BUNDLEDD_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".', + type: 'boolean', + }, + 'rebuild-plugins': { + describe: + 'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.', type: 'boolean', }, 'fast-refresh': { @@ -44,9 +54,9 @@ const argv = yargs 'Enable Fast Refresh - quick reload of UI component changes without restarting Flipper. The flag is disabled by default. Env var FLIPPER_FAST_REFRESH is equivalent to the command-line option "--fast-refresh".', type: 'boolean', }, - 'plugin-auto-update': { + 'plugin-marketplace': { describe: - '[FB-internal only] Enable plugin auto-updates. The flag is disabled by default in dev mode. Env var FLIPPER_NO_PLUGIN_AUTO_UPDATE is equivalent to the command-line option "--no-plugin-auto-update"', + 'Enable plugin marketplace - ability to install plugins from NPM or other sources. Without the flag Flipper will only show default plugins. The flag is disabled by default in dev mode. Env var FLIPPER_NO_PLUGIN_MARKETPLACE is equivalent to the command-line option "--no-plugin-marketplace"', type: 'boolean', }, 'plugin-auto-update-interval': { @@ -105,10 +115,22 @@ if (isFB) { process.env.FLIPPER_FB = 'true'; } -if (argv['embedded-plugins'] === true) { - delete process.env.FLIPPER_NO_EMBEDDED_PLUGINS; -} else if (argv['embedded-plugins'] === false) { - process.env.FLIPPER_NO_EMBEDDED_PLUGINS = 'true'; +if (argv['default-plugins'] === true) { + delete process.env.FLIPPER_NO_DEFAULT_PLUGINS; +} else if (argv['default-plugins'] === false) { + process.env.FLIPPER_NO_DEFAULT_PLUGINS = 'true'; +} + +if (argv['bundled-plugins'] === true) { + delete process.env.FLIPPER_NO_BUNDLED_PLUGINS; +} else if (argv['bundled-plugins'] === false) { + process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true'; +} + +if (argv['rebuild-plugins'] === false) { + process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true'; +} else if (argv['rebuild-plugins'] === true) { + delete process.env.FLIPPER_NO_REBUILD_PLUGINS; } if (argv['fast-refresh'] === true) { @@ -128,16 +150,13 @@ if (argv['public-build'] === true) { delete process.env.FLIPPER_FORCE_PUBLIC_BUILD; } -// By default plugin auto-update is disabled in dev mode, +// By default plugin marketplace is disabled in dev mode, // but it is possible to enable it using this command line // argument or env var. -if ( - argv['plugin-auto-update'] === true || - process.env.FLIPPER_PLUGIN_AUTO_UPDATE -) { - delete process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE; +if (argv['plugin-marketplace'] === true) { + delete process.env.FLIPPER_NO_PLUGIN_MARKETPLACE; } else { - process.env.FLIPPER_DISABLE_PLUGIN_AUTO_UPDATE = 'true'; + process.env.FLIPPER_NO_PLUGIN_MARKETPLACE = 'true'; } if (argv['plugin-auto-update-interval']) { @@ -397,7 +416,7 @@ function checkDevServer() { (async () => { checkDevServer(); - await generatePluginEntryPoints( + await prepareDefaultPlugins( process.env.FLIPPER_RELEASE_CHANNEL === 'insiders', ); await ensurePluginFoldersWatchable(); diff --git a/desktop/scripts/tsc-plugins.ts b/desktop/scripts/tsc-plugins.ts index 47dbb3b83..1909ed78e 100644 --- a/desktop/scripts/tsc-plugins.ts +++ b/desktop/scripts/tsc-plugins.ts @@ -14,6 +14,7 @@ import {EOL} from 'os'; import pmap from 'p-map'; import {rootDir} from './paths'; import yargs from 'yargs'; +import {isPluginJson} from 'flipper-plugin-lib'; const argv = yargs .usage('yarn tsc-plugins [args]') @@ -107,7 +108,7 @@ async function findAffectedPlugins(errors: string[]) { depsByName.set(name, getDependencies(name)); } for (const pkg of allPackages) { - if (!pkg.json?.keywords?.includes('flipper-plugin')) { + if (!isPluginJson(pkg.json)) { continue; } const logFile = path.join(pkg.dir, 'tsc-error.log'); diff --git a/desktop/scripts/workspaces.ts b/desktop/scripts/workspaces.ts index c9c60e7b5..78c087159 100644 --- a/desktop/scripts/workspaces.ts +++ b/desktop/scripts/workspaces.ts @@ -15,6 +15,7 @@ import globImport from 'glob'; import pfilter from 'p-filter'; import pmap from 'p-map'; import {execSync} from 'child_process'; +import {isPluginJson} from 'flipper-plugin-lib'; const glob = promisify(globImport); export interface Package { @@ -52,10 +53,7 @@ async function getWorkspacesByRoot( dir, json, isPrivate: json.private || dir.startsWith(pluginsDir), - isPlugin: - json.keywords && - Array.isArray(json.keywords) && - json.keywords.includes('flipper-plugin'), + isPlugin: isPluginJson(json), }; }, );