From 6ff4abbc6738443cbc5e726fc472ff1173f51391 Mon Sep 17 00:00:00 2001 From: Michel Weststrate Date: Fri, 24 Dec 2021 02:15:25 -0800 Subject: [PATCH] Make flipper-server stand-alone servable Summary: This diff makes flipper-server stand-alone servable as a single package. It basically works as follows: - (default) plugins are build as usually into static folder - static folder is copied into flipper-server/static (as far as relevant) - `flipper-server/src/index.tsx` is bundled into `flipper-server/dist/index.js` - `flipper-ui-browser/src/index.tsx` is bundled into `flipper-server/static/bundle.js` If flipper-server is started without the `--bundler` config, the `bundle.js` will be served statically, rather than starting up the Metro bundler with all the typical transforms (in a distributed version of the package those all wouldn't resolve) Things to be done in next diffs: * make sure plugins actually load in the non bundled setup * make sure flipper-server gets published properly * make sure build is created as part of CI At the end of this stack, running `npx flipper-server` on any machine should basically be a viable approach to get flipper up and running locally :) Reviewed By: nikoant Differential Revision: D33171107 fbshipit-source-id: 4af8ac2699467a0b55283fe084640482700744c2 --- desktop/.gitignore | 1 + desktop/flipper-server/README.md | 24 +++ desktop/flipper-server/package.json | 8 +- desktop/flipper-server/server.js | 1 + desktop/flipper-server/src/index.tsx | 58 ++++-- desktop/package.json | 3 +- .../scripts/build-flipper-server-release.ts | 132 ++++++++++++++ desktop/scripts/build-release.ts | 2 +- desktop/scripts/build-utils.ts | 172 ++++++++++++++++-- desktop/scripts/package.json | 1 - desktop/scripts/paths.ts | 1 + desktop/scripts/start-flipper-server.ts | 70 ++----- desktop/static/index.web.html | 126 +++++++++++++ 13 files changed, 515 insertions(+), 84 deletions(-) mode change 100644 => 100755 desktop/flipper-server/server.js create mode 100644 desktop/scripts/build-flipper-server-release.ts create mode 100644 desktop/static/index.web.html diff --git a/desktop/.gitignore b/desktop/.gitignore index 4fa35ee6d..d0a04bfba 100644 --- a/desktop/.gitignore +++ b/desktop/.gitignore @@ -9,3 +9,4 @@ node_modules/ /coverage .env tsc-error.log +/flipper-server/static/ diff --git a/desktop/flipper-server/README.md b/desktop/flipper-server/README.md index 04b84c5c1..98e60086a 100644 --- a/desktop/flipper-server/README.md +++ b/desktop/flipper-server/README.md @@ -3,3 +3,27 @@ Stand alone Flipper server as NodeJS process, that uses flipper-server-core for device communication and also provides a webserver to serve flipper-ui. Flipper-server can be used as background process, for example on CI servers or to power IDE plugins. + +## Running flipper server + +### From NPM + +TODO: + +### From source + +``` +cd /desktop +yarn install +yarn flipper-server +``` + +### Production build from source + +``` +cd /desktop +yarn install +yarn build:flipper-server +``` + +Pass the `--open` flag to open Flipper server after building diff --git a/desktop/flipper-server/package.json b/desktop/flipper-server/package.json index 250a692bb..1f68f2dbe 100644 --- a/desktop/flipper-server/package.json +++ b/desktop/flipper-server/package.json @@ -5,6 +5,7 @@ "description": "Standalone nodeJS based Flipper server", "repository": "facebook/flipper", "main": "server.js", + "bin": "server.js", "flipperBundlerEntry": "src", "license": "MIT", "bugs": "https://github.com/facebook/flipper/issues", @@ -22,7 +23,9 @@ "flipper-common": "0.0.0", "flipper-pkg-lib": "0.0.0", "flipper-server-core": "0.0.0", - "metro": "^0.66.2" + "metro": "^0.66.2", + "open": "^8.3.0", + "yargs": "^17.0.1" }, "peerDependencies": {}, "scripts": { @@ -31,7 +34,8 @@ "prepack": "yarn reset && yarn build" }, "files": [ - "dist/**/*" + "dist/**/*", + "static/**/*" ], "homepage": "https://github.com/facebook/flipper", "keywords": [ diff --git a/desktop/flipper-server/server.js b/desktop/flipper-server/server.js old mode 100644 new mode 100755 index 84ec34305..9961e35de --- a/desktop/flipper-server/server.js +++ b/desktop/flipper-server/server.js @@ -1,3 +1,4 @@ +#!/usr/bin/env node /** * Copyright (c) Facebook, Inc. and its affiliates. * diff --git a/desktop/flipper-server/src/index.tsx b/desktop/flipper-server/src/index.tsx index 4da3e564d..e4a7d554c 100644 --- a/desktop/flipper-server/src/index.tsx +++ b/desktop/flipper-server/src/index.tsx @@ -12,13 +12,45 @@ import path from 'path'; import {startFlipperServer} from './startFlipperServer'; import {startBaseServer} from './startBaseServer'; import {startSocketServer} from './startSocketServer'; -// TODO: currently flipper-server is only suitable for development, -// needs to be come independently runnable, prebundled, distributed, etc! -// in future require conditionally import {startWebServerDev} from './startWebServerDev'; -const PORT = 52342; -const rootDir = path.resolve(__dirname, '..', '..'); +import yargs from 'yargs'; +import open from 'open'; +import {sleep} from 'flipper-common'; + +const argv = yargs + .usage('yarn flipper-server [args]') + .options({ + port: { + describe: 'Port to serve on', + type: 'number', + default: 52342, + }, + bundler: { + describe: + 'Serve the UI bundle from source. This option only works for source checkouts', + type: 'boolean', + default: false, + }, + open: { + describe: 'Open Flipper in the default browser after starting', + type: 'boolean', + default: true, + }, + }) + .version('DEV') + .help() + .parse(process.argv.slice(1)); + +console.log( + `Starting flipper server with ${ + argv.bundler ? 'UI bundle from source' : 'pre-bundled UI' + }`, +); + +const rootDir = argv.bundler + ? path.resolve(__dirname, '..', '..') + : path.resolve(__dirname, '..'); // in pre packaged versions of the server, static is copied inside the package const staticDir = path.join(rootDir, 'static'); async function start() { @@ -28,14 +60,14 @@ async function start() { }; const {app, server, socket} = await startBaseServer({ - port: PORT, + port: argv.port, staticDir, entry: 'index.web.dev.html', }); const [flipperServer] = await Promise.all([ startFlipperServer(rootDir, staticDir), - startWebServerDev(app, server, socket, rootDir), + argv.bundler ? startWebServerDev(app, server, socket, rootDir) : undefined, ]); startSocketServer(flipperServer, socket); @@ -43,10 +75,14 @@ async function start() { start() .then(() => { - console.log( - 'Flipper DEV server started at ' + - chalk.green(chalk.bold(`http://localhost:${PORT}/index.web.dev.html`)), - ); + const url = `http://localhost:${argv.port}/index.web${ + argv.bundler ? '.dev' : '' + }.html`; + console.log('Flipper server started at ' + chalk.green(chalk.bold(url))); + if (argv.open) { + sleep(1000); + open(url); + } }) .catch((e) => { console.error(chalk.red('Server error: '), e); diff --git a/desktop/package.json b/desktop/package.json index 1b2ecf492..8f4041857 100644 --- a/desktop/package.json +++ b/desktop/package.json @@ -122,6 +122,7 @@ "build:eslint": "cd eslint-plugin-flipper && yarn build", "build:themes": "lessc --js themes/light.less static/themes/light.css && lessc --js themes/dark.less static/themes/dark.css", "build:tsc": "tsc -b tsc-root/tsconfig.json && ./ts-node ./scripts/compute-package-checksum.ts -d ./babel-transformer -o ./lib/checksum.txt", + "build:flipper-server": "yarn build:tsc && ./ts-node scripts/build-flipper-server-release.ts", "bump-versions": "./ts-node scripts/bump-versions.ts", "bundle-all-plugins": "./ts-node scripts/bundle-all-plugins.ts", "predev-server": "yarn build:tsc", @@ -142,7 +143,7 @@ "publish-packages": "./ts-node scripts/publish-packages.ts", "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-modules", "resolve-plugin-dir": "./ts-node scripts/resolve-plugin-dir.ts", - "rm-bundle": "rimraf static/main.bundle.* **/dist/bundle.* **/lib **/*.tsbuildinfo", + "rm-bundle": "rimraf static/main.bundle.* **/dist/bundle.* **/lib **/*.tsbuildinfo flipper-server/static", "rm-dist": "rimraf ../dist", "rm-metro-cache": "rimraf $TMPDIR/metro-cache*", "rm-modules": "rimraf **/*/node_modules node_modules", diff --git a/desktop/scripts/build-flipper-server-release.ts b/desktop/scripts/build-flipper-server-release.ts new file mode 100644 index 000000000..0021ffa50 --- /dev/null +++ b/desktop/scripts/build-flipper-server-release.ts @@ -0,0 +1,132 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +const dotenv = require('dotenv').config(); +import path from 'path'; +import { + buildBrowserBundle, + compileServerMain, + launchServer, + prepareDefaultPlugins, +} from './build-utils'; +import {serverStaticDir, staticDir} from './paths'; +import isFB from './isFB'; +import yargs from 'yargs'; +import {copy, mkdir, remove} from 'fs-extra'; + +const argv = yargs + .usage('yarn build-flipper-server [args]') + .options({ + 'default-plugins': { + describe: + 'Enables embedding of default plugins into Flipper package so they are always available. The flag is enabled by default. Env var FLIPPER_NO_DEFAULT_PLUGINS is equivalent to the command-line option "--no-default-plugins".', + type: 'boolean', + default: true, + }, + 'public-build': { + describe: + '[FB-internal only] Will force using public sources only, to be able to iterate quickly on the public version. If sources are checked out from GitHub this is already the default. Setting env var "FLIPPER_FORCE_PUBLIC_BUILD" is equivalent.', + type: 'boolean', + default: false, + }, + open: { + describe: 'Open Flipper in the default browser after starting', + type: 'boolean', + default: false, + }, + 'rebuild-plugins': { + describe: + 'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.', + type: 'boolean', + default: true, + }, + }) + .version('DEV') + .help() + .parse(process.argv.slice(1)); + +if (isFB) { + process.env.FLIPPER_FB = 'true'; +} + +// Don't bundle any plugins into the UI +process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true'; + +// Don't rebuild default plugins, mostly to speed up testing +if (argv['rebuild-plugins'] === false) { + process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true'; +} else if (argv['rebuild-plugins'] === true) { + delete process.env.FLIPPER_NO_REBUILD_PLUGINS; +} + +if (argv['public-build'] === true) { + // we use a separate env var for forced_public builds, since + // FB_FLIPPER / isFB reflects whether we are running on FB sources / infra + // so changing that will not give the desired result (e.g. incorrect resolve paths, yarn installs) + // this variable purely overrides whether imports are from `fb` or `fb-stubs` + console.log('🐬 Emulating open source build of Flipper'); + process.env.FLIPPER_FORCE_PUBLIC_BUILD = 'true'; +} else if (argv['public-build'] === false) { + delete process.env.FLIPPER_FORCE_PUBLIC_BUILD; +} + +(async () => { + console.log(`⚙️ Starting build-flipper-server-release`); + + if (dotenv && dotenv.parsed) { + console.log('✅ Loaded env vars from .env file: ', dotenv.parsed); + } + + // clear and re-create static dir + await remove(serverStaticDir); + await mkdir(serverStaticDir); + + await prepareDefaultPlugins(false); + + await compileServerMain(false); + await buildBrowserBundle(false); + await copyStaticResources(); + + if (argv.open) { + await launchServer(false, true); + } +})().catch((e) => { + console.error('Failed to build flipper-server', e, e.stack); + process.exit(1); +}); + +async function copyStaticResources() { + console.log(`⚙️ Copying static resources...`); + + // static folder, without the things that are only for Electron + const thingsToCopy = [ + 'defaultPlugins', + 'facebook', + 'icons', + 'native-modules', + 'PortForwardingMacApp.app', + 'themes', + 'vis', + 'icon.icns', + 'icon.ico', + 'icon.png', + 'icons.json', + 'index.web.dev.html', + 'index.web.html', + 'package.json', + 'style.css', + ]; + + await Promise.all( + thingsToCopy.map((e) => + copy(path.join(staticDir, e), path.join(serverStaticDir, e)), + ), + ); + console.log('✅ Copied static resources.'); +} diff --git a/desktop/scripts/build-release.ts b/desktop/scripts/build-release.ts index 7ebd41e30..c94ea6b77 100755 --- a/desktop/scripts/build-release.ts +++ b/desktop/scripts/build-release.ts @@ -352,7 +352,7 @@ async function downloadIcons(buildFolder: string) { if (res.status !== 200) { throw new Error( // eslint-disable-next-line prettier/prettier - `Could not download the icon ${name} from ${url}: got status ${res.status}`, + `Could not download the icon ${icon} from ${url}: got status ${res.status}`, ); } return res; diff --git a/desktop/scripts/build-utils.ts b/desktop/scripts/build-utils.ts index 0ee139845..c2836315d 100644 --- a/desktop/scripts/build-utils.ts +++ b/desktop/scripts/build-utils.ts @@ -9,6 +9,9 @@ // @ts-ignore import Metro from 'metro'; +// provided by Metro +// eslint-disable-next-line +import MetroResolver from 'metro-resolver'; import tmp from 'tmp'; import path from 'path'; import fs from 'fs-extra'; @@ -32,7 +35,11 @@ import { serverDir, rootDir, browserUiDir, + serverStaticDir, } from './paths'; +import pFilter from 'p-filter'; +import child from 'child_process'; +import pMap from 'p-map'; // eslint-disable-next-line flipper/no-relative-imports-across-packages const {version} = require('../package.json'); @@ -170,23 +177,29 @@ async function buildDefaultPlugins(defaultPlugins: InstalledPluginDetails[]) { .join(', ')}`, ); } - for (const plugin of defaultPlugins) { - try { - if (!process.env.FLIPPER_NO_REBUILD_PLUGINS) { - console.log( - `⚙️ Building plugin ${plugin.id} to include it into the default plugins list...`, + await pMap( + defaultPlugins, + async function (plugin) { + try { + if (!process.env.FLIPPER_NO_REBUILD_PLUGINS) { + console.log( + `⚙️ Building plugin ${plugin.id} to include it into the default plugins list...`, + ); + await runBuild(plugin.dir, dev); + } + await fs.ensureSymlink( + plugin.dir, + path.join(defaultPluginsDir, plugin.name), + 'junction', ); - await runBuild(plugin.dir, dev); + } catch (err) { + console.error(`✖ Failed to build plugin ${plugin.id}`, err); } - await fs.ensureSymlink( - plugin.dir, - path.join(defaultPluginsDir, plugin.name), - 'junction', - ); - } catch (err) { - console.error(`✖ Failed to build plugin ${plugin.id}`, err); - } - } + }, + { + concurrency: 16, + }, + ); } const minifierConfig = { @@ -371,7 +384,7 @@ export function genMercurialRevision(): Promise { .catch(() => null); } -export async function compileServerMain() { +export async function compileServerMain(dev: boolean) { await fs.promises.mkdir(path.join(serverDir, 'dist'), {recursive: true}); const out = path.join(serverDir, 'dist', 'index.js'); console.log('⚙️ Compiling server bundle...'); @@ -406,3 +419,130 @@ export async function compileServerMain() { stripSourceMapComment(out); } } + +// TODO: needed? +const uiSourceDirs = [ + 'flipper-ui-browser', + 'flipper-ui-core', + 'flipper-plugin', + 'flipper-common', +]; + +export async function buildBrowserBundle(dev: boolean) { + console.log('⚙️ Compiling browser bundle...'); + const out = path.join(serverStaticDir, 'bundle.js'); + + const electronRequires = path.join( + babelTransformationsDir, + 'electron-requires', + ); + const stubModules = new Set(require(electronRequires).BUILTINS); + if (!stubModules.size) { + throw new Error('Failed to load list of Node builtins'); + } + + const watchFolders = await dedupeFolders([ + ...( + await Promise.all( + uiSourceDirs.map((dir) => getWatchFolders(path.resolve(rootDir, dir))), + ) + ).flat(), + ...(await getPluginSourceFolders()), + ]); + + const baseConfig = await Metro.loadConfig(); + const config = Object.assign({}, baseConfig, { + projectRoot: rootDir, + watchFolders, + transformer: { + ...baseConfig.transformer, + babelTransformerPath: path.join( + babelTransformationsDir, + 'transform-browser', + ), + ...(!dev ? minifierConfig : undefined), + }, + resolver: { + ...baseConfig.resolver, + resolverMainFields: ['flipperBundlerEntry', 'browser', 'module', 'main'], + blacklistRE: [/\.native\.js$/], + sourceExts: ['js', 'jsx', 'ts', 'tsx', 'json', 'mjs', 'cjs'], + resolveRequest(context: any, moduleName: string, ...rest: any[]) { + // flipper is special cased, for plugins that we bundle, + // we want to resolve `impoSrt from 'flipper'` to 'flipper-ui-core', which + // defines all the deprecated exports + if (moduleName === 'flipper') { + return MetroResolver.resolve(context, 'flipper-ui-core', ...rest); + } + if (stubModules.has(moduleName)) { + console.warn( + `Found a reference to built-in module '${moduleName}', which will be stubbed out. Referer: ${context.originModulePath}`, + ); + return { + type: 'empty', + }; + } + return MetroResolver.resolve( + { + ...context, + resolveRequest: null, + }, + moduleName, + ...rest, + ); + }, + }, + }); + await Metro.runBuild(config, { + platform: 'web', + entry: path.join(browserUiDir, 'src', 'index.tsx'), + out, + dev, + minify: !dev, + sourceMap: true, + sourceMapUrl: dev ? 'index.map' : undefined, + inlineSourceMap: false, + }); + console.log('✅ Compiled browser bundle.'); + if (!dev) { + stripSourceMapComment(out); + } +} + +async function dedupeFolders(paths: string[]): Promise { + return pFilter( + paths.filter((value, index, self) => self.indexOf(value) === index), + (f) => fs.pathExists(f), + ); +} + +export function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +let proc: child.ChildProcess | undefined; + +export async function launchServer(startBundler: boolean, open: boolean) { + if (proc) { + console.log('⚙️ Killing old flipper-server...'); + proc.kill(9); + await sleep(1000); + } + console.log('⚙️ Launching flipper-server...'); + proc = child.spawn( + 'node', + [ + '--inspect=9229', + `../flipper-server/server.js`, + startBundler ? `--bundler` : `--no-bundler`, + open ? `--open` : `--no-open`, + ], + { + cwd: serverDir, + env: { + ...process.env, + }, + stdio: 'inherit', + }, + ); +} diff --git a/desktop/scripts/package.json b/desktop/scripts/package.json index 801dd4767..66d3780cc 100644 --- a/desktop/scripts/package.json +++ b/desktop/scripts/package.json @@ -30,7 +30,6 @@ "ignore": "^5.1.4", "metro": "^0.66.2", "metro-minify-terser": "^0.66.2", - "open": "^8.3.0", "p-filter": "^2.1.0", "p-map": "^4.0.0", "promisify-child-process": "^4.1.0", diff --git a/desktop/scripts/paths.ts b/desktop/scripts/paths.ts index 396a3074f..a5e2ef664 100644 --- a/desktop/scripts/paths.ts +++ b/desktop/scripts/paths.ts @@ -14,6 +14,7 @@ export const appDir = path.join(rootDir, 'app'); export const browserUiDir = path.join(rootDir, 'flipper-ui-browser'); export const staticDir = path.join(rootDir, 'static'); export const serverDir = path.join(rootDir, 'flipper-server'); +export const serverStaticDir = path.join(serverDir, 'static'); // for pre-bundled server, static resources are copied here export const defaultPluginsDir = path.join(staticDir, 'defaultPlugins'); export const pluginsDir = path.join(rootDir, 'plugins'); export const fbPluginsDir = path.join(pluginsDir, 'fb'); diff --git a/desktop/scripts/start-flipper-server.ts b/desktop/scripts/start-flipper-server.ts index 17fa3e73b..f5435cca4 100644 --- a/desktop/scripts/start-flipper-server.ts +++ b/desktop/scripts/start-flipper-server.ts @@ -8,19 +8,22 @@ */ const dotenv = require('dotenv').config(); -import child from 'child_process'; import chalk from 'chalk'; import path from 'path'; -import {compileServerMain, prepareDefaultPlugins} from './build-utils'; +import { + compileServerMain, + launchServer, + prepareDefaultPlugins, +} from './build-utils'; import Watchman from './watchman'; -import {serverDir} from './paths'; +import {serverStaticDir} from './paths'; import isFB from './isFB'; import yargs from 'yargs'; -import open from 'open'; import ensurePluginFoldersWatchable from './ensurePluginFoldersWatchable'; +import {remove} from 'fs-extra'; const argv = yargs - .usage('yarn start [args]') + .usage('yarn flipper-server [args]') .options({ 'default-plugins': { describe: @@ -57,11 +60,6 @@ const argv = yargs '[FB-internal only] Will force using public sources only, to be able to iterate quickly on the public version. If sources are checked out from GitHub this is already the default. Setting env var "FLIPPER_FORCE_PUBLIC_BUILD" is equivalent.', type: 'boolean', }, - build: { - describe: - 'Build the server without watching for changing or starting the service', - type: 'boolean', - }, open: { describe: 'Open Flipper in the default browser after starting', type: 'boolean', @@ -122,31 +120,12 @@ if (argv['enabled-plugins'] !== undefined) { process.env.FLIPPER_ENABLED_PLUGINS = argv['enabled-plugins'].join(','); } -let proc: child.ChildProcess | undefined; - -function launchServer() { - console.log('⚙️ Launching flipper-server...'); - proc = child.spawn( - 'node', - ['--inspect=9229', `../flipper-server/server.js`], - { - cwd: serverDir, - env: { - ...process.env, - }, - stdio: 'inherit', - }, - ); -} +let startCount = 0; async function restartServer() { - if (proc) { - proc.kill(9); - await sleep(1000); - } try { - await compileServerMain(); - await launchServer(); + await compileServerMain(true); + await launchServer(true, ++startCount === 1); // only openon the first time } catch (e) { console.error( chalk.red( @@ -193,6 +172,8 @@ async function startWatchChanges() { } (async () => { + await remove(serverStaticDir); + if (dotenv && dotenv.parsed) { console.log('✅ Loaded env vars from .env file: ', dotenv.parsed); } @@ -200,24 +181,9 @@ async function startWatchChanges() { process.env.FLIPPER_RELEASE_CHANNEL === 'insiders', ); - // build? - if (argv['build']) { - await compileServerMain(); - } else { - // watch - await startWatchChanges(); - await ensurePluginFoldersWatchable(); - // builds and starts - await restartServer(); - - if (argv.open) { - await sleep(2000); - // TODO: make port configurable together with flipper-server - open('http://localhost:52342/index.web.dev.html'); - } - } + // watch + await startWatchChanges(); + await ensurePluginFoldersWatchable(); + // builds and starts + await restartServer(); })(); - -function sleep(ms: number) { - return new Promise((r) => setTimeout(r, ms)); -} diff --git a/desktop/static/index.web.html b/desktop/static/index.web.html new file mode 100644 index 000000000..4448c9dab --- /dev/null +++ b/desktop/static/index.web.html @@ -0,0 +1,126 @@ + + + + + + + + Flipper + + + + +
+
+ Loading... +
+
+ + + + + + + + +