Add support to build flipper-server from SandCastle

Summary:
This refactors the flipper-server release script in such a way that it works the same as the normal release script, which solves two problems:

1) the official release script modifies versioned files, as it touches the package.json
2) it was slightly confusing that `flipper-server/static` was filled for release builds only, but not used in dev builds
3) running test:npx without running a release build before it, would fail with hard to comprehend errors. this has been solved now by removing that script and make it an arg of `build:flipper-server`

Also some further minor changes to prepare running a corresponding build / release job from SandCastle (later in this stack)

Reviewed By: nikoant

Differential Revision: D33297214

fbshipit-source-id: f6299aa982c3e59d1cc6479a93c56cbe4b57f85c

desktop/flipper-server-core/src/utils/environmentInfo.tsx

@@ -14,11 +14,11 @@ import path from 'path';
 import {EnvironmentInfo, ReleaseChannel} from 'flipper-common';
 
 export async function getEnvironmentInfo(
-  staticPath: string,
+  packageJsonDir: string,
   isProduction: boolean,
 ): Promise<EnvironmentInfo> {
   const packageJson = await fs.readJSON(
-    path.resolve(staticPath, 'package.json'),
+    path.resolve(packageJsonDir, 'package.json'),
   );
 
   const releaseChannel: ReleaseChannel =

desktop/flipper-server/README.md

@@ -29,12 +29,3 @@ yarn build:flipper-server
 Pass the `--open` flag to open Flipper server after building
 
 Use `--no-rebuild-plugins` to speed up subsequent builds if default plugins have been build already
-
-### Test NPX build
-
-```
-cd <Flipper checkout>/desktop/
-yarn build:flipper-server
-cd flipper-server
-yarn test:npx
-```

desktop/flipper-server/package.json

@@ -6,7 +6,6 @@
   "repository": "facebook/flipper",
   "main": "server.js",
   "bin": "server.js",
-  "flipperBundlerEntry": "src",
   "license": "MIT",
   "bugs": "https://github.com/facebook/flipper/issues",
   "dependenciesComment": "mac-ca is required dynamically for darwin, node-fetch is treated special in electron-requires, not sure why",
@@ -32,12 +31,13 @@
   "peerDependencies": {},
   "scripts": {
     "reset": "rimraf lib *.tsbuildinfo",
-    "build": "tsc -b",
+    "build": "tsc -b"
-    "test:npx": "yarn pack && cd ~/Desktop && rm -rf ~/.npm/_npx/ && mv ~/fbsource/xplat/sonar/desktop/flipper-server/flipper-server-v0.0.0.tgz . && npx flipper-server-v0.0.0.tgz"
   },
   "files": [
     "dist/**/*",
-    "static/**/*"
+    "static/**/*",
+    "README.md",
+    "server.js"
   ],
   "homepage": "https://github.com/facebook/flipper",
   "keywords": [

desktop/flipper-server/src/startFlipperServer.tsx

@@ -69,7 +69,7 @@ export async function startFlipperServer(
     console.error('Failed to load keytar:', e);
   }
 
-  const environmentInfo = await getEnvironmentInfo(staticPath, isProduction);
+  const environmentInfo = await getEnvironmentInfo(appPath, isProduction);
 
   const flipperServer = new FlipperServerImpl(
     {

desktop/package.json

@@ -143,7 +143,7 @@
     "publish-packages": "./ts-node scripts/publish-packages.ts",
     "reset": "yarn rm-dist && yarn rm-temp && yarn rm-metro-cache && yarn cache clean && yarn rm-bundle && yarn rm-modules",
     "resolve-plugin-dir": "./ts-node scripts/resolve-plugin-dir.ts",
-    "rm-bundle": "rimraf static/main.bundle.* **/dist/bundle.* **/lib **/*.tsbuildinfo flipper-server/static",
+    "rm-bundle": "rimraf static/main.bundle.* **/dist/bundle.* **/lib **/*.tsbuildinfo",
     "rm-dist": "rimraf ../dist",
     "rm-metro-cache": "rimraf $TMPDIR/metro-cache*",
     "rm-modules": "rimraf **/*/node_modules node_modules",

desktop/scripts/build-flipper-server-release.ts

@@ -11,23 +11,23 @@ const dotenv = require('dotenv').config();
 import path from 'path';
 import {
   buildBrowserBundle,
+  buildFolder,
   compileServerMain,
-  launchServer,
+  genMercurialRevision,
+  getVersionNumber,
   prepareDefaultPlugins,
 } from './build-utils';
-import {
+import {defaultPluginsDir, distDir, serverDir, staticDir} from './paths';
-  defaultPluginsDir,
-  serverDefaultPluginsDir,
-  serverStaticDir,
-  staticDir,
-} from './paths';
 import isFB from './isFB';
 import yargs from 'yargs';
 import fs from 'fs-extra';
 import {downloadIcons} from './build-icons';
+import {spawn} from 'promisify-child-process';
+import {homedir} from 'os';
 
 const argv = yargs
   .usage('yarn build-flipper-server [args]')
+  .version(false)
   .options({
     'default-plugins': {
       describe:
@@ -62,9 +62,19 @@ const argv = yargs
       choices: ['stable', 'insiders'],
       default: 'stable',
     },
+    'default-plugins-dir': {
+      describe:
+        'Directory with prepared list of default plugins which will be included into the Flipper distribution as "defaultPlugins" dir',
+      type: 'string',
+    },
+    version: {
+      description:
+        'Unique build identifier to be used as the version patch part for the build',
+      type: 'number',
+    },
   })
-  .version('DEV')
   .help()
+  .strict()
   .parse(process.argv.slice(1));
 
 if (isFB) {
@@ -97,35 +107,13 @@ if (argv['enabled-plugins'] !== undefined) {
   process.env.FLIPPER_ENABLED_PLUGINS = argv['enabled-plugins'].join(',');
 }
 
-(async () => {
+if (argv['default-plugins-dir']) {
-  console.log(`⚙️  Starting build-flipper-server-release`);
+  process.env.FLIPPER_DEFAULT_PLUGINS_DIR = argv['default-plugins-dir'];
-
-  if (dotenv && dotenv.parsed) {
-    console.log('✅  Loaded env vars from .env file: ', dotenv.parsed);
 }
 
-  // clear and re-create static dir
+async function copyStaticResources(outDir: string) {
-  await fs.remove(serverStaticDir);
-  await fs.mkdir(serverStaticDir);
-
-  await prepareDefaultPlugins(argv.channel === 'insiders');
-
-  await compileServerMain(false);
-  await buildBrowserBundle(false);
-  await copyStaticResources();
-  await downloadIcons(serverStaticDir);
-
-  if (argv.open) {
-    await launchServer(false, true);
-  }
-})().catch((e) => {
-  console.error('Failed to build flipper-server', e, e.stack);
-  process.exit(1);
-});
-
-async function copyStaticResources() {
   console.log(`⚙️  Copying default plugins...`);
-  await fs.mkdirp(serverDefaultPluginsDir);
+
   const plugins = await fs.readdir(defaultPluginsDir);
   for (const plugin of plugins) {
     let source = path.join(defaultPluginsDir, plugin);
@@ -133,7 +121,7 @@ async function copyStaticResources() {
     while ((await fs.lstat(source)).isSymbolicLink()) {
       source = await fs.readlink(source);
     }
-    const target = path.join(serverDefaultPluginsDir, plugin);
+    const target = path.join(outDir, 'static', 'defaultPlugins', plugin);
     if ((await fs.stat(source)).isDirectory()) {
       // for plugins, only copy package.json & dist, to keep impact minimal
       await fs.copy(
@@ -146,10 +134,21 @@ async function copyStaticResources() {
     }
   }
 
+  console.log(`⚙️  Copying package resources...`);
+
+  // static folder, without the things that are only for Electron
+  const packageFilesToCopy = ['README.md', 'package.json', 'server.js', 'dist'];
+
+  await Promise.all(
+    packageFilesToCopy.map((e) =>
+      fs.copy(path.join(serverDir, e), path.join(outDir, e)),
+    ),
+  );
+
   console.log(`⚙️  Copying static resources...`);
 
   // static folder, without the things that are only for Electron
-  const thingsToCopy = [
+  const staticsToCopy = [
     'facebook',
     'icons',
     'native-modules',
@@ -162,14 +161,86 @@ async function copyStaticResources() {
     'icons.json',
     'index.web.dev.html',
     'index.web.html',
-    'package.json',
     'style.css',
   ];
 
   await Promise.all(
-    thingsToCopy.map((e) =>
+    staticsToCopy.map((e) =>
-      fs.copy(path.join(staticDir, e), path.join(serverStaticDir, e)),
+      fs.copy(path.join(staticDir, e), path.join(outDir, 'static', e)),
     ),
   );
   console.log('✅  Copied static resources.');
 }
+
+async function modifyPackageManifest(
+  buildFolder: string,
+  versionNumber: string,
+  hgRevision: string | null,
+  channel: string,
+) {
+  // eslint-disable-next-line no-console
+  console.log('Creating package.json manifest');
+  // eslint-disable-next-line flipper/no-relative-imports-across-packages
+  const manifest = require('../flipper-server/package.json');
+
+  manifest.version = versionNumber;
+  if (hgRevision != null) {
+    manifest.revision = hgRevision;
+  }
+  manifest.releaseChannel = channel;
+  await fs.writeFile(
+    path.join(buildFolder, 'package.json'),
+    JSON.stringify(manifest, null, '  '),
+  );
+}
+
+(async () => {
+  console.log(`⚙️  Starting build-flipper-server-release`);
+  console.dir(argv);
+  const dir = await buildFolder();
+  console.log('Created build directory', dir);
+
+  if (dotenv && dotenv.parsed) {
+    console.log('✅  Loaded env vars from .env file: ', dotenv.parsed);
+  }
+
+  const versionNumber = getVersionNumber(argv.version);
+  const hgRevision = await genMercurialRevision();
+  console.log(
+    `  Building version / revision ${versionNumber} ${hgRevision ?? ''}`,
+  );
+
+  // create static dir
+  await fs.mkdirp(path.join(dir, 'static', 'defaultPlugins'));
+
+  await prepareDefaultPlugins(argv.channel === 'insiders');
+  await compileServerMain(false);
+  await buildBrowserBundle(path.join(dir, 'static'), false);
+  await copyStaticResources(dir);
+  await downloadIcons(path.join(dir, 'static'));
+  await modifyPackageManifest(dir, versionNumber, hgRevision, argv.channel);
+
+  console.log(`⚙️  Packing flipper-server.tgz`);
+  const archive = path.resolve(distDir, 'flipper-server.tgz');
+  await spawn('yarn', ['pack', '--filename', archive], {
+    cwd: dir,
+    stdio: 'inherit',
+  });
+
+  console.log(
+    `✅  flipper-release-build completed, version ${versionNumber} in ${dir}`,
+  );
+
+  if (argv.open) {
+    // This is a hack, as npx cached very aggressively if package.version
+    // didn't change
+    console.log(`⚙️  Installing flipper-server.tgz using npx`);
+    await fs.remove(path.join(homedir(), '.npm', '_npx'));
+    await spawn('npx', [archive], {
+      stdio: 'inherit',
+    });
+  }
+})().catch((e) => {
+  console.error('Failed to build flipper-server', e, e.stack);
+  process.exit(1);
+});

desktop/scripts/build-utils.ts

@@ -431,9 +431,9 @@ const uiSourceDirs = [
   'flipper-common',
 ];
 
-export async function buildBrowserBundle(dev: boolean) {
+export async function buildBrowserBundle(outDir: string, dev: boolean) {
   console.log('⚙️  Compiling browser bundle...');
-  const out = path.join(serverStaticDir, 'bundle.js');
+  const out = path.join(outDir, 'bundle.js');
 
   const electronRequires = path.join(
     babelTransformationsDir,

desktop/scripts/paths.ts

@@ -14,12 +14,7 @@ export const appDir = path.join(rootDir, 'app');
 export const browserUiDir = path.join(rootDir, 'flipper-ui-browser');
 export const staticDir = path.join(rootDir, 'static');
 export const serverDir = path.join(rootDir, 'flipper-server');
-export const serverStaticDir = path.join(serverDir, 'static'); // for pre-bundled server, static resources are copied here
 export const defaultPluginsDir = path.join(staticDir, 'defaultPlugins');
-export const serverDefaultPluginsDir = path.join(
-  serverStaticDir,
-  'defaultPlugins',
-);
 export const pluginsDir = path.join(rootDir, 'plugins');
 export const fbPluginsDir = path.join(pluginsDir, 'fb');
 export const publicPluginsDir = path.join(pluginsDir, 'public');