Fix flipper server prod build

Summary: Make flipper-server link local flipper-* deps for intern prod build. Update flipper-* deps versions for public builds so `npx` pulls them from npm

Reviewed By: lblasa

Differential Revision: D39497944

fbshipit-source-id: ca2674a4ac8f5b6c3efa9546b631f2526bf48f8e

desktop/scripts/build-flipper-server-release.tsx

@@ -18,7 +18,13 @@ import {
   prepareDefaultPlugins,
   moveServerSourceMaps,
 } from './build-utils';
-import {defaultPluginsDir, distDir, serverDir, staticDir} from './paths';
+import {
+  defaultPluginsDir,
+  distDir,
+  serverDir,
+  staticDir,
+  rootDir,
+} from './paths';
 import isFB from './isFB';
 import yargs from 'yargs';
 import fs from 'fs-extra';
@@ -113,6 +119,13 @@ const argv = yargs
         'Unique build identifier to be used as the version patch part for the build',
       type: 'number',
     },
+    // On intern we ship flipper-server with node_modules (no big internet behind the firewall). yarn.lock makes sure that a CI that builds flipper-server installs the same dependencies all the time.
+    'generate-lock': {
+      describe:
+        'Generate a new yarn.lock file for flipper-server prod build. It is used for reproducible builds of the final artifact for the intern.',
+      type: 'boolean',
+      default: false,
+    },
     mac: {
       describe: 'Build a platform-specific bundle for MacOS.',
       type: 'boolean',
@@ -216,7 +229,7 @@ async function copyStaticResources(outDir: string, versionNumber: string) {
   console.log(`⚙️  Copying package resources...`);
 
   // static folder, without the things that are only for Electron
-  const packageFilesToCopy = ['README.md', 'package.json', 'server.js', 'lib'];
+  const packageFilesToCopy = ['README.md', 'server.js', 'lib'];
 
   await Promise.all(
     packageFilesToCopy.map((e) =>
@@ -253,7 +266,44 @@ async function copyStaticResources(outDir: string, versionNumber: string) {
   console.log('✅  Copied static resources.');
 }
 
-async function modifyPackageManifest(
+async function linkLocalDeps(buildFolder: string) {
+  // eslint-disable-next-line no-console
+  console.log('Creating package.json manifest to link local deps');
+  const manifest = await fs.readJSON(path.resolve(serverDir, 'package.json'));
+
+  const resolutions = {
+    'flipper-doctor': `file:${rootDir}/doctor`,
+    'flipper-common': `file:${rootDir}/flipper-common`,
+    'flipper-frontend-core': `file:${rootDir}/flipper-frontend-core`,
+    'flipper-plugin-core': `file:${rootDir}/flipper-plugin-core`,
+    'flipper-server-companion': `file:${rootDir}/flipper-server-companion`,
+    'flipper-server-core': `file:${rootDir}/flipper-server-core`,
+    'flipper-pkg-lib': `file:${rootDir}/pkg-lib`,
+    'flipper-plugin-lib': `file:${rootDir}/plugin-lib`,
+  };
+  manifest.resolutions = resolutions;
+
+  for (const depName of Object.keys(manifest.dependencies)) {
+    if (depName in resolutions) {
+      manifest.dependencies[depName] =
+        resolutions[depName as keyof typeof resolutions];
+    }
+  }
+
+  delete manifest.scripts;
+  delete manifest.devDependencies;
+
+  await fs.writeFile(
+    path.join(buildFolder, 'package.json'),
+    JSON.stringify(manifest, null, '  '),
+  );
+
+  await yarnInstall(buildFolder);
+
+  console.log('✅ Linked local deps');
+}
+
+async function modifyPackageManifestForPublishing(
   buildFolder: string,
   versionNumber: string,
   hgRevision: string | null,
@@ -261,8 +311,7 @@ async function modifyPackageManifest(
 ) {
   // eslint-disable-next-line no-console
   console.log('Creating package.json manifest');
-  // eslint-disable-next-line flipper/no-relative-imports-across-packages
-  const manifest = require('../flipper-server/package.json');
+  const manifest = await fs.readJSON(path.resolve(serverDir, 'package.json'));
 
   manifest.version = versionNumber;
   manifest.private = false; // make this package npm-publishable
@@ -273,8 +322,15 @@ async function modifyPackageManifest(
   // not needed in public builds
   delete manifest.scripts;
   delete manifest.devDependencies;
-  // TODO: Remove me later
-  delete manifest.dependencies;
+
+  // update local monorepo dependencies' versions
+  // we will need them for npx to work
+  for (const depName of Object.keys(manifest.dependencies)) {
+    if (depName.startsWith('flipper')) {
+      manifest.dependencies[depName] = versionNumber;
+    }
+  }
+
   await fs.writeFile(
     path.join(buildFolder, 'package.json'),
     JSON.stringify(manifest, null, '  '),
@@ -315,7 +371,6 @@ async function runPostBuildAction(archive: string, dir: string) {
     );
   } else if (argv.start) {
     console.log(`⚙️  Starting flipper-server from build dir`);
-    await yarnInstall(dir);
     await spawn(
       './server.js',
       [argv.open ? '--open' : '--no-open', argv.tcp ? '--tcp' : '--no-tcp'],
@@ -328,10 +383,40 @@ async function runPostBuildAction(archive: string, dir: string) {
 }
 
 async function yarnInstall(dir: string) {
-  console.log(`⚙️  Running yarn install in ${dir}`);
-  await spawn('yarn', ['install', '--production', '--no-lockfile'], {
-    cwd: dir,
-  });
+  console.log(
+    `⚙️  Running yarn install in ${dir}. package.json: ${await fs.readFile(
+      path.resolve(dir, 'package.json'),
+    )}`,
+  );
+
+  if (!argv['generate-lock']) {
+    await fs.copyFile(
+      path.resolve(rootDir, 'yarn.flipper-server.lock'),
+      path.resolve(dir, 'yarn.lock'),
+    );
+  }
+
+  await spawn(
+    'yarn',
+    [
+      'install',
+      '--production',
+      ...(process.env.SANDCASTLE ? ['--offline'] : []),
+    ],
+    {
+      cwd: dir,
+      stdio: 'inherit',
+    },
+  );
+
+  if (argv['generate-lock']) {
+    await fs.copyFile(
+      path.resolve(dir, 'yarn.lock'),
+      path.resolve(rootDir, 'yarn.flipper-server.lock'),
+    );
+  }
+
+  await fs.rm(path.resolve(dir, 'yarn.lock'));
 }
 
 async function buildServerRelease() {
@@ -356,10 +441,16 @@ async function buildServerRelease() {
   await prepareDefaultPlugins(argv.channel === 'insiders');
   await compileServerMain(false);
   await copyStaticResources(dir, versionNumber);
+  await linkLocalDeps(dir);
   await downloadIcons(path.join(dir, 'static'));
   await buildBrowserBundle(path.join(dir, 'static'), false);
   await moveServerSourceMaps(dir, argv['source-map-dir']);
-  await modifyPackageManifest(dir, versionNumber, hgRevision, argv.channel);
+  await modifyPackageManifestForPublishing(
+    dir,
+    versionNumber,
+    hgRevision,
+    argv.channel,
+  );
   const archive = await packNpmArchive(dir, versionNumber);
   await runPostBuildAction(archive, dir);
 
@@ -375,9 +466,6 @@ async function buildServerRelease() {
     platforms.push(BuildPlatform.WINDOWS);
   }
 
-  if (platforms.length > 0) {
-    await yarnInstall(dir);
-  }
   platforms.forEach(
     bundleServerReleaseForPlatform.bind(null, dir, versionNumber),
   );

desktop/scripts/build-utils.tsx

@@ -345,8 +345,9 @@ export function genMercurialRevision(): Promise<string | null> {
 }
 
 export async function compileServerMain(dev: boolean) {
+  console.log('⚙️  Compiling server sources...');
   await exec(`cd ${serverDir} && yarn build`);
-  console.log('✅  Compiled server bundle.');
+  console.log('✅  Compiled server sources.');
 }
 
 // TODO: needed?