Bundle operation implementation

Summary: Plugin bundling operation implemented in flipper-pkg

Reviewed By: passy

Differential Revision: D20191845

fbshipit-source-id: 6a7156debf96668c323dcb740b33542f129f0689

.flowconfig

@@ -2,6 +2,7 @@
 .*/scripts/.*
 .*/coverage/.*
 .*/build/.*
+.*/pkg/.*
 .*/dist/.*
 .*/static/.*
 <PROJECT_ROOT>/src/fb/plugins/relaydevtools/relay-devtools/DevtoolsUI.js$

.vscode/launch.json

@@ -29,7 +29,7 @@
     "name": "Launch Current Script",
     "args": ["${file}"],
     "env": {
-      "TS_NODE_FILES": "true",
+      "TS_NODE_FILES": "true"
     },
     "protocol": "inspector",
     "internalConsoleOptions": "openOnSessionStart",

pkg/package.json

@@ -11,21 +11,32 @@
   },
   "bugs": "https://github.com/facebook/flipper/issues",
   "dependencies": {
+    "@babel/core": "^7.8.6",
+    "@babel/generator": "^7.8.6",
+    "@babel/parser": "^7.8.6",
+    "@babel/plugin-proposal-class-properties": "^7.8.3",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
+    "@babel/plugin-proposal-object-rest-spread": "^7.8.3",
+    "@babel/plugin-proposal-optional-chaining": "^7.8.3",
+    "@babel/plugin-transform-flow-strip-types": "^7.8.3",
+    "@babel/plugin-transform-modules-commonjs": "^7.8.3",
+    "@babel/plugin-transform-typescript": "^7.8.3",
+    "@babel/preset-react": "^7.8.3",
     "@oclif/command": "^1",
     "@oclif/config": "^1",
     "@oclif/plugin-help": "^2",
     "@types/fs-extra": "^8.1.0",
     "@types/inquirer": "^6.5.0",
     "@types/node": "^13.7.5",
+    "cli-ux": "^5.4.5",
     "fs-extra": "^8.1.0",
     "inquirer": "^7.0.4",
-    "listr": "^0.14.3",
+    "metro": "^0.58.0",
     "tslib": "^1"
   },
   "devDependencies": {
     "@oclif/dev-cli": "^1",
     "@types/jest": "^24.0.21",
-    "@types/listr": "^0.14.2",
     "globby": "^10",
     "jest": "^24.9.0",
     "prettier": "^1.19.1",

pkg/src/commands/bundle.ts

@@ -9,22 +9,16 @@
 
 import {Command, flags} from '@oclif/command';
 import {promises as fs} from 'fs';
-import {mkdirp, pathExists, readJSON} from 'fs-extra';
+import {mkdirp, pathExists, readJSON, ensureDir} from 'fs-extra';
 import * as inquirer from 'inquirer';
 import * as path from 'path';
-
-interface BundleArgs {
-  inputDirectory: string;
-  outputFile: string;
-}
-
-async function bundle(_args: BundleArgs) {
-  throw new Error('Not implemented.');
-}
+import * as yarn from '../utils/yarn';
+import cli from 'cli-ux';
+import runBuild from '../utils/runBuild';
 
 async function deriveOutputFileName(inputDirectory: string): Promise<string> {
   const packageJson = await readJSON(path.join(inputDirectory, 'package.json'));
-  return `${packageJson.name || ''}-${packageJson.version}.vsix`;
+  return `${packageJson.name || ''}-${packageJson.version}.tgz`;
 }
 
 export default class Bundle extends Command {
@@ -97,11 +91,37 @@ export default class Bundle extends Command {
       output = path.join(dir, file);
     }
 
-    const bundleArgs: BundleArgs = {
-      inputDirectory: args.directory,
-      outputFile: output,
-    };
-    this.log('Bundling: ', bundleArgs);
-    bundle(bundleArgs);
+    const inputDirectory = path.resolve(args.directory);
+    const outputFile = path.resolve(output);
+
+    this.log(`⚙️  Bundling ${inputDirectory} to ${outputFile}...`);
+
+    cli.action.start('Installing dependencies');
+    await yarn.install(inputDirectory);
+    cli.action.stop();
+
+    cli.action.start('Reading package.json');
+    const packageJson = await readJSON(
+      path.join(inputDirectory, 'package.json'),
+    );
+    const entry =
+      packageJson.main ??
+      ((await pathExists(path.join(inputDirectory, 'index.tsx')))
+        ? 'index.tsx'
+        : 'index.jsx');
+    const bundleMain = packageJson.bundleMain ?? path.join('dist', 'index.js');
+    const out = path.resolve(inputDirectory, bundleMain);
+    cli.action.stop(`done. Entry: ${entry}. Bundle main: ${bundleMain}.`);
+
+    cli.action.start(`Compiling`);
+    await ensureDir(path.dirname(out));
+    await runBuild(inputDirectory, entry, out);
+    cli.action.stop();
+
+    cli.action.start(`Packing to ${outputFile}`);
+    await yarn.pack(inputDirectory, outputFile);
+    cli.action.stop();
+
+    this.log(`✅  Bundled ${inputDirectory} to ${outputFile}`);
   }
 }

pkg/src/index.ts

@@ -9,3 +9,4 @@
 
 export {run} from '@oclif/command';
 export const PKG = 'flipper-pkg';
+export {default as runBuild} from './utils/runBuild';

pkg/src/transforms/__tests__/flipper-requires.node.js

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const {parse} = require('@babel/parser');
+const {transformFromAstSync} = require('@babel/core');
+const {default: generate} = require('@babel/generator');
+
+const flipperRequires = require('../flipper-requires');
+
+const babelOptions = {
+  ast: true,
+  plugins: [flipperRequires],
+  filename: 'index.js',
+};
+
+test('transform react requires to global object', () => {
+  const src = 'require("react")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.React;');
+});
+
+test('transform react-dom requires to global object', () => {
+  const src = 'require("react-dom")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.ReactDOM;');
+});
+
+test('transform flipper requires to global object', () => {
+  const src = 'require("flipper")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.Flipper;');
+});
+
+test('transform React identifier to global.React', () => {
+  const src = 'React;';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.React;');
+});
+
+test.skip('throw error when requiring outside the plugin', () => {
+  const src = 'require("../test.js")';
+  const ast = parse(src);
+  expect(() => {
+    transformFromAstSync(ast, src, babelOptions);
+  }).toThrow();
+});
+
+test('allow requiring from parent folder as long as we stay in plugin folder', () => {
+  const src = 'require("../test.js")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, {
+    ...babelOptions,
+    root: '/path/to/plugin',
+    filename: '/path/to/plugin/subfolder/index.js',
+  }).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('require("../test.js");');
+});

pkg/src/transforms/dynamic-requires.js

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+function isDynamicRequire(node) {
+  return (
+    node.type === 'CallExpression' &&
+    node.callee.type === 'Identifier' &&
+    node.callee.name === 'require' &&
+    (node.arguments.length !== 1 || node.arguments[0].type !== 'StringLiteral')
+  );
+}
+
+module.exports = function(babel) {
+  const t = babel.types;
+
+  return {
+    name: 'replace-dynamic-requires',
+    visitor: {
+      CallExpression(path) {
+        if (!isDynamicRequire(path.node)) {
+          return;
+        }
+
+        path.replaceWith(t.identifier('triggerDynamicRequireError'));
+      },
+    },
+  };
+};

pkg/src/transforms/electron-requires.js

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const BUILTINS = [
+  'electron',
+  'buffer',
+  'child_process',
+  'crypto',
+  'dgram',
+  'dns',
+  'fs',
+  'http',
+  'https',
+  'net',
+  'os',
+  'readline',
+  'stream',
+  'string_decoder',
+  'tls',
+  'tty',
+  'zlib',
+  'constants',
+  'events',
+  'url',
+  'assert',
+  'util',
+  'path',
+  'perf_hooks',
+  'punycode',
+  'querystring',
+  'cluster',
+  'console',
+  'module',
+  'process',
+  'vm',
+  'domain',
+  'v8',
+  'repl',
+  'timers',
+  'node-fetch',
+];
+
+const IGNORED_MODULES = [
+  'bufferutil',
+  'utf-8-validate',
+  'spawn-sync',
+  './src/logcat',
+  './src/monkey',
+  './src/adb',
+];
+
+function isRequire(node) {
+  return (
+    node.type === 'CallExpression' &&
+    node.callee.type === 'Identifier' &&
+    node.callee.name === 'require' &&
+    node.arguments.length === 1 &&
+    node.arguments[0].type === 'StringLiteral'
+  );
+}
+
+module.exports = function(babel) {
+  const t = babel.types;
+
+  return {
+    name: 'infinity-import-react',
+    visitor: {
+      CallExpression(path) {
+        if (!isRequire(path.node)) {
+          return;
+        }
+
+        const source = path.node.arguments[0].value;
+
+        if (
+          BUILTINS.includes(source) ||
+          BUILTINS.some(moduleName => source.startsWith(`${moduleName}/`))
+        ) {
+          path.node.callee.name = 'electronRequire';
+        }
+
+        if (IGNORED_MODULES.includes(source)) {
+          path.replaceWith(t.identifier('triggerReferenceError'));
+        }
+      },
+    },
+  };
+};

pkg/src/transforms/electron-stubs.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const babylon = require('@babel/parser');
+const fs = require('fs');
+
+const electronStubs = babylon.parseExpression(
+  fs.readFileSync('static/electron-stubs.notjs').toString(),
+);
+
+module.exports = function(babel) {
+  return {
+    name: 'replace-electron-requires-with-stubs',
+    visitor: {
+      CallExpression(path) {
+        if (
+          path.node.type === 'CallExpression' &&
+          path.node.callee.type === 'Identifier' &&
+          path.node.callee.name === 'require' &&
+          path.node.arguments.length > 0
+        ) {
+          if (path.node.arguments[0].value === 'electron') {
+            path.replaceWith(electronStubs);
+          }
+        }
+      },
+    },
+  };
+};

pkg/src/transforms/fb-stubs.js

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const fs = require('fs');
+const path = require('path');
+const replaceFBStubs = fs.existsSync(
+  path.join(__dirname, '..', '..', 'src', 'fb'),
+);
+
+const requireFromFolder = (folder, path) =>
+  new RegExp(folder + '/[A-Za-z0-9.-_]+(.js)?$', 'g').test(path);
+
+module.exports = function(babel) {
+  return {
+    name: 'replace-dynamic-requires',
+    visitor: {
+      CallExpression(path) {
+        if (
+          replaceFBStubs &&
+          path.node.type === 'CallExpression' &&
+          path.node.callee.type === 'Identifier' &&
+          path.node.callee.name === 'require' &&
+          path.node.arguments.length > 0
+        ) {
+          if (requireFromFolder('fb', path.node.arguments[0].value)) {
+            throw new Error(
+              'Do not require directly from fb/, but rather from fb-stubs/ to not break flow-typing and make sure stubs are up-to-date.',
+            );
+          } else if (
+            requireFromFolder('fb-stubs', path.node.arguments[0].value)
+          ) {
+            path.node.arguments[0].value = path.node.arguments[0].value.replace(
+              '/fb-stubs/',
+              '/fb/',
+            );
+          }
+        }
+      },
+    },
+  };
+};

pkg/src/transforms/flipper-requires.js

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const {resolve, dirname} = require('path');
+
+// do not apply this transform for these paths
+const EXCLUDE_PATHS = [
+  '/node_modules/react-devtools-core/',
+  'relay-devtools/DevtoolsUI',
+];
+
+function isExcludedPath(path) {
+  for (const epath of EXCLUDE_PATHS) {
+    if (path.indexOf(epath) > -1) {
+      return true;
+    }
+  }
+  return false;
+} // $FlowFixMe
+module.exports = ({types: t}) => ({
+  visitor: {
+    // $FlowFixMe
+    CallExpression(path, state) {
+      if (isExcludedPath(state.file.opts.filename)) {
+        return;
+      }
+      const node = path.node;
+      const args = node.arguments || [];
+
+      if (
+        node.callee.name === 'require' &&
+        args.length === 1 &&
+        t.isStringLiteral(args[0])
+      ) {
+        if (args[0].value === 'flipper') {
+          path.replaceWith(t.identifier('global.Flipper'));
+        } else if (args[0].value === 'react') {
+          path.replaceWith(t.identifier('global.React'));
+        } else if (args[0].value === 'react-dom') {
+          path.replaceWith(t.identifier('global.ReactDOM'));
+        } else if (args[0].value === 'adbkit') {
+          path.replaceWith(t.identifier('global.adbkit'));
+        } else if (
+          // require a file not a pacakge
+          args[0].value.indexOf('/') > -1 &&
+          // in the plugin itself and not inside one of its dependencies
+          state.file.opts.filename.indexOf('node_modules') === -1 &&
+          // the resolved path for this file is outside the plugins root
+          !resolve(dirname(state.file.opts.filename), args[0].value).startsWith(
+            state.file.opts.root,
+          ) &&
+          !resolve(dirname(state.file.opts.filename), args[0].value).indexOf(
+            '/static/',
+          ) < 0
+        ) {
+          throw new Error(
+            `Plugins cannot require files from outside their folder. Attempted to require ${resolve(
+              dirname(state.file.opts.filename),
+              args[0].value,
+            )} which isn't inside ${state.file.opts.root}`,
+          );
+        }
+      }
+    },
+    Identifier(path, state) {
+      if (
+        path.node.name === 'React' &&
+        path.parentPath.node.id !== path.node &&
+        !isExcludedPath(state.file.opts.filename)
+      ) {
+        path.replaceWith(t.identifier('global.React'));
+      }
+    },
+  },
+});

pkg/src/transforms/index.js

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const generate = require('@babel/generator').default;
+const babylon = require('@babel/parser');
+const babel = require('@babel/core');
+const fs = require('fs');
+const path = require('path');
+
+function transform({filename, options, src}) {
+  const isTypeScript = filename.endsWith('.tsx') || filename.endsWith('.ts');
+  const presets = [require('@babel/preset-react')];
+
+  const ast = babylon.parse(src, {
+    filename,
+    plugins: isTypeScript
+      ? [
+          'jsx',
+          'typescript',
+          'classProperties',
+          'optionalChaining',
+          'nullishCoalescingOperator',
+        ]
+      : [
+          'jsx',
+          ['flow', {all: true}],
+          'classProperties',
+          'objectRestSpread',
+          'optionalChaining',
+          'nullishCoalescingOperator',
+        ],
+    sourceType: 'module',
+  });
+
+  // run babel
+  const plugins = [];
+
+  if (!isTypeScript) {
+    plugins.push(
+      require('@babel/plugin-transform-modules-commonjs'),
+      require('@babel/plugin-proposal-object-rest-spread'),
+      require('@babel/plugin-proposal-class-properties'),
+      require('@babel/plugin-transform-flow-strip-types'),
+      require('@babel/plugin-proposal-optional-chaining'),
+      require('@babel/plugin-proposal-nullish-coalescing-operator'),
+      require('./dynamic-requires.js'),
+    );
+  } else {
+    plugins.push(
+      require('@babel/plugin-transform-typescript'),
+      require('@babel/plugin-proposal-class-properties'),
+      require('@babel/plugin-transform-modules-commonjs'),
+      require('@babel/plugin-proposal-optional-chaining'),
+      require('@babel/plugin-proposal-nullish-coalescing-operator'),
+    );
+  }
+
+  if (
+    fs.existsSync(
+      path.resolve(path.dirname(path.dirname(__dirname)), 'src', 'fb'),
+    )
+  ) {
+    plugins.push(require('./fb-stubs.js'));
+  }
+  if (!options.isTestRunner) {
+    // Replacing require statements with electronRequire to prevent metro from
+    // resolving them. electronRequire are resolved during runtime by electron.
+    // As the tests are not bundled by metro and run in @jest-runner/electron,
+    // electron imports are working out of the box.
+    plugins.push(require('./electron-requires'));
+  }
+  plugins.push(require('./flipper-requires.js'));
+  const transformed = babel.transformFromAst(ast, src, {
+    ast: true,
+    babelrc: !filename.includes('node_modules'),
+    code: false,
+    comments: false,
+    compact: false,
+    root: options.projectRoot,
+    filename,
+    plugins,
+    presets,
+    sourceMaps: true,
+    retainLines: !!options.isTestRunner,
+  });
+  const result = generate(
+    transformed.ast,
+    {
+      filename,
+      sourceFileName: filename,
+      sourceMaps: true,
+      retainLines: !!options.isTestRunner,
+    },
+    src,
+  );
+  return {
+    ast: transformed.ast,
+    code: result.code,
+    filename,
+    map: result.map,
+  };
+}
+
+module.exports = {
+  transform,
+  // Disable caching of babel transforms all together. We haven't found a good
+  // way to cache our transforms, as they rely on side effects like env vars or
+  // the existence of folders in the file system.
+  getCacheKey: () => Math.random().toString(36),
+  process(src, filename, config, options) {
+    return transform({
+      src,
+      filename,
+      config,
+      options: {...options, isTestRunner: true},
+    });
+  },
+};

pkg/src/utils/runBuild.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const Metro = require('metro'); // no typings :(
+import * as path from 'path';
+
+function hash(string: string) {
+  let hash = 0;
+  if (string.length === 0) {
+    return hash;
+  }
+  let chr;
+  for (let i = 0; i < string.length; i++) {
+    chr = string.charCodeAt(i);
+    hash = (hash << 5) - hash + chr;
+    hash |= 0;
+  }
+  return hash;
+}
+const fileToIdMap = new Map();
+const createModuleIdFactory = () => (filePath: string) => {
+  if (filePath === '__prelude__') {
+    return 0;
+  }
+  let id = fileToIdMap.get(filePath);
+  if (typeof id !== 'number') {
+    id = hash(filePath);
+    fileToIdMap.set(filePath, id);
+  }
+  return id;
+};
+
+export default async function runBuild(
+  inputDirectory: string,
+  entry: string,
+  out: string,
+) {
+  await Metro.runBuild(
+    {
+      reporter: {update: () => {}},
+      projectRoot: inputDirectory,
+      watchFolders: [inputDirectory, path.resolve(__dirname, '..', '..')],
+      serializer: {
+        getRunModuleStatement: (moduleID: string) =>
+          `module.exports = global.__r(${moduleID}).default;`,
+        createModuleIdFactory,
+      },
+      transformer: {
+        babelTransformerPath: path.resolve(
+          __dirname,
+          '..',
+          'transforms',
+          'index.js',
+        ),
+      },
+    },
+    {
+      dev: false,
+      minify: false,
+      resetCache: true,
+      sourceMap: true,
+      entry,
+      out,
+    },
+  );
+}

pkg/src/utils/yarn.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import * as util from 'util';
+import {exec as execImport} from 'child_process';
+const exec = util.promisify(execImport);
+
+const WINDOWS = /^win/.test(process.platform);
+const YARN_PATH = 'yarn' + (WINDOWS ? '.cmd' : '');
+
+export async function install(pkgDir: string) {
+  const {stderr} = await exec(YARN_PATH, {
+    cwd: pkgDir,
+  });
+  if (stderr) {
+    console.warn(stderr);
+  }
+}
+
+export async function pack(pkgDir: string, out: string) {
+  const {stderr} = await exec(
+    [YARN_PATH, 'pack', '--filename', out].join(' '),
+    {
+      cwd: pkgDir,
+    },
+  );
+  if (stderr) {
+    console.warn(stderr);
+  }
+}

pkg/tsconfig.json

@@ -6,7 +6,9 @@
     "outDir": "lib",
     "rootDir": "src",
     "strict": true,
-    "importHelpers": true
+    "importHelpers": true,
+    "esModuleInterop": true,
+    "allowJs": true
   },
   "include": [
     "src/**/*"

static/compilePlugins.ts

@@ -38,6 +38,7 @@ export type PluginManifest = {
   version: string;
   name: string;
   main?: string;
+  bundleMain?: string;
   [key: string]: any;
 };
 
@@ -245,14 +246,12 @@ async function compilePlugin(
   options: DynamicCompileOptions,
 ): Promise<CompiledPluginInfo | null> {
   const {rootDir, manifest, entry, name} = pluginInfo;
-  const isPreBundled = fs.existsSync(path.join(rootDir, 'dist'));
-  if (isPreBundled) {
+  const bundleMain = manifest.bundleMain ?? path.join('dist', 'index.js');
+  const bundlePath = path.join(rootDir, bundleMain);
+  if (fs.existsSync(bundlePath)) {
     // eslint-disable-next-line no-console
-    console.log(`🥫  Using pre-built version of ${name}...`);
-    const out = path.join(
-      rootDir,
-      manifest.main ?? path.join('dist', 'index.js'),
-    );
+    const out = path.join(rootDir, bundleMain);
+    console.log(`🥫  Using pre-built version of ${name}: ${out}...`);
     return Object.assign({}, pluginInfo.manifest, {out});
   } else {
     const out = path.join(