Move desktop-related code to "desktop" subfolder (#872)

Summary:
Pull Request resolved: https://github.com/facebook/flipper/pull/872
Move all the JS code related to desktop app to "desktop" subfolder.

The structure of "desktop" folder:
- `src` - JS code of Flipper desktop app executing in Electron Renderer (Chrome) process. This folder also contains all the Flipper plugins in subfolder "src/plugins".
- `static` - JS code of Flipper desktop app bootstrapping executing in Electron Main (Node.js) process
- `pkg` - Flipper packaging lib and CLI tool
- `doctor` - Flipper diagnostics lib and CLI tool
- `scripts` - Build scripts for Flipper desktop app
- `headless` - Headless version of Flipper app
- `headless-tests` - Integration tests running agains Flipper headless version

Reviewed By: passy

Differential Revision: D20249304

fbshipit-source-id: 9a51c63b51b92b758a02fc8ebf7d3d116770efe9

desktop/pkg/src/__tests__/index.ts

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+test('tests are working', () => {
+  expect(true).toBeTruthy();
+});

desktop/pkg/src/commands/bundle.ts

@@ -0,0 +1,127 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {Command, flags} from '@oclif/command';
+import {promises as fs} from 'fs';
+import {mkdirp, pathExists, readJSON, ensureDir} from 'fs-extra';
+import * as inquirer from 'inquirer';
+import * as path from 'path';
+import * as yarn from '../utils/yarn';
+import cli from 'cli-ux';
+import runBuild from '../utils/runBuild';
+
+async function deriveOutputFileName(inputDirectory: string): Promise<string> {
+  const packageJson = await readJSON(path.join(inputDirectory, 'package.json'));
+  return `${packageJson.name || ''}-${packageJson.version}.tgz`;
+}
+
+export default class Bundle extends Command {
+  public static description =
+    'bundle a plugin folder into a distributable archive';
+
+  public static examples = [`$ flipper-pkg bundle path/to/plugin`];
+
+  public static flags = {
+    output: flags.string({
+      char: 'o',
+      default: '.',
+      description:
+        "Where to output the bundle, file or directory. Defaults to '.'.",
+    }),
+  };
+
+  public static args = [{name: 'directory', required: true}];
+
+  public async run() {
+    const {args, flags: parsedFlags} = this.parse(Bundle);
+
+    const stat = await fs.lstat(args.directory);
+    if (!stat.isDirectory()) {
+      this.error(`Plugin source ${args.directory} is not a directory.`);
+    }
+
+    let output;
+    if (await pathExists(parsedFlags.output)) {
+      const outputStat = await fs.lstat(parsedFlags.output);
+      if (outputStat.isDirectory()) {
+        output = path.resolve(
+          path.join(
+            parsedFlags.output,
+            await deriveOutputFileName(args.directory),
+          ),
+        );
+      } else {
+        output = parsedFlags.output;
+      }
+    } else {
+      let dir;
+      let file = null;
+      // Treat this as a
+      if (parsedFlags.output.slice(-1) === '/') {
+        dir = parsedFlags.output;
+      } else {
+        dir = path.dirname(parsedFlags.output);
+        file = path.basename(parsedFlags.output);
+      }
+
+      if (!(await pathExists(dir))) {
+        const answer: {confirm: boolean} = await inquirer.prompt({
+          default: true,
+          message: `Output directory '${dir}' doesn't exist. Create it?`,
+          name: 'confirm',
+          type: 'confirm',
+        });
+
+        if (answer.confirm) {
+          mkdirp(dir);
+        } else {
+          this.error(`Output directory ${dir} not found.`);
+        }
+      }
+
+      if (file === null) {
+        file = await deriveOutputFileName(args.directory);
+      }
+      output = path.join(dir, file);
+    }
+
+    const inputDirectory = path.resolve(args.directory);
+    const outputFile = path.resolve(output);
+
+    this.log(`⚙️  Bundling ${inputDirectory} to ${outputFile}...`);
+
+    cli.action.start('Installing dependencies');
+    await yarn.install(inputDirectory);
+    cli.action.stop();
+
+    cli.action.start('Reading package.json');
+    const packageJson = await readJSON(
+      path.join(inputDirectory, 'package.json'),
+    );
+    const entry =
+      packageJson.main ??
+      ((await pathExists(path.join(inputDirectory, 'index.tsx')))
+        ? 'index.tsx'
+        : 'index.jsx');
+    const bundleMain = packageJson.bundleMain ?? path.join('dist', 'index.js');
+    const out = path.resolve(inputDirectory, bundleMain);
+    cli.action.stop(`done. Entry: ${entry}. Bundle main: ${bundleMain}.`);
+
+    cli.action.start(`Compiling`);
+    await ensureDir(path.dirname(out));
+    await runBuild(inputDirectory, entry, out);
+    cli.action.stop();
+
+    cli.action.start(`Packing to ${outputFile}`);
+    await yarn.pack(inputDirectory, outputFile);
+    cli.action.stop();
+
+    this.log(`✅  Bundled ${inputDirectory} to ${outputFile}`);
+  }
+}

desktop/pkg/src/index.ts

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+export {run} from '@oclif/command';
+export const PKG = 'flipper-pkg';
+export {default as runBuild} from './utils/runBuild';

desktop/pkg/src/transforms/__tests__/flipper-requires.node.js

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const {parse} = require('@babel/parser');
+const {transformFromAstSync} = require('@babel/core');
+const {default: generate} = require('@babel/generator');
+
+const flipperRequires = require('../flipper-requires');
+
+const babelOptions = {
+  ast: true,
+  plugins: [flipperRequires],
+  filename: 'index.js',
+};
+
+test('transform react requires to global object', () => {
+  const src = 'require("react")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.React;');
+});
+
+test('transform react-dom requires to global object', () => {
+  const src = 'require("react-dom")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.ReactDOM;');
+});
+
+test('transform flipper requires to global object', () => {
+  const src = 'require("flipper")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.Flipper;');
+});
+
+test('transform React identifier to global.React', () => {
+  const src = 'React;';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, babelOptions).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('global.React;');
+});
+
+test.skip('throw error when requiring outside the plugin', () => {
+  const src = 'require("../test.js")';
+  const ast = parse(src);
+  expect(() => {
+    transformFromAstSync(ast, src, babelOptions);
+  }).toThrow();
+});
+
+test('allow requiring from parent folder as long as we stay in plugin folder', () => {
+  const src = 'require("../test.js")';
+  const ast = parse(src);
+  const transformed = transformFromAstSync(ast, src, {
+    ...babelOptions,
+    root: '/path/to/plugin',
+    filename: '/path/to/plugin/subfolder/index.js',
+  }).ast;
+  const {code} = generate(transformed);
+  expect(code).toBe('require("../test.js");');
+});

desktop/pkg/src/transforms/dynamic-requires.js

@@ -0,0 +1,34 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+function isDynamicRequire(node) {
+  return (
+    node.type === 'CallExpression' &&
+    node.callee.type === 'Identifier' &&
+    node.callee.name === 'require' &&
+    (node.arguments.length !== 1 || node.arguments[0].type !== 'StringLiteral')
+  );
+}
+
+module.exports = function(babel) {
+  const t = babel.types;
+
+  return {
+    name: 'replace-dynamic-requires',
+    visitor: {
+      CallExpression(path) {
+        if (!isDynamicRequire(path.node)) {
+          return;
+        }
+
+        path.replaceWith(t.identifier('triggerDynamicRequireError'));
+      },
+    },
+  };
+};

desktop/pkg/src/transforms/electron-requires.js

@@ -0,0 +1,94 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const BUILTINS = [
+  'electron',
+  'buffer',
+  'child_process',
+  'crypto',
+  'dgram',
+  'dns',
+  'fs',
+  'http',
+  'https',
+  'net',
+  'os',
+  'readline',
+  'stream',
+  'string_decoder',
+  'tls',
+  'tty',
+  'zlib',
+  'constants',
+  'events',
+  'url',
+  'assert',
+  'util',
+  'path',
+  'perf_hooks',
+  'punycode',
+  'querystring',
+  'cluster',
+  'console',
+  'module',
+  'process',
+  'vm',
+  'domain',
+  'v8',
+  'repl',
+  'timers',
+  'node-fetch',
+];
+
+const IGNORED_MODULES = [
+  'bufferutil',
+  'utf-8-validate',
+  'spawn-sync',
+  './src/logcat',
+  './src/monkey',
+  './src/adb',
+];
+
+function isRequire(node) {
+  return (
+    node.type === 'CallExpression' &&
+    node.callee.type === 'Identifier' &&
+    node.callee.name === 'require' &&
+    node.arguments.length === 1 &&
+    node.arguments[0].type === 'StringLiteral'
+  );
+}
+
+module.exports = function(babel) {
+  const t = babel.types;
+
+  return {
+    name: 'infinity-import-react',
+    visitor: {
+      CallExpression(path) {
+        if (!isRequire(path.node)) {
+          return;
+        }
+
+        const source = path.node.arguments[0].value;
+
+        if (
+          BUILTINS.includes(source) ||
+          BUILTINS.some(moduleName => source.startsWith(`${moduleName}/`))
+        ) {
+          path.node.callee.name = 'electronRequire';
+        }
+
+        if (IGNORED_MODULES.includes(source)) {
+          path.replaceWith(t.identifier('triggerReferenceError'));
+        }
+      },
+    },
+  };
+};

desktop/pkg/src/transforms/electron-stubs.js

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const babylon = require('@babel/parser');
+const fs = require('fs');
+
+const electronStubs = babylon.parseExpression(
+  fs.readFileSync('static/electron-stubs.notjs').toString(),
+);
+
+module.exports = function(babel) {
+  return {
+    name: 'replace-electron-requires-with-stubs',
+    visitor: {
+      CallExpression(path) {
+        if (
+          path.node.type === 'CallExpression' &&
+          path.node.callee.type === 'Identifier' &&
+          path.node.callee.name === 'require' &&
+          path.node.arguments.length > 0
+        ) {
+          if (path.node.arguments[0].value === 'electron') {
+            path.replaceWith(electronStubs);
+          }
+        }
+      },
+    },
+  };
+};

desktop/pkg/src/transforms/fb-stubs.js

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const fs = require('fs');
+const path = require('path');
+const replaceFBStubs = fs.existsSync(
+  path.join(__dirname, '..', '..', 'src', 'fb'),
+);
+
+const requireFromFolder = (folder, path) =>
+  new RegExp(folder + '/[A-Za-z0-9.-_]+(.js)?$', 'g').test(path);
+
+module.exports = function(babel) {
+  return {
+    name: 'replace-dynamic-requires',
+    visitor: {
+      CallExpression(path) {
+        if (
+          replaceFBStubs &&
+          path.node.type === 'CallExpression' &&
+          path.node.callee.type === 'Identifier' &&
+          path.node.callee.name === 'require' &&
+          path.node.arguments.length > 0
+        ) {
+          if (requireFromFolder('fb', path.node.arguments[0].value)) {
+            throw new Error(
+              'Do not require directly from fb/, but rather from fb-stubs/ to not break flow-typing and make sure stubs are up-to-date.',
+            );
+          } else if (
+            requireFromFolder('fb-stubs', path.node.arguments[0].value)
+          ) {
+            path.node.arguments[0].value = path.node.arguments[0].value.replace(
+              '/fb-stubs/',
+              '/fb/',
+            );
+          }
+        }
+      },
+    },
+  };
+};

desktop/pkg/src/transforms/flipper-requires.js

@@ -0,0 +1,81 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const {resolve, dirname} = require('path');
+
+// do not apply this transform for these paths
+const EXCLUDE_PATHS = [
+  '/node_modules/react-devtools-core/',
+  'relay-devtools/DevtoolsUI',
+];
+
+function isExcludedPath(path) {
+  for (const epath of EXCLUDE_PATHS) {
+    if (path.indexOf(epath) > -1) {
+      return true;
+    }
+  }
+  return false;
+} // $FlowFixMe
+module.exports = ({types: t}) => ({
+  visitor: {
+    // $FlowFixMe
+    CallExpression(path, state) {
+      if (isExcludedPath(state.file.opts.filename)) {
+        return;
+      }
+      const node = path.node;
+      const args = node.arguments || [];
+
+      if (
+        node.callee.name === 'require' &&
+        args.length === 1 &&
+        t.isStringLiteral(args[0])
+      ) {
+        if (args[0].value === 'flipper') {
+          path.replaceWith(t.identifier('global.Flipper'));
+        } else if (args[0].value === 'react') {
+          path.replaceWith(t.identifier('global.React'));
+        } else if (args[0].value === 'react-dom') {
+          path.replaceWith(t.identifier('global.ReactDOM'));
+        } else if (args[0].value === 'adbkit') {
+          path.replaceWith(t.identifier('global.adbkit'));
+        } else if (
+          // require a file not a pacakge
+          args[0].value.indexOf('/') > -1 &&
+          // in the plugin itself and not inside one of its dependencies
+          state.file.opts.filename.indexOf('node_modules') === -1 &&
+          // the resolved path for this file is outside the plugins root
+          !resolve(dirname(state.file.opts.filename), args[0].value).startsWith(
+            state.file.opts.root,
+          ) &&
+          !resolve(dirname(state.file.opts.filename), args[0].value).indexOf(
+            '/static/',
+          ) < 0
+        ) {
+          throw new Error(
+            `Plugins cannot require files from outside their folder. Attempted to require ${resolve(
+              dirname(state.file.opts.filename),
+              args[0].value,
+            )} which isn't inside ${state.file.opts.root}`,
+          );
+        }
+      }
+    },
+    Identifier(path, state) {
+      if (
+        path.node.name === 'React' &&
+        path.parentPath.node.id !== path.node &&
+        !isExcludedPath(state.file.opts.filename)
+      ) {
+        path.replaceWith(t.identifier('global.React'));
+      }
+    },
+  },
+});

desktop/pkg/src/transforms/index.js

@@ -0,0 +1,124 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const generate = require('@babel/generator').default;
+const babylon = require('@babel/parser');
+const babel = require('@babel/core');
+const fs = require('fs');
+const path = require('path');
+
+function transform({filename, options, src}) {
+  const isTypeScript = filename.endsWith('.tsx') || filename.endsWith('.ts');
+  const presets = [require('@babel/preset-react')];
+
+  const ast = babylon.parse(src, {
+    filename,
+    plugins: isTypeScript
+      ? [
+          'jsx',
+          'typescript',
+          'classProperties',
+          'optionalChaining',
+          'nullishCoalescingOperator',
+        ]
+      : [
+          'jsx',
+          ['flow', {all: true}],
+          'classProperties',
+          'objectRestSpread',
+          'optionalChaining',
+          'nullishCoalescingOperator',
+        ],
+    sourceType: 'module',
+  });
+
+  // run babel
+  const plugins = [];
+
+  if (!isTypeScript) {
+    plugins.push(
+      require('@babel/plugin-transform-modules-commonjs'),
+      require('@babel/plugin-proposal-object-rest-spread'),
+      require('@babel/plugin-proposal-class-properties'),
+      require('@babel/plugin-transform-flow-strip-types'),
+      require('@babel/plugin-proposal-optional-chaining'),
+      require('@babel/plugin-proposal-nullish-coalescing-operator'),
+      require('./dynamic-requires.js'),
+    );
+  } else {
+    plugins.push(
+      require('@babel/plugin-transform-typescript'),
+      require('@babel/plugin-proposal-class-properties'),
+      require('@babel/plugin-transform-modules-commonjs'),
+      require('@babel/plugin-proposal-optional-chaining'),
+      require('@babel/plugin-proposal-nullish-coalescing-operator'),
+    );
+  }
+
+  if (
+    fs.existsSync(
+      path.resolve(path.dirname(path.dirname(__dirname)), 'src', 'fb'),
+    )
+  ) {
+    plugins.push(require('./fb-stubs.js'));
+  }
+  if (!options.isTestRunner) {
+    // Replacing require statements with electronRequire to prevent metro from
+    // resolving them. electronRequire are resolved during runtime by electron.
+    // As the tests are not bundled by metro and run in @jest-runner/electron,
+    // electron imports are working out of the box.
+    plugins.push(require('./electron-requires'));
+  }
+  plugins.push(require('./flipper-requires.js'));
+  const transformed = babel.transformFromAst(ast, src, {
+    ast: true,
+    babelrc: !filename.includes('node_modules'),
+    code: false,
+    comments: false,
+    compact: false,
+    root: options.projectRoot,
+    filename,
+    plugins,
+    presets,
+    sourceMaps: true,
+    retainLines: !!options.isTestRunner,
+  });
+  const result = generate(
+    transformed.ast,
+    {
+      filename,
+      sourceFileName: filename,
+      sourceMaps: true,
+      retainLines: !!options.isTestRunner,
+    },
+    src,
+  );
+  return {
+    ast: transformed.ast,
+    code: result.code,
+    filename,
+    map: result.map,
+  };
+}
+
+module.exports = {
+  transform,
+  // Disable caching of babel transforms all together. We haven't found a good
+  // way to cache our transforms, as they rely on side effects like env vars or
+  // the existence of folders in the file system.
+  getCacheKey: () => Math.random().toString(36),
+  process(src, filename, config, options) {
+    return transform({
+      src,
+      filename,
+      config,
+      options: {...options, isTestRunner: true},
+    });
+  },
+};

desktop/pkg/src/utils/runBuild.ts

@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+const Metro = require('metro'); // no typings :(
+import * as path from 'path';
+
+function hash(string: string) {
+  let hash = 0;
+  if (string.length === 0) {
+    return hash;
+  }
+  let chr;
+  for (let i = 0; i < string.length; i++) {
+    chr = string.charCodeAt(i);
+    hash = (hash << 5) - hash + chr;
+    hash |= 0;
+  }
+  return hash;
+}
+const fileToIdMap = new Map();
+const createModuleIdFactory = () => (filePath: string) => {
+  if (filePath === '__prelude__') {
+    return 0;
+  }
+  let id = fileToIdMap.get(filePath);
+  if (typeof id !== 'number') {
+    id = hash(filePath);
+    fileToIdMap.set(filePath, id);
+  }
+  return id;
+};
+
+export default async function runBuild(
+  inputDirectory: string,
+  entry: string,
+  out: string,
+) {
+  await Metro.runBuild(
+    {
+      reporter: {update: () => {}},
+      projectRoot: inputDirectory,
+      watchFolders: [inputDirectory, path.resolve(__dirname, '..', '..')],
+      serializer: {
+        getRunModuleStatement: (moduleID: string) =>
+          `module.exports = global.__r(${moduleID}).default;`,
+        createModuleIdFactory,
+      },
+      transformer: {
+        babelTransformerPath: path.resolve(
+          __dirname,
+          '..',
+          'transforms',
+          'index.js',
+        ),
+      },
+    },
+    {
+      dev: false,
+      minify: false,
+      resetCache: true,
+      sourceMap: true,
+      entry,
+      out,
+    },
+  );
+}

desktop/pkg/src/utils/yarn.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import * as util from 'util';
+import {exec as execImport} from 'child_process';
+const exec = util.promisify(execImport);
+
+const WINDOWS = /^win/.test(process.platform);
+const YARN_PATH = 'yarn' + (WINDOWS ? '.cmd' : '');
+
+export async function install(pkgDir: string) {
+  const {stderr} = await exec(YARN_PATH, {
+    cwd: pkgDir,
+  });
+  if (stderr) {
+    console.warn(stderr);
+  }
+}
+
+export async function pack(pkgDir: string, out: string) {
+  const {stderr} = await exec(
+    [YARN_PATH, 'pack', '--filename', out].join(' '),
+    {
+      cwd: pkgDir,
+    },
+  );
+  if (stderr) {
+    console.warn(stderr);
+  }
+}