Re-use babel transformations

Summary:
SORRY FOR BIG DIFF, but it's really hard to split it as all these changes are cross-dependent and should be made at once:
1. Moved transformations to separate package "flipper-babel-transformer" and linked it using yarn workspaces to "static" and "pkg" packages where they are re-used. Removed double copies of transformations we had before int these two packages.
2. Converted transformations to typescript
3. Refactored transformations to avoid relying on file system paths for customisation (FB stubs and Electron stubs for headless build)
4. As babel transformations must be built before other builds - enabled incremental build for them and changed scripts to invoke the transformations build before other build scripts
5. As we need to deploy all the dependencies including the fresh "flipper-babel-transformer" as a part of "static" - implemented script which copies package with all the dependencies taking in account yarn workspaces (hoisting and symlinks)

Reviewed By: passy, mweststrate

Differential Revision: D20690662

fbshipit-source-id: 38a275b60d3c91e01ec21d1dbd72d03c05cfac0b
This commit is contained in:
Anton Nikolaev
2020-03-27 03:21:58 -07:00
committed by Facebook GitHub Bot
parent 07a6a3b87d
commit c1bb656a0d
59 changed files with 1168 additions and 2690 deletions

View File

@@ -0,0 +1,66 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {transform} from '@babel/core';
const electronProcess = require('../electron-process');
const babelOptions = {
ast: true,
plugins: [electronProcess],
filename: 'index.js',
};
test('transform "process.exit(0);"', () => {
const src = 'process.exit(0);';
const code = transform(src, babelOptions)!.code;
expect(code).toMatchInlineSnapshot(`"electronProcess.exit(0);"`);
});
test('transform "global.process.exit(0);"', () => {
const src = 'global.process.exit(0);';
const code = transform(src, babelOptions)!.code;
expect(code).toMatchInlineSnapshot(`"global.electronProcess.exit(0);"`);
});
test('transform "process.ENV.TEST = "true";"', () => {
const src = 'process.ENV.TEST = "true";';
const code = transform(src, babelOptions)!.code;
expect(code).toMatchInlineSnapshot(
`"electronProcess.ENV.TEST = \\"true\\";"`,
);
});
test('do not transform if process bound in an upper scope', () => {
const src = `
const process = {};
for (const i=0; i<10; i++) {
process.ENV[i] = i;
}
`;
const code = transform(src, babelOptions)!.code;
expect(code).toMatchInlineSnapshot(`
"const process = {};
for (const i = 0; i < 10; i++) {
process.ENV[i] = i;
}"
`);
});
test('do not transform if process bound to the current scope', () => {
const src = `
const process = {};
process.ENV.TEST = "true";
`;
const code = transform(src, babelOptions)!.code;
expect(code).toMatchInlineSnapshot(`
"const process = {};
process.ENV.TEST = \\"true\\";"
`);
});

View File

@@ -0,0 +1,34 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {transform} from '@babel/core';
const electronStubs = require('../electron-stubs');
const babelOptions = {
ast: true,
plugins: [electronStubs],
filename: 'index.js',
};
test('transform electron requires to inlined stubs', () => {
const src = 'require("electron")';
const transformed = transform(src, babelOptions)!.ast;
const body = transformed!.program.body[0];
expect(body.type).toBe('ExpressionStatement');
if (body.type !== 'ExpressionStatement') {
return;
}
expect(body.expression.type).toBe('ObjectExpression');
if (body.expression.type !== 'ObjectExpression') {
return;
}
expect(body.expression.properties.map((p) => (p as any).key.name)).toContain(
'remote',
);
});

View File

@@ -0,0 +1,71 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {parse} from '@babel/parser';
import {transformFromAstSync} from '@babel/core';
import {default as generate} from '@babel/generator';
const flipperRequires = require('../flipper-requires');
const babelOptions = {
ast: true,
plugins: [flipperRequires],
filename: 'index.js',
};
test('transform react requires to global object', () => {
const src = 'require("react")';
const ast = parse(src);
const transformed = transformFromAstSync(ast, src, babelOptions)!.ast;
const {code} = generate(transformed!);
expect(code).toBe('global.React;');
});
test('transform react-dom requires to global object', () => {
const src = 'require("react-dom")';
const ast = parse(src);
const transformed = transformFromAstSync(ast, src, babelOptions)!.ast;
const {code} = generate(transformed!);
expect(code).toBe('global.ReactDOM;');
});
test('transform flipper requires to global object', () => {
const src = 'require("flipper")';
const ast = parse(src);
const transformed = transformFromAstSync(ast, src, babelOptions)!.ast;
const {code} = generate(transformed!);
expect(code).toBe('global.Flipper;');
});
test('transform React identifier to global.React', () => {
const src = 'React;';
const ast = parse(src);
const transformed = transformFromAstSync(ast, src, babelOptions)!.ast;
const {code} = generate(transformed!);
expect(code).toBe('global.React;');
});
test('throw error when requiring outside the plugin', () => {
const src = 'require("../test.js")';
const ast = parse(src);
expect(() => {
transformFromAstSync(ast, src, babelOptions);
}).toThrow();
});
test('allow requiring from parent folder as long as we stay in plugin folder', () => {
const src = 'require("../test.js")';
const ast = parse(src);
const transformed = transformFromAstSync(ast, src, {
...babelOptions,
root: '/path/to/plugin',
filename: '/path/to/plugin/subfolder/index.js',
})!.ast;
const {code} = generate(transformed!);
expect(code).toBe('require("../test.js");');
});

View File

@@ -0,0 +1,33 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {CallExpression, identifier} from '@babel/types';
import {NodePath} from '@babel/traverse';
function isDynamicRequire(node: CallExpression) {
return (
node.type === 'CallExpression' &&
node.callee.type === 'Identifier' &&
node.callee.name === 'require' &&
(node.arguments.length !== 1 || node.arguments[0].type !== 'StringLiteral')
);
}
module.exports = () => ({
name: 'replace-dynamic-requires',
visitor: {
CallExpression(path: NodePath<CallExpression>) {
path.node;
if (!isDynamicRequire(path.node)) {
return;
}
path.replaceWith(identifier('triggerDynamicRequireError'));
},
},
});

View File

@@ -0,0 +1,34 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {MemberExpression} from '@babel/types';
import {NodePath} from '@babel/traverse';
module.exports = () => ({
name: 'change-process-to-electronProcess',
visitor: {
MemberExpression(path: NodePath<MemberExpression>) {
if (
path.node.object.type === 'Identifier' &&
path.node.object.name === 'process' &&
!path.scope.hasBinding('process')
) {
path.node.object.name = 'electronProcess';
} else if (
path.node.object.type === 'MemberExpression' &&
path.node.object.object.type === 'Identifier' &&
path.node.object.object.name === 'global' &&
path.node.object.property.type === 'Identifier' &&
path.node.object.property.name === 'process'
) {
path.node.object.property.name = 'electronProcess';
}
},
},
});

View File

@@ -0,0 +1,32 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {CallExpression} from '@babel/types';
import {NodePath} from '@babel/traverse';
module.exports = () => ({
name: 'change-electron-to-electronRequire-in-main',
visitor: {
CallExpression(path: NodePath<CallExpression>) {
const node = path.node;
if (
node.type === 'CallExpression' &&
node.callee.type === 'Identifier' &&
node.callee.name === 'require' &&
node.arguments.length === 1 &&
node.arguments[0].type === 'StringLiteral'
) {
const source = node.arguments[0].value;
if (!source.startsWith('./')) {
node.callee.name = 'electronRequire';
}
}
},
},
});

View File

@@ -0,0 +1,87 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {CallExpression, identifier} from '@babel/types';
import {NodePath} from '@babel/traverse';
const BUILTINS = [
'electron',
'buffer',
'child_process',
'crypto',
'dgram',
'dns',
'fs',
'http',
'https',
'net',
'os',
'readline',
'stream',
'string_decoder',
'tls',
'tty',
'zlib',
'constants',
'events',
'url',
'assert',
'util',
'path',
'perf_hooks',
'punycode',
'querystring',
'cluster',
'console',
'module',
'process',
'vm',
'domain',
'v8',
'repl',
'timers',
'node-fetch',
];
const IGNORED_MODULES = [
'bufferutil',
'utf-8-validate',
'spawn-sync',
'./src/logcat',
'./src/monkey',
'./src/adb',
];
module.exports = () => ({
name: 'infinity-import-react',
visitor: {
CallExpression(path: NodePath<CallExpression>) {
const node = path.node;
if (
node.type === 'CallExpression' &&
node.callee.type === 'Identifier' &&
node.callee.name === 'require' &&
node.arguments.length === 1 &&
node.arguments[0].type === 'StringLiteral'
) {
const source = node.arguments[0].value;
if (
BUILTINS.includes(source) ||
BUILTINS.some((moduleName) => source.startsWith(`${moduleName}/`))
) {
node.callee.name = 'electronRequire';
}
if (IGNORED_MODULES.includes(source)) {
path.replaceWith(identifier('triggerReferenceError'));
}
}
},
},
});

View File

@@ -0,0 +1,65 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {parseExpression} from '@babel/parser';
import {CallExpression} from '@babel/types';
import {NodePath} from '@babel/traverse';
const electronStubs = parseExpression(
`{
remote: {
process: {
env: {},
},
getCurrentWindow: function() {
return {
isFocused: function() {return true;},
on: function() {return true;}
};
},
app: {
getVersion: function() {return global.__VERSION__ || '1';},
getName: function() {return '';},
getAppPath: function() {return process.cwd();}
},
shell: {
openExternal: function() {}
},
Menu: {
buildFromTemplate: function() {
return {items: []}
},
setApplicationMenu: function() {}
}
},
ipcRenderer: {
on: function() {return true;}
},
}
`,
);
module.exports = () => ({
name: 'replace-electron-requires-with-stubs',
visitor: {
CallExpression(path: NodePath<CallExpression>) {
const node = path.node;
if (
node.type === 'CallExpression' &&
node.callee.type === 'Identifier' &&
node.callee.name === 'require' &&
node.arguments.length > 0 &&
node.arguments[0].type === 'StringLiteral' &&
node.arguments[0].value === 'electron'
) {
path.replaceWith(electronStubs);
}
},
},
});

View File

@@ -0,0 +1,49 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import path from 'path';
import {CallExpression} from '@babel/types';
import {NodePath} from '@babel/traverse';
const isFBFile = (filePath: string) =>
filePath.includes(`${path.sep}fb${path.sep}`);
const requireFromFolder = (folder: string, path: string) =>
new RegExp(folder + '/[A-Za-z0-9.-_]+(.js)?$', 'g').test(path);
module.exports = () => ({
name: 'replace-dynamic-requires',
visitor: {
CallExpression(path: NodePath<CallExpression>, state: any) {
if (
path.node.type === 'CallExpression' &&
path.node.callee.type === 'Identifier' &&
path.node.callee.name === 'require' &&
path.node.arguments.length > 0 &&
path.node.arguments[0].type === 'StringLiteral'
) {
if (
requireFromFolder('fb', path.node.arguments[0].value) &&
!isFBFile(state.file.opts.filename)
) {
throw new Error(
'For files which are not under fb/ do not require directly from fb/, but rather from fb-stubs/ to not break flow-typing and make sure stubs are up-to-date.',
);
} else if (
requireFromFolder('fb-stubs', path.node.arguments[0].value)
) {
path.node.arguments[0].value = path.node.arguments[0].value.replace(
'/fb-stubs/',
'/fb/',
);
}
}
},
},
});

View File

@@ -0,0 +1,89 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {
CallExpression,
Identifier,
isStringLiteral,
identifier,
} from '@babel/types';
import {NodePath} from '@babel/traverse';
import {resolve, dirname, relative} from 'path';
// do not apply this transform for these paths
const EXCLUDE_PATHS = [
'/node_modules/react-devtools-core/',
'relay-devtools/DevtoolsUI',
];
function isExcludedPath(path: string) {
for (const epath of EXCLUDE_PATHS) {
if (path.indexOf(epath) > -1) {
return true;
}
}
return false;
}
module.exports = () => ({
visitor: {
CallExpression(path: NodePath<CallExpression>, state: any) {
if (isExcludedPath(state.file.opts.filename)) {
return;
}
const node = path.node;
const args = node.arguments || [];
if (
node.callee.type === 'Identifier' &&
node.callee.name === 'require' &&
args.length === 1 &&
isStringLiteral(args[0])
) {
if (args[0].value === 'flipper') {
path.replaceWith(identifier('global.Flipper'));
} else if (args[0].value === 'react') {
path.replaceWith(identifier('global.React'));
} else if (args[0].value === 'react-dom') {
path.replaceWith(identifier('global.ReactDOM'));
} else if (args[0].value === 'adbkit') {
path.replaceWith(identifier('global.adbkit'));
} else if (
// require a file not a pacakge
args[0].value.indexOf('/') > -1 &&
// in the plugin itself and not inside one of its dependencies
state.file.opts.filename.indexOf('node_modules') === -1 &&
// the resolved path for this file is outside the plugins root
!resolve(
state.file.opts.root,
relative(state.file.opts.cwd, dirname(state.file.opts.filename)),
args[0].value,
).startsWith(state.file.opts.root)
) {
throw new Error(
`Plugins cannot require files from outside their folder. Attempted to require ${resolve(
state.file.opts.root,
relative(state.file.opts.cwd, dirname(state.file.opts.filename)),
args[0].value,
)} which isn't inside ${state.file.opts.root}`,
);
}
}
},
Identifier(path: NodePath<Identifier>, state: any) {
if (
path.node.name === 'React' &&
(path.parentPath.node as any).id !== path.node &&
!isExcludedPath(state.file.opts.filename)
) {
path.replaceWith(identifier('global.React'));
}
},
},
});

View File

@@ -0,0 +1,15 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
// Disable caching of babel transforms all together. We haven't found a good
// way to cache our transforms, as they rely on side effects like env vars or
// the existence of folders in the file system.
export default function getCacheKey() {
return Math.random().toString(36);
}

View File

@@ -0,0 +1,57 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {
variableDeclarator,
variableDeclaration,
identifier,
callExpression,
stringLiteral,
memberExpression,
Identifier,
Program,
} from '@babel/types';
import {NodePath} from '@babel/traverse';
module.exports = () => ({
name: 'infinity-import-react',
visitor: {
Program: {
exit(path: NodePath<Program>, state: any) {
if (state.get('NEEDS_REACT')) {
path.unshiftContainer('body', [
variableDeclaration('var', [
variableDeclarator(
identifier('React'),
callExpression(identifier('require'), [stringLiteral('react')]),
),
]),
]);
}
},
},
ReferencedIdentifier(path: NodePath<Identifier>, state: any) {
// mark react as needing to be imported
if (path.node.name === 'React' && !path.scope.getBinding('React')) {
state.set('NEEDS_REACT', true);
}
// replace Buffer with require('buffer')
if (path.node.name === 'Buffer' && !path.scope.getBinding('Buffer')) {
path.replaceWith(
memberExpression(
callExpression(identifier('require'), [stringLiteral('buffer')]),
identifier('Buffer'),
),
);
}
},
},
});

View File

@@ -0,0 +1,18 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as transform} from './transform-plugin';
import {default as getCacheKey} from './get-cache-key';
module.exports = {
transform,
getCacheKey,
};
export default transform;

View File

@@ -0,0 +1,38 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as doTransform} from './transform';
import {default as getCacheKey} from './get-cache-key';
module.exports = {
transform,
getCacheKey,
};
function transform({
filename,
options,
src,
}: {
filename: string;
options: any;
src: string;
}) {
const presets = [require('@babel/preset-react')];
const plugins = [];
if (process.env.FLIPPER_FB) {
plugins.push(require('./fb-stubs'));
}
if (process.env.BUILD_HEADLESS) {
plugins.push(require('./electron-stubs'));
}
plugins.push(require('./electron-requires'));
plugins.push(require('./import-react'));
return doTransform({filename, options, src, presets, plugins});
}

View File

@@ -0,0 +1,37 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as doTransform} from './transform';
import {default as getCacheKey} from './get-cache-key';
module.exports = {
transform,
getCacheKey,
};
function transform({
filename,
options,
src,
}: {
filename: string;
options: any;
src: string;
}) {
const presets = [require('@babel/preset-react')];
const plugins = [];
if (process.env.FLIPPER_FB) {
plugins.push(require('./fb-stubs'));
}
if (process.env.BUILD_HEADLESS) {
plugins.push(require('./electron-stubs'));
}
plugins.push(require('./import-react'));
return doTransform({filename, options, src, presets, plugins});
}

View File

@@ -0,0 +1,40 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as doTransform} from './transform';
import {default as getCacheKey} from './get-cache-key';
module.exports = {
transform,
getCacheKey,
};
function transform({
filename,
options,
src,
}: {
filename: string;
options: any;
src: string;
}) {
const presets = [
[
require('@babel/preset-env'),
{targets: {electron: process.env.FLIPPER_ELECTRON_VERSION}},
],
];
const plugins = [];
if (process.env.FLIPPER_FB) {
plugins.push(require('./fb-stubs'));
}
plugins.push(require('./electron-requires-main'));
plugins.push(require('./electron-process'));
return doTransform({filename, options, src, presets, plugins});
}

View File

@@ -0,0 +1,36 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as doTransform} from './transform';
export default function transform({
filename,
options,
src,
presets,
plugins,
}: {
filename: string;
options: any;
src: string;
presets: any[];
plugins: any[];
}) {
presets = presets ?? [require('@babel/preset-react')];
plugins = plugins ?? [];
if (process.env.FLIPPER_FB) {
plugins.push(require('./fb-stubs'));
}
if (process.env.BUILD_HEADLESS) {
plugins.push(require('./electron-stubs'));
}
plugins.push(require('./electron-requires'));
plugins.push(require('./flipper-requires'));
return doTransform({filename, options, src, presets, plugins});
}

View File

@@ -0,0 +1,101 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {default as generate} from '@babel/generator';
import {parse} from '@babel/parser';
import {transformFromAstSync} from '@babel/core';
export default function transform({
filename,
options,
src,
presets,
plugins,
}: {
filename: string;
options: any;
src: string;
presets?: any[];
plugins?: any[];
}) {
presets = presets ?? [require('@babel/preset-react')];
plugins = plugins ?? [];
const isTypeScript = filename.endsWith('.tsx') || filename.endsWith('.ts');
if (!isTypeScript) {
plugins.unshift(
require('@babel/plugin-transform-modules-commonjs'),
require('@babel/plugin-proposal-object-rest-spread'),
require('@babel/plugin-proposal-class-properties'),
require('@babel/plugin-transform-flow-strip-types'),
require('@babel/plugin-proposal-optional-chaining'),
require('@babel/plugin-proposal-nullish-coalescing-operator'),
require('./dynamic-requires'),
);
} else {
plugins.unshift(
require('@babel/plugin-transform-typescript'),
require('@babel/plugin-proposal-class-properties'),
require('@babel/plugin-transform-modules-commonjs'),
require('@babel/plugin-proposal-optional-chaining'),
require('@babel/plugin-proposal-nullish-coalescing-operator'),
);
}
const ast = parse(src, {
sourceFilename: filename,
plugins: isTypeScript
? [
'jsx',
'typescript',
'classProperties',
'optionalChaining',
'nullishCoalescingOperator',
]
: [
'jsx',
['flow', {all: true}],
'classProperties',
'objectRestSpread',
'optionalChaining',
'nullishCoalescingOperator',
],
sourceType: 'module',
});
const transformed = transformFromAstSync(ast, src, {
ast: true,
babelrc: !filename.includes('node_modules'),
code: false,
comments: false,
compact: false,
root: options.projectRoot,
filename,
plugins,
presets,
sourceMaps: true,
retainLines: !!options.isTestRunner,
});
if (!transformed) {
throw new Error('Failed to transform');
}
const result = generate(
transformed.ast!,
{
filename,
sourceFileName: filename,
sourceMaps: true,
retainLines: !!options.isTestRunner,
},
src,
);
return {
ast: transformed.ast,
code: result.code,
filename,
map: result.map,
};
}