Merge branch '0.227' into universalBuild

This commit is contained in:
2023-10-18 10:19:18 +02:00
1013 changed files with 58711 additions and 20312 deletions

View File

@@ -9,6 +9,8 @@
const dotenv = require('dotenv').config();
import path from 'path';
import os from 'os';
import tar from 'tar';
import {
buildBrowserBundle,
buildFolder,
@@ -16,37 +18,56 @@ import {
genMercurialRevision,
getVersionNumber,
prepareDefaultPlugins,
prepareHeadlessPlugins,
moveServerSourceMaps,
} from './build-utils';
import {defaultPluginsDir, distDir, serverDir, staticDir} from './paths';
import {
defaultPluginsDir,
distDir,
serverDir,
staticDir,
rootDir,
} from './paths';
import isFB from './isFB';
import yargs from 'yargs';
import fs from 'fs-extra';
import {downloadIcons} from './build-icons';
import {spawn} from 'promisify-child-process';
import {spawn, exec as execAsync} from 'promisify-child-process';
import {homedir} from 'os';
import {need as pkgFetch} from 'pkg-fetch';
import {exec} from 'child_process';
import fetch from '@adobe/node-fetch-retry';
import plist from 'simple-plist';
// This needs to be tested individually. As of 2022Q2, node17 is not supported.
const SUPPORTED_NODE_PLATFORM = 'node16';
// Node version below is only used for macOS AARCH64 builds as we download
// the binary directly from Node distribution site instead of relying on pkg-fetch.
const NODE_VERSION = 'v16.15.0';
enum BuildPlatform {
LINUX = 'linux',
WINDOWS = 'windows',
MAC_X64 = 'mac-x64',
MAC_AARCH64 = 'mac-aarch64',
}
const LINUX_STARTUP_SCRIPT = `#!/bin/sh
THIS_DIR="$( cd "$( dirname "\${BASH_SOURCE[0]}" )" && pwd )"
cd "$THIS_DIR"
./flipper-runtime ./server "$@"
`;
const WINDOWS_STARTUP_SCRIPT = `@echo off
setlocal
set "THIS_DIR=%~dp0"
cd /d "%THIS_DIR%"
flipper-runtime server %*
`;
const argv = yargs
.usage('yarn build-flipper-server [args]')
.version(false)
.options({
'default-plugins': {
describe:
'Enables embedding of default plugins into Flipper package so they are always available. The flag is enabled by default. Env var FLIPPER_NO_DEFAULT_PLUGINS is equivalent to the command-line option "--no-default-plugins".',
type: 'boolean',
default: true,
},
'public-build': {
describe:
'[FB-internal only] Will force using public sources only, to be able to iterate quickly on the public version. If sources are checked out from GitHub this is already the default. Setting env var "FLIPPER_FORCE_PUBLIC_BUILD" is equivalent.',
@@ -71,11 +92,6 @@ const argv = yargs
type: 'boolean',
default: false,
},
tcp: {
describe: 'Enable TCP connections on flipper-server.',
type: 'boolean',
default: true,
},
'rebuild-plugins': {
describe:
'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.',
@@ -93,12 +109,6 @@ const argv = yargs
choices: ['stable', 'insiders'],
default: 'stable',
},
'bundled-plugins': {
describe:
'Enables bundling of plugins into Flipper bundle. Env var FLIPPER_NO_BUNDLED_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".',
type: 'boolean',
default: false,
},
'default-plugins-dir': {
describe:
'Directory with prepared list of default plugins which will be included into the Flipper distribution as "defaultPlugins" dir',
@@ -114,6 +124,13 @@ const argv = yargs
'Unique build identifier to be used as the version patch part for the build',
type: 'number',
},
// On intern we ship flipper-server with node_modules (no big internet behind the firewall). yarn.lock makes sure that a CI that builds flipper-server installs the same dependencies all the time.
'generate-lock': {
describe:
'Generate a new yarn.lock file for flipper-server prod build. It is used for reproducible builds of the final artifact for the intern.',
type: 'boolean',
default: false,
},
mac: {
describe: 'Build a platform-specific bundle for MacOS.',
type: 'boolean',
@@ -127,6 +144,11 @@ const argv = yargs
linux: {
describe: 'Build a platform-specific bundle for Linux.',
},
dmg: {
describe: 'Package built server as a DMG file (Only for a MacOS build).',
type: 'boolean',
default: false,
},
})
.help()
.parse(process.argv.slice(1));
@@ -137,24 +159,6 @@ if (isFB) {
process.env.FLIPPER_RELEASE_CHANNEL = argv.channel;
if (argv['bundled-plugins'] === false) {
process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true';
} else if (argv['bundled-plugins'] === true) {
delete process.env.FLIPPER_NO_BUNDLED_PLUGINS;
}
if (argv['default-plugins'] === true) {
delete process.env.FLIPPER_NO_DEFAULT_PLUGINS;
} else if (argv['default-plugins'] === false) {
process.env.FLIPPER_NO_DEFAULT_PLUGINS = 'true';
}
// Don't rebuild default plugins, mostly to speed up testing
if (argv['rebuild-plugins'] === false) {
process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true';
} else if (argv['rebuild-plugins'] === true) {
delete process.env.FLIPPER_NO_REBUILD_PLUGINS;
}
if (argv['default-plugins-dir']) {
process.env.FLIPPER_DEFAULT_PLUGINS_DIR = argv['default-plugins-dir'];
}
@@ -172,10 +176,6 @@ if (argv['enabled-plugins'] !== undefined) {
process.env.FLIPPER_ENABLED_PLUGINS = argv['enabled-plugins'].join(',');
}
if (argv['default-plugins-dir']) {
process.env.FLIPPER_DEFAULT_PLUGINS_DIR = argv['default-plugins-dir'];
}
async function copyStaticResources(outDir: string, versionNumber: string) {
console.log(`⚙️ Copying default plugins...`);
@@ -217,7 +217,7 @@ async function copyStaticResources(outDir: string, versionNumber: string) {
console.log(`⚙️ Copying package resources...`);
// static folder, without the things that are only for Electron
const packageFilesToCopy = ['README.md', 'package.json', 'server.js', 'dist'];
const packageFilesToCopy = ['README.md', 'server.js', 'lib'];
await Promise.all(
packageFilesToCopy.map((e) =>
@@ -237,9 +237,14 @@ async function copyStaticResources(outDir: string, versionNumber: string) {
'icon.icns',
'icon.ico',
'icon.png',
'icon_grey.png',
'icons.json',
'index.web.dev.html',
'index.web.html',
'install_desktop.svg',
'loading.html',
'offline.html',
'service-worker.js',
'style.css',
];
if (isFB) {
@@ -251,10 +256,55 @@ async function copyStaticResources(outDir: string, versionNumber: string) {
fs.copy(path.join(staticDir, e), path.join(outDir, 'static', e)),
),
);
// Manifest needs to be copied over to static folder with the correct name.
await fs.copy(
path.join(staticDir, 'manifest.template.json'),
path.join(outDir, 'static', 'manifest.json'),
);
console.log('✅ Copied static resources.');
}
async function modifyPackageManifest(
async function linkLocalDeps(buildFolder: string) {
// eslint-disable-next-line no-console
console.log('Creating package.json manifest to link local deps');
const manifest = await fs.readJSON(path.resolve(serverDir, 'package.json'));
const resolutions = {
'flipper-doctor': `file:${rootDir}/doctor`,
'flipper-common': `file:${rootDir}/flipper-common`,
'flipper-frontend-core': `file:${rootDir}/flipper-frontend-core`,
'flipper-plugin-core': `file:${rootDir}/flipper-plugin-core`,
'flipper-server-client': `file:${rootDir}/flipper-server-client`,
'flipper-server-companion': `file:${rootDir}/flipper-server-companion`,
'flipper-server-core': `file:${rootDir}/flipper-server-core`,
'flipper-pkg-lib': `file:${rootDir}/pkg-lib`,
'flipper-plugin-lib': `file:${rootDir}/plugin-lib`,
};
manifest.resolutions = resolutions;
for (const depName of Object.keys(manifest.dependencies)) {
if (depName in resolutions) {
manifest.dependencies[depName] =
resolutions[depName as keyof typeof resolutions];
}
}
delete manifest.scripts;
delete manifest.devDependencies;
await fs.writeFile(
path.join(buildFolder, 'package.json'),
JSON.stringify(manifest, null, ' '),
);
await yarnInstall(buildFolder);
console.log('✅ Linked local deps');
}
async function modifyPackageManifestForPublishing(
buildFolder: string,
versionNumber: string,
hgRevision: string | null,
@@ -262,8 +312,7 @@ async function modifyPackageManifest(
) {
// eslint-disable-next-line no-console
console.log('Creating package.json manifest');
// eslint-disable-next-line flipper/no-relative-imports-across-packages
const manifest = require('../flipper-server/package.json');
const manifest = await fs.readJSON(path.resolve(serverDir, 'package.json'));
manifest.version = versionNumber;
manifest.private = false; // make this package npm-publishable
@@ -274,6 +323,15 @@ async function modifyPackageManifest(
// not needed in public builds
delete manifest.scripts;
delete manifest.devDependencies;
// update local monorepo dependencies' versions
// we will need them for npx to work
for (const depName of Object.keys(manifest.dependencies)) {
if (depName.startsWith('flipper')) {
manifest.dependencies[depName] = versionNumber;
}
}
await fs.writeFile(
path.join(buildFolder, 'package.json'),
JSON.stringify(manifest, null, ' '),
@@ -286,6 +344,7 @@ async function packNpmArchive(dir: string, versionNumber: any) {
const archive = path.resolve(distDir, 'flipper-server.tgz');
await spawn('yarn', ['pack', '--filename', archive], {
cwd: dir,
shell: true,
stdio: 'inherit',
});
@@ -301,36 +360,56 @@ async function runPostBuildAction(archive: string, dir: string) {
// didn't change
console.log(`⚙️ Installing flipper-server.tgz using npx`);
await fs.remove(path.join(homedir(), '.npm', '_npx'));
await spawn(
'npx',
[
archive,
argv.open ? '--open' : '--no-open',
argv.tcp ? '--tcp' : '--no-tcp',
],
{
stdio: 'inherit',
},
);
await spawn('npx', [archive, argv.open ? '--open' : '--no-open'], {
stdio: 'inherit',
shell: true,
});
} else if (argv.start) {
console.log(`⚙️ Starting flipper-server from build dir`);
await yarnInstall(dir);
await spawn(
'./server.js',
[argv.open ? '--open' : '--no-open', argv.tcp ? '--tcp' : '--no-tcp'],
{
cwd: dir,
stdio: 'inherit',
},
);
await spawn('./server.js', [argv.open ? '--open' : '--no-open'], {
cwd: dir,
stdio: 'inherit',
shell: true,
});
}
}
async function yarnInstall(dir: string) {
console.log(`⚙️ Running yarn install in ${dir}`);
await spawn('yarn', ['install', '--production', '--no-lockfile'], {
cwd: dir,
});
console.log(
`⚙️ Running yarn install in ${dir}. package.json: ${await fs.readFile(
path.resolve(dir, 'package.json'),
)}`,
);
if (!argv['generate-lock']) {
await fs.copyFile(
path.resolve(rootDir, 'yarn.flipper-server.lock'),
path.resolve(dir, 'yarn.lock'),
);
}
await spawn(
'yarn',
[
'install',
'--production',
...(process.env.SANDCASTLE ? ['--offline'] : []),
],
{
cwd: dir,
stdio: 'inherit',
shell: true,
},
);
if (argv['generate-lock']) {
await fs.copyFile(
path.resolve(dir, 'yarn.lock'),
path.resolve(rootDir, 'yarn.flipper-server.lock'),
);
}
await fs.rm(path.resolve(dir, 'yarn.lock'));
}
async function buildServerRelease() {
@@ -352,14 +431,19 @@ async function buildServerRelease() {
// create plugin output dir
await fs.mkdirp(path.join(dir, 'static', 'defaultPlugins'));
await compileServerMain(false);
await prepareDefaultPlugins(argv.channel === 'insiders');
await prepareHeadlessPlugins();
await compileServerMain(false);
await copyStaticResources(dir, versionNumber);
await linkLocalDeps(dir);
await downloadIcons(path.join(dir, 'static'));
await buildBrowserBundle(path.join(dir, 'static'), false);
await moveServerSourceMaps(dir, argv['source-map-dir']);
await modifyPackageManifest(dir, versionNumber, hgRevision, argv.channel);
await modifyPackageManifestForPublishing(
dir,
versionNumber,
hgRevision,
argv.channel,
);
const archive = await packNpmArchive(dir, versionNumber);
await runPostBuildAction(archive, dir);
@@ -367,32 +451,103 @@ async function buildServerRelease() {
if (argv.linux) {
platforms.push(BuildPlatform.LINUX);
}
// TODO: In the future, also cover aarch64 here.
if (argv.mac) {
platforms.push(BuildPlatform.MAC_X64);
platforms.push(BuildPlatform.MAC_AARCH64);
}
if (argv.win) {
platforms.push(BuildPlatform.WINDOWS);
}
if (platforms.length > 0) {
await yarnInstall(dir);
// Instead of parallel builds, these have to be done sequential.
// As we are building a native app, the resulting binary will be
// different per platform meaning that there's a risk of overriding
// intermediate artefacts if done in parallel.
for (const platform of platforms) {
await bundleServerReleaseForPlatform(dir, versionNumber, platform);
}
platforms.forEach(
bundleServerReleaseForPlatform.bind(null, dir, versionNumber),
);
}
function nodeArchFromBuildPlatform(_platform: BuildPlatform): string {
// TODO: Change this as we support aarch64.
function nodeArchFromBuildPlatform(platform: BuildPlatform): string {
if (platform === BuildPlatform.MAC_AARCH64) {
return 'arm64';
}
return 'x64';
}
/**
* Downloads a file located at the given URL and saves it to the destination path..
* @param url - URL of the file to download.
* @param dest - Destination path for the downloaded file.
* @returns - A promise that resolves when the file is downloaded.
* If the file can't be downloaded, it rejects with an error.
*/
async function download(url: string, dest: string): Promise<void> {
// First, check if the file already exists and remove it.
try {
await fs.access(dest, fs.constants.F_OK);
await fs.unlink(dest);
} catch (err) {}
return new Promise<void>((resolve, reject) => {
// Then, download the file and save it to the destination path.
const file: fs.WriteStream = fs.createWriteStream(dest);
fetch(url)
.then((response) => {
response.body.pipe(file);
response.body.on('error', (err) => {
throw err;
});
file.on('finish', () => {
file.close();
console.log(`✅ Download successful ${url} to ${dest}.`);
resolve();
});
})
.catch((error: Error) => {
console.log(`❌ Download failed ${url}. Error: ${error}`);
fs.unlink(dest);
reject(error);
});
});
}
/**
* Unpacks a tarball and extracts the contents to a directory.
* @param source - Source tarball.
* @param dest - Destination directory for the extracted contents.
*/
async function unpack(source: string, destination: string) {
console.log(`⚙️ Extracting ${source} to ${destination}.`);
try {
await fs.access(destination, fs.constants.F_OK);
await fs.rm(destination, {recursive: true, force: true});
} catch (err) {}
await fs.mkdir(destination);
try {
await tar.x({
file: source,
strip: 1,
cwd: destination,
});
console.log(`✅ Extraction completed.`);
} catch (error) {
console.error(
`❌ Error found whilst trying to extract '${source}'. Found: ${error}`,
);
}
}
function nodePlatformFromBuildPlatform(platform: BuildPlatform): string {
switch (platform) {
case BuildPlatform.LINUX:
return 'linux';
case BuildPlatform.MAC_X64:
case BuildPlatform.MAC_AARCH64:
return 'macos';
case BuildPlatform.WINDOWS:
return 'win32';
@@ -401,37 +556,232 @@ function nodePlatformFromBuildPlatform(platform: BuildPlatform): string {
}
}
async function setRuntimeAppIcon(binaryPath: string): Promise<void> {
console.log(`⚙️ Updating runtime icon for MacOS in ${binaryPath}.`);
const iconPath = path.join(staticDir, 'icon.png');
const tempRsrcPath = path.join(os.tmpdir(), 'icon.rsrc');
const deRezCmd = `DeRez -only icns ${iconPath} > ${tempRsrcPath}`;
try {
await execAsync(deRezCmd);
} catch (err) {
console.error(
`❌ Error while extracting icon with '${deRezCmd}'. Error: ${err}`,
);
throw err;
}
const rezCmd = `Rez -append ${tempRsrcPath} -o ${binaryPath}`;
try {
await execAsync(rezCmd);
} catch (err) {
console.error(
`❌ Error while setting icon on executable ${binaryPath}. Error: ${err}`,
);
throw err;
}
const updateCmd = `SetFile -a C ${binaryPath}`;
try {
await execAsync(updateCmd);
} catch (err) {
console.error(
`❌ Error while changing icon visibility on ${binaryPath}. Error: ${err}`,
);
throw err;
}
console.log(`✅ Updated flipper-runtime icon.`);
}
async function installNodeBinary(outputPath: string, platform: BuildPlatform) {
const path = await pkgFetch({
arch: nodeArchFromBuildPlatform(platform),
platform: nodePlatformFromBuildPlatform(platform),
nodeRange: SUPPORTED_NODE_PLATFORM,
});
await fs.copyFile(path, outputPath);
/**
* Below is a temporary patch that doesn't use pkg-fetch to
* download a node binary for macOS arm64.
* This will be removed once there's a properly
* signed binary for macOS arm64 architecture.
*/
if (platform === BuildPlatform.MAC_AARCH64) {
const temporaryDirectory = os.tmpdir();
const name = `node-${NODE_VERSION}-darwin-arm64`;
const downloadOutputPath = path.resolve(
temporaryDirectory,
`${name}.tar.gz`,
);
const unpackedOutputPath = path.resolve(temporaryDirectory, name);
let nodePath = path.resolve(unpackedOutputPath, 'bin', 'node');
console.log(
`⚙️ Downloading node version for ${platform} using temporary patch.`,
);
// Check local cache.
let cached = false;
try {
const cachePath = path.join(homedir(), '.node', name);
await fs.access(cachePath, fs.constants.F_OK);
console.log(`⚙️ Cached artifact found, skip download.`);
nodePath = path.resolve(cachePath, 'bin', 'node');
cached = true;
} catch (err) {}
if (!cached) {
// Download node tarball from the distribution site.
// If this is not present (due to a node update) follow these steps:
// - Update the download URL to `https://nodejs.org/dist/${NODE_VERSION}/${name}.tar.gz`
// - Ensure the Xcode developer tools are installed
// - Build a full MacOS server release locally using `yarn build:flipper-server --mac`
// - Enter the dist folder: dist/flipper-server-mac-aarch64/Flipper.app/Contents/MacOS
// - `mkdir bin && cp flipper-runtime bin/node && tar -czvf node-${NODE_VERSION}-darwin-arm64.tar.gz bin`
// - Upload the resulting tar ball to the Flipper release page as a new tag: https://github.com/facebook/flipper/releases
await download(
`https://github.com/facebook/flipper/releases/download/node-${NODE_VERSION}/${name}.tar.gz`,
downloadOutputPath,
);
// Finally, unpack the tarball to a local folder i.e. outputPath.
await unpack(downloadOutputPath, unpackedOutputPath);
console.log(`✅ Node successfully downloaded and unpacked.`);
}
console.log(`⚙️ Copying node binary from ${nodePath} to ${outputPath}`);
await fs.copyFile(nodePath, outputPath);
} else {
console.log(`⚙️ Downloading node version for ${platform} using pkg-fetch`);
const nodePath = await pkgFetch({
arch: nodeArchFromBuildPlatform(platform),
platform: nodePlatformFromBuildPlatform(platform),
nodeRange: SUPPORTED_NODE_PLATFORM,
});
console.log(`⚙️ Copying node binary from ${nodePath} to ${outputPath}`);
await fs.copyFile(nodePath, outputPath);
}
if (
platform === BuildPlatform.MAC_AARCH64 ||
platform === BuildPlatform.MAC_X64
) {
if (process.platform === 'darwin') {
await setRuntimeAppIcon(outputPath).catch(() => {
console.warn('⚠️ Unable to update runtime icon');
});
} else {
console.warn("⚠️ Skipping icon update as it's only supported on macOS");
}
}
// Set +x on the binary as copyFile doesn't maintain the bit.
await fs.chmod(outputPath, 0o755);
}
async function createMacDMG(
platform: BuildPlatform,
outputPath: string,
destPath: string,
) {
console.log(`⚙️ Create macOS DMG from: ${outputPath}`);
const name = `Flipper-server-${platform}.dmg`;
const temporaryDirectory = os.tmpdir();
const dmgOutputPath = path.resolve(temporaryDirectory, name);
await fs.remove(dmgOutputPath);
const dmgPath = path.resolve(destPath, name);
const cmd = `hdiutil create -format UDZO -srcfolder "${outputPath}/" -volname "Flipper" ${dmgOutputPath}`;
return new Promise<void>((resolve, reject) => {
exec(cmd, async (error, _stdout, stderr) => {
if (error) {
console.error(`❌ Failed to create DMG with error: ${error.message}`);
return reject(error);
}
if (stderr) {
console.error(`❌ Failed to create DMG with error: ${stderr}`);
return reject(new Error(stderr));
}
await fs.move(dmgOutputPath, dmgPath);
await fs.remove(outputPath);
console.log(`✅ DMG successfully created ${dmgPath}`);
resolve();
});
});
}
async function setUpLinuxBundle(outputDir: string) {
console.log(`⚙️ Creating Linux startup script in ${outputDir}/flipper`);
await fs.writeFile(path.join(outputDir, 'flipper'), LINUX_STARTUP_SCRIPT);
// Give the script +x
await fs.chmod(path.join(outputDir, 'flipper'), 0o755);
}
async function setUpWindowsBundle(outputDir: string) {
console.log(`⚙️ Creating Windows bundle in ${outputDir}`);
await fs.writeFile(
path.join(outputDir, 'flipper.bat'),
WINDOWS_STARTUP_SCRIPT,
);
// Give the script +x
await fs.chmod(path.join(outputDir, 'flipper.bat'), 0o755);
}
async function setUpMacBundle(
outputDir: string,
platform: BuildPlatform,
versionNumber: string,
): Promise<{nodePath: string; resourcesPath: string}> {
console.log(`⚙️ Creating Mac bundle in ${outputDir}`);
await fs.copy(path.join(staticDir, 'flipper-server-app-template'), outputDir);
let appTemplate = path.join(staticDir, 'flipper-server-app-template');
if (isFB) {
appTemplate = path.join(
staticDir,
'facebook',
'flipper-server-app-template',
platform,
);
console.info('⚙️ Using internal template from: ' + appTemplate);
}
await fs.copy(appTemplate, outputDir);
function replacePropertyValue(
obj: any,
targetValue: string,
replacementValue: string,
): any {
if (typeof obj === 'object' && !Array.isArray(obj) && obj !== null) {
for (const key in obj) {
if (obj.hasOwnProperty(key)) {
obj[key] = replacePropertyValue(
obj[key],
targetValue,
replacementValue,
);
}
}
} else if (typeof obj === 'string' && obj === targetValue) {
obj = replacementValue;
}
return obj;
}
console.log(`⚙️ Writing plist`);
const pListPath = path.join(
const plistPath = path.join(
outputDir,
'Flipper.app',
'Contents',
'Info.plist',
);
const pListContents = await fs.readFile(pListPath, 'utf-8');
const updatedPlistContents = pListContents.replace(
/* eslint-disable node/no-sync*/
const pListContents: Record<any, any> = plist.readFileSync(plistPath);
replacePropertyValue(
pListContents,
'{flipper-server-version}',
versionNumber,
);
await fs.writeFile(pListPath, updatedPlistContents, 'utf-8');
plist.writeBinaryFileSync(plistPath, pListContents);
/* eslint-enable node/no-sync*/
const resourcesOutputDir = path.join(
outputDir,
@@ -440,12 +790,16 @@ async function setUpMacBundle(
'Resources',
'server',
);
if (!(await fs.exists(resourcesOutputDir))) {
await fs.mkdir(resourcesOutputDir);
}
const nodeOutputPath = path.join(
outputDir,
'Flipper.app',
'Contents',
'MacOS',
'node',
'flipper-runtime',
);
return {resourcesPath: resourcesOutputDir, nodePath: nodeOutputPath};
}
@@ -463,22 +817,40 @@ async function bundleServerReleaseForPlatform(
await fs.mkdirp(outputDir);
let outputPaths = {
nodePath: path.join(outputDir, 'node'),
nodePath: path.join(outputDir, 'flipper-runtime'),
resourcesPath: outputDir,
};
// On the mac, we need to set up a resource bundle which expects paths
// to be in different places from Linux/Windows bundles.
if (platform === BuildPlatform.MAC_X64) {
outputPaths = await setUpMacBundle(outputDir, versionNumber);
if (
platform === BuildPlatform.MAC_X64 ||
platform === BuildPlatform.MAC_AARCH64
) {
outputPaths = await setUpMacBundle(outputDir, platform, versionNumber);
} else if (platform === BuildPlatform.LINUX) {
await setUpLinuxBundle(outputDir);
} else if (platform === BuildPlatform.WINDOWS) {
await setUpWindowsBundle(outputDir);
}
console.log(`⚙️ Copying from ${dir} to ${outputPaths.resourcesPath}`);
await fs.copy(dir, outputPaths.resourcesPath);
await fs.copy(dir, outputPaths.resourcesPath, {
overwrite: true,
dereference: true,
});
console.log(`⚙️ Downloading compatible node version`);
await installNodeBinary(outputPaths.nodePath, platform);
if (
argv.dmg &&
(platform === BuildPlatform.MAC_X64 ||
platform === BuildPlatform.MAC_AARCH64)
) {
await createMacDMG(platform, outputDir, distDir);
}
console.log(`✅ Wrote ${platform}-specific server version to ${outputDir}`);
}

View File

@@ -13,6 +13,8 @@ import fetch from '@adobe/node-fetch-retry';
// eslint-disable-next-line node/no-extraneous-import
import type {Icon} from 'flipper-ui-core';
const AVAILABLE_SIZES: Icon['size'][] = [8, 10, 12, 16, 18, 20, 24, 28, 32];
export type Icons = {
[key: string]: Icon['size'][];
};
@@ -48,45 +50,53 @@ export async function downloadIcons(buildFolder: string) {
[],
);
return Promise.all(
iconURLs.map((icon) => {
const url = getPublicIconUrl(icon);
return fetch(url, {
retryOptions: {
retryMaxDuration: 120 * 1000,
// Be default, only 5xx are retried but we're getting the odd 404
// which goes away on a retry for some reason.
retryOnHttpResponse: (res) => res.status >= 400,
retryOnHttpError: () => true,
},
})
.then((res) => {
if (res.status !== 200) {
throw new Error(
// eslint-disable-next-line prettier/prettier
`Could not download the icon ${icon} from ${url}: got status ${res.status}`,
);
}
return res;
})
.then(
(res) =>
new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(
path.join(buildFolder, buildLocalIconPath(icon)),
);
res.body.pipe(fileStream);
res.body.on('error', reject);
fileStream.on('finish', resolve);
}),
);
await Promise.all(
iconURLs.map(async (icon) => {
const sizeIndex = AVAILABLE_SIZES.indexOf(icon.size);
if (sizeIndex === -1) {
throw new Error('Size unavailable: ' + icon.size);
}
const sizesToTry = AVAILABLE_SIZES.slice(sizeIndex);
while (sizesToTry.length) {
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const size = sizesToTry.shift()!;
const url = getPublicIconUrl({...icon, size});
const res = await fetch(url);
if (res.status !== 200) {
// console.log(
// // eslint-disable-next-line prettier/prettier
// `Could not download the icon ${
// icon.name
// } at size ${size} from ${url}: got status ${
// res.status
// }. Will fallback to one of the sizes: ${sizesToTry.join(' or ')}`,
// );
// not available at this size, pick the next
continue;
}
return new Promise((resolve, reject) => {
const fileStream = fs.createWriteStream(
path.join(buildFolder, buildLocalIconPath(icon)),
);
res.body.pipe(fileStream);
res.body.on('error', reject);
fileStream.on('finish', resolve);
});
}
console.error(
`Could not download the icon ${JSON.stringify(
icon,
)} from ${getPublicIconUrl(icon)}, didn't find any matching size`,
);
}),
);
}
// should match flipper-ui-core/src/utils/icons.tsx
export function getPublicIconUrl({name, variant, size, density}: Icon) {
return `https://facebook.com/assets/?name=${name}&variant=${variant}&size=${size}&set=facebook_icons&density=${density}x`;
return `https://facebook.com/images/assets_DO_NOT_HARDCODE/facebook_icons/${name}_${variant}_${size}.png`;
}
// should match app/src/utils/icons.tsx

View File

@@ -16,6 +16,7 @@ import yargs from 'yargs';
import tmp from 'tmp';
import {execSync} from 'child_process';
import {promisify} from 'util';
import isFB from './isFB';
const argv = yargs
.usage('yarn build-plugin [args]')
@@ -80,7 +81,7 @@ async function buildPlugin() {
const outputSourcemapServerAddOnArg = argv['output-sourcemap-server-addon'];
const packageJsonPath = path.join(pluginDir, 'package.json');
const packageJsonOverridePath = path.join(pluginDir, 'fb', 'package.json');
await runBuild(pluginDir, false, {
await runBuild(pluginDir, false, isFB, {
sourceMapPath: outputSourcemapArg,
sourceMapPathServerAddOn: outputSourcemapServerAddOnArg,
});

View File

@@ -78,16 +78,6 @@ const argv = yargs
choices: ['stable', 'insiders'],
default: 'stable',
},
'bundled-plugins': {
describe:
'Enables bundling of plugins into Flipper bundle. Env var FLIPPER_NO_BUNDLED_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".',
type: 'boolean',
},
'rebuild-plugins': {
describe:
'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.',
type: 'boolean',
},
'default-plugins-dir': {
describe:
'Directory with prepared list of default plugins which will be included into the Flipper distribution as "defaultPlugins" dir',
@@ -121,18 +111,6 @@ if (isFB) {
process.env.FLIPPER_RELEASE_CHANNEL = argv.channel;
if (argv['bundled-plugins'] === false) {
process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true';
} else if (argv['bundled-plugins'] === true) {
delete process.env.FLIPPER_NO_BUNDLED_PLUGINS;
}
if (argv['rebuild-plugins'] === false) {
process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true';
} else if (argv['rebuild-plugins'] === true) {
delete process.env.FLIPPER_NO_REBUILD_PLUGINS;
}
if (argv['default-plugins-dir']) {
process.env.FLIPPER_DEFAULT_PLUGINS_DIR = argv['default-plugins-dir'];
}
@@ -227,9 +205,13 @@ async function buildDist(buildFolder: string) {
if (argv['mac-dmg']) {
targetsRaw.push(Platform.MAC.createTarget(['dmg'], Arch.universal));
}
const macPath = path.join(
distDir,
process.arch === 'arm64' ? 'mac-arm64' : 'mac',
);
postBuildCallbacks.push(() =>
spawn('zip', ['-qyr9', '../Flipper-mac.zip', 'Flipper.app'], {
cwd: path.join(distDir, 'mac-universal'),
cwd: macPath,
encoding: 'utf-8',
}),
);

View File

@@ -16,18 +16,15 @@ import MetroResolver from 'metro-resolver';
import tmp from 'tmp';
import path from 'path';
import fs from 'fs-extra';
import {spawn} from 'promisify-child-process';
import {spawn, exec} from 'promisify-child-process';
import {
buildDefaultPlugins,
getDefaultPlugins,
getWatchFolders,
runBuild,
stripSourceMapComment,
} from 'flipper-pkg-lib';
import getAppWatchFolders from './get-app-watch-folders';
import {getSourcePlugins, getPluginSourceFolders} from 'flipper-plugin-lib';
import type {
BundledPluginDetails,
InstalledPluginDetails,
} from 'flipper-common';
import {getPluginSourceFolders} from 'flipper-plugin-lib';
import {
appDir,
staticDir,
@@ -36,41 +33,13 @@ import {
serverDir,
rootDir,
browserUiDir,
serverCoreDir,
serverCompanionDir,
} from './paths';
import pFilter from 'p-filter';
import child from 'child_process';
import pMap from 'p-map';
// eslint-disable-next-line flipper/no-relative-imports-across-packages
const {version} = require('../package.json');
import isFB from './isFB';
const dev = process.env.NODE_ENV !== 'production';
// For insiders builds we bundle top 5 popular device plugins,
// plus top 10 popular "universal" plugins enabled by more than 100 users.
const hardcodedPlugins = new Set<string>([
// Popular device plugins
'DeviceLogs',
'CrashReporter',
'MobileBuilds',
'Hermesdebuggerrn',
'React',
// Popular client plugins
'Inspector',
'Network',
'AnalyticsLogging',
'GraphQL',
'UIPerf',
'MobileConfig',
'Databases',
'FunnelLogger',
'Navigation',
'Fresco',
'Preferences',
]);
export function die(err: Error) {
console.error('Script termnated.', err);
process.exit(1);
@@ -88,173 +57,19 @@ export async function prepareDefaultPlugins(isInsidersBuild: boolean = false) {
`⚙️ Copying the provided default plugins dir "${forcedDefaultPluginsDir}"...`,
);
await fs.copy(forcedDefaultPluginsDir, defaultPluginsDir, {
recursive: true,
overwrite: true,
dereference: true,
});
console.log('✅ Copied the provided default plugins dir.');
await generateDefaultPluginEntryPoints([]); // calling it here just to generate empty indexes
} else {
const sourcePlugins = process.env.FLIPPER_NO_DEFAULT_PLUGINS
? []
: await getSourcePlugins();
const defaultPlugins = sourcePlugins
// we only include predefined set of plugins into insiders release
.filter((p) => !isInsidersBuild || hardcodedPlugins.has(p.id));
if (process.env.FLIPPER_NO_BUNDLED_PLUGINS) {
await buildDefaultPlugins(defaultPlugins);
await generateDefaultPluginEntryPoints([]); // calling it here just to generate empty indexes
} else {
await generateDefaultPluginEntryPoints(defaultPlugins);
}
}
console.log('✅ Prepared default plugins.');
}
export async function prepareHeadlessPlugins() {
console.log(`⚙️ Preparing headless plugins...`);
const sourcePlugins = await getSourcePlugins();
const headlessPlugins = sourcePlugins.filter((p) => p.headless);
await generateHeadlessPluginEntryPoints(headlessPlugins);
console.log('✅ Prepared headless plugins.');
}
function getGeneratedIndex(pluginRequires: string) {
return `
/* eslint-disable */
// THIS FILE IS AUTO-GENERATED by function "generateDefaultPluginEntryPoints" in "build-utils.ts".
declare const require: any;
// This function exists to make sure that if one require fails in its module initialisation, not everything fails
function tryRequire(module: string, fn: () => any): any {
try {
return fn();
} catch (e) {
console.error(\`Could not require \${module}: \`, e)
return {};
}
}
export default {\n${pluginRequires}\n} as any
`;
}
async function generateDefaultPluginEntryPoints(
defaultPlugins: InstalledPluginDetails[],
) {
console.log(
`⚙️ Generating entry points for ${defaultPlugins.length} bundled plugins...`,
);
const bundledPlugins = defaultPlugins.map(
(p) =>
({
...p,
isBundled: true,
version: p.version === '0.0.0' ? version : p.version,
flipperSDKVersion:
p.flipperSDKVersion === '0.0.0' ? version : p.flipperSDKVersion,
dir: undefined,
entry: undefined,
serverAddOnEntry: undefined,
} as BundledPluginDetails),
);
await fs.writeJSON(
path.join(defaultPluginsDir, 'bundled.json'),
bundledPlugins,
);
const pluginRequires = bundledPlugins
.map(
(x) =>
` '${x.name}': tryRequire('${x.name}', () => require('${x.name}'))`,
)
.join(',\n');
const generatedIndex = getGeneratedIndex(pluginRequires);
await fs.ensureDir(path.join(appDir, 'src', 'defaultPlugins'));
await fs.writeFile(
path.join(appDir, 'src', 'defaultPlugins', 'index.tsx'),
generatedIndex,
);
await fs.ensureDir(path.join(browserUiDir, 'src', 'defaultPlugins'));
await fs.writeFile(
path.join(browserUiDir, 'src', 'defaultPlugins', 'index.tsx'),
generatedIndex,
);
const serverAddOns = defaultPlugins.filter(
({serverAddOnSource}) => !!serverAddOnSource,
);
const serverAddOnRequires = serverAddOns
.map(
(x) =>
` '${x.name}': tryRequire('${x.name}', () => require('${x.name}/${x.serverAddOnSource}'))`,
)
.join(',\n');
const generatedIndexServerAddOns = getGeneratedIndex(serverAddOnRequires);
await fs.ensureDir(path.join(serverCoreDir, 'src', 'defaultPlugins'));
await fs.writeFile(
path.join(serverCoreDir, 'src', 'defaultPlugins', 'index.tsx'),
generatedIndexServerAddOns,
);
console.log('✅ Generated bundled plugin entry points.');
}
async function generateHeadlessPluginEntryPoints(
headlessPlugins: InstalledPluginDetails[],
) {
console.log(
`⚙️ Generating entry points for ${headlessPlugins.length} headless plugins...`,
);
const headlessRequires = headlessPlugins
.map(
(x) =>
` '${x.name}': tryRequire('${x.name}', () => require('${x.name}'))`,
)
.join(',\n');
const generatedIndexHeadless = getGeneratedIndex(headlessRequires);
await fs.ensureDir(path.join(serverCompanionDir, 'src', 'defaultPlugins'));
await fs.writeFile(
path.join(serverCompanionDir, 'src', 'defaultPlugins', 'index.tsx'),
generatedIndexHeadless,
);
console.log('✅ Generated headless plugin entry points.');
}
async function buildDefaultPlugins(defaultPlugins: InstalledPluginDetails[]) {
if (process.env.FLIPPER_NO_REBUILD_PLUGINS) {
console.log(
`⚙️ Including ${
defaultPlugins.length
} plugins into the default plugins list. Skipping rebuilding because "no-rebuild-plugins" option provided. List of default plugins: ${defaultPlugins
.map((p) => p.id)
.join(', ')}`,
const defaultPlugins = await getDefaultPlugins(isInsidersBuild);
await buildDefaultPlugins(
defaultPlugins,
dev,
isFB && !process.env.FLIPPER_FORCE_PUBLIC_BUILD,
);
}
await pMap(
defaultPlugins,
async function (plugin) {
try {
if (!process.env.FLIPPER_NO_REBUILD_PLUGINS) {
console.log(
`⚙️ Building plugin ${plugin.id} to include it into the default plugins list...`,
);
await runBuild(plugin.dir, dev);
}
await fs.ensureSymlink(
plugin.dir,
path.join(defaultPluginsDir, plugin.name),
'junction',
);
} catch (err) {
console.error(`✖ Failed to build plugin ${plugin.id}`, err);
}
},
{
concurrency: 16,
},
);
console.log('✅ Prepared default plugins.');
}
const minifierConfig = {
@@ -307,7 +122,7 @@ async function compile(
},
);
if (!dev) {
stripSourceMapComment(out);
await stripSourceMapComment(out);
}
}
@@ -352,8 +167,7 @@ export async function moveSourceMaps(
// If we don't move them out of the build folders, they'll get included in the ASAR
// which we don't want.
console.log(`⏭ Removing source maps.`);
await fs.remove(mainBundleMap);
await fs.remove(rendererBundleMap);
await Promise.all([fs.remove(mainBundleMap), fs.remove(rendererBundleMap)]);
}
}
@@ -362,13 +176,12 @@ export async function moveServerSourceMaps(
sourceMapFolder: string | undefined,
) {
console.log(`⚙️ Moving server source maps...`);
const mainBundleMap = path.join(buildFolder, 'dist', 'index.map');
const rendererBundleMap = path.join(buildFolder, 'static', 'bundle.map');
if (sourceMapFolder) {
await fs.ensureDir(sourceMapFolder);
await fs.move(mainBundleMap, path.join(sourceMapFolder, 'bundle.map'), {
overwrite: true,
});
// TODO: Remove me
// Create an empty file not satisfy Sandcastle. Remove it once Sandcastle no longer requires the file
await fs.writeFile(path.join(sourceMapFolder, 'bundle.map'), '{}');
await fs.move(
rendererBundleMap,
path.join(sourceMapFolder, 'main.bundle.map'),
@@ -378,7 +191,6 @@ export async function moveServerSourceMaps(
} else {
// Removing so we don't bundle them up as part of the release.
console.log(`⏭ Removing source maps.`);
await fs.remove(mainBundleMap);
await fs.remove(rendererBundleMap);
}
}
@@ -419,7 +231,7 @@ export async function compileMain() {
});
console.log('✅ Compiled main bundle.');
if (!dev) {
stripSourceMapComment(out);
await stripSourceMapComment(out);
}
} catch (err) {
die(err);
@@ -468,41 +280,9 @@ export function genMercurialRevision(): Promise<string | null> {
}
export async function compileServerMain(dev: boolean) {
await fs.promises.mkdir(path.join(serverDir, 'dist'), {recursive: true});
const out = path.join(serverDir, 'dist', 'index.js');
console.log('⚙️ Compiling server bundle...');
const config = Object.assign({}, await Metro.loadConfig(), {
reporter: {update: () => {}},
projectRoot: rootDir,
transformer: {
babelTransformerPath: path.join(
babelTransformationsDir,
'transform-server-' + (dev ? 'dev' : 'prod'),
),
...minifierConfig,
},
resolver: {
// no 'mjs' / 'module'; it caused issues
sourceExts: ['tsx', 'ts', 'js', 'json', 'cjs'],
resolverMainFields: ['flipperBundlerEntry', 'main'],
resolveRequest(context: any, moduleName: string, ...rest: any[]) {
assertSaneImport(context, moduleName);
return defaultResolve(context, moduleName, ...rest);
},
},
});
await Metro.runBuild(config, {
platform: 'node',
entry: path.join(serverDir, 'src', 'index.tsx'),
out,
dev,
minify: false, // !dev,
sourceMap: true,
sourceMapUrl: dev ? 'index.map' : undefined,
inlineSourceMap: false,
resetCache: !dev,
});
console.log('✅ Compiled server bundle.');
console.log('⚙️ Compiling server sources...');
await exec(`cd ${serverDir} && yarn build`);
console.log(' Compiled server sources.');
}
// TODO: needed?
@@ -603,15 +383,10 @@ export function sleep(ms: number) {
let proc: child.ChildProcess | undefined;
export async function launchServer(
startBundler: boolean,
open: boolean,
tcp: boolean,
) {
export async function launchServer(startBundler: boolean, open: boolean) {
if (proc) {
console.log('⚙️ Killing old flipper-server...');
proc.kill(9);
await sleep(1000);
}
console.log('⚙️ Launching flipper-server...');
proc = child.spawn(
@@ -621,7 +396,6 @@ export async function launchServer(
`../flipper-server/server.js`,
startBundler ? `--bundler` : `--no-bundler`,
open ? `--open` : `--no-open`,
tcp ? `--tcp` : `--no-tcp`,
],
{
cwd: serverDir,

View File

@@ -13,6 +13,7 @@ import {pluginsDir} from './paths';
import path from 'path';
import {runBuild} from 'flipper-pkg-lib';
import {getWorkspaces} from './workspaces';
import isFB from './isFB';
async function bundleAllPlugins() {
const plugins = await getWorkspaces().then((workspaces) =>
@@ -24,7 +25,7 @@ async function bundleAllPlugins() {
console.log(`Bundling "${relativeDir}"`);
console.time(`Finished bundling "${relativeDir}"`);
try {
await runBuild(plugin.dir, false);
await runBuild(plugin.dir, false, isFB);
} catch (err) {
console.log(`Failed to bundle "${relativeDir}": ${err.message}`);
errors.set(relativeDir, err);

View File

@@ -59,7 +59,6 @@ async function copyPackageWithDependenciesRecursive(
.then((l: Array<string>) => ignore().add(DEFAULT_BUILD_IGNORES.concat(l)));
await fs.copy(packageDir, targetDir, {
dereference: true,
recursive: true,
filter: (src) => {
const relativePath = path.relative(packageDir, src);
return relativePath === '' || !ignores.ignores(relativePath);

View File

@@ -1,19 +0,0 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
/* eslint-disable flipper/no-console-error-without-context */
import {prepareDefaultPlugins, prepareHeadlessPlugins} from './build-utils';
Promise.all([prepareDefaultPlugins(), prepareHeadlessPlugins()]).catch(
(err) => {
console.error(err);
process.exit(1);
},
);

View File

@@ -0,0 +1,11 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
* @ts-check
*/
module.exports = {};

View File

@@ -22,6 +22,7 @@ import {
ReleaseChannel,
Tristate,
parseEnvironmentVariables,
uuid,
} from 'flipper-common';
// Only import the type!
@@ -63,17 +64,51 @@ console.debug = function () {
};
// make perf tools available in Node (it is available in Browser / Electron just fine)
const {PerformanceObserver, performance} = require('perf_hooks');
Object.freeze(performance);
Object.freeze(Object.getPrototypeOf(performance));
import {PerformanceObserver, performance} from 'perf_hooks';
// Object.freeze(performance);
// Object.freeze(Object.getPrototypeOf(performance));
// Something in our unit tests is messing with the performance global
// This fixes that.....
// This fixes that.
let _performance = performance;
Object.defineProperty(global, 'performance', {
get() {
return performance;
return _performance;
},
set() {
throw new Error('Attempt to overwrite global.performance');
set(value) {
_performance = value;
if (typeof _performance.mark === 'undefined') {
_performance.mark = (_markName: string, _markOptions?) => {
return {
name: '',
detail: '',
duration: 0,
entryType: '',
startTime: 0,
toJSON() {},
};
};
}
if (typeof _performance.clearMarks === 'undefined') {
_performance.clearMarks = () => {};
}
if (typeof _performance.measure === 'undefined') {
_performance.measure = (
_measureName: string,
_startOrMeasureOptions?,
_endMark?: string | undefined,
) => {
return {
name: '',
detail: '',
duration: 0,
entryType: '',
startTime: 0,
toJSON() {},
};
};
}
},
});
@@ -97,6 +132,7 @@ Object.defineProperty(global, 'matchMedia', {
function createStubRenderHost(): RenderHost {
const rootPath = resolve(__dirname, '..');
const stubConfig: FlipperServerConfig = {
sessionId: uuid(),
environmentInfo: {
processId: process.pid,
appVersion: '0.0.0',
@@ -137,6 +173,8 @@ function createStubRenderHost(): RenderHost {
launcherEnabled: false,
launcherMsg: null,
screenCapturePath: `/dev/null`,
updaterEnabled: true,
suppressPluginUpdateNotifications: false,
},
settings: {
androidHome: `/dev/null`,
@@ -161,7 +199,7 @@ function createStubRenderHost(): RenderHost {
return {
readTextFromClipboard() {
return '';
return Promise.resolve('');
},
writeTextToClipboard() {},
async importFile() {
@@ -170,6 +208,9 @@ function createStubRenderHost(): RenderHost {
async exportFile() {
return undefined;
},
async exportFileBinary() {
return undefined;
},
hasFocus() {
return true;
},
@@ -181,15 +222,12 @@ function createStubRenderHost(): RenderHost {
restartFlipper() {},
openLink() {},
serverConfig: stubConfig,
loadDefaultPlugins() {
return {};
},
GK(gk: string) {
return stubConfig.gatekeepers[gk] ?? false;
},
flipperServer: TestUtils.createFlipperServerMock(),
async requirePlugin(path: string) {
return require(path);
return {plugin: require(path)};
},
getStaticResourceUrl(relativePath): string {
return 'file://' + resolve(rootPath, 'static', relativePath);

View File

@@ -18,11 +18,10 @@ process.env.FLIPPER_TEST_RUNNER = 'true';
const {transform} = require('../babel-transformer/lib/transform-jest');
module.exports = {
process(src, filename, config, options) {
process(src, filename, options) {
return transform({
src,
filename,
config,
options: {...options, isTestRunner: true},
});
},

View File

@@ -8,36 +8,39 @@
"bugs": "https://github.com/facebook/flipper/issues",
"devDependencies": {
"@adobe/node-fetch-retry": "^2.2.0",
"@babel/code-frame": "^7.16.7",
"@babel/code-frame": "^7.22.13",
"@types/adobe__node-fetch-retry": "^1.0.4",
"@types/babel__code-frame": "^7.0.3",
"@types/detect-port": "^1.3.2",
"@types/fs-extra": "^9.0.13",
"@types/detect-port": "^1.3.3",
"@types/fs-extra": "^11.0.0",
"@types/node": "^17.0.31",
"ansi-to-html": "^0.7.2",
"app-builder-lib": "23.4.0",
"app-builder-lib": "23.6.0",
"chalk": "^4",
"detect-port": "^1.1.1",
"dotenv": "^14.2.0",
"electron-builder": "23.0.3",
"express": "^4.17.3",
"fb-watchman": "^2.0.1",
"fast-glob": "3.3.1",
"flipper-common": "0.0.0",
"flipper-pkg-lib": "0.0.0",
"flipper-plugin-lib": "0.0.0",
"fs-extra": "^10.1.0",
"fs-extra": "^11.0.0",
"glob": "^8.0.1",
"ignore": "^5.1.4",
"ignore": "^5.2.4",
"metro": "^0.70.2",
"metro-minify-terser": "^0.70.2",
"metro-minify-terser": "^0.75.0",
"p-filter": "^2.1.0",
"p-map": "^4.0.0",
"pkg-fetch": "3.4.1",
"promisify-child-process": "^4.1.0",
"semver": "^7.5.4",
"simple-plist": "^1.3.1",
"socket.io": "^4.5.0",
"tar": "6.1.15",
"tmp": "^0.2.1",
"uuid": "^8.3.2",
"yargs": "^17.4.1"
"yargs": "^17.6.0"
},
"scripts": {},
"files": [

View File

@@ -0,0 +1,35 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
/* eslint-disable flipper/no-console-error-without-context */
import fs from 'fs-extra';
import path from 'path';
import {rootDir} from './paths';
// TODO: Remove me in 2023 when everyone who is building flipper from source have legacy plugin entry points removed by this script
(async () => {
await Promise.all(
[
path.resolve(rootDir, 'app/src/defaultPlugins'),
path.resolve(rootDir, 'flipper-server-companion/src/defaultPlugins'),
path.resolve(rootDir, 'flipper-server-core/src/defaultPlugins'),
path.resolve(rootDir, 'flipper-ui-browser/src/defaultPlugins'),
].map((dir) =>
fs.rm(dir, {
recursive: true,
force: true,
}),
),
);
})().catch((err) => {
console.error(err);
process.exit(1);
});

View File

@@ -20,12 +20,7 @@ import http from 'http';
import path from 'path';
import fs from 'fs-extra';
import {hostname} from 'os';
import {
compileMain,
prepareDefaultPlugins,
prepareHeadlessPlugins,
} from './build-utils';
import Watchman from './watchman';
import {compileMain, prepareDefaultPlugins} from './build-utils';
// @ts-ignore no typings for metro
import Metro from 'metro';
import {staticDir, babelTransformationsDir, rootDir} from './paths';
@@ -33,27 +28,12 @@ import isFB from './isFB';
import getAppWatchFolders from './get-app-watch-folders';
import {getPluginSourceFolders} from 'flipper-plugin-lib';
import ensurePluginFoldersWatchable from './ensurePluginFoldersWatchable';
import startWatchPlugins from './startWatchPlugins';
import yargs from 'yargs';
import {startWatchPlugins, Watchman} from 'flipper-pkg-lib';
const argv = yargs
.usage('yarn start [args]')
.options({
'default-plugins': {
describe:
'Enables embedding of default plugins into Flipper package so they are always available. The flag is enabled by default. Env var FLIPPER_NO_DEFAULT_PLUGINS is equivalent to the command-line option "--no-default-plugins".',
type: 'boolean',
},
'bundled-plugins': {
describe:
'Enables bundling of plugins into Flipper bundle. This is useful for debugging, because it makes Flipper dev mode loading faster and unblocks fast refresh. The flag is enabled by default. Env var FLIPPER_NO_BUNDLED_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".',
type: 'boolean',
},
'rebuild-plugins': {
describe:
'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.',
type: 'boolean',
},
'fast-refresh': {
describe:
'Enable Fast Refresh - quick reload of UI component changes without restarting Flipper. The flag is disabled by default. Env var FLIPPER_FAST_REFRESH is equivalent to the command-line option "--fast-refresh".',
@@ -125,24 +105,6 @@ if (isFB) {
process.env.FLIPPER_FB = 'true';
}
if (argv['default-plugins'] === true) {
delete process.env.FLIPPER_NO_DEFAULT_PLUGINS;
} else if (argv['default-plugins'] === false) {
process.env.FLIPPER_NO_DEFAULT_PLUGINS = 'true';
}
if (argv['bundled-plugins'] === true) {
delete process.env.FLIPPER_NO_BUNDLED_PLUGINS;
} else if (argv['bundled-plugins'] === false) {
process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true';
}
if (argv['rebuild-plugins'] === false) {
process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true';
} else if (argv['rebuild-plugins'] === true) {
delete process.env.FLIPPER_NO_REBUILD_PLUGINS;
}
if (argv['fast-refresh'] === true) {
process.env.FLIPPER_FAST_REFRESH = 'true';
} else if (argv['fast-refresh'] === false) {
@@ -348,11 +310,14 @@ async function startWatchChanges(io: socketIo.Server) {
await Promise.all(
[
'app',
'pkg',
'doctor',
'pkg-lib',
'plugin-lib',
'flipper-plugin',
'flipper-common',
'flipper-frontend-core',
'flipper-plugin',
'flipper-plugin-core',
'flipper-server-core',
'flipper-ui-core',
].map((dir) =>
watchman.startWatchFiles(
@@ -366,9 +331,6 @@ async function startWatchChanges(io: socketIo.Server) {
),
),
);
await startWatchPlugins(() => {
io.emit('refresh');
});
} catch (err) {
console.error(
'Failed to start watching for changes using Watchman, continue without hot reloading',
@@ -441,11 +403,9 @@ function checkDevServer() {
}
(async () => {
const isInsidersBuild = process.env.FLIPPER_RELEASE_CHANNEL === 'insiders';
checkDevServer();
await prepareDefaultPlugins(
process.env.FLIPPER_RELEASE_CHANNEL === 'insiders',
);
await prepareHeadlessPlugins();
await prepareDefaultPlugins(isInsidersBuild);
await ensurePluginFoldersWatchable();
const port = await detect(DEFAULT_PORT);
const {app, server} = await startAssetServer(port);
@@ -453,6 +413,13 @@ function checkDevServer() {
await startMetroServer(app, server);
outputScreen(socket);
await compileMain();
await startWatchPlugins(
isInsidersBuild,
isFB && !process.env.FLIPPER_FORCE_PUBLIC_BUILD,
(changedPlugins) => {
socket.emit('plugins-source-updated', changedPlugins);
},
);
if (dotenv && dotenv.parsed) {
console.log('✅ Loaded env vars from .env file: ', dotenv.parsed);
}

View File

@@ -14,31 +14,16 @@ import {
compileServerMain,
launchServer,
prepareDefaultPlugins,
prepareHeadlessPlugins,
} from './build-utils';
import Watchman from './watchman';
import isFB from './isFB';
import yargs from 'yargs';
import ensurePluginFoldersWatchable from './ensurePluginFoldersWatchable';
import {Watchman} from 'flipper-pkg-lib';
import fs from 'fs-extra';
const argv = yargs
.usage('yarn flipper-server [args]')
.options({
'default-plugins': {
describe:
'Enables embedding of default plugins into Flipper package so they are always available. The flag is enabled by default. Env var FLIPPER_NO_DEFAULT_PLUGINS is equivalent to the command-line option "--no-default-plugins".',
type: 'boolean',
},
'bundled-plugins': {
describe:
'Enables bundling of plugins into Flipper bundle. This is useful for debugging, because it makes Flipper dev mode loading faster and unblocks fast refresh. The flag is enabled by default. Env var FLIPPER_NO_BUNDLED_PLUGINS is equivalent to the command-line option "--no-bundled-plugins".',
type: 'boolean',
},
'rebuild-plugins': {
describe:
'Enables rebuilding of default plugins on Flipper build. Only make sense in conjunction with "--no-bundled-plugins". Enabled by default, but if disabled using "--no-plugin-rebuild", then plugins are just released as is without rebuilding. This can save some time if you know plugin bundles are already up-to-date.',
type: 'boolean',
},
'plugin-marketplace': {
describe:
'Enable plugin marketplace - ability to install plugins from NPM or other sources. Without the flag Flipper will only show default plugins. The flag is disabled by default in dev mode. Env var FLIPPER_NO_PLUGIN_MARKETPLACE is equivalent to the command-line option "--no-plugin-marketplace"',
@@ -59,16 +44,6 @@ const argv = yargs
'[FB-internal only] Will force using public sources only, to be able to iterate quickly on the public version. If sources are checked out from GitHub this is already the default. Setting env var "FLIPPER_FORCE_PUBLIC_BUILD" is equivalent.',
type: 'boolean',
},
open: {
describe: 'Open Flipper in the default browser after starting',
type: 'boolean',
default: true,
},
tcp: {
describe: 'Enable TCP connections on flipper-server.',
type: 'boolean',
default: true,
},
channel: {
description: 'Release channel for the build',
choices: ['stable', 'insiders'],
@@ -85,24 +60,6 @@ if (isFB) {
process.env.FLIPPER_RELEASE_CHANNEL = argv.channel;
if (argv['default-plugins'] === true) {
delete process.env.FLIPPER_NO_DEFAULT_PLUGINS;
} else if (argv['default-plugins'] === false) {
process.env.FLIPPER_NO_DEFAULT_PLUGINS = 'true';
}
if (argv['bundled-plugins'] === true) {
delete process.env.FLIPPER_NO_BUNDLED_PLUGINS;
} else if (argv['bundled-plugins'] === false) {
process.env.FLIPPER_NO_BUNDLED_PLUGINS = 'true';
}
if (argv['rebuild-plugins'] === false) {
process.env.FLIPPER_NO_REBUILD_PLUGINS = 'true';
} else if (argv['rebuild-plugins'] === true) {
delete process.env.FLIPPER_NO_REBUILD_PLUGINS;
}
if (argv['public-build'] === true) {
// we use a separate env var for forced_public builds, since
// FB_FLIPPER / isFB reflects whether we are running on FB sources / infra
@@ -133,10 +90,20 @@ if (argv['enabled-plugins'] !== undefined) {
let startCount = 0;
async function copyStaticResources() {
console.log(`⚙️ Copying necessary static resources`);
const staticDir = path.resolve(__dirname, '..', 'static');
await fs.copy(
path.join(staticDir, 'manifest.template.json'),
path.join(staticDir, 'manifest.json'),
);
}
async function restartServer() {
try {
await compileServerMain(true);
await launchServer(true, argv.open && ++startCount === 1, argv.tcp); // only open on the first time
await launchServer(true, ++startCount === 1); // only open on the first time
} catch (e) {
console.error(
chalk.red(
@@ -155,12 +122,15 @@ async function startWatchChanges() {
// For UI changes, Metro / hot module reloading / fast refresh take care of the changes
await Promise.all(
[
'pkg',
'doctor',
'pkg-lib',
'plugin-lib',
'flipper-server',
'flipper-common',
'flipper-frontend-core',
'flipper-plugin-core',
'flipper-server-companion',
'flipper-server-core',
'flipper-server',
].map((dir) =>
watchman.startWatchFiles(
dir,
@@ -191,11 +161,9 @@ async function startWatchChanges() {
await prepareDefaultPlugins(
process.env.FLIPPER_RELEASE_CHANNEL === 'insiders',
);
await prepareHeadlessPlugins();
// watch
await startWatchChanges();
await copyStaticResources();
await ensurePluginFoldersWatchable();
// builds and starts
await restartServer();
await startWatchChanges();
})();

View File

@@ -1,52 +0,0 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import Watchman from './watchman';
import {getPluginSourceFolders} from 'flipper-plugin-lib';
export default async function startWatchPlugins(
onChanged: () => void | Promise<void>,
) {
// eslint-disable-next-line no-console
console.log('🕵️‍ Watching for plugin changes');
let delayedCompilation: NodeJS.Timeout | undefined;
const kCompilationDelayMillis = 1000;
const onPluginChangeDetected = () => {
if (!delayedCompilation) {
delayedCompilation = setTimeout(() => {
delayedCompilation = undefined;
// eslint-disable-next-line no-console
console.log(`🕵️‍ Detected plugin change`);
onChanged();
}, kCompilationDelayMillis);
}
};
try {
await startWatchingPluginsUsingWatchman(onPluginChangeDetected);
} catch (err) {
console.error(
'Failed to start watching plugin files using Watchman, continue without hot reloading',
err,
);
}
}
async function startWatchingPluginsUsingWatchman(onChange: () => void) {
const pluginFolders = await getPluginSourceFolders();
await Promise.all(
pluginFolders.map(async (pluginFolder) => {
const watchman = new Watchman(pluginFolder);
await watchman.initialize();
await watchman.startWatchFiles('.', () => onChange(), {
excludes: ['**/__tests__/**/*', '**/node_modules/**/*', '**/.*'],
});
}),
);
}

View File

@@ -109,9 +109,6 @@ async function findAffectedPlugins(errors: string[]) {
depsByName.set(name, getDependencies(name));
}
for (const pkg of allPackages) {
if (!isPluginJson(pkg.json)) {
continue;
}
const logFile = path.join(pkg.dir, 'tsc-error.log');
await fs.remove(logFile);
let logStream: fs.WriteStream | undefined;

View File

@@ -3,10 +3,10 @@
"compilerOptions": {
"outDir": "lib",
"rootDir": ".",
"lib": ["ES2019"],
"lib": ["ES2021"],
"noEmit": true,
"esModuleInterop": true,
"types": ["jest"]
"types": ["jest", "node"]
},
"references": [
{

View File

@@ -0,0 +1,162 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import fs from 'fs-extra';
import semver from 'semver';
import fg from 'fast-glob';
/**
* Lists all dependencies that DO NOT have to match their type declaration package major versions
*
* Leave a comment for packages that you list here
*/
const IGNORED_TYPES = new Set(
[
// node is not an installed package
'node',
// we are useing experimental versions of these packages
'react',
'react-dom',
'react-test-renderer',
// these packages do not have new major versions
'async',
'dateformat',
'deep-equal',
'inquirer',
'mock-fs',
'npm-packlist',
].map((x) => `@types/${x}`),
);
type UnmatchedLibType = {
types: readonly [string, string];
lib: readonly [string, string];
};
type PackageJsonResult = {
packageJson: string;
unmatchedTypesPackages: UnmatchedLibType[];
};
function isValidTypesPackageName(x: [name: string, version: string]): boolean {
return x[0].startsWith('@types/') && !IGNORED_TYPES.has(x[0]);
}
async function validatePackageJson(
filepath: string,
): Promise<PackageJsonResult> {
try {
const jsonBuf = await fs.promises.readFile(filepath);
const json = JSON.parse(jsonBuf.toString());
const deps: Record<string, string> = json.dependencies || {};
const devDeps: Record<string, string> = json.devDependencies || {};
const typesPackages: Array<[string, string]> = [
...Object.entries(deps).filter(isValidTypesPackageName),
...Object.entries(devDeps).filter(isValidTypesPackageName),
];
const unmatchedTypesPackages: UnmatchedLibType[] = typesPackages
.map(([rawName, rawVersion]) => {
const name: string | void = rawName.split('/', 2).pop();
if (name == null) {
throw new Error(
`Could not infer package name from types "${rawName}"`,
);
}
const typeVersionParsed = parsePackageVersion(
rawVersion,
rawName,
filepath,
);
const depsWithLib = name in deps ? deps : devDeps || {};
if (depsWithLib[name] == null) {
return null;
}
const targetVersion = parsePackageVersion(
depsWithLib[name],
name,
filepath,
);
if (targetVersion.major !== typeVersionParsed.major) {
return {
types: [rawName, rawVersion] as const,
lib: [name, depsWithLib[name]] as const,
};
}
})
.filter(<T,>(x: T | undefined | null): x is T => x != null);
return {
packageJson: filepath,
unmatchedTypesPackages,
};
} catch (e) {
console.error(`Failed to parse ${filepath}`);
throw e;
}
}
async function main() {
const packageJsons = await fg('**/package.json', {
ignore: ['**/node_modules'],
});
const unmatched = await Promise.all(
packageJsons.map(validatePackageJson),
).then((x) => x.filter((x) => x.unmatchedTypesPackages.length > 0));
if (unmatched.length === 0) {
console.log('No issues found');
return;
}
console.log(
unmatched
.map((x) =>
[
x.packageJson,
...x.unmatchedTypesPackages.map(
(x: UnmatchedLibType) =>
`\t${x.types[0]}: ${x.types[1]} --- ${x.lib[0]}: ${x.lib[1]}`,
),
].join('\n'),
)
.join('\n'),
);
process.exit(1);
}
main().catch((e) => {
console.log(`Unexpected error: ${e}`);
process.exit(1);
});
function parsePackageVersion(
version: string,
pkgName: string,
filepath: string,
): semver.SemVer {
// versions can start with ~ or ^
if (!version.match(/^\d/)) {
version = version.slice(1);
}
const parsed = semver.parse(version);
if (parsed == null) {
throw new Error(
`Could not parse version number from "${version}" for package "${pkgName}" in ${filepath}`,
);
}
return parsed;
}

View File

@@ -1,126 +0,0 @@
/**
* Copyright (c) Meta Platforms, Inc. and affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {Client} from 'fb-watchman';
import {v4 as uuid} from 'uuid';
import path from 'path';
const watchmanTimeout = 60 * 1000;
export default class Watchman {
constructor(private rootDir: string) {}
private client?: Client;
private watch?: any;
private relativeRoot?: string;
async initialize(): Promise<void> {
if (this.client) {
return;
}
this.client = new Client();
this.client.setMaxListeners(250);
await new Promise<void>((resolve, reject) => {
const onError = (err: Error) => {
this.client!.removeAllListeners('error');
reject(err);
this.client!.end();
delete this.client;
};
const timeouthandle = setTimeout(() => {
onError(new Error('Timeout when trying to start Watchman'));
}, watchmanTimeout);
this.client!.once('error', onError);
this.client!.capabilityCheck(
{optional: [], required: ['relative_root']},
(error) => {
if (error) {
onError(error);
return;
}
this.client!.command(
['watch-project', this.rootDir],
(error, resp) => {
if (error) {
onError(error);
return;
}
if ('warning' in resp) {
console.warn(resp.warning);
}
this.watch = resp.watch;
this.relativeRoot = resp.relative_path;
clearTimeout(timeouthandle);
resolve();
},
);
},
);
});
}
async startWatchFiles(
relativeDir: string,
handler: (resp: any) => void,
options: {excludes: string[]},
): Promise<void> {
if (!this.watch) {
throw new Error(
'Watchman is not initialized, please call "initialize" function and wait for the returned promise completion before calling "startWatchFiles".',
);
}
options = Object.assign({excludes: []}, options);
return new Promise((resolve, reject) => {
this.client!.command(['clock', this.watch], (error, resp) => {
if (error) {
return reject(error);
}
try {
const {clock} = resp;
const sub = {
expression: [
'allof',
['not', ['type', 'd']],
...options!.excludes.map((e) => [
'not',
['match', e, 'wholename'],
]),
],
fields: ['name'],
since: clock,
relative_root: this.relativeRoot
? path.join(this.relativeRoot, relativeDir)
: relativeDir,
};
const id = uuid();
this.client!.command(['subscribe', this.watch, id, sub], (error) => {
if (error) {
return reject(error);
}
this.client!.on('subscription', (resp) => {
if (resp.subscription !== id || !resp.files) {
return;
}
handler(resp);
});
resolve();
});
} catch (err) {
reject(err);
}
});
});
}
}

View File

@@ -69,9 +69,12 @@ async function getWorkspacesByRoot(
}
export async function getWorkspaces(): Promise<Workspaces> {
const rootWorkspaces = await getWorkspacesByRoot(rootDir);
const publicPluginsWorkspaces = await getWorkspacesByRoot(publicPluginsDir);
const fbPluginsWorkspaces = await getWorkspacesByRoot(fbPluginsDir);
const [rootWorkspaces, publicPluginsWorkspaces, fbPluginsWorkspaces] =
await Promise.all([
getWorkspacesByRoot(rootDir),
getWorkspacesByRoot(publicPluginsDir),
getWorkspacesByRoot(fbPluginsDir),
]);
const mergedWorkspaces: Workspaces = {
rootPackage: rootWorkspaces!.rootPackage,
packages: [