Make ClientQuery available to certificate provider

Summary:
CertificateProvider is intrinsically related to a client query, make it available to it.

This becomes the optional 'context' of shell executions. When these are recorded, the context is provider to the recorder which can then link an ongoing certificate exchange process with the success or failure of said command.

Reviewed By: antonk52

Differential Revision: D47295894

fbshipit-source-id: 9469d18bda02793d71a6a8b29c93f4af1db23569

desktop/flipper-server-core/src/app-connectivity/ServerController.tsx

@@ -444,6 +444,7 @@ export class ServerController
       assertNotNull(this.flipperServer.android);
       (clientQuery as any).device_id =
         await this.flipperServer.android.certificateProvider.getTargetDeviceId(
+          clientQuery,
           bundleId,
           csr_path,
           csr,

desktop/flipper-server-core/src/app-connectivity/certificate-exchange/CertificateProvider.tsx

@@ -48,6 +48,7 @@ export default abstract class CertificateProvider {
 
     recorder.log(clientQuery, 'Deploy CA certificate to application sandbox');
     await this.deployOrStageFileForDevice(
+      clientQuery,
       appDirectory,
       deviceCAcertFile,
       caCert,
@@ -62,6 +63,7 @@ export default abstract class CertificateProvider {
       'Deploy client certificate to application sandbox',
     );
     await this.deployOrStageFileForDevice(
+      clientQuery,
       appDirectory,
       deviceClientCertFile,
       clientCert,
@@ -75,7 +77,12 @@ export default abstract class CertificateProvider {
       clientQuery,
       'Get target device from CSR and application name',
     );
-    const deviceId = await this.getTargetDeviceId(bundleId, appDirectory, csr);
+    const deviceId = await this.getTargetDeviceId(
+      clientQuery,
+      bundleId,
+      appDirectory,
+      csr,
+    );
 
     recorder.log(
       clientQuery,
@@ -87,12 +94,14 @@ export default abstract class CertificateProvider {
   }
 
   abstract getTargetDeviceId(
+    clientQuery: ClientQuery,
     bundleId: string,
     appDirectory: string,
     csr: string,
   ): Promise<string>;
 
   protected abstract deployOrStageFileForDevice(
+    clientQuery: ClientQuery,
     destination: string,
     filename: string,
     contents: string,

desktop/flipper-server-core/src/devices/android/AndroidCertificateProvider.tsx

@@ -14,6 +14,7 @@ import {
   csrFileName,
   extractBundleIdFromCSR,
 } from '../../app-connectivity/certificate-exchange/certificate-utils';
+import {ClientQuery} from 'flipper-common';
 
 const logTag = 'AndroidCertificateProvider';
 
@@ -26,6 +27,7 @@ export default class AndroidCertificateProvider extends CertificateProvider {
   }
 
   async getTargetDeviceId(
+    clientQuery: ClientQuery,
     appName: string,
     appDirectory: string,
     csr: string,
@@ -93,13 +95,19 @@ export default class AndroidCertificateProvider extends CertificateProvider {
   }
 
   protected async deployOrStageFileForDevice(
+    clientQuery: ClientQuery,
     destination: string,
     filename: string,
     contents: string,
     csr: string,
   ) {
     const appName = await extractBundleIdFromCSR(csr);
-    const deviceId = await this.getTargetDeviceId(appName, destination, csr);
+    const deviceId = await this.getTargetDeviceId(
+      clientQuery,
+      appName,
+      destination,
+      csr,
+    );
     await androidUtil.push(
       this.adb,
       deviceId,

desktop/flipper-server-core/src/devices/desktop/DesktopCertificateProvider.tsx

@@ -9,18 +9,25 @@
 
 import CertificateProvider from '../../app-connectivity/certificate-exchange/CertificateProvider';
 import fs from 'fs-extra';
+import {ClientQuery} from 'flipper-common';
 
 export default class DesktopCertificateProvider extends CertificateProvider {
   name = 'DesktopCertificateProvider';
   medium = 'FS_ACCESS' as const;
+
+  /**
+   * For Desktop devices, we currently return an empty string as the device
+   * identifier. TODO: Is there an actual device serial we could use instead?
+   * - What if some app connects from a remote device?
+   * - What if two apps connect from several different remote devices?
+   * @returns An empty string.
+   */
   async getTargetDeviceId(): Promise<string> {
-    // TODO: Could we use some real device serial? Currently, '' corresponds to a local device.
-    // Whats if some app connects from a remote device?
-    // What if two apps connect from several different remote devices?
     return '';
   }
 
   protected async deployOrStageFileForDevice(
+    _: ClientQuery,
     destination: string,
     filename: string,
     contents: string,

desktop/flipper-server-core/src/devices/ios/iOSCertificateProvider.tsx

@@ -17,6 +17,7 @@ import {
   extractBundleIdFromCSR,
 } from '../../app-connectivity/certificate-exchange/certificate-utils';
 import path from 'path';
+import {ClientQuery} from 'flipper-common';
 
 const tmpDir = promisify(tmp.dir) as (options?: DirOptions) => Promise<string>;
 
@@ -31,6 +32,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
   }
 
   async getTargetDeviceId(
+    clientQuery: ClientQuery,
     appName: string,
     appDirectory: string,
     csr: string,
@@ -45,6 +47,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
     const targets = await iosUtil.targets(
       this.idbConfig.idbPath,
       this.idbConfig.enablePhysicalIOS,
+      clientQuery,
     );
     if (targets.length === 0) {
       throw new Error('No iOS devices found');
@@ -52,6 +55,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
     const deviceMatchList = targets.map(async (target) => {
       try {
         const isMatch = await this.iOSDeviceHasMatchingCSR(
+          clientQuery,
           appDirectory,
           target.udid,
           appName,
@@ -79,6 +83,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
   }
 
   protected async deployOrStageFileForDevice(
+    clientQuery: ClientQuery,
     destination: string,
     filename: string,
     contents: string,
@@ -100,8 +105,14 @@ export default class iOSCertificateProvider extends CertificateProvider {
 
       console.debug(`[conn] Relative path '${relativePathInsideApp}'`);
 
-      const udid = await this.getTargetDeviceId(bundleId, destination, csr);
+      const udid = await this.getTargetDeviceId(
+        clientQuery,
+        bundleId,
+        destination,
+        csr,
+      );
       await this.pushFileToiOSDevice(
+        clientQuery,
         udid,
         bundleId,
         relativePathInsideApp,
@@ -122,6 +133,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
   }
 
   private async pushFileToiOSDevice(
+    clientQuery: ClientQuery,
     udid: string,
     bundleId: string,
     destination: string,
@@ -138,10 +150,12 @@ export default class iOSCertificateProvider extends CertificateProvider {
       bundleId,
       destination,
       this.idbConfig.idbPath,
+      clientQuery,
     );
   }
 
   private async iOSDeviceHasMatchingCSR(
+    clientQuery: ClientQuery,
     directory: string,
     deviceId: string,
     bundleId: string,
@@ -153,7 +167,14 @@ export default class iOSCertificateProvider extends CertificateProvider {
     const dst = await tmpDir({unsafeCleanup: true});
 
     try {
-      await iosUtil.pull(deviceId, src, bundleId, dst, this.idbConfig.idbPath);
+      await iosUtil.pull(
+        deviceId,
+        src,
+        bundleId,
+        dst,
+        this.idbConfig.idbPath,
+        clientQuery,
+      );
     } catch (e) {
       console.warn(
         `[conn] Original idb pull failed. Most likely it is a physical device
@@ -167,6 +188,7 @@ export default class iOSCertificateProvider extends CertificateProvider {
         bundleId,
         path.join(dst, csrFileName),
         this.idbConfig.idbPath,
+        clientQuery,
       );
       console.info(
         '[conn] Subsequent idb pull succeeded. Nevermind previous wranings.',

desktop/flipper-server-core/src/devices/ios/iOSContainerUtility.tsx

@@ -68,7 +68,9 @@ async function safeExec(
   return await unsafeExec(command).finally(release);
 }
 
-async function queryTargetsWithXcode(): Promise<Array<DeviceTarget>> {
+async function queryTargetsWithXcode(
+  context: any,
+): Promise<Array<DeviceTarget>> {
   const cmd = 'xcrun xctrace list devices';
   const description = 'Query available devices with Xcode';
   const troubleshoot = `Xcode command line tools are not installed.
@@ -77,7 +79,13 @@ async function queryTargetsWithXcode(): Promise<Array<DeviceTarget>> {
   try {
     const {stdout} = await safeExec(cmd);
     if (!stdout) {
-      recorder.event('cmd', {cmd, description, success: false, troubleshoot});
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        troubleshoot,
+        context,
+      });
       throw new Error('No output from command');
     }
     recorder.event('cmd', {
@@ -85,6 +93,7 @@ async function queryTargetsWithXcode(): Promise<Array<DeviceTarget>> {
       description,
       success: true,
       stdout: stdout.toString(),
+      context,
     });
     return stdout
       .toString()
@@ -104,6 +113,7 @@ async function queryTargetsWithXcode(): Promise<Array<DeviceTarget>> {
       success: false,
       troubleshoot,
       stderr: e.toString(),
+      context,
     });
     return [];
   }
@@ -111,16 +121,24 @@ async function queryTargetsWithXcode(): Promise<Array<DeviceTarget>> {
 
 async function queryTargetsWithIdb(
   idbPath: string,
+  context: any,
 ): Promise<Array<DeviceTarget>> {
   const cmd = `${idbPath} list-targets --json`;
-  const description = 'Query available devices with idb';
+  const description = `Query available devices with idb. idb is aware of the companions that you have
+    manually connected, as well as other iOS targets that do not yet have companions.`;
   const troubleshoot = `Either idb is not installed or needs to be reset.
     Run 'idb kill' from terminal.`;
 
   try {
     const {stdout} = await safeExec(cmd);
     if (!stdout) {
-      recorder.event('cmd', {cmd, description, success: false, troubleshoot});
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        troubleshoot,
+        context,
+      });
       throw new Error('No output from command');
     }
 
@@ -129,6 +147,7 @@ async function queryTargetsWithIdb(
       description,
       success: true,
       stdout: stdout.toString(),
+      context,
     });
 
     return parseIdbTargets(stdout.toString());
@@ -139,6 +158,7 @@ async function queryTargetsWithIdb(
       success: false,
       troubleshoot,
       stderr: e.toString(),
+      context,
     });
     return [];
   }
@@ -147,16 +167,36 @@ async function queryTargetsWithIdb(
 async function queryTargetsWithIdbCompanion(
   idbCompanionPath: string,
   isPhysicalDeviceEnabled: boolean,
+  context: any,
 ): Promise<Array<DeviceTarget>> {
-  if (await isAvailable(idbCompanionPath)) {
   const cmd = `${idbCompanionPath} --list 1 --only device`;
-    recorder.rawLog(`Query devices with idb companion '${cmd}'`);
+  const description = `Query available devices with idb companion. Lists all available devices and simulators
+    in the current context. If Xcode is not correctly installed, only devices will be listed.`;
+  const troubleshoot = `Unable to locate idb_companion in '${idbCompanionPath}'.
+    Try running sudo yum install -y fb-idb`;
+
+  if (await isAvailable(idbCompanionPath)) {
     try {
       const {stdout} = await safeExec(cmd);
       if (!stdout) {
+        recorder.event('cmd', {
+          cmd,
+          description,
+          success: false,
+          troubleshoot,
+          context,
+        });
         throw new Error('No output from command');
       }
 
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: true,
+        stdout: stdout.toString(),
+        context,
+      });
+
       const devices = parseIdbTargets(stdout.toString());
       if (devices.length > 0 && !isPhysicalDeviceEnabled) {
         recorder.rawError(
@@ -166,14 +206,24 @@ async function queryTargetsWithIdbCompanion(
       }
       return devices;
     } catch (e) {
-      recorder.rawError(`Failed to query devices using '${cmd}'`, e);
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        troubleshoot,
+        stderr: e.toString(),
+        context,
+      });
       return [];
     }
   } else {
-    recorder.rawError(
+    recorder.event('cmd', {
-      `Unable to locate idb_companion in '${idbCompanionPath}'.
+      cmd,
-      Try running sudo yum install -y fb-idb`,
+      description,
-    );
+      success: false,
+      troubleshoot,
+      context,
+    });
     return [];
   }
 }
@@ -212,17 +262,53 @@ function parseIdbTargets(lines: string): Array<DeviceTarget> {
 
 async function idbDescribeTarget(
   idbPath: string,
+  context: any,
 ): Promise<DeviceTarget | undefined> {
   const cmd = `${idbPath} describe --json`;
-  recorder.rawLog(`Describe target '${cmd}'`);
+  const description = `Returns metadata about the specified target, including:
+    UDID,
+    Name,
+    Screen dimensions and density,
+    State (booted/...),
+    Type (simulator/device),
+    iOS version,
+    Architecture,
+    Information about its companion,
+  `;
+  const troubleshoot = `Either idb is not installed or needs to be reset.
+    Run 'idb kill' from terminal.`;
+
   try {
     const {stdout} = await safeExec(cmd);
     if (!stdout) {
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        troubleshoot,
+        context,
+      });
       throw new Error('No output from command');
     }
+
+    recorder.event('cmd', {
+      cmd,
+      description,
+      success: true,
+      stdout: stdout.toString(),
+      context,
+    });
+
     return parseIdbTarget(stdout.toString());
   } catch (e) {
-    recorder.rawError(`Failed to execute '${cmd}' to describe a target.`, e);
+    recorder.event('cmd', {
+      cmd,
+      description,
+      success: false,
+      troubleshoot,
+      stderr: e.toString(),
+      context,
+    });
     return undefined;
   }
 }
@@ -230,6 +316,7 @@ async function idbDescribeTarget(
 async function targets(
   idbPath: string,
   isPhysicalDeviceEnabled: boolean,
+  context?: any,
 ): Promise<Array<DeviceTarget>> {
   if (process.platform !== 'darwin') {
     return [];
@@ -240,7 +327,7 @@ async function targets(
   // use that instead and do not query other devices.
   // See stack of D36315576 for details
   if (process.env.IDB_COMPANION) {
-    const target = await idbDescribeTarget(idbPath);
+    const target = await idbDescribeTarget(idbPath, context);
     return target ? [target] : [];
   }
 
@@ -255,6 +342,7 @@ async function targets(
     return queryTargetsWithIdbCompanion(
       idbCompanionPath,
       isPhysicalDeviceEnabled,
+      context,
     );
   }
 
@@ -264,9 +352,9 @@ async function targets(
   // when installed, use it. This still holds true
   // with the move from instruments to xcrun.
   if (await memoize(isAvailable)(idbPath)) {
-    return await queryTargetsWithIdb(idbPath);
+    return await queryTargetsWithIdb(idbPath, context);
   } else {
-    return queryTargetsWithXcode();
+    return queryTargetsWithXcode(context);
   }
 }
 
@@ -276,17 +364,34 @@ async function push(
   bundleId: string,
   dst: string,
   idbPath: string,
+  context?: any,
 ): Promise<void> {
   await memoize(checkIdbIsInstalled)(idbPath);
 
   const push_ = async () => {
     const cmd = `${idbPath} file push --log ${IDB_LOG_LEVEL} --udid ${udid} --bundle-id ${bundleId} '${src}' '${dst}'`;
+    const description = `idb push file to device`;
+    const troubleshoot = `Either idb is not installed or needs to be reset.
+    Run 'idb kill' from terminal.`;
+
     try {
-      recorder.rawLog(`Push file to device '${cmd}'`);
       await safeExec(cmd);
-      recorder.rawLog(`Successfully pushed file to device`);
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: true,
+        troubleshoot,
+        context,
+      });
     } catch (e) {
-      recorder.rawError(`Failed to push file to device`, e);
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        stdout: e.toString(),
+        troubleshoot,
+        context,
+      });
       handleMissingIdb(e, idbPath);
       throw e;
     }
@@ -301,17 +406,34 @@ async function pull(
   bundleId: string,
   dst: string,
   idbPath: string,
+  context?: any,
 ): Promise<void> {
   await memoize(checkIdbIsInstalled)(idbPath);
 
   const pull_ = async () => {
     const cmd = `${idbPath} file pull --log ${IDB_LOG_LEVEL} --udid ${udid} --bundle-id ${bundleId} '${src}' '${dst}'`;
+    const description = `idb pull file from device`;
+    const troubleshoot = `Either idb is not installed or needs to be reset.
+    Run 'idb kill' from terminal.`;
+
     try {
-      recorder.rawLog(`Pull file from device '${cmd}'`);
       await safeExec(cmd);
-      recorder.rawLog(`Successfully pulled file from device`);
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: true,
+        troubleshoot,
+        context,
+      });
     } catch (e) {
-      recorder.rawError(`Failed to pull file from device`, e);
+      recorder.event('cmd', {
+        cmd,
+        description,
+        success: false,
+        stdout: e.toString(),
+        troubleshoot,
+        context,
+      });
       handleMissingIdb(e, idbPath);
       handleMissingPermissions(e);
       throw e;

desktop/flipper-server-core/src/fb-stubs/WWWCertificateProvider.tsx

@@ -13,6 +13,7 @@ import CertificateProvider from '../app-connectivity/certificate-exchange/Certif
 export default class WWWCertificateProvider extends CertificateProvider {
   name = 'WWWCertificateProvider';
   medium = 'WWW' as const;
+
   constructor(private keytarManager: KeytarManager) {
     super();
   }

desktop/flipper-server-core/src/recorder.tsx

@@ -7,7 +7,12 @@
  * @format
  */
 
-import {ClientQuery} from 'flipper-common';
+import {
+  ClientQuery,
+  ConnectionRecordEntry,
+  CommandRecordEntry,
+} from 'flipper-common';
+import {FlipperServerImpl} from './FlipperServerImpl';
 
 type CommandEventPayload = {
   cmd: string;
@@ -16,6 +21,7 @@ type CommandEventPayload = {
   stdout?: string;
   stderr?: string;
   troubleshoot?: string;
+  context?: any;
 };
 
 type ConnectionRecorderEvents = {
@@ -23,11 +29,29 @@ type ConnectionRecorderEvents = {
 };
 
 class Recorder {
+  private flipperServer: FlipperServerImpl | undefined;
+
   private handler_ = {
-    cmd: (_payload: CommandEventPayload) => {
+    cmd: (payload: CommandEventPayload) => {
-      // The output from logging the whole command can be quite
+      if (this.flipperServer && payload.context) {
-      // verbose. So, disable it as is.
+        const clientQuery = payload.context as ClientQuery;
-      // this.rawLog(_payload);
+        const entry: CommandRecordEntry = {
+          time: new Date(),
+          type: 'cmd',
+          device: clientQuery.device,
+          app: clientQuery.app,
+          message: payload.cmd,
+          medium: clientQuery.medium,
+          cmd: payload.cmd,
+          description: payload.description,
+          success: payload.success,
+          stdout: payload.stdout,
+          stderr: payload.stderr,
+          troubleshoot: payload.troubleshoot,
+        };
+
+        this.flipperServer.emit('connectivity-troubleshoot-cmd', entry);
+      }
     },
   };
 
@@ -42,11 +66,20 @@ class Recorder {
     handler(payload);
   }
 
-  rawLog(...args: any[]) {
-    console.log('[conn]', ...args);
-  }
   log(clientQuery: ClientQuery, ...args: any[]) {
     console.log('[conn]', ...args);
+    if (this.flipperServer) {
+      const entry: ConnectionRecordEntry = {
+        time: new Date(),
+        type: 'info',
+        device: clientQuery.device,
+        app: clientQuery.app,
+        message: args.join(' '),
+        medium: clientQuery.medium,
+      };
+
+      this.flipperServer.emit('connectivity-troubleshoot-log', entry);
+    }
   }
   rawError(...args: any[]) {
     console.error('[conn]', ...args);
@@ -54,6 +87,10 @@ class Recorder {
   error(clientQuery: ClientQuery, ...args: any[]) {
     console.error('[conn]', ...args);
   }
+
+  enable(flipperServer: FlipperServerImpl) {
+    this.flipperServer = flipperServer;
+  }
 }
 
 const recorder = new Recorder();