Support physical device logging

Summary:
Changelog: Logs plugin now supports physical iOS devices

As reported in https://github.com/facebook/flipper/issues/262 and linked papercut.

This diff adds support for iOS device logs through idb. Since idb doesn't respect `--json` flag at the moment, we perform the parsing in Flipper itself.

Reviewed By: passy

Differential Revision: D27346262

fbshipit-source-id: 3b314716f48bb9a7fe709370303396a51893359c

desktop/app/package.json

@@ -72,6 +72,7 @@
     "rsocket-tcp-server": "^0.0.19",
     "rsocket-types": "^0.0.16",
     "semver": "^7.3.5",
+    "split2": "^3.2.2",
     "string-natural-compare": "^3.0.0",
     "tmp": "^0.2.1",
     "uuid": "^8.3.2",
@@ -86,6 +87,7 @@
     "@testing-library/dom": "^7.30.1",
     "@testing-library/react": "^11.2.5",
     "@types/lodash.memoize": "^4.1.6",
+    "@types/split2": "^2.1.6",
     "flipper-test-utils": "0.0.0",
     "metro-runtime": "^0.65.2",
     "mock-fs": "^4.13.0",

desktop/app/src/devices/IOSDevice.tsx

@@ -20,6 +20,7 @@ import {promisify} from 'util';
 import {exec} from 'child_process';
 import {default as promiseTimeout} from '../utils/promiseTimeout';
 import {IOSBridge} from '../utils/IOSBridge';
+import split2 from 'split2';
 
 type IOSLogLevel = 'Default' | 'Info' | 'Debug' | 'Error' | 'Fault';
 
@@ -40,6 +41,10 @@ type RawLogEntry = {
   traceID: string;
 };
 
+// https://regex101.com/r/rrl03T/1
+// Mar 25 17:06:38 iPhone symptomsd(SymptomEvaluator)[125] <Notice>: Stuff
+const logRegex = /(^.{15}) ([^ ]+?) ([^\[]+?)\[(\d+?)\] <(\w+?)>: (.*)$/s;
+
 export default class IOSDevice extends BaseDevice {
   log?: child_process.ChildProcessWithoutNullStreams;
   buffer: string;
@@ -97,8 +102,13 @@ export default class IOSDevice extends BaseDevice {
     }
 
     const logListener = iOSBridge.startLogListener;
-    if (!this.log && logListener) {
+    if (
-      this.log = logListener(this.serial);
+      !this.log &&
+      logListener &&
+      (this.deviceType === 'emulator' ||
+        (this.deviceType === 'physical' && iOSBridge.idbAvailable))
+    ) {
+      this.log = logListener(this.serial, this.deviceType);
       this.log.on('error', (err: Error) => {
         console.error('iOS log tailer error', err);
       });
@@ -112,13 +122,24 @@ export default class IOSDevice extends BaseDevice {
       });
 
       try {
-        this.log.stdout
+        if (this.deviceType === 'physical') {
-          .pipe(new StripLogPrefix())
+          this.log.stdout.pipe(split2('\0')).on('data', (line: string) => {
-          .pipe(JSONStream.parse('*'))
+            const parsed = IOSDevice.parseLogLine(line);
-          .on('data', (data: RawLogEntry) => {
+            if (parsed) {
-            const entry = IOSDevice.parseLogEntry(data);
+              this.addLogEntry(parsed);
-            this.addLogEntry(entry);
+            } else {
+              console.warn('Failed to parse iOS log line: ', line);
+            }
           });
+        } else {
+          this.log.stdout
+            .pipe(new StripLogPrefix())
+            .pipe(JSONStream.parse('*'))
+            .on('data', (data: RawLogEntry) => {
+              const entry = IOSDevice.parseJsonLogEntry(data);
+              this.addLogEntry(entry);
+            });
+        }
       } catch (e) {
         console.error('Could not parse iOS log stream.', e);
         // restart log stream
@@ -129,15 +150,42 @@ export default class IOSDevice extends BaseDevice {
     }
   }
 
-  static parseLogEntry(entry: RawLogEntry): DeviceLogEntry {
+  static getLogLevel(level: string): LogLevel {
-    const LOG_MAPPING: Map<IOSLogLevel, LogLevel> = new Map([
+    switch (level) {
-      ['Default' as IOSLogLevel, 'debug' as LogLevel],
+      case 'Default':
-      ['Info' as IOSLogLevel, 'info' as LogLevel],
+        return 'debug';
-      ['Debug' as IOSLogLevel, 'debug' as LogLevel],
+      case 'Info':
-      ['Error' as IOSLogLevel, 'error' as LogLevel],
+        return 'info';
-      ['Fault' as IOSLogLevel, 'fatal' as LogLevel],
+      case 'Debug':
-    ]);
+        return 'debug';
-    let type: LogLevel = LOG_MAPPING.get(entry.messageType) || 'unknown';
+      case 'Error':
+        return 'error';
+      case 'Notice':
+        return 'verbose';
+      case 'Fault':
+        return 'fatal';
+      default:
+        return 'unknown';
+    }
+  }
+
+  static parseLogLine(line: string): DeviceLogEntry | undefined {
+    const matches = line.match(logRegex);
+    if (matches) {
+      return {
+        date: new Date(Date.parse(matches[1])),
+        tag: matches[3],
+        tid: 0,
+        pid: parseInt(matches[4], 10),
+        type: IOSDevice.getLogLevel(matches[5]),
+        message: matches[6],
+      };
+    }
+    return undefined;
+  }
+
+  static parseJsonLogEntry(entry: RawLogEntry): DeviceLogEntry {
+    let type: LogLevel = IOSDevice.getLogLevel(entry.messageType);
 
     // when Apple log levels are not used, log messages can be prefixed with
     // their loglevel.

desktop/app/src/dispatcher/__tests__/iOSDevice.node.tsx

@@ -64,7 +64,9 @@ test('test getAllPromisesForQueryingDevices when xcode detected', () => {
   const promises = getAllPromisesForQueryingDevices(
     mockStore,
     logger,
-    {},
+    {
+      idbAvailable: false,
+    },
     true,
   );
   expect(promises.length).toEqual(3);
@@ -74,7 +76,9 @@ test('test getAllPromisesForQueryingDevices when xcode is not detected', () => {
   const promises = getAllPromisesForQueryingDevices(
     mockStore,
     logger,
-    {},
+    {
+      idbAvailable: true,
+    },
     false,
   );
   expect(promises.length).toEqual(1);

desktop/app/src/utils/IOSBridge.tsx

@@ -9,10 +9,13 @@
 
 import fs from 'fs';
 import child_process from 'child_process';
+import {DeviceType} from 'flipper-plugin-lib';
 
 export interface IOSBridge {
+  idbAvailable: boolean;
   startLogListener?: (
     udid: string,
+    deviceType: DeviceType,
   ) => child_process.ChildProcessWithoutNullStreams;
 }
 
@@ -26,27 +29,36 @@ async function isAvailable(idbPath: string): Promise<boolean> {
     .catch((_) => false);
 }
 
-const LOG_EXTRA_ARGS = [
+function getLogExtraArgs(deviceType: DeviceType) {
-  '--style',
+  if (deviceType === 'physical') {
-  'json',
+    return [
-  '--predicate',
+      // idb has a --json option, but that doesn't actually work!
-  'senderImagePath contains "Containers"',
+    ];
-  '--debug',
+  } else {
-  '--info',
+    return [
-];
+      '--style',
+      'json',
+      '--predicate',
+      'senderImagePath contains "Containers"',
+      '--debug',
+      '--info',
+    ];
+  }
+}
 
 export function idbStartLogListener(
   idbPath: string,
   udid: string,
+  deviceType: DeviceType,
 ): child_process.ChildProcessWithoutNullStreams {
   return child_process.spawn(
     idbPath,
-    ['log', '--udid', udid, '--', ...LOG_EXTRA_ARGS],
+    ['log', '--udid', udid, '--', ...getLogExtraArgs(deviceType)],
     {},
   );
 }
 
-export function xcrunStartLogListener(udid: string) {
+export function xcrunStartLogListener(udid: string, deviceType: DeviceType) {
   const deviceSetPath = process.env.DEVICE_SET_PATH
     ? ['--set', process.env.DEVICE_SET_PATH]
     : [];
@@ -59,7 +71,7 @@ export function xcrunStartLogListener(udid: string) {
       udid,
       'log',
       'stream',
-      ...LOG_EXTRA_ARGS,
+      ...getLogExtraArgs(deviceType),
     ],
     {},
   );
@@ -70,18 +82,23 @@ export async function makeIOSBridge(
   isXcodeDetected: boolean,
   isAvailableFn: (idbPath: string) => Promise<boolean> = isAvailable,
 ): Promise<IOSBridge> {
-  if (!isXcodeDetected) {
+  // prefer idb
-    // iOS Physical Device can still get detected without Xcode. In this case there is no way to setup log listener yet.
-    // This will not be the case, idb team is working on making idb log work without XCode atleast for physical device.
-    return {};
-  }
   if (await isAvailableFn(idbPath)) {
     return {
+      idbAvailable: true,
       startLogListener: idbStartLogListener.bind(null, idbPath),
     };
   }
 
+  // no idb, if it's a simulator and xcode is available, we can use xcrun
+  if (isXcodeDetected) {
+    return {
+      idbAvailable: false,
+      startLogListener: xcrunStartLogListener,
+    };
+  }
+  // no idb, and not a simulator, we can't log this device
   return {
-    startLogListener: xcrunStartLogListener,
+    idbAvailable: false,
   };
 }

desktop/app/src/utils/__tests__/IOSBridge.node.tsx

@@ -18,7 +18,7 @@ test('uses xcrun with no idb when xcode is detected', async () => {
   const ib = await makeIOSBridge('', true);
   expect(ib.startLogListener).toBeDefined();
 
-  ib.startLogListener!('deadbeef');
+  ib.startLogListener!('deadbeef', 'emulator');
 
   expect(spawn).toHaveBeenCalledWith(
     'xcrun',
@@ -43,7 +43,7 @@ test('uses idb when present and xcode detected', async () => {
   const ib = await makeIOSBridge('/usr/local/bin/idb', true, async (_) => true);
   expect(ib.startLogListener).toBeDefined();
 
-  ib.startLogListener!('deadbeef');
+  ib.startLogListener!('deadbeef', 'emulator');
 
   expect(spawn).toHaveBeenCalledWith(
     '/usr/local/bin/idb',
@@ -63,6 +63,31 @@ test('uses idb when present and xcode detected', async () => {
   );
 });
 
+test('uses idb when present and xcode detected and physical device connected', async () => {
+  const ib = await makeIOSBridge('/usr/local/bin/idb', true, async (_) => true);
+  expect(ib.startLogListener).toBeDefined();
+
+  ib.startLogListener!('deadbeef', 'physical');
+
+  expect(spawn).toHaveBeenCalledWith(
+    '/usr/local/bin/idb',
+    [
+      'log',
+      '--udid',
+      'deadbeef',
+      '--',
+      // no further args; not supported by idb atm
+    ],
+    {},
+  );
+});
+
+test("without idb physical devices can't log", async () => {
+  const ib = await makeIOSBridge('', true);
+  expect(ib.idbAvailable).toBeFalsy();
+  expect(ib.startLogListener).toBeDefined(); // since we have xcode
+});
+
 test('uses no log listener when xcode is not detected', async () => {
   const ib = await makeIOSBridge('', false);
   expect(ib.startLogListener).toBeUndefined();

desktop/plugins/logs/index.tsx

@@ -35,7 +35,7 @@ export type ExtendedLogEntry = DeviceLogEntry & {
 };
 
 function createColumnConfig(
-  os: 'iOS' | 'Android' | 'Metro',
+  _os: 'iOS' | 'Android' | 'Metro',
 ): DataTableColumn<ExtendedLogEntry>[] {
   return [
     {
@@ -74,7 +74,7 @@ function createColumnConfig(
       key: 'pid',
       title: 'PID',
       width: 60,
-      visible: os === 'Android',
+      visible: true,
     },
     {
       key: 'tid',

desktop/plugins/logs/package.json

@@ -4,10 +4,25 @@
   "id": "DeviceLogs",
   "pluginType": "device",
   "supportedDevices": [
-    {"os": "Android", "type": "emulator"},
+    {
-    {"os": "Android", "type": "physical"},
+      "os": "Android",
-    {"os": "iOS", "type": "emulator"},
+      "type": "emulator"
-    {"os": "Metro"}
+    },
+    {
+      "os": "Android",
+      "type": "physical"
+    },
+    {
+      "os": "iOS",
+      "type": "emulator"
+    },
+    {
+      "os": "iOS",
+      "type": "physical"
+    },
+    {
+      "os": "Metro"
+    }
   ],
   "version": "0.0.0",
   "main": "dist/bundle.js",

desktop/yarn.lock

@@ -2981,6 +2981,13 @@
     "@types/node" "*"
     "@types/socket.io-parser" "*"
 
+"@types/split2@^2.1.6":
+  version "2.1.6"
+  resolved "https://registry.yarnpkg.com/@types/split2/-/split2-2.1.6.tgz#b095c9e064853824b22c67993d99b066777402b1"
+  integrity sha512-ddaFSOMuy2Rp97l6q/LEteQygvTQJuEZ+SRhxFKR0uXGsdbFDqX/QF2xoGcOqLQ8XV91v01SnAv2vpgihNgW/Q==
+  dependencies:
+    "@types/node" "*"
+
 "@types/sql-formatter@^2.3.0":
   version "2.3.0"
   resolved "https://registry.yarnpkg.com/@types/sql-formatter/-/sql-formatter-2.3.0.tgz#d2584c54f865fd57a7fe7e88ee8ed3623b23da33"
@@ -11502,7 +11509,7 @@ readable-stream@^2.0.0, readable-stream@^2.0.5, readable-stream@^2.2.2, readable
     string_decoder "~1.1.1"
     util-deprecate "~1.0.1"
 
-readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
+readable-stream@^3.0.0, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198"
   integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==
@@ -12440,6 +12447,13 @@ split-string@^3.0.1, split-string@^3.0.2:
   dependencies:
     extend-shallow "^3.0.0"
 
+split2@^3.2.2:
+  version "3.2.2"
+  resolved "https://registry.yarnpkg.com/split2/-/split2-3.2.2.tgz#bf2cf2a37d838312c249c89206fd7a17dd12365f"
+  integrity sha512-9NThjpgZnifTkJpzTZ7Eue85S49QwpNhZTq6GRJwObb6jnLFNGB7Qm73V5HewTROPyxD0C29xqmaI68bQtV+hg==
+  dependencies:
+    readable-stream "^3.0.0"
+
 split@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/split/-/split-1.0.1.tgz#605bd9be303aa59fb35f9229fbea0ddec9ea07d9"