Add protobuf support to network inspector (#2080)

Summary:
Protobuf based APIs are becoming more common (i.e. gRPC) but are difficult to inspect. Unlike plain text data formats (JSON), Protobuf calls transmit binary data requiring the format to be known ahead of time, making ad-hoc inspection impossible. This PR allows for those format definitions (messages in protobuf terminology) to be transmitted from the client to the network inspector plugin. These definitions are then imported into ProtobufJS which enables the binary data transmitted to be inspected as easily as JSON data.

See Retrofit PR in https://github.com/facebook/flipper/pull/2084

## Changelog

* Add ProtobufJS library to network plugin
* New `ProtobufFormatter` UI in `RequestDetails`
* `ProtobufDefinitionsRepository` to cache and load protobuf defintions
* `addProtobufDefinitions` call in the Android network plugin

Pull Request resolved: https://github.com/facebook/flipper/pull/2080

Test Plan: ![screenshot](https://user-images.githubusercontent.com/745166/111652068-001a5e80-87c4-11eb-8c94-e19b46dd074c.png)

Reviewed By: mweststrate

Differential Revision: D27507451

Pulled By: passy

fbshipit-source-id: 586d891b74f2b17d28fe7a2a99074da755851f38

android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java

@@ -127,6 +127,19 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw
     job.run();
   }
 
+  public void addProtobufDefinitions(
+      final String baseUrl, final FlipperArray callNestedMessagesPayloads) {
+    (new ErrorReportingRunnable(getConnection()) {
+          @Override
+          protected void runOrThrow() throws Exception {
+            send(
+                "addProtobufDefinitions",
+                new FlipperObject.Builder().put(baseUrl, callNestedMessagesPayloads).build());
+          }
+        })
+        .run();
+  }
+
   private String toBase64(@Nullable byte[] bytes) {
     if (bytes == null) {
       return null;

desktop/plugins/public/network/ProtobufDefinitionsRepository.tsx

@@ -0,0 +1,93 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ *
+ * @format
+ */
+
+import {ProtobufDefinition} from './types';
+import protobuf, {Type} from 'protobufjs';
+
+export class ProtobufDefinitionsRepository {
+  private static instance: ProtobufDefinitionsRepository;
+  private rawDefinitions: {[path: string]: ProtobufDefinition} = {};
+  private cachedDecodedDefinitions: {
+    [path: string]: DecodedProtobufDefinition;
+  } = {};
+
+  private constructor() {}
+
+  public static getInstance(): ProtobufDefinitionsRepository {
+    if (!ProtobufDefinitionsRepository.instance) {
+      ProtobufDefinitionsRepository.instance = new ProtobufDefinitionsRepository();
+    }
+    return ProtobufDefinitionsRepository.instance;
+  }
+
+  public addDefinitions(baseUrl: string, definitions: ProtobufDefinition[]) {
+    for (const d of definitions) {
+      if (!baseUrl.endsWith('/') && d.path.substr(0, 1) != '/') {
+        this.rawDefinitions[this.key(d.method, baseUrl + '/' + d.path)] = d;
+      } else {
+        this.rawDefinitions[this.key(d.method, baseUrl + d.path)] = d;
+      }
+    }
+  }
+
+  public getResponseType(method: string, path: string): Type | undefined {
+    const key = this.key(method, path);
+    this.generateRoots(key);
+    const messageFullName = this.rawDefinitions[key]?.responseMessageFullName;
+    if (messageFullName) {
+      return this.cachedDecodedDefinitions[key]?.responseRoot?.lookupType(
+        messageFullName,
+      );
+    } else {
+      return undefined;
+    }
+  }
+
+  public getRequestType(method: string, path: string): Type | undefined {
+    const key = this.key(method, path);
+    this.generateRoots(key);
+    const messageFullName = this.rawDefinitions[key]?.requestMessageFullName;
+    if (messageFullName) {
+      return this.cachedDecodedDefinitions[key]?.requestRoot?.lookupType(
+        messageFullName,
+      );
+    } else {
+      return undefined;
+    }
+  }
+
+  private generateRoots(key: string) {
+    if (key in this.cachedDecodedDefinitions) {
+      return;
+    }
+    const rawDefinition = this.rawDefinitions[key];
+    if (rawDefinition === undefined) return;
+
+    let responseRoot = undefined;
+    if (rawDefinition.responseDefinitions) {
+      responseRoot = protobuf.Root.fromJSON(rawDefinition.responseDefinitions);
+    }
+
+    let requestRoot = undefined;
+    if (rawDefinition.requestDefinitions) {
+      requestRoot = protobuf.Root.fromJSON(rawDefinition.requestDefinitions);
+    }
+
+    this.cachedDecodedDefinitions[key] = {responseRoot, requestRoot};
+  }
+
+  private key(method: string, path: string): string {
+    return method + '::' + path.split('?')[0];
+  }
+}
+
+type DecodedProtobufDefinition = {
+  responseRoot: protobuf.Root | undefined;
+  requestRoot: protobuf.Root | undefined;
+};

desktop/plugins/public/network/RequestDetails.tsx

@@ -27,6 +27,8 @@ import React from 'react';
 
 import querystring from 'querystring';
 import xmlBeautifier from 'xml-beautifier';
+import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository';
+import {Base64} from 'js-base64';
 
 const WrappingText = styled(Text)({
   wordWrap: 'break-word',
@@ -804,6 +806,75 @@ class BinaryFormatter {
   }
 }
 
+class ProtobufFormatter {
+  private protobufDefinitionRepository = ProtobufDefinitionsRepository.getInstance();
+
+  formatRequest(request: Request) {
+    if (
+      getHeaderValue(request.headers, 'content-type') ===
+      'application/x-protobuf'
+    ) {
+      const protobufDefinition = this.protobufDefinitionRepository.getRequestType(
+        request.method,
+        request.url,
+      );
+      if (protobufDefinition == undefined) {
+        return (
+          <Text>
+            Could not locate protobuf definition for request body of{' '}
+            {request.url}
+          </Text>
+        );
+      }
+
+      if (request?.data) {
+        const data = protobufDefinition.decode(
+          Base64.toUint8Array(request.data),
+        );
+        return <JSONText>{data.toJSON()}</JSONText>;
+      } else {
+        return (
+          <Text>Could not locate request body data for {request.url}</Text>
+        );
+      }
+    }
+    return undefined;
+  }
+
+  formatResponse(request: Request, response: Response) {
+    if (
+      getHeaderValue(response.headers, 'content-type') ===
+        'application/x-protobuf' ||
+      request.url.endsWith('.proto')
+    ) {
+      const protobufDefinition = this.protobufDefinitionRepository.getResponseType(
+        request.method,
+        request.url,
+      );
+      if (protobufDefinition == undefined) {
+        return (
+          <Text>
+            Could not locate protobuf definition for response body of{' '}
+            {request.url}
+          </Text>
+        );
+      }
+
+      if (response?.data) {
+        const data = protobufDefinition.decode(
+          Base64.toUint8Array(response.data),
+        );
+        return <JSONText>{data.toJSON()}</JSONText>;
+      } else {
+        return (
+          <Text>Could not locate response body data for {request.url}</Text>
+        );
+      }
+    }
+    return undefined;
+  }
+}
+
 const BodyFormatters: Array<BodyFormatter> = [
   new ImageFormatter(),
   new VideoFormatter(),
@@ -813,6 +884,7 @@ const BodyFormatters: Array<BodyFormatter> = [
   new JSONFormatter(),
   new FormUrlencodedFormatter(),
   new XMLTextFormatter(),
+  new ProtobufFormatter(),
   new BinaryFormatter(),
 ];
 

desktop/plugins/public/network/index.tsx

@@ -36,8 +36,10 @@ import {
   ResponseFollowupChunk,
   Header,
   MockRoute,
+  AddProtobufEvent,
   PartialResponses,
 } from './types';
+import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository';
 import {convertRequestToCurlCommand, getHeaderValue, decodeBody} from './utils';
 import RequestDetails from './RequestDetails';
 import {clipboard} from 'electron';
@@ -68,6 +70,7 @@ type Events = {
   newRequest: Request;
   newResponse: Response;
   partialResponse: Response | ResponseFollowupChunk;
+  addProtobufDefinitions: AddProtobufEvent;
 };
 
 type Methods = {
@@ -228,6 +231,13 @@ export function plugin(client: PluginClient<Events, Methods>) {
     });
   });
 
+  client.onMessage('addProtobufDefinitions', (data) => {
+    const repository = ProtobufDefinitionsRepository.getInstance();
+    for (const [baseUrl, definitions] of Object.entries(data)) {
+      repository.addDefinitions(baseUrl, definitions);
+    }
+  });
+
   client.onMessage('partialResponse', (data) => {
     /* Some clients (such as low end Android devices) struggle to serialise large payloads in one go, so partial responses allow them
         to split payloads into chunks and serialise each individually.
@@ -240,7 +250,7 @@ export function plugin(client: PluginClient<Events, Methods>) {
         The remaining chunks will be sent in ResponseFollowupChunks, which each contain another piece of the payload, along with their index from 1 onwards.
         The payload of each chunk is individually encoded in the same way that full responses are.
 
-        The order that initialResponse, and followup chunks are recieved is not guaranteed to be in index order.
+        The order that initialResponse, and followup chunks are received is not guaranteed to be in index order.
     */
     const message: Response | ResponseFollowupChunk = data as
       | Response

desktop/plugins/public/network/package.json

@@ -19,6 +19,7 @@
   "dependencies": {
     "lodash": "^4.17.21",
     "pako": "^2.0.3",
+    "protobufjs": "^6.10.2",
     "xml-beautifier": "^0.4.0"
   },
   "peerDependencies": {

desktop/plugins/public/network/types.tsx

@@ -7,6 +7,8 @@
  * @format
  */
 
+import {AnyNestedObject} from 'protobufjs';
+
 export type RequestId = string;
 
 export type Request = {
@@ -31,6 +33,17 @@ export type Response = {
   index?: number;
 };
 
+export type ProtobufDefinition = {
+  path: string;
+  method: string;
+  requestMessageFullName: string | null | undefined;
+  requestDefinitions: {[k: string]: AnyNestedObject} | null | undefined;
+  responseMessageFullName: string | null | undefined;
+  responseDefinitions: {[k: string]: AnyNestedObject} | null | undefined;
+};
+
+export type AddProtobufEvent = {[baseUrl: string]: ProtobufDefinition[]};
+
 export type ResponseFollowupChunk = {
   id: string;
   totalChunks: number;

desktop/plugins/public/yarn.lock

@@ -56,6 +56,59 @@
     "@types/yargs" "^15.0.0"
     chalk "^4.0.0"
 
+"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf"
+  integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78=
+
+"@protobufjs/base64@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735"
+  integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==
+
+"@protobufjs/codegen@^2.0.4":
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb"
+  integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==
+
+"@protobufjs/eventemitter@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70"
+  integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A=
+
+"@protobufjs/fetch@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45"
+  integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU=
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.1"
+    "@protobufjs/inquire" "^1.1.0"
+
+"@protobufjs/float@^1.0.2":
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1"
+  integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E=
+
+"@protobufjs/inquire@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089"
+  integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik=
+
+"@protobufjs/path@^1.1.2":
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d"
+  integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0=
+
+"@protobufjs/pool@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54"
+  integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q=
+
+"@protobufjs/utf8@^1.1.0":
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570"
+  integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA=
+
 "@testing-library/dom@^7.28.1":
   version "7.29.4"
   resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-7.29.4.tgz#1647c2b478789621ead7a50614ad81ab5ae5b86c"
@@ -146,11 +199,21 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
+"@types/long@^4.0.1":
+  version "4.0.1"
+  resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9"
+  integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==
+
 "@types/node@*":
   version "14.14.31"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.31.tgz#72286bd33d137aa0d152d47ec7c1762563d34055"
   integrity sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==
 
+"@types/node@^13.7.0":
+  version "13.13.48"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.48.tgz#46a3df718aed5217277f2395a682e055a487e341"
+  integrity sha512-z8wvSsgWQzkr4sVuMEEOvwMdOQjiRY2Y/ZW4fDfjfe3+TfQrZqFKOthBgk2RnVEmtOKrkwdZ7uTvsxTBLjKGDQ==
+
 "@types/pako@^1.0.1":
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/@types/pako/-/pako-1.0.1.tgz#33b237f3c9aff44d0f82fe63acffa4a365ef4a61"
@@ -964,6 +1027,11 @@ lodash@^4.17.21:
   resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
   integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
 
+long@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28"
+  integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==
+
 loose-envify@^1.1.0, loose-envify@^1.4.0:
   version "1.4.0"
   resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf"
@@ -1176,6 +1244,25 @@ prop-types@^15.6.2, prop-types@^15.7.2:
     object-assign "^4.1.1"
     react-is "^16.8.1"
 
+protobufjs@^6.10.2:
+  version "6.10.2"
+  resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.10.2.tgz#b9cb6bd8ec8f87514592ba3fdfd28e93f33a469b"
+  integrity sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ==
+  dependencies:
+    "@protobufjs/aspromise" "^1.1.2"
+    "@protobufjs/base64" "^1.1.2"
+    "@protobufjs/codegen" "^2.0.4"
+    "@protobufjs/eventemitter" "^1.1.0"
+    "@protobufjs/fetch" "^1.1.0"
+    "@protobufjs/float" "^1.0.2"
+    "@protobufjs/inquire" "^1.1.0"
+    "@protobufjs/path" "^1.1.2"
+    "@protobufjs/pool" "^1.1.0"
+    "@protobufjs/utf8" "^1.1.0"
+    "@types/long" "^4.0.1"
+    "@types/node" "^13.7.0"
+    long "^4.0.0"
+
 raf@^3.4.0:
   version "3.4.1"
   resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"