From 4d262c0da4bad2cd8f475599775491a373734ef4 Mon Sep 17 00:00:00 2001 From: Harold Martin Date: Tue, 20 Apr 2021 05:08:42 -0700 Subject: [PATCH] Add protobuf support to network inspector (#2080) Summary: Protobuf based APIs are becoming more common (i.e. gRPC) but are difficult to inspect. Unlike plain text data formats (JSON), Protobuf calls transmit binary data requiring the format to be known ahead of time, making ad-hoc inspection impossible. This PR allows for those format definitions (messages in protobuf terminology) to be transmitted from the client to the network inspector plugin. These definitions are then imported into ProtobufJS which enables the binary data transmitted to be inspected as easily as JSON data. See Retrofit PR in https://github.com/facebook/flipper/pull/2084 ## Changelog * Add ProtobufJS library to network plugin * New `ProtobufFormatter` UI in `RequestDetails` * `ProtobufDefinitionsRepository` to cache and load protobuf defintions * `addProtobufDefinitions` call in the Android network plugin Pull Request resolved: https://github.com/facebook/flipper/pull/2080 Test Plan: ![screenshot](https://user-images.githubusercontent.com/745166/111652068-001a5e80-87c4-11eb-8c94-e19b46dd074c.png) Reviewed By: mweststrate Differential Revision: D27507451 Pulled By: passy fbshipit-source-id: 586d891b74f2b17d28fe7a2a99074da755851f38 --- .../plugins/network/NetworkFlipperPlugin.java | 13 +++ .../network/ProtobufDefinitionsRepository.tsx | 93 +++++++++++++++++++ .../plugins/public/network/RequestDetails.tsx | 72 ++++++++++++++ desktop/plugins/public/network/index.tsx | 12 ++- desktop/plugins/public/network/package.json | 1 + desktop/plugins/public/network/types.tsx | 13 +++ desktop/plugins/public/yarn.lock | 87 +++++++++++++++++ 7 files changed, 290 insertions(+), 1 deletion(-) create mode 100644 desktop/plugins/public/network/ProtobufDefinitionsRepository.tsx diff --git a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java index 35f1db1aa..f2a381800 100644 --- a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java +++ b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java @@ -127,6 +127,19 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw job.run(); } + public void addProtobufDefinitions( + final String baseUrl, final FlipperArray callNestedMessagesPayloads) { + (new ErrorReportingRunnable(getConnection()) { + @Override + protected void runOrThrow() throws Exception { + send( + "addProtobufDefinitions", + new FlipperObject.Builder().put(baseUrl, callNestedMessagesPayloads).build()); + } + }) + .run(); + } + private String toBase64(@Nullable byte[] bytes) { if (bytes == null) { return null; diff --git a/desktop/plugins/public/network/ProtobufDefinitionsRepository.tsx b/desktop/plugins/public/network/ProtobufDefinitionsRepository.tsx new file mode 100644 index 000000000..94c76318a --- /dev/null +++ b/desktop/plugins/public/network/ProtobufDefinitionsRepository.tsx @@ -0,0 +1,93 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {ProtobufDefinition} from './types'; +import protobuf, {Type} from 'protobufjs'; + +export class ProtobufDefinitionsRepository { + private static instance: ProtobufDefinitionsRepository; + private rawDefinitions: {[path: string]: ProtobufDefinition} = {}; + private cachedDecodedDefinitions: { + [path: string]: DecodedProtobufDefinition; + } = {}; + + private constructor() {} + + public static getInstance(): ProtobufDefinitionsRepository { + if (!ProtobufDefinitionsRepository.instance) { + ProtobufDefinitionsRepository.instance = new ProtobufDefinitionsRepository(); + } + return ProtobufDefinitionsRepository.instance; + } + + public addDefinitions(baseUrl: string, definitions: ProtobufDefinition[]) { + for (const d of definitions) { + if (!baseUrl.endsWith('/') && d.path.substr(0, 1) != '/') { + this.rawDefinitions[this.key(d.method, baseUrl + '/' + d.path)] = d; + } else { + this.rawDefinitions[this.key(d.method, baseUrl + d.path)] = d; + } + } + } + + public getResponseType(method: string, path: string): Type | undefined { + const key = this.key(method, path); + this.generateRoots(key); + const messageFullName = this.rawDefinitions[key]?.responseMessageFullName; + if (messageFullName) { + return this.cachedDecodedDefinitions[key]?.responseRoot?.lookupType( + messageFullName, + ); + } else { + return undefined; + } + } + + public getRequestType(method: string, path: string): Type | undefined { + const key = this.key(method, path); + this.generateRoots(key); + const messageFullName = this.rawDefinitions[key]?.requestMessageFullName; + if (messageFullName) { + return this.cachedDecodedDefinitions[key]?.requestRoot?.lookupType( + messageFullName, + ); + } else { + return undefined; + } + } + + private generateRoots(key: string) { + if (key in this.cachedDecodedDefinitions) { + return; + } + const rawDefinition = this.rawDefinitions[key]; + if (rawDefinition === undefined) return; + + let responseRoot = undefined; + if (rawDefinition.responseDefinitions) { + responseRoot = protobuf.Root.fromJSON(rawDefinition.responseDefinitions); + } + + let requestRoot = undefined; + if (rawDefinition.requestDefinitions) { + requestRoot = protobuf.Root.fromJSON(rawDefinition.requestDefinitions); + } + + this.cachedDecodedDefinitions[key] = {responseRoot, requestRoot}; + } + + private key(method: string, path: string): string { + return method + '::' + path.split('?')[0]; + } +} + +type DecodedProtobufDefinition = { + responseRoot: protobuf.Root | undefined; + requestRoot: protobuf.Root | undefined; +}; diff --git a/desktop/plugins/public/network/RequestDetails.tsx b/desktop/plugins/public/network/RequestDetails.tsx index d43113869..7807ae55f 100644 --- a/desktop/plugins/public/network/RequestDetails.tsx +++ b/desktop/plugins/public/network/RequestDetails.tsx @@ -27,6 +27,8 @@ import React from 'react'; import querystring from 'querystring'; import xmlBeautifier from 'xml-beautifier'; +import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository'; +import {Base64} from 'js-base64'; const WrappingText = styled(Text)({ wordWrap: 'break-word', @@ -804,6 +806,75 @@ class BinaryFormatter { } } +class ProtobufFormatter { + private protobufDefinitionRepository = ProtobufDefinitionsRepository.getInstance(); + + formatRequest(request: Request) { + if ( + getHeaderValue(request.headers, 'content-type') === + 'application/x-protobuf' + ) { + const protobufDefinition = this.protobufDefinitionRepository.getRequestType( + request.method, + request.url, + ); + if (protobufDefinition == undefined) { + return ( + + Could not locate protobuf definition for request body of{' '} + {request.url} + + ); + } + + if (request?.data) { + const data = protobufDefinition.decode( + Base64.toUint8Array(request.data), + ); + return {data.toJSON()}; + } else { + return ( + Could not locate request body data for {request.url} + ); + } + } + return undefined; + } + + formatResponse(request: Request, response: Response) { + if ( + getHeaderValue(response.headers, 'content-type') === + 'application/x-protobuf' || + request.url.endsWith('.proto') + ) { + const protobufDefinition = this.protobufDefinitionRepository.getResponseType( + request.method, + request.url, + ); + if (protobufDefinition == undefined) { + return ( + + Could not locate protobuf definition for response body of{' '} + {request.url} + + ); + } + + if (response?.data) { + const data = protobufDefinition.decode( + Base64.toUint8Array(response.data), + ); + return {data.toJSON()}; + } else { + return ( + Could not locate response body data for {request.url} + ); + } + } + return undefined; + } +} + const BodyFormatters: Array = [ new ImageFormatter(), new VideoFormatter(), @@ -813,6 +884,7 @@ const BodyFormatters: Array = [ new JSONFormatter(), new FormUrlencodedFormatter(), new XMLTextFormatter(), + new ProtobufFormatter(), new BinaryFormatter(), ]; diff --git a/desktop/plugins/public/network/index.tsx b/desktop/plugins/public/network/index.tsx index 58b25e9bb..f32363a2c 100644 --- a/desktop/plugins/public/network/index.tsx +++ b/desktop/plugins/public/network/index.tsx @@ -36,8 +36,10 @@ import { ResponseFollowupChunk, Header, MockRoute, + AddProtobufEvent, PartialResponses, } from './types'; +import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository'; import {convertRequestToCurlCommand, getHeaderValue, decodeBody} from './utils'; import RequestDetails from './RequestDetails'; import {clipboard} from 'electron'; @@ -68,6 +70,7 @@ type Events = { newRequest: Request; newResponse: Response; partialResponse: Response | ResponseFollowupChunk; + addProtobufDefinitions: AddProtobufEvent; }; type Methods = { @@ -228,6 +231,13 @@ export function plugin(client: PluginClient) { }); }); + client.onMessage('addProtobufDefinitions', (data) => { + const repository = ProtobufDefinitionsRepository.getInstance(); + for (const [baseUrl, definitions] of Object.entries(data)) { + repository.addDefinitions(baseUrl, definitions); + } + }); + client.onMessage('partialResponse', (data) => { /* Some clients (such as low end Android devices) struggle to serialise large payloads in one go, so partial responses allow them to split payloads into chunks and serialise each individually. @@ -240,7 +250,7 @@ export function plugin(client: PluginClient) { The remaining chunks will be sent in ResponseFollowupChunks, which each contain another piece of the payload, along with their index from 1 onwards. The payload of each chunk is individually encoded in the same way that full responses are. - The order that initialResponse, and followup chunks are recieved is not guaranteed to be in index order. + The order that initialResponse, and followup chunks are received is not guaranteed to be in index order. */ const message: Response | ResponseFollowupChunk = data as | Response diff --git a/desktop/plugins/public/network/package.json b/desktop/plugins/public/network/package.json index e35eb5b5e..4e0c437f3 100644 --- a/desktop/plugins/public/network/package.json +++ b/desktop/plugins/public/network/package.json @@ -19,6 +19,7 @@ "dependencies": { "lodash": "^4.17.21", "pako": "^2.0.3", + "protobufjs": "^6.10.2", "xml-beautifier": "^0.4.0" }, "peerDependencies": { diff --git a/desktop/plugins/public/network/types.tsx b/desktop/plugins/public/network/types.tsx index d3fda6a53..4a5ed3710 100644 --- a/desktop/plugins/public/network/types.tsx +++ b/desktop/plugins/public/network/types.tsx @@ -7,6 +7,8 @@ * @format */ +import {AnyNestedObject} from 'protobufjs'; + export type RequestId = string; export type Request = { @@ -31,6 +33,17 @@ export type Response = { index?: number; }; +export type ProtobufDefinition = { + path: string; + method: string; + requestMessageFullName: string | null | undefined; + requestDefinitions: {[k: string]: AnyNestedObject} | null | undefined; + responseMessageFullName: string | null | undefined; + responseDefinitions: {[k: string]: AnyNestedObject} | null | undefined; +}; + +export type AddProtobufEvent = {[baseUrl: string]: ProtobufDefinition[]}; + export type ResponseFollowupChunk = { id: string; totalChunks: number; diff --git a/desktop/plugins/public/yarn.lock b/desktop/plugins/public/yarn.lock index 4b952bdd6..04ecce5b0 100644 --- a/desktop/plugins/public/yarn.lock +++ b/desktop/plugins/public/yarn.lock @@ -56,6 +56,59 @@ "@types/yargs" "^15.0.0" chalk "^4.0.0" +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha1-m4sMxmPWaafY9vXQiToU00jzD78= + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha1-NVy8mLr61ZePntCV85diHx0Ga3A= + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha1-upn7WYYUr2VwDBYZ/wbUVLDYTEU= + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha1-Xp4avctz/Ap8uLKR33jIy9l7h9E= + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha1-/yAOPnzyQp4tyvwRQIKOjMY48Ik= + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha1-bMKyDFya1q0NzP0hynZz2Nf79o0= + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha1-Cf0V8tbTq/qbZbw2ZQbWrXhG/1Q= + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha1-p3c2C1s5oaLlEG+OhY8v0tBgxXA= + "@testing-library/dom@^7.28.1": version "7.29.4" resolved "https://registry.yarnpkg.com/@testing-library/dom/-/dom-7.29.4.tgz#1647c2b478789621ead7a50614ad81ab5ae5b86c" @@ -146,11 +199,21 @@ dependencies: "@types/istanbul-lib-report" "*" +"@types/long@^4.0.1": + version "4.0.1" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.1.tgz#459c65fa1867dafe6a8f322c4c51695663cc55e9" + integrity sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w== + "@types/node@*": version "14.14.31" resolved "https://registry.yarnpkg.com/@types/node/-/node-14.14.31.tgz#72286bd33d137aa0d152d47ec7c1762563d34055" integrity sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g== +"@types/node@^13.7.0": + version "13.13.48" + resolved "https://registry.yarnpkg.com/@types/node/-/node-13.13.48.tgz#46a3df718aed5217277f2395a682e055a487e341" + integrity sha512-z8wvSsgWQzkr4sVuMEEOvwMdOQjiRY2Y/ZW4fDfjfe3+TfQrZqFKOthBgk2RnVEmtOKrkwdZ7uTvsxTBLjKGDQ== + "@types/pako@^1.0.1": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/pako/-/pako-1.0.1.tgz#33b237f3c9aff44d0f82fe63acffa4a365ef4a61" @@ -964,6 +1027,11 @@ lodash@^4.17.21: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -1176,6 +1244,25 @@ prop-types@^15.6.2, prop-types@^15.7.2: object-assign "^4.1.1" react-is "^16.8.1" +protobufjs@^6.10.2: + version "6.10.2" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.10.2.tgz#b9cb6bd8ec8f87514592ba3fdfd28e93f33a469b" + integrity sha512-27yj+04uF6ya9l+qfpH187aqEzfCF4+Uit0I9ZBQVqK09hk/SQzKa2MUqUpXaVa7LOFRg1TSSr3lVxGOk6c0SQ== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" "^13.7.0" + long "^4.0.0" + raf@^3.4.0: version "3.4.1" resolved "https://registry.yarnpkg.com/raf/-/raf-3.4.1.tgz#0742e99a4a6552f445d73e3ee0328af0ff1ede39"