From 9efcbdceaf7ec6d699172facd56231ed5b46186e Mon Sep 17 00:00:00 2001 From: John Knox Date: Mon, 10 Aug 2020 08:46:06 -0700 Subject: [PATCH] Send large responses in chunks Summary: It's common for responses to be completely missing in the network inspector. This is because they are larger than can be serialized in one go on some devices, so we drop all messages larger than 1MB. This changes the android client to send large responses in individually serialized batches. This way we avoid running out of memory and can still send arbitrarily large payloads. Changelog: Android network inspector can now handle responses large than 1MB. Reviewed By: passy Differential Revision: D22999905 fbshipit-source-id: ff4eb8fa72a7e42ea90d12ffe0f20c6d1e58b7e5 --- .../plugins/network/NetworkFlipperPlugin.java | 54 +++++-- .../plugins/network/NetworkReporter.java | 3 +- .../plugins/network/__tests__/chunks.node.tsx | 127 ++++++++++++++++ desktop/plugins/network/chunks.tsx | 28 ++++ desktop/plugins/network/index.tsx | 141 ++++++++++++++++-- desktop/plugins/network/types.tsx | 20 +++ yarn.lock | 4 + 7 files changed, 348 insertions(+), 29 deletions(-) create mode 100644 desktop/plugins/network/__tests__/chunks.node.tsx create mode 100644 desktop/plugins/network/chunks.tsx create mode 100644 yarn.lock diff --git a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java index 5ede7e4fb..6b78c9a2b 100644 --- a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java +++ b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java @@ -12,7 +12,9 @@ import com.facebook.flipper.core.ErrorReportingRunnable; import com.facebook.flipper.core.FlipperArray; import com.facebook.flipper.core.FlipperObject; import com.facebook.flipper.plugins.common.BufferingFlipperPlugin; +import java.util.Arrays; import java.util.List; +import javax.annotation.Nullable; public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements NetworkReporter { public static final String ID = "Network"; @@ -64,18 +66,42 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw responseInfo.body = null; } - final FlipperObject response = - new FlipperObject.Builder() - .put("id", responseInfo.requestId) - .put("timestamp", responseInfo.timeStamp) - .put("status", responseInfo.statusCode) - .put("reason", responseInfo.statusReason) - .put("headers", toFlipperObject(responseInfo.headers)) - .put("isMock", responseInfo.isMock) - .put("data", toBase64(responseInfo.body)) - .build(); + int numChunks = + responseInfo.body == null + ? 1 + : (int) Math.ceil((double) responseInfo.body.length / MAX_BODY_SIZE_IN_BYTES); - send("newResponse", response); + for (int i = 0; i < numChunks; i++) { + byte[] chunk = + responseInfo.body == null + ? null + : Arrays.copyOfRange( + responseInfo.body, + i * MAX_BODY_SIZE_IN_BYTES, + Math.min((i + 1) * MAX_BODY_SIZE_IN_BYTES, responseInfo.body.length)); + final FlipperObject response = + i == 0 + ? new FlipperObject.Builder() + .put("id", responseInfo.requestId) + .put("timestamp", responseInfo.timeStamp) + .put("status", responseInfo.statusCode) + .put("reason", responseInfo.statusReason) + .put("headers", toFlipperObject(responseInfo.headers)) + .put("isMock", responseInfo.isMock) + .put("data", toBase64(chunk)) + .put("totalChunks", numChunks) + .put("index", i) + .build() + : new FlipperObject.Builder() + .put("id", responseInfo.requestId) + .put("timestamp", responseInfo.timeStamp) + .put("totalChunks", numChunks) + .put("index", i) + .put("data", toBase64(chunk)) + .build(); + + send(numChunks == 1 ? "newResponse" : "partialResponse", response); + } } }; @@ -99,7 +125,7 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw job.run(); } - private String toBase64(byte[] bytes) { + private String toBase64(@Nullable byte[] bytes) { if (bytes == null) { return null; } @@ -122,10 +148,6 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw return false; } - if (responseInfo.body != null && responseInfo.body.length > MAX_BODY_SIZE_IN_BYTES) { - return true; - } - return contentType.value.contains("image/") || contentType.value.contains("video/") || contentType.value.contains("application/zip"); diff --git a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkReporter.java b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkReporter.java index 5218d24ca..785c527eb 100644 --- a/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkReporter.java +++ b/android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkReporter.java @@ -9,6 +9,7 @@ package com.facebook.flipper.plugins.network; import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; public interface NetworkReporter { void reportRequest(RequestInfo requestInfo); @@ -54,7 +55,7 @@ public interface NetworkReporter { public int statusCode; public String statusReason; public List
headers = new ArrayList<>(); - public byte[] body; + public @Nullable byte[] body; public boolean isMock = false; public Header getFirstHeader(final String name) { diff --git a/desktop/plugins/network/__tests__/chunks.node.tsx b/desktop/plugins/network/__tests__/chunks.node.tsx new file mode 100644 index 000000000..3d46a4fb1 --- /dev/null +++ b/desktop/plugins/network/__tests__/chunks.node.tsx @@ -0,0 +1,127 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {combineBase64Chunks} from '../chunks'; +import network from '../index'; +import {PersistedState} from '../types'; + +test('Test assembling base64 chunks', () => { + const message = 'wassup john?'; + const chunks = message.match(/.{1,2}/g)?.map(btoa); + + if (chunks === undefined) { + throw new Error('invalid chunks'); + } + + const output = combineBase64Chunks(chunks); + expect(output).toBe('wassup john?'); +}); + +test('Reducer correctly adds initial chunk', () => { + const state: PersistedState = { + requests: {}, + responses: {}, + partialResponses: {}, + }; + const result = network.persistedStateReducer(state, 'partialResponse', { + id: '1', + timestamp: 123, + status: 200, + data: 'hello', + reason: 'nothing', + headers: [], + isMock: false, + insights: null, + index: 0, + totalChunks: 2, + }); + expect(result.partialResponses['1']).toMatchInlineSnapshot(` + Object { + "followupChunks": Object {}, + "initialResponse": Object { + "data": "hello", + "headers": Array [], + "id": "1", + "index": 0, + "insights": null, + "isMock": false, + "reason": "nothing", + "status": 200, + "timestamp": 123, + "totalChunks": 2, + }, + } + `); +}); + +test('Reducer correctly adds followup chunk', () => { + const state: PersistedState = { + requests: {}, + responses: {}, + partialResponses: {}, + }; + const result = network.persistedStateReducer(state, 'partialResponse', { + id: '1', + totalChunks: 2, + index: 1, + data: 'hello', + }); + expect(result.partialResponses['1']).toMatchInlineSnapshot(` + Object { + "followupChunks": Object { + "1": "hello", + }, + } + `); +}); + +test('Reducer correctly combines initial response and followup chunk', () => { + const state: PersistedState = { + requests: {}, + responses: {}, + partialResponses: { + '1': { + followupChunks: {}, + initialResponse: { + data: 'aGVs', + headers: [], + id: '1', + insights: null, + isMock: false, + reason: 'nothing', + status: 200, + timestamp: 123, + index: 0, + totalChunks: 2, + }, + }, + }, + }; + const result = network.persistedStateReducer(state, 'partialResponse', { + id: '1', + totalChunks: 2, + index: 1, + data: 'bG8=', + }); + expect(result.partialResponses).toEqual({}); + expect(result.responses['1']).toMatchInlineSnapshot(` + Object { + "data": "aGVsbG8=", + "headers": Array [], + "id": "1", + "index": 0, + "insights": null, + "isMock": false, + "reason": "nothing", + "status": 200, + "timestamp": 123, + "totalChunks": 2, + } + `); +}); diff --git a/desktop/plugins/network/chunks.tsx b/desktop/plugins/network/chunks.tsx new file mode 100644 index 000000000..cda54e2b6 --- /dev/null +++ b/desktop/plugins/network/chunks.tsx @@ -0,0 +1,28 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + * @format + */ + +import {TextDecoder} from 'util'; + +export function combineBase64Chunks(chunks: string[]): string { + const byteArray = chunks.map( + (b64Chunk) => + Uint8Array.from(atob(b64Chunk), (c) => c.charCodeAt(0)).buffer, + ); + const size = byteArray + .map((b) => b.byteLength) + .reduce((prev, curr) => prev + curr, 0); + const buffer = new Uint8Array(size); + let offset = 0; + for (let i = 0; i < byteArray.length; i++) { + buffer.set(new Uint8Array(byteArray[i]), offset); + offset += byteArray[i].byteLength; + } + const data = new TextDecoder('utf-8').decode(buffer); + return data; +} diff --git a/desktop/plugins/network/index.tsx b/desktop/plugins/network/index.tsx index 871d13736..d575ca5d9 100644 --- a/desktop/plugins/network/index.tsx +++ b/desktop/plugins/network/index.tsx @@ -30,21 +30,24 @@ import { TableBodyRow, produce, } from 'flipper'; -import {Request, RequestId, Response, Route} from './types'; +import { + Request, + RequestId, + Response, + Route, + ResponseFollowupChunk, + PersistedState, +} from './types'; import {convertRequestToCurlCommand, getHeaderValue, decodeBody} from './utils'; import RequestDetails from './RequestDetails'; import {clipboard} from 'electron'; import {URL} from 'url'; import {DefaultKeyboardAction} from 'app/src/MenuBar'; import {MockResponseDialog} from './MockResponseDialog'; +import {combineBase64Chunks} from './chunks'; const LOCALSTORAGE_MOCK_ROUTE_LIST_KEY = '__NETWORK_CACHED_MOCK_ROUTE_LIST'; -type PersistedState = { - requests: {[id: string]: Request}; - responses: {[id: string]: Response}; -}; - type State = { selectedIds: Array; searchTerm: string; @@ -126,9 +129,10 @@ export const NetworkRouteContext = createContext( export default class extends FlipperPlugin { static keyboardActions: Array = ['clear']; static subscribed = []; - static defaultPersistedState = { + static defaultPersistedState: PersistedState = { requests: {}, responses: {}, + partialResponses: {}, }; networkRouteManager: NetworkRouteManager = nullNetworkRouteManager; @@ -146,7 +150,7 @@ export default class extends FlipperPlugin { static persistedStateReducer( persistedState: PersistedState, method: string, - data: Request | Response, + data: Request | Response | ResponseFollowupChunk, ) { switch (method) { case 'newRequest': @@ -154,17 +158,130 @@ export default class extends FlipperPlugin { requests: {...persistedState.requests, [data.id]: data as Request}, }); case 'newResponse': + const response: Response = data as Response; return Object.assign({}, persistedState, { - responses: {...persistedState.responses, [data.id]: data as Response}, + responses: { + ...persistedState.responses, + [response.id]: response, + }, }); + case 'partialResponse': + /* Some clients (such as low end Android devices) struggle to serialise large payloads in one go, so partial responses allow them + to split payloads into chunks and serialise each individually. + + Such responses will be distinguished between normal responses by both: + * Being sent to the partialResponse method. + * Having a totalChunks value > 1. + + The first chunk will always be included in the initial response. This response must have index 0. + The remaining chunks will be sent in ResponseFollowupChunks, which each contain another piece of the payload, along with their index from 1 onwards. + The payload of each chunk is individually encoded in the same way that full responses are. + + The order that initialResponse, and followup chunks are recieved is not guaranteed to be in index order. + */ + const message: Response | ResponseFollowupChunk = data as + | Response + | ResponseFollowupChunk; + if (message.index !== undefined && message.index > 0) { + // It's a follow up chunk + const followupChunk: ResponseFollowupChunk = message as ResponseFollowupChunk; + const partialResponseEntry = persistedState.partialResponses[ + followupChunk.id + ] ?? {followupChunks: []}; + const newPartialResponseEntry = { + ...partialResponseEntry, + followupChunks: { + ...partialResponseEntry.followupChunks, + [followupChunk.index]: followupChunk.data, + }, + }; + const newPersistedState = { + ...persistedState, + partialResponses: { + ...persistedState.partialResponses, + [followupChunk.id]: newPartialResponseEntry, + }, + }; + return this.assembleChunksIfResponseIsComplete( + newPersistedState, + followupChunk.id, + ); + } + // It's an initial chunk + const partialResponse: Response = message as Response; + const partialResponseEntry = persistedState.partialResponses[ + partialResponse.id + ] ?? { + followupChunks: {}, + }; + const newPartialResponseEntry = { + ...partialResponseEntry, + initialResponse: partialResponse, + }; + const newPersistedState = { + ...persistedState, + partialResponses: { + ...persistedState.partialResponses, + [partialResponse.id]: newPartialResponseEntry, + }, + }; + return this.assembleChunksIfResponseIsComplete( + newPersistedState, + partialResponse.id, + ); default: return persistedState; } } - static serializePersistedState = (persistedState: PersistedState) => { - return Promise.resolve(JSON.stringify(persistedState)); - }; + static assembleChunksIfResponseIsComplete( + persistedState: PersistedState, + responseId: string, + ): PersistedState { + const partialResponseEntry = persistedState.partialResponses[responseId]; + const numChunks = partialResponseEntry.initialResponse?.totalChunks; + if ( + !partialResponseEntry.initialResponse || + !numChunks || + Object.keys(partialResponseEntry.followupChunks).length + 1 < numChunks + ) { + // Partial response not yet complete, do nothing. + return persistedState; + } + // Partial response has all required chunks, convert it to a full Response. + + const response: Response = partialResponseEntry.initialResponse; + const allChunks: string[] = + response.data != null + ? [ + response.data, + ...Object.entries(partialResponseEntry.followupChunks) + // It's important to parseInt here or it sorts lexicographically + .sort((a, b) => parseInt(a[0], 10) - parseInt(b[0], 10)) + .map(([_k, v]: [string, string]) => v), + ] + : []; + const data = combineBase64Chunks(allChunks); + + const newResponse = { + ...response, + // Currently data is always decoded at render time, so re-encode it to match the single response format. + data: btoa(data), + }; + + return { + ...persistedState, + responses: { + ...persistedState.responses, + [newResponse.id]: newResponse, + }, + partialResponses: Object.fromEntries( + Object.entries(persistedState.partialResponses).filter( + ([k, _v]: [string, unknown]) => k !== newResponse.id, + ), + ), + }; + } static deserializePersistedState = (serializedString: string) => { return JSON.parse(serializedString); diff --git a/desktop/plugins/network/types.tsx b/desktop/plugins/network/types.tsx index 836af26eb..588ddbff6 100644 --- a/desktop/plugins/network/types.tsx +++ b/desktop/plugins/network/types.tsx @@ -27,6 +27,15 @@ export type Response = { data: string | null | undefined; isMock: boolean; insights: Insights | null | undefined; + totalChunks?: number; + index?: number; +}; + +export type ResponseFollowupChunk = { + id: string; + totalChunks: number; + index: number; + data: string; }; export type Header = { @@ -62,3 +71,14 @@ export type Route = { responseHeaders: {[id: string]: Header}; responseStatus: string; }; + +export type PersistedState = { + requests: {[id: string]: Request}; + responses: {[id: string]: Response}; + partialResponses: { + [id: string]: { + initialResponse?: Response; + followupChunks: {[id: number]: string}; + }; + }; +}; diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 000000000..fb57ccd13 --- /dev/null +++ b/yarn.lock @@ -0,0 +1,4 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + +