jens/flipper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Send large responses in chunks

Browse Source 
Summary:
It's common for responses to be completely missing in the network inspector. This is because they are larger than can be serialized in one go on some devices, so we drop all messages larger than 1MB.

This changes the android client to send large responses in individually serialized batches. This way we avoid running out of memory and can still send arbitrarily large payloads.

Changelog: Android network inspector can now handle responses large than 1MB.

Reviewed By: passy

Differential Revision: D22999905

fbshipit-source-id: ff4eb8fa72a7e42ea90d12ffe0f20c6d1e58b7e5
This commit is contained in:
John Knox
2020-08-10 08:46:06 -07:00
committed by Facebook GitHub Bot
parent 0065ddedd7
commit 9efcbdceaf
7 changed files with 348 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkFlipperPlugin.java
View File

@@ -12,7 +12,9 @@ import com.facebook.flipper.core.ErrorReportingRunnable;
import com.facebook.flipper.core.FlipperArray;
import com.facebook.flipper.core.FlipperObject;
import com.facebook.flipper.plugins.common.BufferingFlipperPlugin;
import java.util.Arrays;
import java.util.List;
import javax.annotation.Nullable;
public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements NetworkReporter {
public static final String ID = "Network";
@@ -64,18 +66,42 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw
responseInfo.body = null;
}
int numChunks =
responseInfo.body == null
? 1
: (int) Math.ceil((double) responseInfo.body.length / MAX_BODY_SIZE_IN_BYTES);
for (int i = 0; i < numChunks; i++) {
byte[] chunk =
responseInfo.body == null
? null
: Arrays.copyOfRange(
responseInfo.body,
i * MAX_BODY_SIZE_IN_BYTES,
Math.min((i + 1) * MAX_BODY_SIZE_IN_BYTES, responseInfo.body.length));
final FlipperObject response =
new FlipperObject.Builder()
i == 0
? new FlipperObject.Builder()
.put("id", responseInfo.requestId)
.put("timestamp", responseInfo.timeStamp)
.put("status", responseInfo.statusCode)
.put("reason", responseInfo.statusReason)
.put("headers", toFlipperObject(responseInfo.headers))
.put("isMock", responseInfo.isMock)
.put("data", toBase64(responseInfo.body))
.put("data", toBase64(chunk))
.put("totalChunks", numChunks)
.put("index", i)
.build()
: new FlipperObject.Builder()
.put("id", responseInfo.requestId)
.put("timestamp", responseInfo.timeStamp)
.put("totalChunks", numChunks)
.put("index", i)
.put("data", toBase64(chunk))
.build();
send("newResponse", response);
send(numChunks == 1 ? "newResponse" : "partialResponse", response);
}
}
};
@@ -99,7 +125,7 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw
job.run();
}
private String toBase64(byte[] bytes) {
private String toBase64(@Nullable byte[] bytes) {
if (bytes == null) {
return null;
}
@@ -122,10 +148,6 @@ public class NetworkFlipperPlugin extends BufferingFlipperPlugin implements Netw
return false;
}
if (responseInfo.body != null && responseInfo.body.length > MAX_BODY_SIZE_IN_BYTES) {
return true;
}
return contentType.value.contains("image/")
|| contentType.value.contains("video/")
|| contentType.value.contains("application/zip");

3
android/plugins/network/src/main/java/com/facebook/flipper/plugins/network/NetworkReporter.java
View File

@@ -9,6 +9,7 @@ package com.facebook.flipper.plugins.network;
import java.util.ArrayList;
import java.util.List;
import javax.annotation.Nullable;
public interface NetworkReporter {
void reportRequest(RequestInfo requestInfo);
@@ -54,7 +55,7 @@ public interface NetworkReporter {
public int statusCode;
public String statusReason;
public List<Header> headers = new ArrayList<>();
public byte[] body;
public @Nullable byte[] body;
public boolean isMock = false;
public Header getFirstHeader(final String name) {

127
desktop/plugins/network/__tests__/chunks.node.tsx Normal file
View File

@@ -0,0 +1,127 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {combineBase64Chunks} from '../chunks';
import network from '../index';
import {PersistedState} from '../types';
test('Test assembling base64 chunks', () => {
const message = 'wassup john?';
const chunks = message.match(/.{1,2}/g)?.map(btoa);
if (chunks === undefined) {
throw new Error('invalid chunks');
}
const output = combineBase64Chunks(chunks);
expect(output).toBe('wassup john?');
});
test('Reducer correctly adds initial chunk', () => {
const state: PersistedState = {
requests: {},
responses: {},
partialResponses: {},
};
const result = network.persistedStateReducer(state, 'partialResponse', {
id: '1',
timestamp: 123,
status: 200,
data: 'hello',
reason: 'nothing',
headers: [],
isMock: false,
insights: null,
index: 0,
totalChunks: 2,
});
expect(result.partialResponses['1']).toMatchInlineSnapshot(`
Object {
"followupChunks": Object {},
"initialResponse": Object {
"data": "hello",
"headers": Array [],
"id": "1",
"index": 0,
"insights": null,
"isMock": false,
"reason": "nothing",
"status": 200,
"timestamp": 123,
"totalChunks": 2,
},
}
`);
});
test('Reducer correctly adds followup chunk', () => {
const state: PersistedState = {
requests: {},
responses: {},
partialResponses: {},
};
const result = network.persistedStateReducer(state, 'partialResponse', {
id: '1',
totalChunks: 2,
index: 1,
data: 'hello',
});
expect(result.partialResponses['1']).toMatchInlineSnapshot(`
Object {
"followupChunks": Object {
"1": "hello",
},
}
`);
});
test('Reducer correctly combines initial response and followup chunk', () => {
const state: PersistedState = {
requests: {},
responses: {},
partialResponses: {
'1': {
followupChunks: {},
initialResponse: {
data: 'aGVs',
headers: [],
id: '1',
insights: null,
isMock: false,
reason: 'nothing',
status: 200,
timestamp: 123,
index: 0,
totalChunks: 2,
},
},
},
};
const result = network.persistedStateReducer(state, 'partialResponse', {
id: '1',
totalChunks: 2,
index: 1,
data: 'bG8=',
});
expect(result.partialResponses).toEqual({});
expect(result.responses['1']).toMatchInlineSnapshot(`
Object {
"data": "aGVsbG8=",
"headers": Array [],
"id": "1",
"index": 0,
"insights": null,
"isMock": false,
"reason": "nothing",
"status": 200,
"timestamp": 123,
"totalChunks": 2,
}
`);
});

28
desktop/plugins/network/chunks.tsx Normal file
View File

@@ -0,0 +1,28 @@
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
* This source code is licensed under the MIT license found in the
* LICENSE file in the root directory of this source tree.
*
* @format
*/
import {TextDecoder} from 'util';
export function combineBase64Chunks(chunks: string[]): string {
const byteArray = chunks.map(
(b64Chunk) =>
Uint8Array.from(atob(b64Chunk), (c) => c.charCodeAt(0)).buffer,
);
const size = byteArray
.map((b) => b.byteLength)
.reduce((prev, curr) => prev + curr, 0);
const buffer = new Uint8Array(size);
let offset = 0;
for (let i = 0; i < byteArray.length; i++) {
buffer.set(new Uint8Array(byteArray[i]), offset);
offset += byteArray[i].byteLength;
}
const data = new TextDecoder('utf-8').decode(buffer);
return data;
}

139
desktop/plugins/network/index.tsx
View File

@@ -30,21 +30,24 @@ import {
TableBodyRow,
produce,
} from 'flipper';
import {Request, RequestId, Response, Route} from './types';
import {
Request,
RequestId,
Response,
Route,
ResponseFollowupChunk,
PersistedState,
} from './types';
import {convertRequestToCurlCommand, getHeaderValue, decodeBody} from './utils';
import RequestDetails from './RequestDetails';
import {clipboard} from 'electron';
import {URL} from 'url';
import {DefaultKeyboardAction} from 'app/src/MenuBar';
import {MockResponseDialog} from './MockResponseDialog';
import {combineBase64Chunks} from './chunks';
const LOCALSTORAGE_MOCK_ROUTE_LIST_KEY = '__NETWORK_CACHED_MOCK_ROUTE_LIST';
type PersistedState = {
requests: {[id: string]: Request};
responses: {[id: string]: Response};
};
type State = {
selectedIds: Array<RequestId>;
searchTerm: string;
@@ -126,9 +129,10 @@ export const NetworkRouteContext = createContext<NetworkRouteManager>(
export default class extends FlipperPlugin<State, any, PersistedState> {
static keyboardActions: Array<DefaultKeyboardAction> = ['clear'];
static subscribed = [];
static defaultPersistedState = {
static defaultPersistedState: PersistedState = {
requests: {},
responses: {},
partialResponses: {},
};
networkRouteManager: NetworkRouteManager = nullNetworkRouteManager;
@@ -146,7 +150,7 @@ export default class extends FlipperPlugin<State, any, PersistedState> {
static persistedStateReducer(
persistedState: PersistedState,
method: string,
data: Request | Response,
data: Request | Response | ResponseFollowupChunk,
) {
switch (method) {
case 'newRequest':
@@ -154,18 +158,131 @@ export default class extends FlipperPlugin<State, any, PersistedState> {
requests: {...persistedState.requests, [data.id]: data as Request},
});
case 'newResponse':
const response: Response = data as Response;
return Object.assign({}, persistedState, {
responses: {...persistedState.responses, [data.id]: data as Response},
responses: {
...persistedState.responses,
[response.id]: response,
},
});
case 'partialResponse':
/* Some clients (such as low end Android devices) struggle to serialise large payloads in one go, so partial responses allow them
to split payloads into chunks and serialise each individually.
Such responses will be distinguished between normal responses by both:
* Being sent to the partialResponse method.
* Having a totalChunks value > 1.
The first chunk will always be included in the initial response. This response must have index 0.
The remaining chunks will be sent in ResponseFollowupChunks, which each contain another piece of the payload, along with their index from 1 onwards.
The payload of each chunk is individually encoded in the same way that full responses are.
The order that initialResponse, and followup chunks are recieved is not guaranteed to be in index order.
*/
const message: Response | ResponseFollowupChunk = data as
| Response
| ResponseFollowupChunk;
if (message.index !== undefined && message.index > 0) {
// It's a follow up chunk
const followupChunk: ResponseFollowupChunk = message as ResponseFollowupChunk;
const partialResponseEntry = persistedState.partialResponses[
followupChunk.id
] ?? {followupChunks: []};
const newPartialResponseEntry = {
...partialResponseEntry,
followupChunks: {
...partialResponseEntry.followupChunks,
[followupChunk.index]: followupChunk.data,
},
};
const newPersistedState = {
...persistedState,
partialResponses: {
...persistedState.partialResponses,
[followupChunk.id]: newPartialResponseEntry,
},
};
return this.assembleChunksIfResponseIsComplete(
newPersistedState,
followupChunk.id,
);
}
// It's an initial chunk
const partialResponse: Response = message as Response;
const partialResponseEntry = persistedState.partialResponses[
partialResponse.id
] ?? {
followupChunks: {},
};
const newPartialResponseEntry = {
...partialResponseEntry,
initialResponse: partialResponse,
};
const newPersistedState = {
...persistedState,
partialResponses: {
...persistedState.partialResponses,
[partialResponse.id]: newPartialResponseEntry,
},
};
return this.assembleChunksIfResponseIsComplete(
newPersistedState,
partialResponse.id,
);
default:
return persistedState;
}
}
static serializePersistedState = (persistedState: PersistedState) => {
return Promise.resolve(JSON.stringify(persistedState));
static assembleChunksIfResponseIsComplete(
persistedState: PersistedState,
responseId: string,
): PersistedState {
const partialResponseEntry = persistedState.partialResponses[responseId];
const numChunks = partialResponseEntry.initialResponse?.totalChunks;
if (
!partialResponseEntry.initialResponse ||
!numChunks ||
Object.keys(partialResponseEntry.followupChunks).length + 1 < numChunks
) {
// Partial response not yet complete, do nothing.
return persistedState;
}
// Partial response has all required chunks, convert it to a full Response.
const response: Response = partialResponseEntry.initialResponse;
const allChunks: string[] =
response.data != null
? [
response.data,
...Object.entries(partialResponseEntry.followupChunks)
// It's important to parseInt here or it sorts lexicographically
.sort((a, b) => parseInt(a[0], 10) - parseInt(b[0], 10))
.map(([_k, v]: [string, string]) => v),
]
: [];
const data = combineBase64Chunks(allChunks);
const newResponse = {
...response,
// Currently data is always decoded at render time, so re-encode it to match the single response format.
data: btoa(data),
};
return {
...persistedState,
responses: {
...persistedState.responses,
[newResponse.id]: newResponse,
},
partialResponses: Object.fromEntries(
Object.entries(persistedState.partialResponses).filter(
([k, _v]: [string, unknown]) => k !== newResponse.id,
),
),
};
}
static deserializePersistedState = (serializedString: string) => {
return JSON.parse(serializedString);
};

20
desktop/plugins/network/types.tsx
View File

@@ -27,6 +27,15 @@ export type Response = {
data: string | null | undefined;
isMock: boolean;
insights: Insights | null | undefined;
totalChunks?: number;
index?: number;
};
export type ResponseFollowupChunk = {
id: string;
totalChunks: number;
index: number;
data: string;
};
export type Header = {
@@ -62,3 +71,14 @@ export type Route = {
responseHeaders: {[id: string]: Header};
responseStatus: string;
};
export type PersistedState = {
requests: {[id: string]: Request};
responses: {[id: string]: Response};
partialResponses: {
[id: string]: {
initialResponse?: Response;
followupChunks: {[id: number]: string};
};
};
};

4
yarn.lock Normal file
View File

@@ -0,0 +1,4 @@
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
# yarn lockfile v1